diff options
Diffstat (limited to 'schneider-openembedded/builders.sh')
| -rwxr-xr-x | schneider-openembedded/builders.sh | 157 |
1 files changed, 109 insertions, 48 deletions
diff --git a/schneider-openembedded/builders.sh b/schneider-openembedded/builders.sh index aeb399d307..ff174923f0 100755 --- a/schneider-openembedded/builders.sh +++ b/schneider-openembedded/builders.sh @@ -33,7 +33,7 @@ if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then sleep 15 sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true fi -pkg_list="python-pip android-tools-fsutils chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-requests texinfo vim-tiny whiptail" +pkg_list="python-pip android-tools-fsutils chrpath cpio diffstat gawk gfortran libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-requests texinfo vim-tiny whiptail" if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then echo "INFO: apt install error - try again in a moment" sleep 15 @@ -41,7 +41,8 @@ if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; th fi # Install ruamel.yaml (version pinned for Python-2.7 compat) -pip install --user --force-reinstall 'ruamel.yaml<0.17' +pip install --user 'ruamel.yaml.clib==0.2.2' +pip install --user 'ruamel.yaml<0.17' set -ex @@ -62,7 +63,7 @@ if [ ! -e ".repo/manifest.xml" ]; then if [[ "${IMAGES}" == *clean* ]]; then rm -rf ${sstatecache} fi - mkdir -p ${HOME}/srv/oe/downloads ${sstatecache} + mkdir -p ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} ${sstatecache} #DEL mkdir -p build #DEL ln -s ${HOME}/srv/oe/downloads #DEL ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} sstate-cache @@ -127,6 +128,9 @@ case "${MACHINE}" in *soca9*) MACHINE=snarc-soca9 ;; + *am64x*) + MACHINE=am64xx-evm + ;; esac # SUBMODULES is set to: @@ -138,21 +142,28 @@ if [[ ${MANIFEST_BRANCH} == linaro-* ]]; then if [[ "${SUBMODULES}" != "none" ]]; then ./setup-environment -s build-${machine_orig}/ + + # Temporary 2021/12/07 go back to pinned version of tensorflow + # due to upstream switch to cmake, conflicts with our local fix + # for segfaults on LCES2 due to incorrect VFP mode + git submodule update sources/meta-tensorflow-lite + + # Also avoid updating meta-ti layer for two reasons: + # 1) want to use older tagged version to avoid SPL boot failure + # 2) latest version is not compatible with dunfell branch + git submodule update sources/meta-ti fi fi source ./setup-environment build-${machine_orig}/ -ln -s ${HOME}/srv/oe/downloads +ln -s ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} downloads ln -s ${sstatecache} sstate-cache -# Add job BUILD_NUMBER to output files names +# Add job BUILD_NUMBER to output files names, overriding the +# default suffix of "-${DATETIME}" from bitbake.conf cat << EOF >> conf/auto.conf -IMAGE_NAME_append = "-${BUILD_NUMBER}" -KERNEL_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -MODULE_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -DT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -BOOT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +IMAGE_VERSION_SUFFIX = "-${BUILD_NUMBER}" EOF # get build stats to make sure that we use sstate properly @@ -187,6 +198,7 @@ postfile=$(mktemp /tmp/postfile.XXXXX.conf) echo KERNEL_VERSION_PATCHLEVEL = \"${KERNEL_VERSION_PATCHLEVEL}\" > ${postfile} echo PREFERRED_VERSION_linux-rzn1 = \"${KERNEL_VERSION_PATCHLEVEL}.%\" >> ${postfile} echo PREFERRED_VERSION_linux-socfpga = \"${KERNEL_VERSION_PATCHLEVEL}.%\" >> ${postfile} +echo PREFERRED_VERSION_linux-ti-staging = \"${KERNEL_VERSION_PATCHLEVEL}.%\" >> ${postfile} cat ${postfile} bbopt="-R ${postfile}" @@ -195,6 +207,15 @@ if [ "${clean_packages}" != "" ]; then bitbake ${bbopt} ${build_packages} fi +# Cleanup mbedtls/edgeagent repos, the gitsm fetcher gets confused easily +#rm -rf ${HOME}/srv/oe/downloads/git2/*mbedtls* +#rm -rf ${HOME}/srv/oe/downloads/git2/*optiga* +#rm -rf ${HOME}/srv/oe/downloads/git2/*EdgeAgent* +#rm -rf ${HOME}/srv/oe/downloads/git2/*Azure* +#rm -rf ${HOME}/srv/oe/downloads/git2/*Microsoft* +#rm -rf ${HOME}/srv/oe/downloads/git2/*kgabis.parson* +#bitbake ${bbopt} -c cleansstate mbedtls edgeagent + # Build all ${IMAGES} dipimg="prod-image" devimg="dev-image" @@ -208,7 +229,13 @@ if [[ "${IMAGES}" == *"${dipimg}"* ]]; then grep -c ^processor /proc/cpuinfo grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}' - time bitbake ${bbopt} ${dipimg} + time bitbake ${bbopt} ${dipimg} || true + ls -ld tmp/*/*/edgeagent downloads/git2/*EdgeAgent* downloads/git2/*cmocka* downloads/git2/*Azure* downloads/git2/*Microsoft* downloads/git2/*kgabis* || true + + # Make a copy of the CVE report using a fixed filename, because: + # 1) later invocations of bitbake may overwrite the report, and + # 2) facilitate later retrieval from snapshots.linaro.org via "latest" link. + cp ${DEPLOY_DIR_IMAGE}/${dipimg}-${MACHINE}.cve ${DEPLOY_DIR_IMAGE}/${dipimg}-${MACHINE}.rootfs.cve case "${MACHINE}" in *rzn1*) @@ -216,9 +243,6 @@ if [[ "${IMAGES}" == *"${dipimg}"* ]]; then ;; esac - # Generate pn-buildlist containing names of recipes, for CVE check below - time bitbake ${bbopt} ${dipimg} -g - ls -al ${DEPLOY_DIR_IMAGE} || true ls -al ${DEPLOY_DIR_IMAGE}/optee || true ls -al ${DEPLOY_DIR_IMAGE}/cm3 || true @@ -231,7 +255,11 @@ fi if [[ "${IMAGES}" == *"${devimg}"* ]]; then replace_dmverity_var "" - time bitbake ${bbopt} ${devimg} + time bitbake ${bbopt} ${devimg} || true + ls -ld tmp/*/*/edgeagent downloads/git2/*EdgeAgent* downloads/git2/*cmocka* downloads/git2/*Azure* downloads/git2/*Microsoft* downloads/git2/*kgabis* || true + + # Make a copy of the CVE report using a fixed filename + cp ${DEPLOY_DIR_IMAGE}/${devimg}-${MACHINE}.cve ${DEPLOY_DIR_IMAGE}/${devimg}-${MACHINE}.rootfs.cve ls -al ${DEPLOY_DIR_IMAGE} || true ls -al ${DEPLOY_DIR_IMAGE}/cm3 || true @@ -240,6 +268,10 @@ if [[ "${IMAGES}" == *"${devimg}"* ]]; then ls -al ${DEPLOY_DIR_IMAGE}/optee || true time bitbake ${bbopt} ${sdkimg} + + # Make a copy of the CVE report using a fixed filename + cp ${DEPLOY_DIR_IMAGE}/${sdkimg}-${MACHINE}.cve ${DEPLOY_DIR_IMAGE}/${sdkimg}-${MACHINE}.rootfs.cve + DEPLOY_DIR_SDK=$(bitbake -e | grep "^DEPLOY_DIR="| cut -d'=' -f2 | tr -d '"')/sdk cp -aR ${DEPLOY_DIR_SDK} ${DEPLOY_DIR_IMAGE} fi @@ -252,16 +284,10 @@ find ${DEPLOY_DIR_IMAGE} -type l -delete ### Begin CVE check -# Combine CVE reports for the recipes used in prod-image task. -CVE_CHECK_DIR=$(bitbake -e | grep "^CVE_CHECK_DIR="| cut -d'=' -f2 | tr -d '"') -if [ -e pn-buildlist ] && [ -e "${CVE_CHECK_DIR}" ]; then - sort pn-buildlist | while read r ; do - cat ${CVE_CHECK_DIR}/${r} 2>/dev/null || true - done >cve-${MACHINE}.new +if [ -e ${DEPLOY_DIR_IMAGE}/${dipimg}-${MACHINE}.rootfs.cve ] ; then - # Generate CVE listing with a fixed filename, so it can be retrieved - # from snapshots.linaro.org by subsequent builds using a known URL. - cp cve-${MACHINE}.new ${DEPLOY_DIR_IMAGE}/prod-image-${MACHINE}.rootfs.cve + # Get the current CVE report + cp ${DEPLOY_DIR_IMAGE}/${dipimg}-${MACHINE}.rootfs.cve cve-${MACHINE}.new # Fetch previous CVE report LATEST_DEST=$(echo $PUB_DEST | sed -e "s#/$BUILD_NUMBER/#/latest/#") @@ -286,27 +312,30 @@ if [ -e pn-buildlist ] && [ -e "${CVE_CHECK_DIR}" ]; then EOF fi - if [ -e cve-${MACHINE}.old ]; then - # Do diffs between old and current CVE report. - wget -nv -O diff-cve https://git.linaro.org/ci/job/configs.git/plain/schneider-openembedded/diff-cve - gawk -f diff-cve cve-${MACHINE}.old cve-${MACHINE}.new | tee ${WORKSPACE}/cve-${MACHINE}.txt + # Do diffs between old and current CVE report. + wget -nv -O diff-cve https://git.linaro.org/ci/job/configs.git/plain/schneider-openembedded/diff-cve + gawk -f diff-cve cve-${MACHINE}.old cve-${MACHINE}.new | tee ${WORKSPACE}/cve-${MACHINE}.txt - # Same thing, but against arbitrary (but fixed) baseline - case "${MACHINE}" in - *rzn1*) - wget -nv -O cve-${MACHINE}.base https://releases.linaro.org/members/schneider/openembedded/2020.09-dunfell/rzn1d-5.4/dip-image-rzn1-snarc.rootfs.cve - ;; - *soca9*) - wget -nv -O cve-${MACHINE}.base https://releases.linaro.org/members/schneider/openembedded/2020.09-dunfell/soca9-5.4/dip-image-snarc-soca9.rootfs.cve - ;; - esac - gawk -f diff-cve cve-${MACHINE}.base cve-${MACHINE}.new > ${WORKSPACE}/base-cve-${MACHINE}.txt - else - echo "CVE check skipped because no previous build was found" - fi - ### End CVE check + # Same thing, but against arbitrary (but fixed) baseline + case "${MACHINE}" in + *rzn1*) + wget -nv -O cve-${MACHINE}.base https://releases.linaro.org/members/schneider/openembedded/2021.08.dunfell/rzn1d-5.10/prod-image-rzn1d400-bestla.rootfs.cve + ;; + *soca9*) + wget -nv -O cve-${MACHINE}.base https://releases.linaro.org/members/schneider/openembedded/2021.08.dunfell/soca9-5.10/prod-image-snarc-soca9.rootfs.cve + ;; + *am64x*) + # TODO: update this URL for baseline on am64x. + #wget -nv -O cve-${MACHINE}.base https://releases.linaro.org/members/schneider/openembedded/2021.08.dunfell/soca9-5.10/prod-image-snarc-soca9.rootfs.cve + # For now just copy the current report as the baseline + cp cve-${MACHINE}.new cve-${MACHINE}.base + ;; + esac + gawk -f diff-cve cve-${MACHINE}.base cve-${MACHINE}.new > ${WORKSPACE}/base-cve-${MACHINE}.txt fi +### End CVE check + # FIXME: IMAGE_FSTYPES_remove doesn't work rm -f ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4 \ ${DEPLOY_DIR_IMAGE}/*.rootfs.iso \ @@ -332,6 +361,11 @@ case "${MACHINE}" in rm -f *socfpga_cyclone5_socdk*.dtb || true popd ;; + *am64x*) + pushd ${DEPLOY_DIR_IMAGE} + # TODO delete unnecessary files + popd + ;; juno|stih410-b2260|orangepi-i96) ;; *) @@ -404,9 +438,9 @@ case "${MACHINE}" in WIC_BMAP=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-rzn1*-${BUILD_NUMBER}.rootfs.wic.bmap" | xargs -r basename) # The following images will have their size reported to SQUAD - UBOOT=$(find ${DEPLOY_DIR_IMAGE}/u-boot -type f -name "u-boot-${MACHINE}.bin.spkg") + UBOOT=$(find ${DEPLOY_DIR_IMAGE}/u-boot -type f -name "u-boot-${MACHINE}-${BUILD_NUMBER}.bin.spkg") UBOOT_IMG=$(basename ${UBOOT}) - UBOOT_FIT=$(find ${DEPLOY_DIR_IMAGE}/u-boot -type f -name "u-boot-${MACHINE}.itb") + UBOOT_FIT=$(find ${DEPLOY_DIR_IMAGE}/u-boot -type f -name "u-boot-${MACHINE}-${BUILD_NUMBER}.itb") UBOOT_FIT_IMG=$(basename ${UBOOT_FIT}) DTB=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*rzn1*bestla*.dtb") DTB_IMG=$(basename ${DTB}) @@ -414,9 +448,9 @@ case "${MACHINE}" in KERNEL_IMG=$(basename ${KERNEL}) KERNEL_FIT=$(find ${DEPLOY_DIR_IMAGE} -type f -name "fitImage*.itb") KERNEL_FIT_IMG=$(basename ${KERNEL_FIT}) - FSBL=$(find ${DEPLOY_DIR_IMAGE}/fsbl -type f -name "fsbl-fip-${MACHINE}.spkg") + FSBL=$(find ${DEPLOY_DIR_IMAGE}/fsbl -type f -name "fsbl-fip-${MACHINE}-${BUILD_NUMBER}.spkg") FSBL_IMG=$(basename ${FSBL}) - OPTEE_FIT=$(find ${DEPLOY_DIR_IMAGE}/optee -type f -name "optee-os-${MACHINE}.itb") + OPTEE_FIT=$(find ${DEPLOY_DIR_IMAGE}/optee -type f -name "optee-os-${MACHINE}-${BUILD_NUMBER}.itb") OPTEE_FIT_IMG=$(basename ${OPTEE_FIT}) UBI=$(find ${DEPLOY_DIR_IMAGE} -type f -name "prod-image-${MACHINE}-${BUILD_NUMBER}.rootfs.fitubi") UBI_IMG=$(basename ${UBI}) @@ -430,9 +464,9 @@ case "${MACHINE}" in WIC_BMAP=$(find ${DEPLOY_DIR_IMAGE} -type f -name "prod-image-snarc-soca9-${BUILD_NUMBER}.rootfs.wic.bmap" | xargs -r basename) # The following images will have their size reported to SQUAD - UBOOT=$(find ${DEPLOY_DIR_IMAGE} -type f -name "u-boot-with-spl.sfp") + UBOOT=$(find ${DEPLOY_DIR_IMAGE} -type f -name "u-boot-with-spl-${BUILD_NUMBER}.sfp") UBOOT_IMG=$(basename ${UBOOT}) - DTB=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*soca9*_qspi_micronN25Q_bestla_512m.dtb") + DTB=$(find ${DEPLOY_DIR_IMAGE} -type f -name "zImage-*soca9*.dtb") DTB_IMG=$(basename ${DTB}) KERNEL=$(find ${DEPLOY_DIR_IMAGE} -type f -name "zImage--*soca9*.bin") KERNEL_IMG=$(basename ${KERNEL}) @@ -442,6 +476,33 @@ case "${MACHINE}" in WIC_DEV_IMG=$(basename ${WIC_DEV}) WIC_DEV_BMAP=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-snarc-soca9-${BUILD_NUMBER}.rootfs.wic.bmap" | xargs -r basename) ;; + *am64x*) + ROOTFS_TAR_BZ2=$(find ${DEPLOY_DIR_IMAGE} -type f -name "prod-image-${MACHINE}-${BUILD_NUMBER}.rootfs.tar.bz2" | xargs -r basename) + ROOTFS_DEV_TAR_BZ2=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-${MACHINE}-${BUILD_NUMBER}.rootfs.tar.bz2" | xargs -r basename) + WIC_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-${MACHINE}-${BUILD_NUMBER}.rootfs.wic.bz2" | xargs -r basename) + WIC_BMAP=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-${MACHINE}-${BUILD_NUMBER}.rootfs.wic.bmap" | xargs -r basename) + + # The following images will have their size reported to SQUAD + UBOOT=$(find ${DEPLOY_DIR_IMAGE} -type f -name "u-boot-${MACHINE}-*.img") + UBOOT_IMG=$(basename ${UBOOT}) + #UBOOT_FIT=$(find ${DEPLOY_DIR_IMAGE}/u-boot -type f -name "u-boot-${MACHINE}-*.img") + #UBOOT_FIT_IMG=$(basename ${UBOOT_FIT}) + DTB=$(find ${DEPLOY_DIR_IMAGE} -type f -name "k3-am642-evm-*.dtb") + DTB_IMG=$(basename ${DTB}) + KERNEL=$(find ${DEPLOY_DIR_IMAGE} -type f -name "Image-*.bin") + KERNEL_IMG=$(basename ${KERNEL}) + #KERNEL_FIT=$(find ${DEPLOY_DIR_IMAGE} -type f -name "fitImage*.itb") + #KERNEL_FIT_IMG=$(basename ${KERNEL_FIT}) + #FSBL=$(find ${DEPLOY_DIR_IMAGE}/fsbl -type f -name "fsbl-fip-${MACHINE}-${BUILD_NUMBER}.spkg") + #FSBL_IMG=$(basename ${FSBL}) + OPTEE_FIT=$(find ${DEPLOY_DIR_IMAGE}/optee -type f -name "bl32.bin") + OPTEE_FIT_IMG=$(basename ${OPTEE_FIT}) + #UBI=$(find ${DEPLOY_DIR_IMAGE} -type f -name "prod-image-${MACHINE}-${BUILD_NUMBER}.rootfs.fitubi") + #UBI_IMG=$(basename ${UBI}) + WIC_DEV=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-${MACHINE}-${BUILD_NUMBER}.rootfs.wic.bz2") + WIC_DEV_IMG=$(basename ${WIC_DEV}) + WIC_DEV_BMAP=$(find ${DEPLOY_DIR_IMAGE} -type f -name "dev-image-${MACHINE}-${BUILD_NUMBER}.rootfs.wic.bmap" | xargs -r basename) + ;; *) DTB_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*-${MACHINE}-*-${BUILD_NUMBER}.dtb" | xargs -r basename) ;; |