diff options
Diffstat (limited to 'rpb-openembedded/builders.sh')
| -rwxr-xr-x | rpb-openembedded/builders.sh | 106 |
1 files changed, 76 insertions, 30 deletions
diff --git a/rpb-openembedded/builders.sh b/rpb-openembedded/builders.sh index e26cc634c3..bde6b0a6d3 100755 --- a/rpb-openembedded/builders.sh +++ b/rpb-openembedded/builders.sh @@ -1,6 +1,9 @@ #!/bin/bash -set -e +set -ex + +# install jflog client tool, v1, used for publishing artifacts +(mkdir -p $HOME/bin && cd $HOME/bin && curl -fL https://getcli.jfrog.io | sh) # workaround EDK2 is confused by the long path used during the build # and truncate files name expected by VfrCompile @@ -30,7 +33,7 @@ if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then sleep 15 sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true fi -pkg_list="python-pip $android_tools chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-requests texinfo vim-tiny whiptail pigz" +pkg_list="python-pip python3-pip python3-setuptools $android_tools chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev lz4 python-requests texinfo vim-tiny whiptail pigz zstd" if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then echo "INFO: apt install error - try again in a moment" sleep 15 @@ -38,7 +41,8 @@ if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; th fi # Install ruamel.yaml==0.16.13 -pip install --user --force-reinstall ruamel.yaml==0.16.13 +pip install --user ruamel.yaml.clib==0.2.2 +pip install --user ruamel.yaml==0.16.13 set -ex @@ -51,20 +55,23 @@ export PATH=${HOME}/bin:${PATH} if [ ! -e ".repo/manifest.xml" ]; then repo init -u ${MANIFEST_URL} -b ${MANIFEST_BRANCH_PREFIX}${MANIFEST_BRANCH} - # link to shared downloads on persistent disk - # our builds config is expecting downloads and sstate-cache, here. - # DL_DIR = "${OEROOT}/sources/downloads" - # SSTATE_DIR = "${OEROOT}/build/sstate-cache" - mkdir -p ${HOME}/srv/oe/downloads ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} - mkdir -p build - ln -s ${HOME}/srv/oe/downloads - ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} sstate-cache + # When set skip persistent disk space, useful for debug + if [ "${SKIP_PERSIST}" != "True" ]; then + # link to shared downloads on persistent disk + # our builds config is expecting downloads and sstate-cache, here. + # DL_DIR = "${OEROOT}/sources/downloads" + # SSTATE_DIR = "${OEROOT}/build/sstate-cache" + mkdir -p ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} + mkdir -p build + ln -s ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} downloads + ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} sstate-cache + fi fi if [ "${ghprbPullId}" ]; then echo "Applying Github pull-request #${ghprbPullId} from ${ghprbGhRepository}" - sed -i -e "s|name=\"${ghprbGhRepository}\"|name=\"${ghprbGhRepository}\" revision=\"refs/pull/${ghprbPullId}/head\"|" .repo/manifest.xml + sed -i -e "s|name=\"${ghprbGhRepository}\"|name=\"${ghprbGhRepository}\" revision=\"refs/pull/${ghprbPullId}/head\"|" .repo/manifests/default.xml fi repo sync @@ -74,7 +81,7 @@ MANIFEST_COMMIT=$(cd .repo/manifests && git rev-parse --short HEAD) # XXX: Disable manifest diff because at this moment link latest is not working so not latest pinned-manifest is available # record changes since last build, if available -#if wget -q ${BASE_URL}${PUB_DEST/\/${BUILD_NUMBER}\//\/latest\/}/pinned-manifest.xml -O pinned-manifest-latest.xml; then +#if wget -q ${PUBLISH_SERVER}${PUB_DEST/\/${BUILD_NUMBER}\//\/latest\/}/pinned-manifest.xml -O pinned-manifest-latest.xml; then # # https://github.com/96boards/oe-rpb-manifest/commit/0be354483a124903982103dc937f9e5c1a094a3a # if grep -q ".*linkfile.*\.\./\.\./\.repo/manifests/setup-environment" pinned-manifest-latest.xml ; then # echo "detected old style symlink with relative path, skipping diff report" @@ -107,6 +114,13 @@ export EULA_dragonboard410c=1 export EULA_stih410b2260=1 source setup-environment build +# Support both pre-honister and honister+ override syntax +if grep -q 'CONF_VERSION.*"1"' conf/local.conf ; then + BB_OVERRIDE='_' +else + BB_OVERRIDE=':' +fi + # Accept freescale EULA cat << EOF >> conf/local.conf ACCEPT_FSL_EULA = "1" @@ -114,11 +128,11 @@ EOF # Add job BUILD_NUMBER to output files names cat << EOF >> conf/auto.conf -IMAGE_NAME_append = "-${BUILD_NUMBER}" -KERNEL_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -MODULE_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -DT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" -BOOT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +IMAGE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}" +KERNEL_IMAGE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}" +MODULE_TARBALL_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}" +DT_IMAGE_BASE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}" +BOOT_IMAGE_BASE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}" EOF # get build stats to make sure that we use sstate properly @@ -128,7 +142,7 @@ EOF # Make sure we don't use rm_work in CI slaves since they are non persistent build nodes cat << EOF >> conf/auto.conf -INHERIT_remove = "rm_work" +INHERIT${BB_OVERRIDE}remove = "rm_work" EOF # allow the top level job to append to auto.conf @@ -149,9 +163,38 @@ case "${MACHINE}" in ;; esac +# allow top level job to run a preprocessing command +if [ -x ${WORKSPACE}/bitbake-pre ]; then + ${WORKSPACE}/bitbake-pre +fi + time bitbake ${IMAGES} +# allow top level job to run a postprocessing command +if [ -x ${WORKSPACE}/bitbake-post ]; then + ${WORKSPACE}/bitbake-post +fi + +set +x DEPLOY_DIR_IMAGE=$(bitbake -e | grep "^DEPLOY_DIR_IMAGE="| cut -d'=' -f2 | tr -d '"') +set -x + +if [ "${BUILD_SDK}" ]; then + time bitbake ${BUILD_SDK} -c populate_sdk + set +x + SDK_DEPLOY=$(bitbake -e ${BUILD_SDK} -c populate_sdk | grep "^SDK_DEPLOY="| cut -d'=' -f2 | tr -d '"') + set -x + + cp $SDK_DEPLOY/* $DEPLOY_DIR_IMAGE/ +fi + +# Get Kernel repo, branch and revision +set +x +KERNEL_ENV=$(bitbake -e virtual/kernel) +KERNEL_REPO=$(echo "$KERNEL_ENV" | awk 'match($0, /^SRC_URI=(.*)/, a) {print a[1]}' | cut -d';' -f 1 | tr -d '"') +KERNEL_BRANCH=$(echo "$KERNEL_ENV" | grep "^SRC_URI=" | awk 'match($0, /branch=(.*)/, a) {print a[1]}' | tr -d '"') +KERNEL_COMMIT=$(echo "$KERNEL_ENV" | awk 'match($0, /^SRCREV=(.*)/, a) {print a[1]}' | tr -d '"') +set -x # Prepare files to publish rm -f ${DEPLOY_DIR_IMAGE}/*.txt @@ -210,7 +253,7 @@ EOF fi # Need different files for each machine -BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}*.img" | sort | xargs -r basename) +BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}*.img" | sort | head -n 1 | xargs -r basename) ROOTFS_EXT4_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.ext4.gz" | xargs -r basename) ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename) ROOTFS_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.img.gz" | xargs -r basename) @@ -237,21 +280,24 @@ case "${MACHINE}" in esac # Note: the main job script allows to override the default value for -# BASE_URL and PUB_DEST, typically used for OE RPB builds +# PUBLISH_SERVER and PUB_DEST, typically used for OE RPB builds cat << EOF > ${WORKSPACE}/post_build_lava_parameters DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE} MANIFEST_COMMIT=${MANIFEST_COMMIT} -BOOT_URL=${BASE_URL}${PUB_DEST}/${BOOT_IMG} -ROOTFS_BUILD_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_EXT4_IMG} -ROOTFS_SPARSE_BUILD_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_IMG} -ROOTFS_DESKTOP_SPARSE_BUILD_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_DESKTOP_IMG} -SYSTEM_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_EXT4_IMG} -KERNEL_URL=${BASE_URL}${PUB_DEST}/${KERNEL_IMG} -DTB_URL=${BASE_URL}${PUB_DEST}/${DTB_IMG} -NFSROOTFS_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_TARXZ_IMG} -RECOVERY_IMAGE_URL=${BASE_URL}${PUB_DEST}/juno-oe-uboot.zip +BOOT_URL=${PUBLISH_SERVER}${PUB_DEST}/${BOOT_IMG} +ROOTFS_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_EXT4_IMG} +ROOTFS_SPARSE_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_IMG} +ROOTFS_DESKTOP_SPARSE_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_DESKTOP_IMG} +SYSTEM_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_EXT4_IMG} +KERNEL_URL=${PUBLISH_SERVER}${PUB_DEST}/${KERNEL_IMG} +DTB_URL=${PUBLISH_SERVER}${PUB_DEST}/${DTB_IMG} +NFSROOTFS_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_TARXZ_IMG} +RECOVERY_IMAGE_URL=${PUBLISH_SERVER}${PUB_DEST}/juno-oe-uboot.zip LXC_BOOT_IMG=${BOOT_IMG} LXC_ROOTFS_IMG=$(basename ${ROOTFS_IMG} .gz) INITRD_URL="${INITRD_URL}" KERNEL_ARGS="" +KERNEL_REPO=$KERNEL_REPO +KERNEL_BRANCH=$KERNEL_BRANCH +KERNEL_COMMIT=$KERNEL_COMMIT EOF |