path: root/rpb-openembedded/builders.sh

#!/bin/bash

set -ex

# install jflog client tool, v1, used for publishing artifacts
(mkdir -p $HOME/bin && cd $HOME/bin && curl -fL https://getcli.jfrog.io | sh)

# workaround EDK2 is confused by the long path used during the build
# and truncate files name expected by VfrCompile
sudo mkdir -p /srv/oe
sudo chown buildslave:buildslave /srv/oe
cd /srv/oe

trap cleanup_exit INT TERM EXIT

cleanup_exit()
{
    echo "Running cleanup_exit..."
}

# XXX: In Debian buster use newer tool img2simg because in certain cases ext2img corrupt rootfs images,
# this workaround is meanwhile all RPB builds are switch to use Debian buster.
if [ "$label" == "docker-buster-amd64" ]; then
  android_tools="img2simg"
  simg_cmd="img2simg"
else
  android_tools="android-tools-fsutils"
  simg_cmd="ext2simg -v"
fi

if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then
  echo "INFO: apt update error - try again in a moment"
  sleep 15
  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true
fi
pkg_list="python-pip python3-pip python3-setuptools $android_tools chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev lz4 python-requests texinfo vim-tiny whiptail pigz zstd"
if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then
  echo "INFO: apt install error - try again in a moment"
  sleep 15
  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}
fi

# Install ruamel.yaml==0.16.13
pip install --user ruamel.yaml.clib==0.2.2
pip install --user ruamel.yaml==0.16.13

set -ex

mkdir -p ${HOME}/bin
curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
chmod a+x ${HOME}/bin/repo
export PATH=${HOME}/bin:${PATH}

# initialize repo if not done already
if [ ! -e ".repo/manifest.xml" ]; then
   repo init -u ${MANIFEST_URL} -b ${MANIFEST_BRANCH_PREFIX}${MANIFEST_BRANCH}

   # When set skip persistent disk space, useful for debug
   if [ "${SKIP_PERSIST}" != "True" ]; then
       # link to shared downloads on persistent disk
       # our builds config is expecting downloads and sstate-cache, here.
       # DL_DIR = "${OEROOT}/sources/downloads"
       # SSTATE_DIR = "${OEROOT}/build/sstate-cache"
       mkdir -p ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH}
       mkdir -p build
       ln -s ${HOME}/srv/oe/downloads-${MANIFEST_BRANCH} downloads
       ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO}-${MANIFEST_BRANCH} sstate-cache
   fi
fi


if [ "${ghprbPullId}" ]; then
    echo "Applying Github pull-request #${ghprbPullId} from ${ghprbGhRepository}"
    sed -i -e "s|name=\"${ghprbGhRepository}\"|name=\"${ghprbGhRepository}\" revision=\"refs/pull/${ghprbPullId}/head\"|" .repo/manifests/default.xml
fi

repo sync
cp .repo/manifest.xml source-manifest.xml
repo manifest -r -o pinned-manifest.xml
MANIFEST_COMMIT=$(cd .repo/manifests && git rev-parse --short HEAD)

# XXX: Disable manifest diff because at this moment link latest is not working so not latest pinned-manifest is available
# record changes since last build, if available
#if wget -q ${PUBLISH_SERVER}${PUB_DEST/\/${BUILD_NUMBER}\//\/latest\/}/pinned-manifest.xml -O pinned-manifest-latest.xml; then
#    # https://github.com/96boards/oe-rpb-manifest/commit/0be354483a124903982103dc937f9e5c1a094a3a
#    if grep -q ".*linkfile.*\.\./\.\./\.repo/manifests/setup-environment" pinned-manifest-latest.xml ; then
#	echo "detected old style symlink with relative path, skipping diff report"
#    else
#	repo diffmanifests ${PWD}/pinned-manifest-latest.xml ${PWD}/pinned-manifest.xml > manifest-changes.txt
#    fi
#else
    echo "latest build published does not have pinned-manifest.xml, skipping diff report"
#fi

if [ -n "$GERRIT_PROJECT" ] && [ $GERRIT_EVENT_TYPE == "patchset-created" ]; then
    GERRIT_URL="http://${GERRIT_HOST}/${GERRIT_PROJECT}"
    cd `repo manifest | grep -ni $GERRIT_PROJECT\" | grep -Po 'path="\K[^"]*'`
    if git pull ${GERRIT_URL} ${GERRIT_REFSPEC} | grep -q "Automatic merge failed"; then
        git reset --hard
        echo "Error: *** Error patch merge failed"
        exit 1
    fi
    cd -
fi

# the setup-environment will create auto.conf and site.conf
# make sure we get rid of old config.
# let's remove the previous TMPDIR as well.
# we want to preserve build/buildhistory though.
rm -rf build/conf build/tmp-*glibc/

# Accept EULA if/when needed
export EULA_dragonboard410c=1
export EULA_stih410b2260=1
source setup-environment build

# Support both pre-honister and honister+ override syntax
if grep -q 'CONF_VERSION.*"1"' conf/local.conf ; then
    BB_OVERRIDE='_'
else
    BB_OVERRIDE=':'
fi

# Accept freescale EULA
cat << EOF >> conf/local.conf
ACCEPT_FSL_EULA = "1"
EOF

# Add job BUILD_NUMBER to output files names
cat << EOF >> conf/auto.conf
IMAGE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}"
KERNEL_IMAGE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}"
MODULE_TARBALL_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}"
DT_IMAGE_BASE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}"
BOOT_IMAGE_BASE_NAME${BB_OVERRIDE}append = "-${BUILD_NUMBER}"
EOF

# get build stats to make sure that we use sstate properly
cat << EOF >> conf/auto.conf
INHERIT += "buildstats buildstats-summary"
EOF

# Make sure we don't use rm_work in CI slaves since they are non persistent build nodes
cat << EOF >> conf/auto.conf
INHERIT${BB_OVERRIDE}remove = "rm_work"
EOF

# allow the top level job to append to auto.conf
if [ -f ${WORKSPACE}/auto.conf ]; then
    cat ${WORKSPACE}/auto.conf >> conf/auto.conf
fi

# add useful debug info
cat conf/{site,auto}.conf

[ "${DISTRO}" = "rpb" ] && IMAGES+=" ${IMAGES_RPB}"
[ "${DISTRO}" = "rpb-wayland" ] && IMAGES+=" ${IMAGES_RPB_WAYLAND}"

# These machines only build the basic rpb-console-image
case "${MACHINE}" in
  am57xx-evm|intel-core2-32|intel-corei7-64)
     IMAGES="rpb-console-image"
     ;;
esac

# allow top level job to run a preprocessing command
if [ -x ${WORKSPACE}/bitbake-pre ]; then
    ${WORKSPACE}/bitbake-pre
fi

time bitbake ${IMAGES}

# allow top level job to run a postprocessing command
if [ -x ${WORKSPACE}/bitbake-post ]; then
    ${WORKSPACE}/bitbake-post
fi

set +x
DEPLOY_DIR_IMAGE=$(bitbake -e | grep "^DEPLOY_DIR_IMAGE="| cut -d'=' -f2 | tr -d '"')
set -x

if [ "${BUILD_SDK}" ]; then
  time bitbake ${BUILD_SDK} -c populate_sdk
  set +x
  SDK_DEPLOY=$(bitbake -e ${BUILD_SDK} -c populate_sdk | grep "^SDK_DEPLOY="| cut -d'=' -f2 | tr -d '"')
  set -x

  cp $SDK_DEPLOY/* $DEPLOY_DIR_IMAGE/
fi

# Get Kernel repo, branch and revision
set +x
KERNEL_ENV=$(bitbake -e virtual/kernel)
KERNEL_REPO=$(echo "$KERNEL_ENV" | awk 'match($0, /^SRC_URI=(.*)/, a) {print a[1]}' | cut -d';' -f 1 | tr -d '"')
KERNEL_BRANCH=$(echo "$KERNEL_ENV" | grep "^SRC_URI=" | awk 'match($0, /branch=(.*)/, a) {print a[1]}' | tr -d '"')
KERNEL_COMMIT=$(echo "$KERNEL_ENV" | awk 'match($0, /^SRCREV=(.*)/, a) {print a[1]}' | tr -d '"')
set -x

# Prepare files to publish
rm -f ${DEPLOY_DIR_IMAGE}/*.txt
find ${DEPLOY_DIR_IMAGE} -type l -delete
mv /srv/oe/{source,pinned}-manifest.xml ${DEPLOY_DIR_IMAGE}
cat ${DEPLOY_DIR_IMAGE}/pinned-manifest.xml

# FIXME: IMAGE_FSTYPES_remove doesn't work
rm -f ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4 \
      ${DEPLOY_DIR_IMAGE}/*.rootfs.iso \
      ${DEPLOY_DIR_IMAGE}/*.rootfs.wic \
      ${DEPLOY_DIR_IMAGE}/*.iso \
      ${DEPLOY_DIR_IMAGE}/*.stimg

# FIXME: Sparse images here, until it gets done by OE
case "${MACHINE}" in
  juno|stih410-b2260|orangepi-i96)
    ;;
  *)
    for rootfs in $(find ${DEPLOY_DIR_IMAGE} -type f -name *.rootfs.ext4.gz); do
      pigz -d -k ${rootfs}
      sudo $simg_cmd ${rootfs%.gz} ${rootfs%.ext4.gz}.img
      rm -f ${rootfs%.gz}
      pigz -9 ${rootfs%.ext4.gz}.img
    done
    ;;
esac

# Create MD5SUMS file
find ${DEPLOY_DIR_IMAGE} -type f | xargs md5sum > MD5SUMS.txt
sed -i "s|${DEPLOY_DIR_IMAGE}/||" MD5SUMS.txt
mv MD5SUMS.txt ${DEPLOY_DIR_IMAGE}

# Build information
cat > ${DEPLOY_DIR_IMAGE}/HEADER.textile << EOF

h4. Reference Platform Build - CE OpenEmbedded

Build description:
* Build URL: "$BUILD_URL":$BUILD_URL
* Manifest URL: "${MANIFEST_URL}":${MANIFEST_URL}
* Manifest branch: ${MANIFEST_BRANCH_PREFIX}${MANIFEST_BRANCH}
* Manifest commit: "${MANIFEST_COMMIT}":${MANIFEST_URL/.git/\/commit}/${MANIFEST_COMMIT}
EOF

if [ -e "/srv/oe/manifest-changes.txt" ]; then
  # the space after pre.. tag is on purpose
  cat > ${DEPLOY_DIR_IMAGE}/README.textile << EOF

h4. Manifest changes

pre.. 
EOF
  cat /srv/oe/manifest-changes.txt >> ${DEPLOY_DIR_IMAGE}/README.textile
  mv /srv/oe/manifest-changes.txt ${DEPLOY_DIR_IMAGE}
fi

# Need different files for each machine
BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}*.img" | sort | head -n 1 | xargs -r basename)
ROOTFS_EXT4_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.ext4.gz" | xargs -r basename)
ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename)
ROOTFS_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.img.gz" | xargs -r basename)
ROOTFS_DESKTOP_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-desktop-image-test-${MACHINE}-*-${BUILD_NUMBER}.rootfs.img.gz" | xargs -r basename)
KERNEL_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*Image-*-${MACHINE}-*-${BUILD_NUMBER}.bin" | xargs -r basename)
DISK_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "${IMAGES}-${MACHINE}-*-${BUILD_NUMBER}.rootfs.wic.gz" | xargs -r basename)

case "${MACHINE}" in
  am57xx-evm)
    # LAVA image is too big for am57xx-evm
    ROOTFS_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-${MACHINE}-*-${BUILD_NUMBER}.rootfs.img.gz" | xargs -r basename)
    # FIXME: several dtb files case
    ;;
  intel-core2-32|intel-corei7-64)
    # No LAVA testing on intel-core* machines
    ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename)
    ;;
  juno)
    # FIXME: several dtb files case
    ;;
  *)
    DTB_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*-${MACHINE}-*-${BUILD_NUMBER}.dtb" | xargs -r basename)
    ;;
esac

# Note: the main job script allows to override the default value for
#       PUBLISH_SERVER and PUB_DEST, typically used for OE RPB builds
cat << EOF > ${WORKSPACE}/post_build_lava_parameters
DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE}
MANIFEST_COMMIT=${MANIFEST_COMMIT}
BOOT_URL=${PUBLISH_SERVER}${PUB_DEST}/${BOOT_IMG}
ROOTFS_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_EXT4_IMG}
ROOTFS_SPARSE_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_IMG}
ROOTFS_DESKTOP_SPARSE_BUILD_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_DESKTOP_IMG}
SYSTEM_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_EXT4_IMG}
KERNEL_URL=${PUBLISH_SERVER}${PUB_DEST}/${KERNEL_IMG}
DTB_URL=${PUBLISH_SERVER}${PUB_DEST}/${DTB_IMG}
NFSROOTFS_URL=${PUBLISH_SERVER}${PUB_DEST}/${ROOTFS_TARXZ_IMG}
RECOVERY_IMAGE_URL=${PUBLISH_SERVER}${PUB_DEST}/juno-oe-uboot.zip
LXC_BOOT_IMG=${BOOT_IMG}
LXC_ROOTFS_IMG=$(basename ${ROOTFS_IMG} .gz)
INITRD_URL="${INITRD_URL}"
KERNEL_ARGS=""
KERNEL_REPO=$KERNEL_REPO
KERNEL_BRANCH=$KERNEL_BRANCH
KERNEL_COMMIT=$KERNEL_COMMIT
EOF