96boards-hikey-aosp-master.yaml

- job:
    name: 96boards-hikey-aosp-master
    project-type: freestyle
    defaults: global
    logrotate:
        daysToKeep: 60
        numToKeep: 60
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            everyone-flat:
                - job-build
                - job-cancel
    parameters:
        - string:
            name: BUILD_CONFIG_FILENAME
            default: 'hikey-aosp-master'
        - string:
            name: QA_SERVER
            default: 'https://qa-reports.linaro.org'
        - string:
            name: QA_SERVER_PROJECT
            default: 'aosp-master-tracking'
        - string:
            name: ANDROID_VERSION
            default: 'AOSP MASTER'
        - string:
            name: KERNEL_DESCRIBE
            default: 'dummy'
        - string:
            name: SRCREV_kernel
            default: 'dummy'
        - string:
            name: TOOLCHAIN
            default: 'clang-r353983d'
    disabled: true
    node: docker-xenial-aosp
    display-name: '96boards - AOSP Master - HiKey'
    concurrent: true
    triggers:
        - timed : '@daily'
    wrappers:
        - timestamps
        - timeout:
            timeout: 800
        - credentials-binding:
            - text:
                credential-id: QA_REPORTS_TOKEN
                variable: QA_REPORTS_TOKEN
            - text:
                credential-id: ARTIFACTORIAL_TOKEN
                variable: ARTIFACTORIAL_TOKEN
            - text:
                credential-id: AP_SSID
                variable: AP_SSID
            - text:
                credential-id: AP_KEY
                variable: AP_KEY
    builders:
        - shell: |
            #!/bin/bash

            export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8

            java -version

            sudo apt-get update
            sudo apt-get install -y gcc-4.9-multilib bison git gperf libxml2-utils python-mako zip time python-requests genisoimage patch mtools python-pip pxz libssl-dev

            wget -q http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre-headless_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jdk_8u45-b14-1_amd64.deb
            sudo dpkg -i --force-all *.deb

            mkdir -p ${HOME}/bin ${WORKSPACE}/build/out
            curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
            chmod a+x ${HOME}/bin/*
            export PATH=${HOME}/bin:${PATH}

            if [ ! -d "aosp-master" ]; then
              sudo mkdir -p /home/buildslave/srv/aosp-master
              sudo chmod 777 /home/buildslave/srv/aosp-master
            fi
            cd /home/buildslave/srv/aosp-master

            git config --global user.email "ci_notify@linaro.org"
            git config --global user.name "Linaro CI"

            # Runs as ubuntu
            set -xe
            rm -rf .repo/manifests* .repo/local_manifests build-tools jenkins-tools build/out build/android-patchsets build-configs
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-android-build-tools.git build-tools
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-jenkins-tools.git jenkins-tools
            git clone --depth 1 http://android-git.linaro.org/git/android-build-configs.git build-configs

            export BUILD_CONFIG_FILENAME=${BUILD_CONFIG_FILENAME:-${JOB_NAME#android-*}}

            # Early test
            if [ ! -f build-configs/${BUILD_CONFIG_FILENAME} ]; then
              echo "No config file named ${BUILD_CONFIG_FILENAME} exists"
              echo "in android-build-configs.git"
              exit 1
            fi

            cat << EOF > config.txt
            BUILD_CONFIG_REPO=http://android-git.linaro.org/git/android-build-configs.git
            BUILD_CONFIG_BRANCH=master
            TARGET_KERNEL_USE=4.19
            EOF
            echo config.txt
            export CONFIG=`base64 -w 0 config.txt`

            mkdir -p build
            ANDROID_ROOT=/home/buildslave/srv/aosp-master/build

            # Delete sources before build to save space
            cd build/
            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro* clang-src/ hikey-linaro/configs hikey-linaro-4.19
            rm -fr optee/optee_os
            # delete this kernel repository as we have patches applied
            rm -fr hikey-linaro
            cd -

            # Build Android
            build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "${CONFIG}"
            cp -a /home/buildslave/srv/aosp-master/build/out/*.xml /home/buildslave/srv/aosp-master/build/out/*.json ${WORKSPACE}/

            cd build/out
            for image in "boot.img" "vendor.img" "boot_fat.uefi.img" "system.img" "userdata.img"  "cache.img"; do
              if [ -f ${image} ]; then
                echo "Compressing ${image}"
                xz ${image}
              fi
            done
            cd -
            pushd build/
            source build/envsetup.sh
            lunch hikey-userdebug
            pushd test/suite_harness
            git fetch http://android-review.linaro.org/platform/test/suite_harness refs/changes/23/19623/1 && git cherry-pick FETCH_HEAD 
            popd
            lunch aosp_arm64-userdebug
            make -j"$(nproc)" vts cts
            cp out/host/linux-x86/vts/android-vts.zip out/
            cp out/host/linux-x86/cts/android-cts.zip out/
            rm -rf out/host/

            popd

            rm -rf build/out/BUILD-INFO.txt
            wget https://git.linaro.org/ci/job/configs.git/blob_plain/HEAD:/android-lcr/hikey/build-info/aosp-master-template.txt -O build/out/BUILD-INFO.txt

            # Build latest boot.img with latest kernel from source for 4.4, 4.9, 4.14, 4.19 versions
            cd build/
            export PATH=${PWD}/prebuilts/clang/host/linux-x86/${TOOLCHAIN}/bin/:${PWD}/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/bin/:${PATH}

            export CLANG_TRIPLE=aarch64-linux-gnu-
            export CROSS_COMPILE=aarch64-linux-android-
            CMDLINE="console=ttyAMA3,115200 androidboot.console=ttyAMA3"
            CMDLINE="${CMDLINE} androidboot.hardware=hikey firmware_class.path=/vendor/firmware efi=noruntime"
            CMDLINE="${CMDLINE} init=/init androidboot.boot_devices=soc/f723d000.dwmmc0"
            CMDLINE="${CMDLINE} printk.devkmsg=on buildvariant=userdebug"

            ######################################################################
            # common for 4.9/4.14/4.19
            if [ ! -d hikey-linaro ]; then
               git clone https://android.googlesource.com/kernel/hikey-linaro
            fi
            cd hikey-linaro

            # need to use fetch to get all the new changes
            git fetch --all

            # get the mkbootimg command which will be used to create the boot image by all kernel versions
            wget -q https://android-git.linaro.org/platform/system/core.git/plain/mkbootimg/mkbootimg.py -O mkbootimg

            # Build 4.19 kernel
            git checkout android-hikey-linaro-4.19
            git pull
            export KERNEL_DESCRIBE_4_19=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_4_19=$(make kernelversion)

            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            python mkbootimg \
               --kernel arch/arm64/boot/Image.gz-dtb \
               --ramdisk ../out/ramdisk.img \
               --cmdline "${CMDLINE}" \
               --os_version Q \
               --os_patch_level 2019-01-05 \
               --output ../out/boot-4.19-${KERNEL_DESCRIBE_4_19}.img
            cp vmlinux ../out/4.19_vmlinux
            cp .config ../out/4.19_defconfig
            xz ../out/boot-4.19-${KERNEL_DESCRIBE_4_19}.img

            # common for 4.9 and 4.14
            # set verity and system-as-root kernel boot parameters for 4.14 and 4.9
            CMDLINE="${CMDLINE} overlay_mgr.overlay_dt_entry=hardware_cfg_enable_android_fstab_v2"
            CMDLINE="${CMDLINE} rootwait ro root=/dev/dm-0"
            CMDLINE="${CMDLINE} dm=\"system none ro,0 1 android-verity 179:9\""
            CMDLINE="${CMDLINE} veritykeyid=id:7e4333f9bba00adfe0ede979e28ed1920492b40f"

            # Build 4.14 kernel
            git checkout android-hikey-linaro-4.14
            git pull
            export KERNEL_DESCRIBE_4_14=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_4_14=$(make kernelversion)

            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            python mkbootimg \
              --kernel arch/arm64/boot/Image.gz-dtb \
              --cmdline "${CMDLINE}" \
              --os_version Q \
              --os_patch_level 2019-01-05 \
              --output ../out/boot-4.14-${KERNEL_DESCRIBE_4_14}.img
            cp vmlinux ../out/4.14_vmlinux
            cp .config ../out/4.14_defconfig
            xz ../out/boot-4.14-${KERNEL_DESCRIBE_4_14}.img

            # Build 4.9 kernel
            git checkout android-hikey-linaro-4.9
            git pull
            export KERNEL_DESCRIBE_4_9=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_4_9=$(make kernelversion)
            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            python mkbootimg \
               --kernel arch/arm64/boot/Image.gz-dtb \
               --cmdline "${CMDLINE}" \
               --os_version Q \
               --os_patch_level 2019-01-05 \
               --output ../out/boot-4.9-${KERNEL_DESCRIBE_4_9}.img
            cp vmlinux ../out/4.9_vmlinux
            cp .config ../out/4.9_defconfig
            xz ../out/boot-4.9-${KERNEL_DESCRIBE_4_9}.img

            rm -rf configs
            cd ../
            rm -fr hikey-linaro

            # build kernel for x15
            X15_KERNEL_DIR=${ANDROID_ROOT}/kernel/ti/x15
            rm -fr ${X15_KERNEL_DIR} && mkdir -p ${X15_KERNEL_DIR}
            git clone -b android-beagle-x15-4.14 https://android.googlesource.com/kernel/omap ${X15_KERNEL_DIR}

            # compile x15 4.14 kernel
            cd ${X15_KERNEL_DIR}
            export PATH=~/srv/toolchain/gcc-linaro-7.2.1-2017.11-x86_64_arm-eabi/bin:$PATH
            export KERNEL_DESCRIBE_X15_4_14=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_X15_4_14=$(make kernelversion)

            make ARCH=arm CROSS_COMPILE=arm-eabi- ti_sdk_am57x_android_release_defconfig
            make ARCH=arm CROSS_COMPILE=arm-eabi- -j$(nproc) zImage dtbs modules
            cd -

            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-beagle-x15-revc.dtb device/ti/beagle_x15-kernel/4.14/am57xx-beagle-x15-revc.dtb
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/zImage device/ti/beagle_x15-kernel/4.14/zImage
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-evm-common.dtbo device/ti/beagle_x15-kernel/4.14/am57xx-evm-common.dtbo
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtbo device/ti/beagle_x15-kernel/4.14/am57xx-evm-reva3.dtbo

            # compile pvrsrvkm.ko
            # defined OUT according to info from modules/eurasia_km/eurasiacon/build/linux2/config/core.mk
            PVR_MODULE_OUT=${X15_KERNEL_DIR}/modules/eurasia_km/eurasiacon/binary2_omap_android_release
            PVRSRVKM_KO_PATH=${PVR_MODULE_OUT}/target_armv7-a/pvrsrvkm.ko
            KERNEL_PVR_DIR=${X15_KERNEL_DIR}/modules/eurasia_km/eurasiacon/build/linux2/omap_android
            unset CROSS_COMPILE
            PVR_CROSS_COMPILE=${ANDROID_ROOT}/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9/bin/arm-linux-androideabi-
            cd ${KERNEL_PVR_DIR} && \
                make -j$(nproc) \
                     JAVAC=/bin/true \
                     ARCH=arm \
                     TARGET_DEVICE=am57xevm \
                     CROSS_COMPILE=${PVR_CROSS_COMPILE} \
                     KERNEL_CROSS_COMPILE=${PVR_CROSS_COMPILE} \
                     EXCLUDED_APIS="composerhal,camerahal,unittests,sensorhal" \
                     BUILD_PROP=${X15_KERNEL_DIR}/modules/eurasia_km/build.prop \
                     OUT=${PVR_MODULE_OUT} \
                     KERNELDIR=${X15_KERNEL_DIR} && \
                cd -

            cp -v ${PVRSRVKM_KO_PATH} device/ti/beagle_x15-kernel/4.14/pvrsrvkm.ko

            source build/envsetup.sh
            lunch beagle_x15-userdebug
            make -j$(nproc) droidcore bootfitimage
            wget https://git.linaro.org/ci/job/configs.git/blob_plain/HEAD:/android-lcr/hikey/build-info/aosp-master-template.txt -O ${PWD}/out/target/product/beagle_x15/BUILD-INFO.txt

            X15_KERNEL_FILES="
            arch/arm/boot/zImage
            arch/arm/boot/dts/ti/am57xx-beagle-x15-revc.dtb
            arch/arm/boot/dts/ti/am57xx-evm-common.dtbo
            arch/arm/boot/dts/ti/am57xx-evm-reva3.dtbo
            vmlinux
            System.map
            "
            for f in ${X15_KERNEL_FILES}; do
                if [ -f ${X15_KERNEL_DIR}/${f} ]; then
                    cp -vf ${X15_KERNEL_DIR}/${f} out/x15-4.14-$(basename ${f})
                fi
            done

            X15_IMAGE_FILE="
            MLO
            u-boot.img
            boot_fit.img
            userdata.img
            system.img
            vendor.img
            recovery.img
            ramdisk.img
            "
            for f in ${X15_IMAGE_FILE}; do
                if [ -f out/target/product/beagle_x15/${f} ]; then
                    cp -vf out/target/product/beagle_x15/${f} out/x15-${f}
                fi
            done
            mv out/x15-boot_fit.img out/x15-boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img
            mv out/x15-vendor.img out/x15-vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img

            #######################################
            # compile x15 4.19 kernel
            #######################################
            DIR_PREBUILT_KERNEL=device/ti/beagle_x15-kernel/4.19
            mkdir -p ${DIR_PREBUILT_KERNEL}
            cd ${X15_KERNEL_DIR}
            git checkout android-beagle-x15-4.19 && git pull
            make mrproper

            export KERNEL_DESCRIBE_X15_4_19=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_X15_4_19=$(make kernelversion)


            ## revert the change of Revert "usb: gadget: ffs: Fix BUG when userland exits with submitted AIO transfers"
            git revert --no-edit 8b87305f816cf58b7a4c840b2bedfd37a8c06254

            make ARCH=arm CROSS_COMPILE=arm-eabi- ti_sdk_am57x_android_release_defconfig
            make ARCH=arm CROSS_COMPILE=arm-eabi- -j$(nproc) zImage dtbs modules
            cd -

            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-beagle-x15-revc.dtb ${DIR_PREBUILT_KERNEL}/am57xx-beagle-x15-revc.dtb
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/zImage ${DIR_PREBUILT_KERNEL}/zImage
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-evm-common.dtbo ${DIR_PREBUILT_KERNEL}/am57xx-evm-common.dtbo
            cp -vf ${X15_KERNEL_DIR}/arch/arm/boot/dts/ti/am57xx-evm-reva3.dtbo ${DIR_PREBUILT_KERNEL}/am57xx-evm-reva3.dtbo
            cp -v ${X15_KERNEL_DIR}/drivers/gpu/pvr/pvrsrvkm.ko ${DIR_PREBUILT_KERNEL}/pvrsrvkm.ko

            source build/envsetup.sh
            lunch beagle_x15-userdebug
            TARGET_KERNEL_USE=4.19 make -j$(nproc) bootfitimage vendorimage

            X15_KERNEL_FILES="
            arch/arm/boot/zImage
            arch/arm/boot/dts/ti/am57xx-beagle-x15-revc.dtb
            arch/arm/boot/dts/ti/am57xx-evm-common.dtbo
            arch/arm/boot/dts/ti/am57xx-evm-reva3.dtbo
            vmlinux
            System.map
            "
            for f in ${X15_KERNEL_FILES}; do
                if [ -f ${X15_KERNEL_DIR}/${f} ]; then
                    cp -vf ${X15_KERNEL_DIR}/${f} out/x15-4.19-$(basename ${f})
                fi
            done

            X15_IMAGE_FILE="
            boot_fit.img
            vendor.img
            "
            for f in ${X15_IMAGE_FILE}; do
                if [ -f out/target/product/beagle_x15/${f} ]; then
                    cp -vf out/target/product/beagle_x15/${f} out/x15-${f}
                fi
            done
            mv out/x15-boot_fit.img out/x15-boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img
            mv out/x15-vendor.img out/x15-vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img

            # Delete sources after build to save space
            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro*
            rm -fr ${X15_KERNEL_DIR}

            # Publish parameters
            cat << EOF > ${WORKSPACE}/publish_parameters
            PUB_DEST=96boards/hikey/linaro/aosp-master/${BUILD_NUMBER}
            PUB_SRC=${PWD}/out
            PUB_EXTRA_INC=^[^/]+\.(dtb|dtbo|zip)$|MLO|vmlinux|System.map

            KERNEL_DESCRIBE_4_19=${KERNEL_DESCRIBE_4_19}
            KERNEL_DESCRIBE_4_14=${KERNEL_DESCRIBE_4_14}
            KERNEL_DESCRIBE_4_9=${KERNEL_DESCRIBE_4_9}
            KERNEL_DESCRIBE_4_4=${KERNEL_DESCRIBE_4_4}
            KERNEL_DESCRIBE_X15_4_14=${KERNEL_DESCRIBE_X15_4_14}
            KERNEL_DESCRIBE_X15_4_19=${KERNEL_DESCRIBE_X15_4_19}

            KERNEL_VERSION_4_19=${KERNEL_VERSION_4_19}
            KERNEL_VERSION_4_14=${KERNEL_VERSION_4_14}
            KERNEL_VERSION_4_9=${KERNEL_VERSION_4_9}
            KERNEL_VERSION_4_4=${KERNEL_VERSION_4_4}
            KERNEL_VERSION_X15_4_14=${KERNEL_VERSION_X15_4_14}
            KERNEL_VERSION_X15_4_19=${KERNEL_VERSION_X15_4_19}
            EOF
        - inject:
            properties-file: publish_parameters
        - linaro-publish-token
        - shell:
            !include-raw:
                - android/linaro-publisher.sh
        - inject:
            properties-file: publish_parameters
        - shell: |
            # Install ruamel.yaml==0.16.13
            pip install --user --force-reinstall ruamel.yaml==0.16.13
            pip install --user --force-reinstall Jinja2
            export PUB_DEST=96boards/hikey/linaro/aosp-master/${BUILD_NUMBER}
            export VTS_URL=http://snapshots.linaro.org/${PUB_DEST}
            export CTS_URL=http://snapshots.linaro.org/${PUB_DEST}
            export DEVICE_TYPE=hi6220-hikey
            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
            export DOWNLOAD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export REFERENCE_BUILD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export KERNEL_COMMIT=${BUILD_NUMBER}
            export TOOLCHAIN=${TOOLCHAIN}
            export KERNEL_REPO=hikey-linaro
            export ANDROID_VERSION=aosp-master-$(date --iso-8601)
            export VTS_VERSION=$(echo $VTS_URL | awk -F"/" '{print$(NF-1)}')
            export CTS_VERSION=$(echo $CTS_URL | awk -F"/" '{print$(NF-1)}')
            export QA_BUILD_VERSION=${BUILD_NUMBER}
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_4}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_4}

            rm -rf configs
            git clone --depth 1 http://git.linaro.org/ci/job/configs.git

            # Submit jobs for 4.9 kernel
            cd configs
            git reset --hard
            cd -
            sed -i "s|boot.img.xz|boot-4.9-${KERNEL_DESCRIBE_4_9}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-${KERNEL_VERSION_4_9}-${KERNEL_DESCRIBE_4_9}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_9}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_9}
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.9" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel.yaml template-cts.yaml \
                --quiet

            # Submit jobs for 4.14 kernel
            cd configs
            git reset --hard
            cd -
            sed -i "s|boot.img.xz|boot-4.14-${KERNEL_DESCRIBE_4_14}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-${KERNEL_VERSION_4_14}-${KERNEL_DESCRIBE_4_14}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_14}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_14}
            # submit 4.14 jobs to v.l.o to reduce load on production lkft
            export LAVA_SERVER=https://validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.14" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel.yaml template-cts.yaml \
                --quiet

            # Submit jobs for 4.19 kernel
            cd configs && git reset --hard && cd -
            sed -i "s|boot.img.xz|boot-4.19-${KERNEL_DESCRIBE_4_19}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-${KERNEL_VERSION_4_19}-${KERNEL_DESCRIBE_4_19}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_19}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_19}
            # submit 4.19 jobs to v.l.o to reduce load on production lkft
            export LAVA_SERVER=https://validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.19" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel.yaml template-cts.yaml \
                --quiet

            # submit jobs for X15 4.14 kernel build
            export DEVICE_TYPE=x15
            export TOOLCHAIN=gcc-linaro-7.2.1-2017.11-x86_64_arm-eabi
            export KERNEL_REPO=omap
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_X15_4_14}
            export SRCREV_kernel=${KERNEL_DESCRIBE_X15_4_14}
            export KERNEL_BRANCH="android-beagle-x15-${KERNEL_VERSION_X15_4_14}-${KERNEL_DESCRIBE_X15_4_14}"
            # submit x15 jobs to lkf.v.l.o instance
            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/

            # reset the changes done before
            cd configs && git reset --hard && cd -

            # update templates to use the new image names
            sed -i "s|boot_fit.img|x15-boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|vendor.img|x15-vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|system.img|x15-system.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|userdata.img|x15-userdata.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.14" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --testplan-path configs/lkft/lava-job-definitions/x15 \
                --test-plan template-boot.yaml template-vts-kernel.yaml template-cts.yaml \
                --quiet

            # submit jobs for x15 4.19 kernel build
            # reset the changes done before
            cd configs && git reset --hard && cd -

            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_X15_4_19}
            export SRCREV_kernel=${KERNEL_DESCRIBE_X15_4_19}
            export KERNEL_BRANCH="android-beagle-x15-${KERNEL_VERSION_X15_4_19}-${KERNEL_DESCRIBE_X15_4_19}"

            # update templates to use the new image names
            sed -i "s|boot_fit.img|x15-boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|vendor.img|x15-vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|system.img|x15-system.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
            sed -i "s|userdata.img|x15-userdata.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.19" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --testplan-path configs/lkft/lava-job-definitions/x15 \
                --test-plan template-boot.yaml template-vts-kernel.yaml template-cts.yaml \
                --quiet

    publishers:
        - archive:
            artifacts: '*.xml,*.json'
            latest-only: true
        - logparser:
            parse-rules: '/var/jenkins_home/userContent/android.parse'
            unstable-on-warning: false
            fail-on-error: false
        - fingerprint:
            files: 'build/fingerprints/*'
        - email:
            recipients: 'tom.gall@linaro.org bero@linaro.org yongqin.liu@linaro.org'