path: root/lkft-aosp-master-tracking.yaml

- job:
    name: lkft-aosp-master-tracking
    project-type: freestyle
    defaults: global
    logrotate:
        daysToKeep: 60
        numToKeep: 60
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            linaro:
                - job-build
                - job-cancel
    parameters:
        - string:
            name: BUILD_CONFIG_FILENAME
            default: 'hikey-aosp-master'
        - string:
            name: QA_SERVER
            default: 'https://qa-reports.linaro.org'
        - string:
            name: QA_SERVER_PROJECT
            default: 'aosp-master-tracking'
        - string:
            name: ANDROID_VERSION
            default: 'AOSP MASTER'
        - string:
            name: KERNEL_DESCRIBE
            default: 'dummy'
        - string:
            name: SRCREV_kernel
            default: 'dummy'
        - string:
            name: TOOLCHAIN
            default: 'clang-r353983c'
    disabled: false
    node: docker-xenial-aosp
    display-name: 'LKFT AOSP Master HiKey builds'
    concurrent: true
    wrappers:
        - timestamps
        - timeout:
            timeout: 1000
        - credentials-binding:
            - text:
                credential-id: QA_REPORTS_TOKEN
                variable: QA_REPORTS_TOKEN
            - text:
                credential-id: ARTIFACTORIAL_TOKEN
                variable: ARTIFACTORIAL_TOKEN
            - text:
                credential-id: AP_SSID
                variable: AP_SSID
            - text:
                credential-id: AP_KEY
                variable: AP_KEY
    builders:
        - shell: |
            #!/bin/bash

            export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8

            java -version

            sudo apt-get update
            sudo apt-get install -y gcc-4.9-multilib bison git gperf libxml2-utils python-mako zip time python-requests genisoimage patch mtools python-pip pxz libssl-dev

            wget -q http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre-headless_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jdk_8u45-b14-1_amd64.deb
            sudo dpkg -i --force-all *.deb

            mkdir -p ${HOME}/bin ${WORKSPACE}/build/out
            curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
            chmod a+x ${HOME}/bin/*
            export PATH=${HOME}/bin:${PATH}

            DIR_SRV_AOSP_MASTER="/home/buildslave/srv/aosp-master"
            if [ ! -d "${DIR_SRV_AOSP_MASTER}" ]; then
              sudo mkdir -p "${DIR_SRV_AOSP_MASTER}"
              sudo chmod 777 "${DIR_SRV_AOSP_MASTER}"
            fi
            cd "${DIR_SRV_AOSP_MASTER}"

            git config --global user.email "ci_notify@linaro.org"
            git config --global user.name "Linaro CI"

            # Runs as ubuntu
            set -xe
            rm -rf .repo/manifests* .repo/local_manifests build-tools jenkins-tools build-configs
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-android-build-tools.git build-tools
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-jenkins-tools.git jenkins-tools
            git clone --depth 1 http://android-git.linaro.org/git/android-build-configs.git build-configs

            ANDROID_ROOT="${DIR_SRV_AOSP_MASTER}/build"
            DIR_PUB_SRC="${ANDROID_ROOT}/out"
            DIR_PUB_SRC_PRODUCT="${DIR_PUB_SRC}/target/product/hikey"

            # clean the out directory as this workspace is used accross multiple builds
            rm -fr ${DIR_SRV_AOSP_MASTER}/.repo-backup
            if [ -d "${ANDROID_ROOT}/.repo" ]; then
                mv -f ${ANDROID_ROOT}/.repo ${DIR_SRV_AOSP_MASTER}/.repo-backup
            fi
            rm -fr ${ANDROID_ROOT}/ && mkdir -p ${ANDROID_ROOT}
            if [ -d "${DIR_SRV_AOSP_MASTER}/.repo-backup" ]; then
                mv -f ${DIR_SRV_AOSP_MASTER}/.repo-backup ${ANDROID_ROOT}/.repo
            fi

            export BUILD_CONFIG_FILENAME=${BUILD_CONFIG_FILENAME:-${JOB_NAME#android-*}}

            # Early test
            if [ ! -f ${DIR_SRV_AOSP_MASTER}/build-configs/${BUILD_CONFIG_FILENAME} ]; then
              echo "No config file named ${BUILD_CONFIG_FILENAME} exists"
              echo "in android-build-configs.git"
              exit 1
            fi

            # Build Android for HiKey
            cd ${ANDROID_ROOT}
            ${DIR_SRV_AOSP_MASTER}/build-configs/linaro-build.sh -c ${BUILD_CONFIG_FILENAME}
            cd -

            cp -a ${ANDROID_ROOT}/out/pinned-manifest/*-pinned-manifest.xml ${ANDROID_ROOT}/out
            cp -a ${ANDROID_ROOT}/out/pinned-manifest/*-pinned-manifest.xml ${WORKSPACE}/ || true


            rm -rf ${DIR_PUB_SRC}/BUILD-INFO.txt
            wget https://git.linaro.org/ci/job/configs.git/blob_plain/HEAD:/android-lcr/hikey/build-info/aosp-master-template.txt -O ${DIR_PUB_SRC}/BUILD-INFO.txt

            IMAGE_FILES="boot.img boot-debug.img userdata.img system.img vendor.img"
            for f in ${IMAGE_FILES}; do
                if [ -f ${DIR_PUB_SRC_PRODUCT}/${f} ]; then
                    mv -vf ${DIR_PUB_SRC_PRODUCT}/${f} ${DIR_PUB_SRC}/${f}
                    xz ${DIR_PUB_SRC}/${f}
                fi
            done

            IMAGE_FILES="ramdisk.img ramdisk-debug.img"
            for f in ${IMAGE_FILES}; do
                if [ -f ${DIR_PUB_SRC_PRODUCT}/${f} ]; then
                    mv -vf ${DIR_PUB_SRC_PRODUCT}/${f} ${DIR_PUB_SRC}/${f}
                fi
            done

            [ -n "${DIR_PUB_SRC}/system.img.xz" ] && SYSTEM_IMG_SHA256SUM=$(sha256sum ${DIR_PUB_SRC}/system.img.xz | cut -d\  -f1)
            [ -n "${DIR_PUB_SRC}/userdata.img.xz" ] && USERDATA_IMG_SHA256SUM=$(sha256sum ${DIR_PUB_SRC}/userdata.img.xz | cut -d\  -f1)
            [ -n "${DIR_PUB_SRC}/vendor.img.xz" ] && VENDOR_IMG_SHA256SUM=$(sha256sum ${DIR_PUB_SRC}/vendor.img.xz | cut -d\  -f1)

            # Build latest boot.img with latest kernel from source for 4.14, 4.19 versions
            cd ${ANDROID_ROOT}
            export PATH=${PWD}/prebuilts/clang/host/linux-x86/${TOOLCHAIN}/bin/:${PWD}/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/bin/:${PATH}

            export CLANG_TRIPLE=aarch64-linux-gnu-
            export CROSS_COMPILE=aarch64-linux-android-
            CMDLINE="console=ttyAMA3,115200 androidboot.console=ttyAMA3"
            CMDLINE="${CMDLINE} androidboot.hardware=hikey firmware_class.path=/vendor/firmware efi=noruntime"
            CMDLINE="${CMDLINE} init=/init androidboot.boot_devices=soc/f723d000.dwmmc0"
            CMDLINE="${CMDLINE} printk.devkmsg=on buildvariant=userdebug"

            ######################################################################
            # common for 4.14/4.19
            if [ ! -d hikey-linaro ]; then
               git clone https://android.googlesource.com/kernel/hikey-linaro
            fi
            cd hikey-linaro

            # need to use fetch to get all the new changes
            git fetch --all

            # get the mkbootimg command which will be used to create the boot image by all kernel versions
            wget https://android-git.linaro.org/platform/system/core.git/plain/mkbootimg/mkbootimg.py?id=1e40c30b0cc0f2ae007dacc64fe08af52cc731e3 -O mkbootimg

            # Build 4.19 kernel
            git checkout android-hikey-linaro-4.19
            git pull
            export KERNEL_DESCRIBE_4_19=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_4_19=$(make kernelversion)

            make mrproper
            make -j1 ARCH=arm64 CC=clang HOSTCC=clang hikey_defconfig
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) Image.gz-dtb
            python mkbootimg \
               --kernel arch/arm64/boot/Image.gz-dtb \
               --ramdisk ${DIR_PUB_SRC}/ramdisk.img \
               --cmdline "${CMDLINE}" \
               --os_version Q \
               --os_patch_level 2019-01-05 \
               --output ${DIR_PUB_SRC}/boot-4.19-${KERNEL_DESCRIBE_4_19}.img
            cp vmlinux ${DIR_PUB_SRC}/4.19_vmlinux
            cp .config ${DIR_PUB_SRC}/4.19_defconfig
            xz ${DIR_PUB_SRC}/boot-4.19-${KERNEL_DESCRIBE_4_19}.img
            BOOT_4_19_IMG_SHA256SUM=$(sha256sum ${DIR_PUB_SRC}/boot-4.19-${KERNEL_DESCRIBE_4_19}.img.xz | cut -d\  -f1)

            # Build 4.14 kernel
            git checkout android-hikey-linaro-4.14
            git pull
            export KERNEL_DESCRIBE_4_14=$(git rev-parse --short HEAD)
            export KERNEL_VERSION_4_14=$(make kernelversion)

            make mrproper
            make -j1 ARCH=arm64 CC=clang HOSTCC=clang hikey_defconfig
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) Image.gz-dtb
            python mkbootimg \
              --kernel arch/arm64/boot/Image.gz-dtb \
              --ramdisk ${DIR_PUB_SRC}/ramdisk.img \
              --cmdline "${CMDLINE}" \
              --os_version Q \
              --os_patch_level 2019-01-05 \
              --output ${DIR_PUB_SRC}/boot-4.14-${KERNEL_DESCRIBE_4_14}.img
            cp vmlinux ${DIR_PUB_SRC}/4.14_vmlinux
            cp .config ${DIR_PUB_SRC}/4.14_defconfig
            xz ${DIR_PUB_SRC}/boot-4.14-${KERNEL_DESCRIBE_4_14}.img
            BOOT_4_14_IMG_SHA256SUM=$(sha256sum ${DIR_PUB_SRC}/boot-4.14-${KERNEL_DESCRIBE_4_14}.img.xz | cut -d\  -f1)

            cd ../
            rm -fr hikey-linaro

            # Publish parameters
            cat << EOF > ${WORKSPACE}/publish_parameters
            PUB_DEST=android/lkft/lkft-aosp-master/${BUILD_NUMBER}
            PUB_SRC=${DIR_PUB_SRC}
            PUB_EXTRA_INC=^[^/]+\.(dtb|dtbo|zip)$|MLO|vmlinux|System.map

            KERNEL_DESCRIBE_4_19=${KERNEL_DESCRIBE_4_19}
            KERNEL_DESCRIBE_4_14=${KERNEL_DESCRIBE_4_14}

            KERNEL_VERSION_4_19=${KERNEL_VERSION_4_19}
            KERNEL_VERSION_4_14=${KERNEL_VERSION_4_14}
            BOOT_4_19_IMG_SHA256SUM=${BOOT_4_19_IMG_SHA256SUM}
            BOOT_4_14_IMG_SHA256SUM=${BOOT_4_14_IMG_SHA256SUM}
            SYSTEM_IMG_SHA256SUM=${SYSTEM_IMG_SHA256SUM}
            USERDATA_IMG_SHA256SUM=${USERDATA_IMG_SHA256SUM}
            VENDOR_IMG_SHA256SUM=${VENDOR_IMG_SHA256SUM}
            EOF
        - inject:
            properties-file: publish_parameters
        - linaro-publish-token
        - shell:
            !include-raw:
                - android/linaro-publisher.sh
        - inject:
            properties-file: publish_parameters
        - shell: |
            # Install ruamel.yaml==0.16.13
            pip install --user --force-reinstall ruamel.yaml==0.16.13
            pip install --user --force-reinstall Jinja2
            export PUB_DEST=android/lkft/lkft-aosp-master/${BUILD_NUMBER}
            export DEVICE_TYPE=hi6220-hikey
            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
            export DOWNLOAD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export REFERENCE_BUILD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export KERNEL_COMMIT=${BUILD_NUMBER}
            export TOOLCHAIN=${TOOLCHAIN}
            export KERNEL_REPO=hikey-linaro
            export ANDROID_VERSION=aosp-master-$(date --iso-8601)
            export VTS_VERSION=$(echo $VTS_URL | awk -F"/" '{print$(NF-1)}')
            export CTS_VERSION=$(echo $CTS_URL | awk -F"/" '{print$(NF-1)}')
            export QA_BUILD_VERSION=${BUILD_NUMBER}
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_4}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_4}

            rm -rf configs
            git clone --depth 1 http://git.linaro.org/ci/job/configs.git

            # Submit jobs for 4.14 kernel
            cd configs
            git reset --hard
            cd -
            sed -i "s|boot.img.xz|boot-4.14-${KERNEL_DESCRIBE_4_14}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-${KERNEL_VERSION_4_14}-${KERNEL_DESCRIBE_4_14}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_14}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_14}
            export BOOT_IMG_SHA256SUM=${BOOT_4_14_IMG_SHA256SUM}
            # submit 4.14 jobs to v.l.o to reduce load on production lkft
            export LAVA_SERVER=https://validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.14" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-vts-kernel.yaml \
                --quiet

            # Submit jobs for 4.19 kernel
            cd configs && git reset --hard && cd -
            sed -i "s|boot.img.xz|boot-4.19-${KERNEL_DESCRIBE_4_19}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-${KERNEL_VERSION_4_19}-${KERNEL_DESCRIBE_4_19}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_19}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_19}
            export BOOT_IMG_SHA256SUM=${BOOT_4_19_IMG_SHA256SUM}
            # submit 4.19 jobs to v.l.o to reduce load on production lkft
            export LAVA_SERVER=https://validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.19" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-vts-kernel.yaml \
                --quiet

    publishers:
        - archive:
            artifacts: '*.xml,*.json'
            latest-only: true
        - logparser:
            parse-rules: '/var/jenkins_home/userContent/android.parse'
            unstable-on-warning: false
            fail-on-error: false
        - fingerprint:
            files: 'build/fingerprints/*'
        - email:
            recipients: 'tom.gall@linaro.org yongqin.liu@linaro.org'