path: root/96boards-hikey-aosp-master.yaml

- job:
    name: 96boards-hikey-aosp-master
    project-type: freestyle
    defaults: global
    logrotate:
        daysToKeep: 60
        numToKeep: 60
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            linaro:
                - job-build
                - job-cancel
    parameters:
        - string:
            name: BUILD_CONFIG_FILENAME
            default: 'hikey-aosp-master'
        - string:
            name: QA_SERVER
            default: 'https://qa-reports.linaro.org'
        - string:
            name: QA_SERVER_PROJECT
            default: 'aosp-master-tracking'
        - string:
            name: ANDROID_VERSION
            default: 'AOSP MASTER'
        - string:
            name: KERNEL_DESCRIBE
            default: 'dummy'
        - string:
            name: SRCREV_kernel
            default: 'dummy'
    disabled: false
    node: docker-xenial-aosp
    display-name: '96boards - AOSP Master - HiKey'
    concurrent: true
    triggers:
        - timed : 'H  H H/2 * *'
    wrappers:
        - timestamps
        - timeout:
            timeout: 500
        - credentials-binding:
            - text:
                credential-id: QA_REPORTS_TOKEN
                variable: QA_REPORTS_TOKEN
            - text:
                credential-id: ARTIFACTORIAL_TOKEN
                variable: ARTIFACTORIAL_TOKEN
            - text:
                credential-id: AP_SSID
                variable: AP_SSID
            - text:
                credential-id: AP_KEY
                variable: AP_KEY
    builders:
        - shell: |
            #!/bin/bash

            export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8

            java -version

            sudo apt-get update
            sudo apt-get install -y gcc-4.9-multilib bison git gperf libxml2-utils python-mako zip time python-requests genisoimage patch mtools python-pip pxz libssl-dev

            wget -q http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre-headless_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre_8u45-b14-1_amd64.deb \
              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jdk_8u45-b14-1_amd64.deb
            sudo dpkg -i --force-all *.deb

            mkdir -p ${HOME}/bin ${WORKSPACE}/build/out
            curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
            chmod a+x ${HOME}/bin/*
            export PATH=${HOME}/bin:${PATH}

            if [ ! -d "aosp-master" ]; then
              sudo mkdir -p /home/buildslave/srv/aosp-master
              sudo chmod 777 /home/buildslave/srv/aosp-master
            fi
            cd /home/buildslave/srv/aosp-master

            git config --global user.email "ci_notify@linaro.org"
            git config --global user.name "Linaro CI"

            # Runs as ubuntu
            set -xe
            rm -rf .repo/manifests* .repo/local_manifests build-tools jenkins-tools build/out build/android-patchsets build-configs
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-android-build-tools.git build-tools
            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-jenkins-tools.git jenkins-tools
            git clone --depth 1 http://android-git.linaro.org/git/android-build-configs.git build-configs

            export BUILD_CONFIG_FILENAME=${BUILD_CONFIG_FILENAME:-${JOB_NAME#android-*}}

            # Early test
            if [ ! -f build-configs/${BUILD_CONFIG_FILENAME} ]; then
              echo "No config file named ${BUILD_CONFIG_FILENAME} exists"
              echo "in android-build-configs.git"
              exit 1
            fi

            cat << EOF > config.txt
            BUILD_CONFIG_REPO=http://android-git.linaro.org/git/android-build-configs.git
            BUILD_CONFIG_BRANCH=master
            EOF
            echo config.txt
            export CONFIG=`base64 -w 0 config.txt`

            mkdir -p build
            # Delete sources before build to save space
            cd build/
            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro* clang-src/
            cd -

            # Build Android
            build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "${CONFIG}"
            cp -a /home/buildslave/srv/aosp-master/build/out/*.xml /home/buildslave/srv/aosp-master/build/out/*.json ${WORKSPACE}/

            cd build/out
            for image in "boot.img" "vendor.img" "boot_fat.uefi.img" "system.img" "userdata.img"  "cache.img"; do
              if [ -f ${image} ]; then
                echo "Compressing ${image}"
                xz ${image}
              fi
            done
            cd -
            pushd build/
            source build/envsetup.sh
            lunch hikey-userdebug
            pushd test/suite_harness
            git fetch http://android-review.linaro.org/platform/test/suite_harness refs/changes/23/19623/1 && git cherry-pick FETCH_HEAD 
            popd
            lunch aosp_arm64-userdebug
            make -j"$(nproc)" vts cts
            cp out/host/linux-x86/vts/android-vts.zip out/
            cp out/host/linux-x86/cts/android-cts.zip out/
            rm -rf out/host/

            popd

            rm -rf build/out/BUILD-INFO.txt
            wget https://git.linaro.org/ci/job/configs.git/blob_plain/HEAD:/android-lcr/hikey/build-info/aosp-master-template.txt -O build/out/BUILD-INFO.txt

            # Build latest boot.img with latest 4.9 kernel from source and 4.14 kernel from source
            cd build/
            export PATH=${PWD}/prebuilts/clang/host/linux-x86/clang-4679922/bin/:${PWD}/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/bin/:${PATH}

            if [ ! -d hikey-linaro ]; then
              git clone https://android.googlesource.com/kernel/hikey-linaro
            fi
            cd hikey-linaro
            git checkout android-hikey-linaro-4.14
            git pull
            rm -rf configs
            git clone --depth=1 https://android.googlesource.com/kernel/configs
            wget -q https://android-git.linaro.org/platform/system/core.git/plain/mkbootimg/mkbootimg -O mkbootimg
            export KERNEL_DESCRIBE_4_14=$(git rev-parse --short HEAD)
            export CLANG_TRIPLE=aarch64-linux-gnu-
            export CROSS_COMPILE=aarch64-linux-android-
            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig configs/android-4.14/android-base.config configs/android-4.14/android-base-arm64.config
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            CMD="androidboot.console=ttyFIQ0 androidboot.hardware=hikey firmware_class.path=/vendor/firmware efi=noruntime printk.devkmsg=on buildvariant=userdebug overlay_mgr.overlay_dt_entry=hardware_cfg_enable_android_fstab"
            python mkbootimg \
              --kernel arch/arm64/boot/Image.gz-dtb \
              --cmdline console="${CMD}" \
              --os_version O \
              --os_patch_level 2016-11-05 \
              --ramdisk ../out/ramdisk.img \
              --output ../out/boot-4.14-${KERNEL_DESCRIBE_4_14}.img
            cp vmlinux ../out/4.14_vmlinux
            cp .config ../out/4.14_defconfig
            xz ../out/boot-4.14-${KERNEL_DESCRIBE_4_14}.img

            # Build 4.9 kernel
            git checkout android-hikey-linaro-4.9
            git pull
            export KERNEL_DESCRIBE_4_9=$(git rev-parse --short HEAD)
            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig configs/android-4.9/android-base.config configs/android-4.9/android-base-arm64.config
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            python mkbootimg \
               --kernel arch/arm64/boot/Image.gz-dtb \
               --cmdline console="${CMD}" \
               --os_version O \
               --os_patch_level 2016-11-05 \
               --ramdisk ../out/ramdisk.img \
               --output ../out/boot-4.9-${KERNEL_DESCRIBE_4_9}.img
            cp vmlinux ../out/4.9_vmlinux
            cp .config ../out/4.9_defconfig
            xz ../out/boot-4.9-${KERNEL_DESCRIBE_4_9}.img

            # Build 4.4 kernel
            git checkout android-hikey-linaro-4.4
            git pull
            export KERNEL_DESCRIBE_4_4=$(git rev-parse --short HEAD)
            ARCH=arm64 scripts/kconfig/merge_config.sh arch/arm64/configs/hikey_defconfig configs/android-4.4/android-base.config configs/android-4.4/android-base-arm64.config
            make ARCH=arm64 CC=clang HOSTCC=clang -j$(nproc) -s Image.gz-dtb
            python mkbootimg \
               --kernel arch/arm64/boot/Image.gz-dtb \
               --cmdline console="${CMD}" \
               --os_version O \
               --os_patch_level 2016-11-05 \
               --ramdisk ../out/ramdisk.img \
               --output ../out/boot-4.4-${KERNEL_DESCRIBE_4_4}.img
            cp vmlinux ../out/4.4_vmlinux
            cp .config ../out/4.4_defconfig
            xz ../out/boot-4.4-${KERNEL_DESCRIBE_4_4}.img
            rm -rf configs
            cd ../

            # Delete sources after build to save space
            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro*

            # Publish parameters
            cat << EOF > ${WORKSPACE}/publish_parameters
            PUB_DEST=96boards/hikey/linaro/aosp-master/${BUILD_NUMBER}
            PUB_SRC=${PWD}/out
            PUB_EXTRA_INC=^[^/]+zip
            KERNEL_DESCRIBE_4_14=${KERNEL_DESCRIBE_4_14}
            KERNEL_DESCRIBE_4_9=${KERNEL_DESCRIBE_4_9}
            KERNEL_DESCRIBE_4_4=${KERNEL_DESCRIBE_4_4}
            EOF
        - inject:
            properties-file: publish_parameters
        - linaro-publish-token
        - shell:
            !include-raw:
                - android/linaro-publisher.sh
        - inject:
            properties-file: publish_parameters
        - shell: |
            # Install ruamel.yaml
            pip install --user --force-reinstall ruamel.yaml
            pip install --user --force-reinstall Jinja2
            export PUB_DEST=96boards/hikey/linaro/aosp-master/${BUILD_NUMBER}
            export VTS_URL=http://snapshots.linaro.org/${PUB_DEST}
            export CTS_URL=http://snapshots.linaro.org/${PUB_DEST}
            export DEVICE_TYPE=hi6220-hikey
            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
            export DOWNLOAD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export REFERENCE_BUILD_URL=http://snapshots.linaro.org/${PUB_DEST}
            export KERNEL_COMMIT=${BUILD_NUMBER}
            export TOOLCHAIN=clang
            export KERNEL_REPO=hikey-linaro
            export ANDROID_VERSION=aosp-master-$(date --iso-8601)
            export VTS_VERSION=$(echo $VTS_URL | awk -F"/" '{print$(NF-1)}')
            export CTS_VERSION=$(echo $CTS_URL | awk -F"/" '{print$(NF-1)}')
            export QA_BUILD_VERSION=${BUILD_NUMBER}
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_4}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_4}

            rm -rf configs
            git clone --depth 1 http://git.linaro.org/ci/job/configs.git

            # Submit jobs for 4.4 kernel
            sed -i "s|boot.img.xz|boot-4.4-${KERNEL_DESCRIBE_4_4}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-4.4-${KERNEL_DESCRIBE_4_4}"
            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.4" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel-part1.yaml template-vts-kernel-part2.yaml template-vts-kernel-part3.yaml template-vts-kernel-part4.yaml template-cts-armeabi-v7a.yaml template-cts-arm64-v8a.yaml template-cts-displaytestcases.yaml \
                --quiet

            # Submit jobs for 4.9 kernel
            cd configs
            git reset --hard
            cd -
            sed -i "s|boot.img.xz|boot-4.9-${KERNEL_DESCRIBE_4_9}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-4.9-${KERNEL_DESCRIBE_4_9}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_9}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_9}
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.9" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel-part1.yaml template-vts-kernel-part2.yaml template-vts-kernel-part3.yaml template-vts-kernel-part4.yaml template-cts-armeabi-v7a.yaml template-cts-arm64-v8a.yaml template-cts-displaytestcases.yaml \
                --quiet

            # Submit jobs for 4.14 kernel
            cd configs
            git reset --hard
            cd -
            sed -i "s|boot.img.xz|boot-4.14-${KERNEL_DESCRIBE_4_14}.img.xz|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml

            export KERNEL_BRANCH="android-hikey-linaro-4.14-${KERNEL_DESCRIBE_4_14}"
            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_4_14}
            export SRCREV_kernel=${KERNEL_DESCRIBE_4_14}
            # submit 4.14 jobs to v.l.o to reduce load on production lkft
            export LAVA_SERVER=https://validation.linaro.org/RPC2/
            python configs/openembedded-lkft/submit_for_testing.py \
                --device-type ${DEVICE_TYPE} \
                --build-number ${BUILD_NUMBER} \
                --lava-server ${LAVA_SERVER} \
                --qa-server ${QA_SERVER} \
                --qa-server-team android-lkft \
                --env-suffix "_4.14" \
                --qa-server-project ${QA_SERVER_PROJECT} \
                --git-commit ${QA_BUILD_VERSION} \
                --template-path configs/lkft/lava-job-definitions \
                --template-names template-boot.yaml template-vts-kernel-part2.yaml template-vts-kernel-part3.yaml template-vts-kernel-part4.yaml template-cts-armeabi-v7a.yaml template-cts-arm64-v8a.yaml template-cts-displaytestcases.yaml template-vts-kernel-part1.yaml \
                --quiet

    publishers:
        - archive:
            artifacts: '*.xml,*.json'
            latest-only: true
        - logparser:
            parse-rules: '/var/lib/jenkins/userContent/android.parse'
            unstable-on-warning: false
            fail-on-error: false
        - fingerprint:
            files: 'build/fingerprints/*'
        - email:
            recipients: 'vishal.bhoj@linaro.org fathi.boudra@linaro.org'