diff options
| -rwxr-xr-x | openembedded-lkft/builders-sumits.sh | 370 | ||||
| -rw-r--r-- | sumits-oe.yaml | 144 |
2 files changed, 514 insertions, 0 deletions
diff --git a/openembedded-lkft/builders-sumits.sh b/openembedded-lkft/builders-sumits.sh new file mode 100755 index 0000000000..d58b5f237d --- /dev/null +++ b/openembedded-lkft/builders-sumits.sh @@ -0,0 +1,370 @@ +#!/bin/bash + +set -e + +# workaround EDK2 is confused by the long path used during the build +# and truncate files name expected by VfrCompile +sudo mkdir -p /srv/oe +sudo chown buildslave:buildslave /srv/oe +cd /srv/oe + +trap cleanup_exit INT TERM EXIT + +cleanup_exit() +{ + echo "Running cleanup_exit..." +} + +if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then + echo "INFO: apt update error - try again in a moment" + sleep 15 + sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true +fi +pkg_list="python-pip android-tools-fsutils chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-requests texinfo vim-tiny whiptail libelf-dev" +if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then + echo "INFO: apt install error - try again in a moment" + sleep 15 + sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list} +fi + +# Install jinja2-cli +pip install --user --force-reinstall jinja2-cli + +set -ex + +mkdir -p ${HOME}/bin +curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo +chmod a+x ${HOME}/bin/repo +export PATH=${HOME}/bin:${PATH} + +# initialize repo if not done already +if [ ! -e ".repo/manifest.xml" ]; then + repo init -u https://github.com/96boards/oe-rpb-manifest.git -b ${MANIFEST_BRANCH} + + # link to shared downloads on persistent disk + # our builds config is expecting downloads and sstate-cache, here. + # DL_DIR = "${OEROOT}/sources/downloads" + # SSTATE_DIR = "${OEROOT}/build/sstate-cache" + mkdir -p ${HOME}/srv/oe/downloads ${HOME}/srv/oe/sstate-cache-${DISTRO} + mkdir -p build + ln -s ${HOME}/srv/oe/downloads + ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO} sstate-cache +fi + +repo sync +cp .repo/manifest.xml source-manifest.xml +repo manifest -r -o pinned-manifest.xml +MANIFEST_COMMIT=$(cd .repo/manifests && git rev-parse --short HEAD) + +# the setup-environment will create auto.conf and site.conf +# make sure we get rid of old config. +# let's remove the previous TMPDIR as well. +# we want to preserve build/buildhistory though. +rm -rf conf build/conf build/tmp-*glibc/ + +# Accept EULA if/when needed +export EULA_dragonboard410c=1 +export EULA_stih410b2260=1 +source setup-environment build + +# Add job BUILD_NUMBER to output files names +cat << EOF >> conf/auto.conf +IMAGE_NAME_append = "-${BUILD_NUMBER}" +KERNEL_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +MODULE_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +DT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +BOOT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}" +EOF + +# get build stats to make sure that we use sstate properly +cat << EOF >> conf/auto.conf +INHERIT += "buildstats buildstats-summary" +EOF + +# Set the kernel to use +distro_conf=$(find ../layers/meta-rpb/conf/distro -name rpb.inc) +cat << EOF >> ${distro_conf} +PREFERRED_PROVIDER_virtual/kernel = "${KERNEL_RECIPE}" +EOF + +case "${KERNEL_RECIPE}" in + linux-hikey-aosp|linux-generic-android-common-o*|linux-generic-lsk*|linux-generic-stable*) + cat << EOF >> ${distro_conf} +PREFERRED_VERSION_${KERNEL_RECIPE} = "${KERNEL_VERSION}+git%" +EOF + ;; +esac + +# Use meta-oe version, required for mosh +cat << EOF >> ${distro_conf} +PREFERRED_VERSION_protobuf = "2.6.1+git%" +PREFERRED_VERSION_protobuf-native = "2.6.1+git%" +EOF + +# Set the image types to use +cat << EOF >> ${distro_conf} +IMAGE_FSTYPES_remove = "ext4 iso wic" +EOF + +case "${KERNEL_RECIPE}" in + linux-*-aosp|linux-*-android-*) + cat << EOF >> ${distro_conf} +CONSOLE = "ttyFIQ0" +EOF + ;; +esac + +cat << EOF >> ${distro_conf} +KERNEL_ALT_IMAGETYPE_remove_stih410-b2260 = "vmlinux" +EOF + +# Mali GPU driver fails to build +# error: implicit declaration of function 'copy_from_user' +# [-Werror=implicit-function-declaration] +# Ignore the whole gpu MACHINE_FEATURES mechanism +stih410_b2260_conf=$(find ../layers/meta-st-cannes2/conf/machine -name stih410-b2260.conf) +sed -i -e '/gpu/d' ${stih410_b2260_conf} + +# Include additional recipes in the image +[ "${MACHINE}" = "am57xx-evm" ] || extra_pkgs="numactl" +[ "${MACHINE}" = "intel-core2-32" ] || extra_pkgs="cpupower" +cat << EOF >> conf/local.conf +CORE_IMAGE_BASE_INSTALL_append = " kernel-selftests kselftests-mainline kselftests-next libhugetlbfs-tests ltp ${extra_pkgs}" +CORE_IMAGE_BASE_INSTALL_append = " python python-misc python-modules python-numpy python-pexpect python-pyyaml" +CORE_IMAGE_BASE_INSTALL_append = " git mosh-server parted packagegroup-core-buildessential packagegroup-core-tools-debug" +EOF + +# Override cmdline +cat << EOF >> conf/local.conf +CMDLINE_remove = "quiet" +EOF + +# Remove recipes: +# - docker to reduce image size +cat << EOF >> conf/local.conf +RDEPENDS_packagegroup-rpb_remove = "docker" +EOF + +cat << EOF >> conf/local.conf +DEFAULTTUNE_intel-core2-32 = "core2-64" +SERIAL_CONSOLES_remove_intel-core2-32 = "115200;ttyPCH0" +SERIAL_CONSOLES_append_dragonboard-410c = " 115200;ttyMSM1" +SERIAL_CONSOLES_append_hikey = " 115200;ttyAMA2" +EOF + +# Enable lkft-metadata class +cat << EOF >> conf/local.conf +INHERIT += "lkft-metadata" +LKFTMETADATA_COMMIT = "1" +EOF + +# Remove systemd firstboot and machine-id file +# Backport serialization change from v234 to avoid systemd tty race condition +mkdir -p ../layers/meta-96boards/recipes-core/systemd/systemd +wget -q http://people.linaro.org/~fathi.boudra/backport-v234-e266c06-v230.patch \ + -O ../layers/meta-96boards/recipes-core/systemd/systemd/backport-v234-e266c06-v230.patch +cat << EOF >> ../layers/meta-96boards/recipes-core/systemd/systemd/e2fsck.conf +[options] +# This will prevent e2fsck from stopping boot just because the clock is wrong +broken_system_clock = 1 +EOF +cat << EOF >> ../layers/meta-96boards/recipes-core/systemd/systemd_%.bbappend +FILESEXTRAPATHS_prepend := "\${THISDIR}/\${PN}:" + +SRC_URI += "\\ + file://backport-v234-e266c06-v230.patch \\ + file://e2fsck.conf \\ +" + +PACKAGECONFIG_remove = "firstboot" + +do_install_append() { + # Install /etc/e2fsck.conf to avoid boot stuck by wrong clock time + install -m 644 -p -D \${WORKDIR}/e2fsck.conf \${D}\${sysconfdir}/e2fsck.conf + + rm -f \${D}\${sysconfdir}/machine-id +} + +FILES_\${PN} += "\${sysconfdir}/e2fsck.conf " +EOF + +# Update kernel recipe SRCREV +kernel_recipe=$(find ../layers/meta-96boards -name ${KERNEL_RECIPE}_${KERNEL_VERSION}.bb) +sed -i "s|^SRCREV_kernel = .*|SRCREV_kernel = \"${SRCREV_kernel}\"|" ${kernel_recipe} +# Update kernel recipe SRC_URI +SRC_URI_repo="${KERNEL_REPO/https/git}" +SRC_URI_branch="branch=${KERNEL_BRANCH}" +sed -i "s|^ git://.*| ${SRC_URI_repo};protocol=https;${SRC_URI_branch};name=kernel \\\|" ${kernel_recipe} + +# add useful debug info +cat conf/{site,auto}.conf +cat ${distro_conf} + +# Temporary sstate cleanup to get lkft metadata generated +bitbake -c cleansstate kselftests-mainline kselftests-next ltp libhugetlbfs + +time bitbake ${IMAGES} + +DEPLOY_DIR_IMAGE=$(bitbake -e | grep "^DEPLOY_DIR_IMAGE="| cut -d'=' -f2 | tr -d '"') + +# Prepare files to publish +rm -f ${DEPLOY_DIR_IMAGE}/*.txt +rm -rf ${DEPLOY_DIR_IMAGE}/bootloader +find ${DEPLOY_DIR_IMAGE} -type l -delete +mv /srv/oe/{source,pinned}-manifest.xml ${DEPLOY_DIR_IMAGE} +cat ${DEPLOY_DIR_IMAGE}/pinned-manifest.xml + +# FIXME: IMAGE_FSTYPES_remove doesn't work +rm -f ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4 \ + ${DEPLOY_DIR_IMAGE}/*.rootfs.iso \ + ${DEPLOY_DIR_IMAGE}/*.rootfs.wic \ + ${DEPLOY_DIR_IMAGE}/*.stimg + +# FIXME: Sparse images here, until it gets done by OE +case "${MACHINE}" in + juno|stih410-b2260|intel-core2-32) + ;; + *) + for rootfs in ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4.gz; do + gunzip -k ${rootfs} + sudo ext2simg -v ${rootfs%.gz} ${rootfs%.ext4.gz}.img + rm -f ${rootfs%.gz} + gzip -9 ${rootfs%.ext4.gz}.img + done + ;; +esac + +# Move HiKey's bootloader related files into its own subdir +[ "${MACHINE}" = "hikey" ] && { + mkdir -p ${DEPLOY_DIR_IMAGE}/bootloader + (cd ${DEPLOY_DIR_IMAGE} && mv fip.bin hisi-idt.py l-loader.bin nvme.img ptable-linux-*.img bootloader/) +} + +# Create MD5SUMS file +find ${DEPLOY_DIR_IMAGE} -type f | xargs md5sum > MD5SUMS.txt +sed -i "s|${DEPLOY_DIR_IMAGE}/||" MD5SUMS.txt +mv MD5SUMS.txt ${DEPLOY_DIR_IMAGE} + +# Build information +cat > ${DEPLOY_DIR_IMAGE}/HEADER.textile << EOF + +h4. LKFT - OpenEmbedded + +Build description: +* Build URL: "$BUILD_URL":$BUILD_URL +* Manifest URL: "https://github.com/96boards/oe-rpb-manifest.git":https://github.com/96boards/oe-rpb-manifest.git +* Manifest branch: ${MANIFEST_BRANCH} +* Manifest commit: "${MANIFEST_COMMIT}":https://github.com/96boards/oe-rpb-manifest/commit/${MANIFEST_COMMIT} +EOF + +# FIXME handle properly the publishing URL +case "${KERNEL_RECIPE}" in + linux-generic-stable) + PUB_DEST="linux-stable-${KERNEL_VERSION}" + ;; + linux-generic-stable-rc) + PUB_DEST="linux-stable-rc-${KERNEL_VERSION}" + ;; + linux-generic-mainline) + PUB_DEST="linux-mainline" + ;; + linux-generic-next) + PUB_DEST="linux-next" + ;; + linux-hikey-lts) + PUB_DEST="linaro-hikey-stable-4.4" + ;; + linux-hikey-lts-rc) + PUB_DEST="linaro-hikey-stable-rc-4.4" + ;; + linux-generic-lsk) + PUB_DEST="lsk-${KERNEL_VERSION}" + ;; + linux-generic-lsk-test) + PUB_DEST="lsk-${KERNEL_VERSION}-test" + ;; + linux-generic-lsk-rt) + PUB_DEST="lsk-${KERNEL_VERSION}-rt" + ;; + linux-generic-lsk-rt-test) + PUB_DEST="lsk-${KERNEL_VERSION}-rt-test" + ;; + linux-generic-stable-rt) + PUB_DEST="linux-stable-rt-${KERNEL_VERSION}" + ;; + *) + PUB_DEST="${KERNEL_VERSION}" + ;; +esac + +GCCVERSION=$(bitbake -e | grep "^GCCVERSION="| cut -d'=' -f2 | tr -d '"') +TARGET_SYS=$(bitbake -e | grep "^TARGET_SYS="| cut -d'=' -f2 | tr -d '"') +TUNE_FEATURES=$(bitbake -e | grep "^TUNE_FEATURES="| cut -d'=' -f2 | tr -d '"') +STAGING_KERNEL_DIR=$(bitbake -e | grep "^STAGING_KERNEL_DIR="| cut -d'=' -f2 | tr -d '"') + +# lkft-metadata class generates metadata file, which can be sourced +for recipe in kselftests-mainline kselftests-next ltp libhugetlbfs; do + source lkftmetadata/packages/*/${recipe}/metadata +done + +SNAPSHOTS_URL=http://snapshots.linaro.org +# BASE_URL=openembedded/lkft/${MANIFEST_BRANCH}/${MACHINE}/${DISTRO}/${PUB_DEST}/${BUILD_NUMBER} +BASE_URL=people/sumits/oe/${MANIFEST_BRANCH}/${MACHINE}/${DISTRO}/${PUB_DEST}/${BUILD_NUMBER} +BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}*.img" | xargs -r basename) +ROOTFS_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-${MACHINE}-*-${BUILD_NUMBER}.rootfs.img.gz" | xargs -r basename) +ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename) +KERNEL_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*Image-*-${MACHINE}-*-${BUILD_NUMBER}.bin" | xargs -r basename) +case "${MACHINE}" in + juno) + DTB_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*Image-*-${MACHINE}-r2-*-${BUILD_NUMBER}.dtb" | xargs -r basename) + ;; +esac + +cat > ${DEPLOY_DIR_IMAGE}/build_config.json <<EOF +{ + "kernel_repo" : "${KERNEL_REPO}", + "kernel_commit_id" : "${SRCREV_kernel}", + "make_kernelversion" : "${MAKE_KERNELVERSION}", + "kernel_branch" : "${KERNEL_BRANCH}", + "kernel_describe" : "${KERNEL_DESCRIBE}", + "kselftest_mainline_url" : "${KSELFTESTS_MAINLINE_URL}", + "kselftest_mainline_version" : "${KSELFTESTS_MAINLINE_VERSION}", + "kselftest_next_url" : "${KSELFTESTS_NEXT_URL}", + "kselftest_next_version" : "${KSELFTESTS_NEXT_VERSION}", + "ltp_url" : "${LTP_URL}", + "ltp_version" : "${LTP_VERSION}", + "ltp_revision" : "${LTP_REVISION}", + "libhugetlbfs_url" : "${LIBHUGETLBFS_URL}", + "libhugetlbfs_version" : "${LIBHUGETLBFS_VERSION}", + "libhugetlbfs_revision" : "${LIBHUGETLBFS_REVISION}", + "build_arch" : "${TUNE_FEATURES}", + "compiler" : "${TARGET_SYS} ${GCCVERSION}", + "build_location" : "${SNAPSHOTS_URL}/${BASE_URL}" +} +EOF + +cat << EOF > ${WORKSPACE}/post_build_lava_parameters +DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE} +SNAPSHOTS_URL=${SNAPSHOTS_URL} +BASE_URL=${BASE_URL} +BOOT_URL=${SNAPSHOTS_URL}/${BASE_URL}/${BOOT_IMG} +SYSTEM_URL=${SNAPSHOTS_URL}/${BASE_URL}/${ROOTFS_IMG} +KERNEL_URL=${SNAPSHOTS_URL}/${BASE_URL}/${KERNEL_IMG} +DTB_URL=${SNAPSHOTS_URL}/${BASE_URL}/${DTB_IMG} +RECOVERY_IMAGE_URL=${SNAPSHOTS_URL}/${BASE_URL}/juno-oe-uboot.zip +NFSROOTFS_URL=${SNAPSHOTS_URL}/${BASE_URL}/${ROOTFS_TARXZ_IMG} +KERNEL_COMMIT=${SRCREV_kernel} +KERNEL_CONFIG_URL=${SNAPSHOTS_URL}/${BASE_URL}/config +KERNEL_DEFCONFIG_URL=${SNAPSHOTS_URL}/${BASE_URL}/defconfig +KSELFTESTS_MAINLINE_URL=${KSELFTESTS_MAINLINE_URL} +KSELFTESTS_MAINLINE_VERSION=${KSELFTESTS_MAINLINE_VERSION} +KSELFTESTS_NEXT_URL=${KSELFTESTS_NEXT_URL} +KSELFTESTS_NEXT_VERSION=${KSELFTESTS_NEXT_VERSION} +LTP_URL=${LTP_URL} +LTP_VERSION=${LTP_VERSION} +LTP_REVISION=${LTP_REVISION} +LIBHUGETLBFS_URL=${LIBHUGETLBFS_URL} +LIBHUGETLBFS_VERSION=${LIBHUGETLBFS_VERSION} +LIBHUGETLBFS_REVISION=${LIBHUGETLBFS_REVISION} +MAKE_KERNELVERSION=${MAKE_KERNELVERSION} +EOF diff --git a/sumits-oe.yaml b/sumits-oe.yaml new file mode 100644 index 0000000000..1b7c893602 --- /dev/null +++ b/sumits-oe.yaml @@ -0,0 +1,144 @@ +- job: + name: sumits-oe + project-type: matrix + defaults: global + properties: + - authorization: + anonymous: + - job-read + - job-extended-read + - job-workspace + linaro: + - job-read + - job-extended-read + - job-build + - job-cancel + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + artifact-num-to-keep: 1 + parameters: + - string: + name: KERNEL_BRANCH + default: 'hikey-new-rebase-trial' + - string: + name: KERNEL_VERSION + default: '4.4' + - string: + name: KERNEL_RECIPE + default: 'linux-generic-lts' + - string: + name: KERNEL_REPO + default: 'https://git.linaro.org/people/sumit.semwal/linux-3.x.git' + - string: + name: KERNEL_DESCRIBE + - string: + name: SRCREV_kernel + - string: + name: MAKE_KERNELVERSION + - string: + name: IMAGES + default: 'rpb-console-image' + - string: + name: MANIFEST_BRANCH + default: 'morty' + - string: + name: LAVA_SERVER + default: 'https://lkft.validation.linaro.org/RPC2/' + - string: + name: QA_SERVER + default: 'https://qa-reports.linaro.org' + - string: + name: QA_SERVER_PROJECT + default: 'linaro-hikey-stable-4.4-oe' + - string: + name: SKIP_LAVA + default: '1' + - string: + name: LAVA_JOB_PRIORITY + default: 'low' + disabled: false + node: master + display-name: 'Sumit Semwal (OpenEmbedded/morty)' + axes: + - axis: + type: user-defined + name: MACHINE + values: + - 'hikey' + - axis: + type: user-defined + name: DISTRO + values: + - rpb + - axis: + type: slave + name: label + values: + - docker-stretch-amd64 + execution-strategy: + sequential: false + wrappers: + - timestamps + - ssh-agent-credentials: + users: + - 'OE_COMMIT_BOT_KEY' + - credentials-binding: + - text: + credential-id: QA_REPORTS_TOKEN + variable: QA_REPORTS_TOKEN + builders: + - shell: | + #!/bin/bash + + # Mapping for MACHINE -> DEVICE_TYPE + case "${MACHINE}" in + hikey) + DEVICE_TYPE=hi6220-hikey + ;; + esac + + echo "DEVICE_TYPE=${DEVICE_TYPE}" > device_type_parameters + echo "#${BUILD_NUMBER}-${SRCREV_kernel:0:8}" > ${WORKSPACE}/version.txt + - build-name-setter: + name: 'version.txt' + file: true + - inject: + properties-file: device_type_parameters + - shell: + !include-raw: openembedded-lkft/builders-sumits.sh + - inject: + properties-file: post_build_lava_parameters + - linaro-publish-token + - shell: | + #!/bin/bash + + # Publish + test -d ${HOME}/bin || mkdir ${HOME}/bin + wget -q https://git.linaro.org/ci/publishing-api.git/blob_plain/HEAD:/linaro-cp.py -O ${HOME}/bin/linaro-cp.py + time python ${HOME}/bin/linaro-cp.py \ + --server ${PUBLISH_SERVER} \ + --link-latest \ + ${DEPLOY_DIR_IMAGE}/ ${BASE_URL} + - shell: + !include-raw: openembedded-lkft/submit_for_testing.sh + publishers: + - groovy-postbuild: + script: + !include-raw: openembedded-lkft/postbuild.groovy + - email-ext: + recipients: 'sumit.semwal@linaro.org' + matrix-trigger: only-configurations + subject: '[CI] ${MACHINE}: ${KERNEL_BRANCH} {{SRCREV_kernel}} ${BUILD_STATUS}' + body: | + Build URL: ${PROJECT_URL}${BUILD_NUMBER}/ + Full log: ${PROJECT_URL}${BUILD_NUMBER}/consoleText + Parsed warnings/errors: ${PROJECT_URL}${BUILD_NUMBER}/parsed_console + Git branch: ${KERNEL_BRANCH} + Git URL: ${KERNEL_REPO} + Git commit: ${SRCREV_kernel} + + Errors: + ${BUILD_LOG_REGEX, regex="^ERROR:.* ", linesBefore=0, linesAfter=0, showTruncatedLines=false} + presend-script: + !include-raw: openembedded-lkft/presend-script.groovy |