- job: name: tcwg-bkk16-buildfarm-benchmark project-type: matrix defaults: global properties: - authorization: anonymous: - job-read - job-extended-read linaro: - job-build - job-cancel - build-discarder: days-to-keep: 30 num-to-keep: 30 artifact-days-to-keep: 2 artifact-num-to-keep: 40 parameters: - string: name: override default: '' description: 'Versions of components to build; e.g. binutils=binutils-gdb.git/linaro_binutils-2_25-branch gcc=gcc.git~linaro/gcc-5-branch glibc=glibc.git~release/2.21/master' - string: name: host_x86_64_languages default: 'default' description: 'Languages to build for x86_64-hosted toolchains, e.g., "c,c++" or "all" or "default"' - string: name: host_aarchXX_languages default: 'c,c++' description: 'Languages to build for AArchXX-hosted toolchains, e.g., "c,c++" or "all" or "default"' - string: name: runtests default: 'yes' description: 'Run toolchain testsuites: "yes", "no", "x86_64-host"' - bool: name: try_bootstrap default: 'true' description: 'Attempt to bootstrap GCC for compatible host and target combinations' - string: name: host_x86_64_excludecheck default: 'gdb' description: 'Do not run "make check" on these components on x86_64' - string: name: host_aarchXX_excludecheck default: 'gcc' description: 'Do not run "make check" on these components on AArchXX' - bool: name: rebuild default: true description: 'Rebuild the toolchain even if results are already on logserver' - string: name: log_name default: '' description: 'Logname directory on logserver; leave empty for default' - bool: name: dont_fail default: false description: 'Do not fail the build' - string: name: log_server default: dev-01.tcwglab:/home/maxim.kuvyrkov/logs description: 'Log_Server' - string: name: targets default: 'cross_main cross_aux native_x86 native_aarch64 native_aarch32' description: 'Which target toolchains to build: cross_main cross_aux native_x86 native_aarch64 native_aarch32' - string: name: abe_branch default: refs/remotes/origin/stable description: 'ABE revision to test' - string: name: scripts_branch default: master description: 'Scripts branch to use' - bool: name: run_benchmark default: false description: 'Whether to run a benchmark for benchmarkable targets.' disabled: false #Change to node would break the use of copy-to-master at end of file. #What we are really doing there is copy-to-matrix-parent, and that only #works if matrix parent == jenkins master node. node: master #Change to child workspace may need to be reflected in the build triggers #under publishers, at end of file. child-workspace: $target concurrent: true display-name: 'TCWG BKK16 BuildFarm Benchmark' scm: - git: url: https://git-us.linaro.org/toolchain/abe.git refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/remotes/changes/* branches: - $abe_branch skip-tag: true shallow-clone: true wipe-workspace: false clean: before: true #Any changes in these axes may need to be reflected in the build triggers #under publishers, at end of file. axes: - axis: type: slave name: label values: - tcwg-x86_64-build - tcwg-aarch64-build - axis: type: user-defined name: target values: - aarch64-linux-gnu - armv8l-linux-gnueabihf - arm-linux-gnueabi - arm-linux-gnueabihf - armeb-linux-gnueabihf - aarch64-none-elf - aarch64_be-none-elf - native - schroot-i386-native - schroot-armhf-native execution-strategy: combination-filter: | (label=="tcwg-x86_64-build" && target!="schroot-armhf-native") || (label=="tcwg-aarch64-build" && (target=="native" || target=="schroot-armhf-native")) sequential: false wrappers: - timeout: timeout: 600 - timestamps - ssh-agent-credentials: # tcwg-buildslave user id users: - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a' - build-name: name: '#${BUILD_NUMBER}' builders: - shell: | #!/bin/bash set -e set -x #Ensure that artifact_parameters is empty - must exist to be used #by inject step, but if it is empty we will not publish any #artifacts. Must exist before first exit. rm -f artifact_parameters touch artifact_parameters #Ensure that the properties file isn't lying around from some old #build - stops us from kicking off the wrong benchmark. rm -f bench_parameters touch bench_parameters case "$target--$(uname -m)" in native--aarch64) target_kind=native_aarch64 ;; schroot-armhf-native--*) target_kind=native_aarch32 ;; native--x86_64|schroot-i386-native--*) target_kind=native_x86 ;; armeb-linux-gnueabihf--*|*-none-elf--*) target_kind=cross_aux ;; *-linux-gnu*--*) target_kind=cross_main ;; *) echo "ERROR: Cannot handle target: $target"; exit 1 ;; esac if ! echo "$targets" | grep -q "$target_kind"; then echo "NOTE: Skipping target $target because $target_kind is not in list: $targets" exit 0 fi case ${target} in schroot-*) schroot_arch=$(echo ${target} | sed -e "s/^schroot-\([^-]*\)-\(.*\)/\1/") target=$(echo ${target} | sed -e "s/^schroot-\([^-]*\)-\(.*\)/\2/") ;; *) case $(uname -m) in x86_64) schroot_arch="amd64" ;; aarch64) schroot_arch="arm64" ;; *) echo "Unexpected architecture $(uname -m)"; exit 1 ;; esac ;; esac schroot_image="tcwg-build-${schroot_arch}-trusty" case $(uname -m) in x86_64) languages="$host_x86_64_languages" ;; *) languages="$host_aarchXX_languages" ;; esac case "$runtests-$(uname -m)" in "yes"-*) runtests_opt="--runtests" ;; "x86_64-host"-x86_64) runtests_opt="--runtests" ;; *) runtests_opt="" ;; esac if $try_bootstrap; then bootstrap="--bootstrap" else bootstrap="" fi if [ x"$log_name" != x"" ]; then eval "logname_opt=\"--logname $log_name\"" fi if $rebuild; then norebuild="" else norebuild="--norebuild" fi case $(uname -m) in x86_64) excludecheck="$host_x86_64_excludecheck" ;; *) excludecheck="$host_aarchXX_excludecheck" ;; esac excludecheck_opt="" for testsuite in $excludecheck; do excludecheck_opt="$excludecheck_opt --excludecheck $testsuite" done ulimit -u 5000 session_id=$(schroot -b -c chroot:$schroot_image --preserve-environment) BUILD_SHELL="schroot -r -c session:$session_id --preserve-environment -- bash" $BUILD_SHELL -c "echo \"Build session is up; ulimit config:\"; ulimit -a" # Remove schroot session on exit trap "schroot -f -e -c session:$session_id" 0 git clone -b $scripts_branch --depth 1 https://git-us.linaro.org/toolchain/jenkins-scripts result="0" ${BUILD_SHELL} -x ${WORKSPACE}/jenkins-scripts/jenkins.sh --abedir ${WORKSPACE} --target ${target} --override "$override" --languages ${languages} $bootstrap $runtests_opt $excludecheck_opt --logserver $log_server $logname_opt $norebuild -o '--tarbin' || result=$? echo "Result of jenkins.sh: $result" if test ${result} -eq 0; then #Cases for which we want to trigger a benchmark #Any changes in this list may need mirroring in the parameterized #builds triggers at end of file. if test "x${target}" = xarm-linux-gnueabihf || test "x${target}" = xaarch64-linux-gnu || test "x${target}" = xarmv8l-linux-gnueabihf; then #Only trigger if jenkins.sh exits without error _and_ the #artifacts exist. (If --norebuild is set and logs already #exist, there will be no error but also no artifacts.) toolchain="`ls snapshots/gcc-linaro-*.tar.xz`" sysroot="`ls snapshots/sysroot-*-linaro-*.tar.xz`" #Repeat ls, because this way is robust to filenames with spaces if test `ls snapshots/gcc-linaro-*.tar.xz | wc -l` -eq 1 && test `ls snapshots/sysroot-*-linaro-*.tar.xz | wc -l` -eq 1; then cat > bench_parameters < artifact_parameters << EOF TOOLCHAIN=${toolchain} SYSROOT=${sysroot} EOF elif test `ls snapshots/gcc-linaro-*.tar.xz | wc -l` -ne 0 || test `ls snapshots/sysroot-*-linaro-*.tar.xz | wc -l` -ne 0; then echo "Multiple matches for toolchain and/or sysroot tarball, cannot dispatch benchmark" >&2 result=2 fi fi fi if $dont_fail; then result="0" fi exit $result - inject: #Create env vars used by achive step, below. properties-file: 'artifact_parameters' publishers: - archive: artifacts: '${TOOLCHAIN},${SYSROOT},source.tar.xz' only-if-success: true fingerprint: true allow-empty: true #Because JJB does not expose 'Use files in matrix child builds', we have #to copy the files to master and trigger from the copies. (We make sure #to initialize the files to empty so that we don't retrigger an old #benchmark by mistake.) #TODO: When we upgrade to a JJB that includes a fix for https://storyboard.openstack.org/#!/story/2000444 #we can simplify all this, and also stop hard-coding paths that Jenkins #should be deriving from the matrix. And we should then delete the comments #that this commit has inserted next to axes and the list of benchmarkable #targets in the shell, as they will be misleading. And we can switch #the 'node' parameter back from 'master' to 'tcwg-x86_64-build' and delete #the comment above 'node', too. - copy-to-master: includes: - bench_parameters - trigger-parameterized-builds: - project: tcwg-benchmark property-file: aarch64-linux-gnu/bench_parameters condition: 'SUCCESS' - project: tcwg-benchmark property-file: armv8l-linux-gnueabihf/bench_parameters condition: 'SUCCESS' - project: tcwg-benchmark property-file: arm-linux-gnueabihf/bench_parameters condition: 'SUCCESS' - workspace-cleanup: include: - "**/.git" dirmatch: true