path: root/tcwg-buildfarm.yaml

- job:
    name: tcwg-buildfarm
    project-type: matrix
    defaults: global
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            linaro:
                - job-build
                - job-cancel
        - build-discarder:
            days-to-keep: 30
            num-to-keep: 100
    parameters:
        - string:
            name: override
            default: ''
            description: 'Versions of components to build; e.g. binutils=binutils-gdb.git/linaro_binutils-2_25-branch gcc=gcc.git~linaro/gcc-5-branch glibc=glibc.git~release/2.21/master'
        - string:
            name: host_x86_64_languages
            default: 'default'
            description: 'Languages to build for x86_64-hosted toolchains, e.g., "c,c++" or "all" or "default"'
        - string:
            name: host_aarchXX_languages
            default: 'c,c++'
            description: 'Languages to build for AArchXX-hosted toolchains, e.g., "c,c++" or "all" or "default"'
        - string:
            name: runtests
            default: 'x86_64-host'
            description: 'Run toolchain testsuites: "yes", "no", "x86_64-host"'
        - bool:
            name: try_bootstrap
            default: 'true'
            description: 'Attempt to bootstrap GCC for compatible host and target combinations'
        - string:
            name: host_x86_64_excludecheck
            default: 'gdb'
            description: 'Do not run "make check" on these components on x86_64'
        - string:
            name: host_aarchXX_excludecheck
            default: 'gcc'
            description: 'Do not run "make check" on these components on AArchXX'
        - string:
            name: extraconfig
            default: ''
            description: 'Extra configuration files, syntax tool=path'
        - bool:
            name: rebuild
            default: true
            description: 'Rebuild the toolchain even if results are already on logserver'
        - string:
            name: log_name
            default: '${JOB_NAME}-${BUILD_NUMBER}/$(uname -m).$target'
            description: 'Logname directory on logserver'
        - bool:
            name: dont_fail
            default: false
            description: 'Do not fail the build'
        - string:
            name: log_server
            default: dev-01.tcwglab:/home/tcwg-buildslave/logs
            description: 'Log_Server'
        - string:
            name: targets
            default: 'cross_main cross_aux native_x86'
            description: 'Which target toolchains to build: cross_main cross_aux native_x86 native_armv8'
        - string:
            name: abe_branch
            default: refs/remotes/origin/stable
            description: 'ABE revision to test'
    disabled: false
    node: tcwg-x86_64-cam
    child-workspace: $target
    concurrent: true
    display-name: 'TCWG AAA BuildFarm'
    scm:
        - git:
            url: https://git-us.linaro.org/toolchain/abe.git
            refspec: +refs/changes/*:refs/remotes/changes/*
            branches:
                - $abe_branch
            skip-tag: true
            shallow-clone: true
            wipe-workspace: true
    axes:
        - axis:
            type: slave
            name: label
            values:
                - docker-trusty-amd64-tcwg-build
                - docker-trusty-i386-tcwg-build
                - docker-trusty-arm64-tcwg-build
                - docker-trusty-armhf-tcwg-build
        - axis:
            type: user-defined
            name: target
            values:
                - aarch64-linux-gnu
                - armv8l-linux-gnueabihf
                - arm-linux-gnueabi
                - arm-linux-gnueabihf
                - armeb-linux-gnueabihf
                - aarch64-none-elf
                - aarch64_be-none-elf
                - native
    execution-strategy:
        combination-filter: |
            label=="docker-trusty-amd64-tcwg-build" || target=="native"
        sequential: false
    wrappers:
        - timeout:
            timeout: 600
        - timestamps
        - ssh-agent-credentials:
            # tcwg-buildslave user id
            users:
                - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a'
        - build-name:
            name: '#${BUILD_NUMBER}'
    builders:
        - shell: |
            #!/bin/bash

            set -ex

            mkdir -p artifacts/

            case "$target--$(uname -m)" in
              native--aarch64) target_kind=native_aarch64 ;;
              native--arm*) target_kind=native_aarch32 ;;
              native--x86_64|native--i686) target_kind=native_x86 ;;
              armeb-linux-gnueabihf--*|*-none-elf--*) target_kind=cross_aux ;;
              *-linux-gnu*--*) target_kind=cross_main ;;
              *) echo "ERROR: Cannot handle target: $target"; exit 1 ;;
            esac

            if ! echo "$targets" | grep -q "$target_kind"; then
              echo "NOTE: Skipping target $target because $target_kind is not in list: $targets"
              # Create at least an (empty) artifact so that Jenkins does not complain
              echo > artifacts/target-skipped.txt
              exit 0
            fi

            case "$target_kind-$runtests-$(uname -m)" in
              "cross_main"-"yes"-*|"cross_main"-"x86_64-host"-"x86_64"|"cross_main"-"x86_64-host"-"i686")
                case "$target" in
                  aarch64*) container_arch=arm64 ;;
                  arm*) container_arch=armhf ;;
                  *) echo "ERROR: Wrong target $target"; exit 1 ;;
                esac
                docker_host="$(echo $DOCKER_HOST | sed -e "s#^tcp://\(.*\):.*#\1#")"
                ssh_port="$(ssh $docker_host docker port $DOCKER_CONTAINER_ID 22)"
                ssh_port="$(echo $ssh_port | cut -d: -f 2)"
                cat > test-container.param << EOF
            container_name=docker-trusty-$container_arch-tcwg-test
            upstream_dest=$docker_host:$ssh_port:$WORKSPACE/test-container.data
            watchdog=upstream
            EOF
                ;;
            esac
            if echo "$override" | grep -q -e "--extraconfigdir ../config/gcc5" \
               || true \
               && [ x"$target" = x"aarch64-linux-gnu" ]; then
              # Sanitizers in GCC-5 and GCC-6 toolchains for aarch64-linux-gnu
              # don't support 48-bit VA.  Therefore don't run them on Xenial tester.
              rm -f test-container.param
            fi
        - conditional-step:
            condition-kind: file-exists
            condition-filename: test-container.param
            steps:
                - trigger-builds:
                    - project: tcwg-test-container-matrix
                      property-file: test-container.param
                      block: false
        - conditional-step:
            condition-kind: not
            condition-operand:
                condition-kind: file-exists
                condition-filename: artifacts/target-skipped.txt
            steps:
              - shell: |
                  #!/bin/bash

                  set -ex

                  if [ -f test-container.param ]; then
                    runtests_opt="--runtests --testcontainerfile $WORKSPACE/test-container.data"
                  else
                    case "$target_kind-$runtests-$(uname -m)" in
                      *-"yes"-*|*-"x86_64-host"-"x86_64"|*-"x86_64-host"-"i686")
                        runtests_opt="--runtests"
                      ;;
                      *)
                        runtests_opt=""
                      ;;
                    esac
                  fi

                  case $(uname -m) in
                    x86_64|i686) languages="$host_x86_64_languages" ;;
                    *) languages="$host_aarchXX_languages" ;;
                  esac

                  if $try_bootstrap; then
                    bootstrap="--bootstrap"
                  else
                    bootstrap=""
                  fi

                  if [ x"$log_name" != x"" ]; then
                    eval "logname_opt=\"--logname $log_name\""
                  fi

                  if $rebuild; then
                    norebuild=""
                  else
                    norebuild="--norebuild"
                  fi

                  case $(uname -m) in
                    x86_64|i686) excludecheck="$host_x86_64_excludecheck" ;;
                    *) excludecheck="$host_aarchXX_excludecheck" ;;
                  esac

                  excludecheck_opt=""
                  for testsuite in $excludecheck; do
                    excludecheck_opt="$excludecheck_opt --excludecheck $testsuite"
                  done

                  # If there were extraconfig options, extract the
                  # corresponding repo
                  if test -n "$extraconfig" ; then
                    git clone http://git.linaro.org/people/christophe.lyon/abe-extraconfigs.git
                  fi

                  extraconfig_opt=""
                  for config in $extraconfig; do
                    case ${extraconfig} in
                      *=*)
                        tool=${extraconfig%=*}
                        value=${extraconfig#*=}
                        ;;
                      *)
                        echo ERROR: wrong extraconfig: $extraconfig
                        exit 1
                        ;;
                    esac
                    extraconfig_opt="${extraconfig_opt} --extraconfig $tool=$PWD/abe-extraconfigs/$value"
                  done

                  ulimit -u 5000

                  result="0"
                  /bin/bash -x ./jenkins.sh --target ${target} --override "$override" --languages ${languages} $bootstrap $runtests_opt $excludecheck_opt ${extraconfig_opt} --logserver $log_server $logname_opt $norebuild || result=$?

                  # If we actually built a toolchain, save its manifest.
                  # If the build was a no-op (results already present on log_server),
                  # copy the existing ones.
                  if [ -d _build/builds ]; then
                    manifest="`find _build/builds/ -name destdir -prune -o -name \*manifest.txt -print`"
                  else
                    manifest=""
                  fi
                  if [ x"${manifest}" != x ]; then
                    cp ${manifest} artifacts/
                  else
                    # We need to do the same name expansion as jenkins.sh
                    eval dir="$log_name"
                    scp "$log_server/$dir/*manifest.txt" artifacts/
                  fi

                  # Save logs and their directory structure
                  rsync -a --exclude="/logs/" --include="*/" --include="*.log" --include="*.err" --include="*.out" --include="*.txt" --exclude="*" --prune-empty-dirs ./ ./logs/
                  if ! tar -I pxz -cf ./artifacts/logs.tar.xz ./logs/ >/dev/null 2>&1; then
                    # PXZ can [rarely] fail due to high RAM usage, so fallback to normal XZ.
                    rm -f ./artifacts/logs.tar.xz
                    tar -cJf ./artifacts/logs.tar.xz ./logs/
                  fi

                  echo "Result of jenkins.sh: $result"
                  if $dont_fail; then
                    result="0"
                  fi
                  exit $result
    publishers:
        - archive:
            artifacts: 'artifacts/*'
            latest-only: false