tcwg-benchmark.yaml.in

#include tcwg/default.yaml.inc

- job:
    name: tcwg-benchmark#{BACKEND}
    project-type: freestyle
    defaults: global
    properties:
      - default-properties
    parameters:
        - choice:
            name: bmk_hw
            choices:
              - apm
              - sq
              - stm32
              - tk1
              - tx1
              - fx
              - qc
            description: 'Hardware to use for benchmarking'
        - string:
            name: toolchain_url
            default: ''
            description: |
              Toolchain URL. Either http[s]://xxx.tar.xz,
              rsync://machine:/path/bin/target-triplet-, or
              ssh://machine/path/compiler.  You need to specify 'builder'
              setting below when using non-ssh:// toolchain_urls.
        - choice:
            name: toolchain_type
            choices:
              - auto
              - gnu
              - llvm
        - string:
            name: sysroot
            default: 'tarball'
            description: 'Rsync-friendly path to the sysroot to run against; use libs from toolchain_url tarball/rsync if "tarball"; use system libs if empty'
        - string:
           name: bench_list
           default: 'all'
           description: 'List of benchmarks to run. Can be for ex : all (i.e. spec2006), spec2017_speed, or coremark'
        - string:
            name: cflags
            default: ''
            description: 'CFLAGS to use'
        - string:
            name: ldflags
            default: 'default'
            description: 'LDFLAGS to use'
        - choice:
            name: extension
            choices:
              - default
              - O2
              - O3
              - O2g
              - O3g
              - Ofast
              - Os
            description: 'Compilation profile'
        - choice:
            name: testmode
            choices:
              - verify
              - build
              - benchmark
            description: 'Whether to just build or run full benchmarks'
        - string:
            name: iterations
            default: '1'
            description: 'Number of benchmark iterations'
        - string:
            name: results_dest
            default: 'bkp-01.tcwglab:/home/tcwg-benchmark/results-@hw_tag@/tcwg-benchmark/@build_num@'
            description: 'Destination where to push results..'
        - bool:
            name: forceinstall
            default: 'false'
            description: 'Force clean re-install of benchmark sources'
        - choice:
            name: run_profile
            choices:
              - parallel_0x4
              - parallel
              - serial
            description: |
              parallel_0x4 - Build benchmark in parallel (all CPU cores), Run benchmark in parallel (max 4)
              parallel - Run benchmarks in parallel on all CPU cores.
              serial - Run benchmarks serially on a single CPU core.
        - choice:
            name: board_selection
            choices:
              - balance
              - one_board
              - all_boards
            description: |
              balance - Distribute benchmarks across all available boards.
              1board - Run all benchmarks on a single board.
              allboards - Run all benchmarks on all boards (e.g., to check that all boards give the same performance result).
        - choice:
            name: image_arch
            choices:
              - arm64
              - armhf
              - amd64
            description: 'Docker image architecture'
        - bool:
            name: reboot
            default: 'false'
            description: 'Reboot the board before building and running the benchmarks.'
        - string:
            name: displaytag
            default: '$bmk_hw'
            description: 'Tag to display in the Jenkins console, so that the build name is more meaningful'
        - bool:
            name: ignore_errors
            default: 'false'
            description: 'Ignore build errors as much as possible'
        - string:
            name: clean_older_than
            default: '60'
            description: 'Delete previous results older than X days (on board)'
        - string:
            name: builder
            default: 'bmk_board'
            description: |
              Jenkins label or node to procure a builder from and the docker
              image distro tag to use for the build environment.
              This is needed only for non-ssh:// toolchain_urls.
              Special value of bmk_board uses benchmarking board as the builder
        - string:
            name: scripts_branch
            default: master
            description: 'Jenkins-scripts branch to use'
        - string:
            name: bmk_branch
            default: master
            description: 'Bmk-scripts branch to use'
        - node:
            name: bmk_node
            default-slaves:
              - tcwg-bmk-bkp-01
            allowed-slaves:
              - tcwg-bmk-bkp-01
              - tcwg-bmk-apm
              - tcwg-bmk-sq
              - tcwg-bmk-stm32
              - tcwg-bmk-tk1
              - tcwg-bmk-tx1
              - tcwg-bmk-fx
              - tcwg-bmk-qc
            description: 'Bmk node; used internally, do not touch'
    disabled: false
    concurrent: true
    display-name: 'TCWG Benchmark#{BACKEND}'
    workspace: workspace/tcwg-benchmark-${NODE_NAME}_$EXECUTOR_NUMBER
    scm:
      - jenkins-scripts
      - bmk-scripts
    wrappers:
        - timeout:
            timeout: 1200
        - timestamps
        - ssh-agent-credentials:
            users:
                - 'tcwg-benchmark'
        - build-name:
            name: '#$BUILD_NUMBER-$displaytag ($bmk_hw:$image_arch)'
    builders:
#if !BACKEND__backend
      - shell:
          command: |
            #!/bin/bash

            set -euf -o pipefail
            set -x

            rm -rf artifacts
            mkdir -p artifacts/jenkins

            has_perf_logs=yes
            case "$bmk_hw:$image_arch" in
              apm:arm64) hw_tag=apm_64 ;;
              apm:armhf) hw_tag=apm_32 ;;
              sq:arm64) hw_tag=sq_64 ;;
              sq:armhf) hw_tag=sq_32 ;;
              stm32:amd64) hw_tag=stm32; has_perf_logs=no ;;
              tk1:armhf) hw_tag=tk1_32 ;;
              tx1:arm64) hw_tag=tx1_64 ;;
              tx1:armhf) hw_tag=tx1_32 ;;
              fx:arm64) hw_tag=fx_64 ;;
              fx:armhf) hw_tag=fx_32 ;;
              qc:arm64) hw_tag=qc_64 ;;
              qc:armhf) hw_tag=qc_32 ;;
              *)
                echo "ERROR: Unsupported bmk_hw:image_arch: $bmk_hw:$image_arch"
                exit 1
                ;;
            esac

            results_dest=$(echo "$results_dest" | sed \
                           -e "s/@build_num@/$BUILD_NUMBER/g" \
                           -e "s/@hw_tag@/$hw_tag/g")

            idle_boards="0"
            total_boards="0"
            for lock in $(set +f; ls -tr $HOME/boards/tcwg-bmk-$bmk_hw-*.lock); do
                (
                  flock -en -E 124 9
                ) 9<$lock &
                res=0 && wait $! || res=$?
                total_boards=$(($total_boards + 1))
                if [ $res = 124 ]; then
                  continue
                fi
                idle_boards=$(($idle_boards + 1))
            done

            n_boards=$((($idle_boards + $total_boards) / 2))
            # Distribute benchmarks among idle boards.
            if [ $n_boards -lt 1 ]; then
              n_boards="1"
            fi

            count=1
            while read -a bmks; do
              cat > artifacts/bmk-$count-parameters <<EOF
            bench_list=${bmks[@]}
            displaytag=$BUILD_NUMBER+$count-$displaytag
            bmk_node=tcwg-bmk-$bmk_hw
            results_dest=$results_dest
            EOF
              count=$(($count+1))
            done < <(./bmk-scripts/break-up-bmks.sh \
                       --bench_list "$bench_list" \
                       --bmk_hw "$bmk_hw" \
                       --n_boards "$n_boards" \
                       --run_profile "$run_profile" \
                       --board_selection "$board_selection")
            cat > artifacts/bmk-info <<EOF
            results_dest=$results_dest
            hw_tag=$hw_tag
            has_perf_logs=$has_perf_logs
            EOF
      - trigger-builds:
          - project: tcwg-benchmark_backend
            block: true
            current-parameters: true
            parameter-factories:
              - factory: filebuild
                file-pattern: artifacts/bmk-*-parameters
#else
      - shell:
          command: |
            #!/bin/bash

            set -ef -o pipefail
            set -x

            rm -rf artifacts
            mkdir -p artifacts/jenkins

            case "$bmk_hw:$image_arch" in
              apm:arm64) hw_tag=apm_64 ;;
              apm:armhf) hw_tag=apm_32 ;;
              sq:arm64) hw_tag=sq_64 ;;
              sq:armhf) hw_tag=sq_32 ;;
              stm32:amd64) hw_tag=stm32;;
              tk1:armhf) hw_tag=tk1_32 ;;
              tx1:arm64) hw_tag=tx1_64 ;;
              tx1:armhf) hw_tag=tx1_32 ;;
              fx:arm64) hw_tag=fx_64 ;;
              fx:armhf) hw_tag=fx_32 ;;
              qc:arm64) hw_tag=qc_64 ;;
              qc:armhf) hw_tag=qc_32 ;;
              *)
                echo "ERROR: Unsupported bmk_hw:image_arch: $bmk_hw:$image_arch"
                exit 1
                ;;
            esac

            # Retry benchmarking once, just in case we selected a board that
            # recently went offline.
            bmk_retry=true

            declare -gA bad_boards

            while true; do
              good_boards=()
              for lock in $(set +f; ls -tr $HOME/boards/$NODE_NAME-*.lock); do
                if [ x"${bad_boards[$lock]-unset}" = x"unset" ]; then
                  good_boards+=("$lock")
                fi
              done

              if [ ${#good_boards[@]} = 0 ]; then
                echo "Ran out of boards to try -- giving up." >> artifacts/jenkins/mail-body.txt
                exit 125
              fi

              for lock in "${good_boards[@]}"; do
                bmk_start=$(date +%s)
                (
                  flock -en -E 124 9
                  touch $lock
                  boardname=$(cat <&9)
                  export boardname
                  echo "#$BUILD_NUMBER-$displaytag-$boardname" > artifacts/jenkins/build-name
                  exec ./jenkins-scripts/tcwg-benchmark.sh \
                    --boardname "$boardname" \
                    --toolchain_url "$toolchain_url" \
                    --toolchain_type "$toolchain_type" \
                    --sysroot "$sysroot" \
                    --bench_list "$bench_list" \
                    --cflags "$cflags" \
                    --ldflags "$ldflags" \
                    --extension "$extension" \
                    --testmode "$testmode" \
                    --iterations "$iterations" \
                    --results_dest "$results_dest" \
                    --forceinstall "$forceinstall" \
                    --run_profile "$run_profile" \
                    --image_arch "$image_arch" \
                    --hw_tag "$hw_tag" \
                    --reboot "$reboot" \
                    --ignore_errors "$ignore_errors" \
                    --clean_older_than "$clean_older_than" \
                    --builder "$builder" \
                    --WORKSPACE "$WORKSPACE"
                ) 9<$lock &
                res=0 && wait $! || res=$?
                if [ $res != 0 ]; then
                  if [ $res = 124 ]; then
                    # Try to lock next board.
                    continue
                  elif $bmk_retry; then
                    bad_boards[$lock]="1"
                    bmk_retry=false
                    if [ $res = 125 ]; then
                      echo "Failed to prepare board $lock for benchmarking, retry with a different board." >> artifacts/jenkins/mail-body.txt
                      continue
                    elif [ $(($(date +%s) - $bmk_start)) -le 300 ]; then
                      echo "Benchmarking on board $lock failed due to /something/ within 5 minutes -- give it another chance." >> artifacts/jenkins/mail-body.txt
                      continue
                    fi
                  fi
                  echo "Benchmarking on board $lock failed -- giving up." >> artifacts/jenkins/mail-body.txt
                fi
                if [ -f artifacts/jenkins/mail-body.txt ]; then
                  if [ $res = 0 ]; then
                    echo "Benchmarking succeeded" >> artifacts/jenkins/mail-body.txt
                  fi
                  echo "$BUILD_URL" >> artifacts/jenkins/mail-body.txt
                  echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" > artifacts/jenkins/mail-recipients.txt
                fi
                exit $res
              done
              sleep 30
            done
          unstable-return: 125
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
#endif
    publishers:
      - archive:
          artifacts: 'artifacts/**'
          latest-only: false
          allow-empty: true
      - email-ext:
          recipients: |
            ${FILE,path="artifacts/jenkins/mail-recipients.txt"}
          content-type: text
          body: |
            ${FILE,path="artifacts/jenkins/mail-body.txt"}
          failure: true
          success: true
          aborted: true
          send-to:
            - recipients