path: root/tcwg-benchmark.yaml.in

#include tcwg/default.yaml.inc

- job:
    name: tcwg-benchmark#{BACKEND}
    project-type: freestyle
    defaults: global
    properties:
      - default-properties
    parameters:
        - choice:
            name: bmk_hw
            choices:
              - apm
              - sq
              - stm32
              - tk1
              - tx1
            description: 'Hardware to use for benchmarking'
        - string:
            name: toolchain_url
            default: ''
            description: |
              Toolchain URL. Either http[s]://xxx.tar.xz,
              rsync://machine:/path/bin/target-triplet-, or
              ssh://machine/path/compiler.  You need to specify 'builder'
              setting below when using non-ssh:// toolchain_urls.
        - choice:
            name: toolchain_type
            choices:
              - auto
              - gnu
              - llvm
        - string:
            name: sysroot
            default: 'tarball'
            description: 'Rsync-friendly path to the sysroot to run against; use libs from toolchain_url tarball/rsync if "tarball"; use system libs if empty'
        - string:
           name: bench_list
           default: 'all'
           description: 'List of benchmarks to run (from spec2k6). Use "all" to run all of them.'
        - string:
            name: cflags
            default: ''
            description: 'CFLAGS to use'
        - string:
            name: ldflags
            default: 'default'
            description: 'LDFLAGS to use'
        - choice:
            name: extension
            choices:
              - default
              - O2
              - O3
              - O2g
              - O3g
              - Ofast
              - Os
            description: 'Compilation profile'
        - choice:
            name: testmode
            choices:
              - verify
              - build
              - benchmark
            description: 'Whether to just build or run full benchmarks'
        - string:
            name: iterations
            default: '1'
            description: 'Number of benchmark iterations'
        - string:
            name: results_id
#if !BACKEND__backend
            default: '@hw_tag@/tcwg-benchmark/@build_num@'
#endif
            description: 'ID for benchmarking results; results will appear in bkp-01.tcwglab:/home/tcwg-benchmark/results-$results_id/'
        - bool:
            name: forceinstall
            default: 'false'
            description: 'Force clean re-install of benchmark sources'
        - choice:
            name: run_profile
            choices:
              - parallel
              - serial
            description: |
              parallel - Run benchmarks in parallel on all CPU cores.
              serial - Run benchmarks serially on a single CPU core.
        - choice:
            name: board_selection
            choices:
              - balance
              - one_board
              - all_boards
            description: |
              balance - Distribute benchmarks across all available boards.
              1board - Run all benchmarks on a single board.
              allboards - Run all benchmarks on all boards (e.g., to check that all boards give the same performance result).
        - choice:
            name: image_arch
            choices:
              - arm64
              - armhf
              - amd64
            description: 'Docker image architecture'
        - bool:
            name: reboot
            default: 'false'
            description: 'Reboot the board before building and running the benchmarks.'
        - string:
            name: displaytag
            default: '$bmk_hw'
            description: 'Tag to display in the Jenkins console, so that the build name is more meaningful'
        - bool:
            name: ignore_errors
            default: 'false'
            description: 'Ignore build errors as much as possible'
        - string:
            name: clean_older_than
            default: '60'
            description: 'Delete previous results older than X days (on board)'
        - string:
            name: builder
            default: 'bmk_board'
            description: |
              Jenkins label or node to procure a builder from and the docker
              image distro tag to use for the build environment.
              This is needed only for non-ssh:// toolchain_urls.
              Special value of bmk_board uses benchmarking board as the builder
        - string:
            name: scripts_branch
            default: master
            description: 'Jenkins-scripts branch to use'
        - string:
            name: bmk_branch
            default: master
            description: 'Bmk-scripts branch to use'
        - node:
            name: bmk_node
            default-slaves:
              - tcwg-bmk-bkp-01
            allowed-slaves:
              - tcwg-bmk-bkp-01
              - tcwg-bmk-apm
              - tcwg-bmk-sq
              - tcwg-bmk-stm32
              - tcwg-bmk-tk1
              - tcwg-bmk-tx1
            description: 'Bmk node; used internally, do not touch'
    disabled: false
    concurrent: true
    display-name: 'TCWG Benchmark#{BACKEND}'
    workspace: workspace/tcwg-benchmark-${NODE_NAME}_$EXECUTOR_NUMBER
    scm:
      - jenkins-scripts
      - bmk-scripts
    wrappers:
        - timeout:
            timeout: 900
        - timestamps
        - ssh-agent-credentials:
            users:
                - 'tcwg-benchmark'
        - build-name:
            name: '#$BUILD_NUMBER-$displaytag'
    builders:
#if !BACKEND__backend
      - shell:
          command: |
            #!/bin/bash

            set -euf -o pipefail
            set -x

            rm -rf artifacts
            mkdir artifacts

            case "$bmk_hw:$image_arch" in
              apm:arm64) hw_tag=apm_64 ;;
              apm:armhf) hw_tag=apm_32 ;;
              sq:arm64) hw_tag=sq_64 ;;
              sq:armhf) hw_tag=sq_32 ;;
              stm32:amd64) hw_tag=stm32 ;;
              tk1:armhf) hw_tag=tk1_32 ;;
              tx1:arm64) hw_tag=tx1_64 ;;
              tx1:armhf) hw_tag=tx1_32 ;;
              *)
                echo "ERROR: Unsupported bmk_hw:image_arch: $bmk_hw:$image_arch"
                exit 1
                ;;
            esac

            results_id=$(echo "$results_id" | sed \
                           -e "s/@build_num@/$BUILD_NUMBER/g" \
                           -e "s/@hw_tag@/$hw_tag/g")

            n_boards=$(set +f; ls -tr $HOME/boards/tcwg-bmk-$bmk_hw-*.lock | wc -l)
            count=1
            while read -a bmks; do
              cat > artifacts/bmk-$count-parameters <<EOF
            bench_list=${bmks[@]}
            displaytag=$BUILD_NUMBER+$count-$displaytag
            bmk_node=tcwg-bmk-$bmk_hw
            results_id=$results_id
            EOF
              count=$(($count+1))
            done < <(./bmk-scripts/break-up-bmks.sh \
                       --bench_list "$bench_list" \
                       --bmk_hw "$bmk_hw" \
                       --n_boards "$n_boards" \
                       --run_profile "$run_profile" \
                       --board_selection "$board_selection")
      - trigger-builds:
          - project: tcwg-benchmark_backend
            block: true
            current-parameters: true
            parameter-factories:
              - factory: filebuild
                file-pattern: artifacts/bmk-*-parameters
#else
      - shell:
          command: |
            #!/bin/bash

            set -ef -o pipefail
            set -x

            rm -rf artifacts
            mkdir -p artifacts/jenkins

            # Retry benchmarking once, just in case we selected a board that
            # recently went offline.
            bmk_retry=true

            while true; do
              for lock in $(set +f; ls -tr $HOME/boards/$NODE_NAME-*.lock); do
                bmk_start=$(date +%s)
                (
                  flock -en -E 124 9
                  touch $lock
                  boardname=$(cat <&9)
                  export boardname
                  echo "#$BUILD_NUMBER-$displaytag-$boardname" > artifacts/jenkins/build-name
                  exec ./jenkins-scripts/tcwg-benchmark.sh \
                    --boardname "$boardname" \
                    --toolchain_url "$toolchain_url" \
                    --toolchain_type "$toolchain_type" \
                    --sysroot "$sysroot" \
                    --bench_list "$bench_list" \
                    --cflags "$cflags" \
                    --ldflags "$ldflags" \
                    --extension "$extension" \
                    --testmode "$testmode" \
                    --iterations "$iterations" \
                    --results_id "$results_id" \
                    --forceinstall "$forceinstall" \
                    --run_profile "$run_profile" \
                    --image_arch "$image_arch" \
                    --reboot "$reboot" \
                    --ignore_errors "$ignore_errors" \
                    --clean_older_than "$clean_older_than" \
                    --builder "$builder" \
                    --WORKSPACE "$WORKSPACE"
                ) 9<$lock &
                res=0 && wait $! || res=$?
                if [ $res != 0 ]; then
                  if [ $res = 124 ]; then
                    # Try to lock next board.
                    continue
                  elif $bmk_retry; then
                    bmk_retry=false
                    if [ $res = 125 ]; then
                      echo "Failed to prepare board $lock for benchmarking, retry with a different board." >> artifacts/jenkins/mail-body.txt
                      continue
                    elif [ $(($(date +%s) - $bmk_start)) -le 300 ]; then
                      echo "Benchmarking on board $lock failed due to /something/ within 5 minutes -- give it another chance." >> artifacts/jenkins/mail-body.txt
                      continue
                    fi
                  fi
                  echo "Benchmarking on board $lock failed -- giving up." >> artifacts/jenkins/mail-body.txt
                fi
                if [ -f artifacts/jenkins/mail-body.txt ]; then
                  if [ $res = 0 ]; then
                    echo "Benchmarking succeeded" >> artifacts/jenkins/mail-body.txt
                  fi
                  echo "$BUILD_URL" >> artifacts/jenkins/mail-body.txt
                  echo "maxim.kuvyrkov@linaro.org" > artifacts/jenkins/mail-recipients.txt
                fi
                exit $res
              done
              sleep 30
            done
          unstable-return: 125
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
#endif
    publishers:
      - archive:
          artifacts: 'artifacts/**'
          latest-only: false
          allow-empty: true
      - email-ext:
          recipients: |
            ${FILE,path="artifacts/jenkins/mail-recipients.txt"}
          content-type: text
          body: |
            ${FILE,path="artifacts/jenkins/mail-body.txt"}
          failure: true
          success: true
          aborted: true
          send-to:
            - recipients