tcwg-benchmark: New version of benchmarking job

In this experimental version we move away from matrix job.  Instead we
trigger the top-level job on tcwg-bmk-coordinator, which
1. splits up benchmarks into board-size chunks
2. triggers sub-jobs for per-board benchmarking

Unlike in tcwg-benchmark-{hw} jobs, boards are dynamically assigned
to benchmarking sub-jobs.  For this we create board lock files on
bkp-01.tcwglab:/home/tcwg-benchmark/boards/ , and grab the locks
before starting benchmarks on a particular board.

In this approach benchmarking boards are represented as executors on
tcwg-bmk-{hw} jenkins nodes, rather then individual tcwg-bmk-{hw}-N
jenkins nodes.  This allows flexible addition and removal of benchmarking
boards -- just add/remove additional executors and create/delete
board lock file under bkp-01.tcwglab:/home/tcwg-benchmark/boards/ .

Change-Id: I152da6ea6ae5b9665f32e0aca19900320345a538

1 files changed, 250 insertions, 0 deletions

diff --git a/tcwg-benchmark.yaml b/tcwg-benchmark.yaml
new file mode 100644
index 0000000000..db22846283
--- /dev/null
+++ b/tcwg-benchmark.yaml
@@ -0,0 +1,250 @@
+# Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg-benchmark.yaml.in and tcwg-benchmark/tcwg-benchmark.def. Do not edit.
+#BEGIN: tcwg/default.yaml.inc
+# -*- mode: Yaml -*-
+
+- property:
+    name: default-properties
+    properties:
+      - authorization:
+          tcwg-benchmark:
+            - job-read
+            - job-extended-read
+            - job-build
+            - job-cancel
+      - build-discarder:
+          days-to-keep: 30
+          num-to-keep: 100
+
+- scm:
+    name: jenkins-scripts
+    scm:
+      - git:
+          url: https://git.linaro.org/toolchain/jenkins-scripts.git
+          refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*
+          branches:
+            - $scripts_branch
+          basedir: jenkins-scripts
+          skip-tag: true
+          reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git
+          wipe-workspace: false
+          clean:
+            before: true
+          prune: true
+
+- scm:
+    name: bmk-scripts
+    scm:
+      - git:
+          url: https://git.linaro.org/toolchain/bmk-scripts.git
+          refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*
+          branches:
+            - $bmk_branch
+          basedir: bmk-scripts
+          skip-tag: true
+          reference-repo: /home/tcwg-buildslave/snapshots-ref/bmk-scripts.git
+          wipe-workspace: false
+          clean:
+            before: true
+          prune: true
+#END:   tcwg/default.yaml.inc
+
+- job:
+    name: tcwg-benchmark
+    project-type: freestyle
+    defaults: global
+    properties:
+      - default-properties
+    parameters:
+        - choice:
+            name: bmk_hw
+            choices:
+              - sq
+              - tk1
+              - tx1
+            description: 'Hardware to use for benchmarking'
+        - string:
+            name: toolchain_url
+            default: ''
+            description: 'Toolchain URL. Either http[s]://xxx.tar.xz, rsync://machine:/path/bin/target-triplet-, or ssh://machine/path/compiler'
+        - choice:
+            name: toolchain_type
+            choices:
+              - auto
+              - gnu
+              - llvm
+        - string:
+            name: sysroot
+            default: 'tarball'
+            description: 'Rsync-friendly path to the sysroot to run against; use libs from toolchain_url tarball/rsync if "tarball"; use system libs if empty'
+        - string:
+           name: bench_list
+           default: 'bzip2'
+           description: 'List of benchmarks to run (from spec2k6). Use "all" to run all of them.'
+        - string:
+            name: cflags
+            default: ''
+            description: 'CFLAGS to use'
+        - string:
+            name: ldflags
+            default: 'default'
+            description: 'LDFLAGS to use'
+        - choice:
+            name: extension
+            choices:
+              - default
+              - O2
+              - O3
+              - O2g
+              - O3g
+              - Ofast
+              - Os
+            description: 'Compilation profile'
+        - choice:
+            name: testmode
+            choices:
+              - verify
+              - build
+              - benchmark
+            description: 'Whether to just build or run full benchmarks'
+        - string:
+            name: iterations
+            default: '1'
+            description: 'Number of benchmark iterations'
+        - string:
+            name: results_id
+            default: '@hw_tag@/tcwg-benchmark/@build_num@'
+            description: 'ID for benchmarking results; results will appear in bkp-01.tcwglab:/home/tcwg-benchmark/results-$results_id/'
+        - bool:
+            name: forceinstall
+            default: 'false'
+            description: 'Force clean re-install of benchmark sources'
+        - choice:
+            name: run_profile
+            choices:
+              - parallel
+              - serial
+            description: 'Benchmark profile to use: parallel3x3 runs on 3 boards and 3 CPUs in parallel; parallel3x1 runs on 3 boards on 1 CPU in parallel; serial1 runs on a single board on 1 CPU.'
+        - string:
+            name: image_arch
+            default: 'arm64'
+            description: 'Docker image architecture -- arm64 or armhf'
+        - bool:
+            name: reboot
+            default: 'false'
+            description: 'Reboot the board before building and running the benchmarks.'
+        - string:
+            name: displaytag
+            default: ''
+            description: 'Tag to display in the Jenkins console, so that the build name is more meaningful'
+        - bool:
+            name: ignore_errors
+            default: 'false'
+            description: 'Ignore build errors as much as possible'
+        - string:
+            name: clean_older_than
+            default: '60'
+            description: 'Delete previous results older than X days (on board)'
+        - string:
+            name: builder
+            default: 'tcwg-x86_64-build'
+            description: 'Builder -- jenkins label or node'
+        - string:
+            name: scripts_branch
+            default: master
+            description: 'Jenkins-scripts branch to use'
+        - string:
+            name: bmk_branch
+            default: master
+            description: 'Bmk-scripts branch to use'
+        - node:
+            name: bmk_node
+            default-slaves:
+              - tcwg-bmk-coordinator
+            allowed-slaves:
+              - tcwg-bmk-coordinator
+              - tcwg-bmk-sq
+            description: 'Bmk node; used internally, do not touch'
+    disabled: false
+    concurrent: true
+    display-name: 'TCWG BBB Benchmark'
+    scm:
+      - jenkins-scripts
+      - bmk-scripts
+    wrappers:
+        - timeout:
+            timeout: 900
+        - timestamps
+        - ssh-agent-credentials:
+            users:
+                - 'tcwg-benchmark'
+        - build-name:
+            name: '#$BUILD_NUMBER-$displaytag'
+    builders:
+      - conditional-step:
+          condition-kind: execution-node
+          nodes:
+            - tcwg-bmk-coordinator
+          steps:
+            - shell:
+                command: |
+                  #!/bin/bash
+
+                  set -euf -o pipefail
+                  set -x
+
+                  rm -rf artifacts
+                  mkdir artifacts
+
+                  count=0
+                  while read -a bmks; do
+                    echo "bench_list=${bmks[@]}" > artifacts/bmk-$count-parameters
+                    count=$(($count+1))
+                  done < <(./bmk-scripts/break-up-bmks.sh \
+                             --bench_list "$bench_list" \
+                             --bmk_hw "$bmk_hw" \
+                             --run_profile "$run_profile")
+            - trigger-builds:
+                - project: tcwg-benchmark
+                  node-label-name: bmk_node
+                  nodel-label: tcwg-bmk-${bmk_hw}
+                  block: true
+                  parameter-factories:
+                    - factory: filebuild
+                      file-pattern: artifacts/bmk-*-parameters
+                      current-parameters: true
+                      node-label-name: bmk_node
+                      node-label: tcwg-bmk-$bmk_hw
+      - conditional-step:
+          condition-kind: not
+          condition-operand:
+            condition-kind: execution-node
+            nodes:
+              - tcwg-bmk-coordinator
+          steps:
+            - shell:
+                command: |
+                  #!/bin/bash
+
+                  set -euf -o pipefail
+                  set -x
+
+                  rm -rf artifacts
+                  mkdir artifacts
+
+                  while true; do
+                    for lock in $(set +f; ls -tr $HOME/boards/$NODE_NAME-*); do
+                      (
+                        flock -en -E 123 9 || exit 1
+                        NODE_NAME=$(cat $lock)
+                        export NODE_NAME
+                        exec ./jenkins-scripts/tcwg-benchmark.sh
+                      ) 9>$HOME/boards/$lock &
+                      res=0 && wait $! || res=$?
+                      if [ $res != 123 ]; then
+                        exit $res
+                      fi
+                    done
+                    sleep 30
+                  done
+                unstable-return: 125
+# checksum: 2dd91d0feb588fd69b481bb59490ccc0