aboutsummaryrefslogtreecommitdiff
path: root/tcwg-benchmark.yaml
blob: 49b70f5abee5622ea6eb7b78822d90d40f53c084 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
#See http://docs.openstack.org/infra/jenkins-job-builder/genindex.html
- job:
    name: tcwg-benchmark
    project-type: freestyle
    node: master
    display-name: 'TCWG BBB Benchmark'
    description: 'Dispatch TCWG benchmark jobs to LAVA'
    block-downstream: true
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            tcwg:
                - job-build
                - job-cancel
        - build-discarder:
            days-to-keep: 30
            num-to-keep: 30
            artifact-days-to-keep: 2
            artifact-num-to-keep: 1
    parameters:
        - string:
            name: LAVA_JOB_NAME
            description: |
              Name of job from LAVA's point-of view, in other words what you
              will see in the LAVA UI.
              If left blank, will be ${BENCHMARK}-${TARGET_CONFIG}-${TRIPLE}-jenkins.
        - choice:
            name: BENCHMARK
            choices:
              - Coremark-Pro
              - CPU2000
              - CPU2006
              - fakebench
        - string:
            name: TARGET_CONFIG
            description: |
              Space-separated list of target configuration(s) to run on.
              At time of writing, may be:<br/>
              <ul>
                <li>Juno-A53</li>
                <li>Juno-A57</li>
                <li>KVM</li>
              </ul>
              Where 'KVM' is an x86.
        - choice:
            name: TRIPLE
            choices:
              - (Derive from TOOLCHAIN)
              - (Native)
              - arm-linux-gnueabihf
              - aarch64-linux-gnu
              - armv8l-linux-gnueabihf
              - arm-linux-gnueabi
              - armeb-linux-gnueabihf
              - aarch64-none-elf
              - aarch64_be-none-elf
            description: |
              A triple describing the target (and its execution state).
        - string:
            name: PREBUILT
            description: |
              A prebuilt benchmark tarball as produced by abe or direct make invocation.
              <ul>
                <li>If produced by abe, tar up the contents of <code>build/&lt;host&gt;/&lt;target&gt;</code>.</li>
                <li>If produced by direct make, tar up the top-level git repository (but you can exclude the <code>.git</code> dir).</li>
              </ul>
              Leave blank if you have supplied <code>TOOLCHAIN</code>.<br/>
              If this option is given, no build will be performed.
        - string:
            name: TOOLCHAIN
            description: |
              Toolchain to use. Give local filesystem path, or formats interpretable by wget or scp/rsync.<br/>
              Leave blank if you have supplied <code>PREBUILT</code>.<br/>
              <br/>
              <b>Examples</b><br/>
              <code>/usr/bin/gcc</code><br/>
              <code>https://releases.linaro.org/components/toolchain/binaries/latest-5.1/aarch64-linux-gnu/gcc-linaro-5.1-2015.08-x86_64_aarch64-linux-gnu.tar.xz</code><br/>
              <code>my.name@my.server:location_of_my_toolchain</code>
        - string:
            name: SYSROOT
            description: |
              Sysroot to use. Give local filesystem path, or formats interpretable by wget or scp/rsync.<br/>
              Leave blank if you have supplied <code>PREBUILT</code>, or to let toolchain use whatever sysroot it uses by default.<br/>
              <br/>
              <b>Examples</b><br/>
              <code>/home/foo/sysroots/sysroot_bar/</code><br/>
              <code>https://releases.linaro.org/components/toolchain/binaries/latest-5.1/arm-linux-gnueabihf/sysroot-linaro-glibc-gcc5.1-2015.08-arm-linux-gnueabihf.tar.xz</code><br/>
              <code>my.name@my.server:location_of_my_sysroot</code>
        - string:
            name: COMPILER_FLAGS
            description: |
              Flags to pass to compiler when building the benchmark.<br/>
              Leave blank if you have supplied <code>PREBUILT</code>.<br/>
              Nested quotes must be triple-escaped, for example:<br/>
              <code>-O3 -DFOO=\\\"FOO BAR\\\"</code>
        - string:
            name: MAKE_FLAGS
            description: |
              Flags to pass to make when building the benchmark.<br/>
              Leave blank if you have supplied <code>PREBUILT</code>.<br/>
              Nested quotes must be triple-escaped, for example:<br/>
              <code>-O3 FOO=\\\"FOO BAR\\\"</code>
        - string:
            name: RUN_FLAGS
            description: |
              Flags to pass to benchmark framework at runtime.<br/>
              <dl>
                <dt>SPEC Example</dt>
                <dd><code>--size test --iterations 1 int</code>
                    (run 1 iteration of the int suite using the test workload)
                </dd>
              </dl>
        - string:
            name: BUNDLE_STREAM
            default: '/anonymous/benchmark/'
            description: |
              Stream to dispatch results bundles to.<br/>
              If planning to run a series of related benchmarks, you might want
              to direct them all to the same stream. This stream can be created
              with lava-tool and managed in the LAVA web interface. It must
              be accessible to the user 'benchmark'. If you're dispatching the
              benchmark from here, it should be fine to make the stream
              accessible to everyone.
        - string:
            name: Overrides
            description: |
              Key=Value arguments, for expert users.<br/>
              Separate with whitespace, for example <code>TIMEOUT=3600 HOST_DEVICE_TYPE=juno</code>.
              Does not cope well if the Value side of Key=Value contains spaces.
    wrappers:
        - timeout:
            timeout: 120
        - workspace-cleanup
    builders:
      - shell: |
          #!/bin/bash
          set -eux
          set -o pipefail

          #Process TARGET_CONFIG
          TARGET_CONFIG="${TARGET_CONFIG,,}"

          #Process TRIPLE
          if test x"${TRIPLE:-}" = x"(Derive from TOOLCHAIN)"; then
            if test -z "${TOOLCHAIN:-}"; then
              echo "Cannot derive TRIPLE from TOOLCHAIN if no TOOLCHAIN is given!" >&2
              exit 1
            fi
            TRIPLE=
            for y in aarch64-linux-gnu arm-linux-gnueabihf \
                     armv8l-linux-gnueabihf arm-linux-gnueabi \
                     armeb-linux-gnueabihf aarch64-none-elf aarch64_be-none-elf
            do
              if echo "${TOOLCHAIN}" | grep -q "${y}"; then
              if test -z "${TRIPLE:-}"; then
                  TRIPLE="${y}"
                else
                  echo "Cannot derive unique TRIPLE from ambiguous TOOLCHAIN '${TOOLCHAIN}'" >&2
                  exit 1
                fi
              fi
            done
          #If TRIPLE is still empty then it means 'Native', which is what we want
          elif test x"${TRIPLE:-}" = x"(Native)"; then
            TRIPLE=
          fi

          #Defaults that user is unlikely to want to twiddle
          #TODO: If benchmarking ever merges back to mainline abe, set this to a suitable branch
          export TESTDEF_REVISION=benchmarking

          for x in ${Overrides:-}; do
            export "${x}"
          done

          #Force this _after_ overrides
          export LAVA_SERVER=192.168.16.2/RPC2/
          export BUNDLE_SERVER=lava.tcwglab/RPC2/

          #Process LAVA_JOB_NAME
          #Have to do this after overrides in case one of the inputs is overridden
          #If LAVA_JOB_NAME itself was overridden then it will not be touched.
          if test -z "${LAVA_JOB_NAME:-}"; then
            export LAVA_JOB_NAME="${BENCHMARK:-}_${TARGET_CONFIG:-}_${TRIPLE:-native}_jenkins"
          fi

          git clone --depth 1 -b ${TESTDEF_REVISION} https://git.linaro.org/toolchain/abe

          #Generate YAML, which will be converted to JSON
          yaml=`mktemp`
          bash -x ./abe/scripts/Benchmark.sh > "${yaml}"
          #Must be exported to be visible to Benchmark.sh - env vars that
          #happen to be job params are already exported, so don't need this

          #Parameters for post-build-lava
          cat << EOF > post_build_lava_parameters
          CUSTOM_JSON_URL=${BUILD_URL}/artifact/lava-jobdef.json

          #Server to dispatch to, as opposed to server where bundle stream is.
          #post-build-lava takes responsibility for specifying a protocol.
          LAVA_SERVER=${LAVA_SERVER}

          #DEVICE_TYPE to placate pbl - we don't actually use it
          DEVICE_TYPE=dummy-ssh
          EOF

          #Convert YAML to JSON
          git clone --depth 1 https://git.linaro.org/ci/lci-build-tools
          ./lci-build-tools/yaml-to-json.py "${yaml}" > lava-jobdef.json
          rm "${yaml}"

    publishers:
      - archive:
          artifacts: lava-jobdef.json
      - trigger-parameterized-builds:
        - project: tcwg-post-build-lava
          condition: SUCCESS
          property-file: post_build_lava_parameters
      - email:
          recipients: 'bernie.ogden@linaro.org'