blob: 16606d73c2e9a666fe7b80cd2e1a4a36bd0804ee (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
# Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg-update-containers.yaml.in and tcwg-update-containers/tcwg-update-bmk-containers.def. Do not edit.
#BEGIN: tcwg/default.yaml.inc
# -*- mode: Yaml -*-
- property:
name: default-properties
properties:
- authorization:
anonymous:
- job-read
- job-extended-read
everyone-flat:
- job-build
- job-cancel
- build-discarder:
days-to-keep: 30
- scm:
name: jenkins-scripts
scm:
- git:
url: https://git.linaro.org/toolchain/jenkins-scripts.git
refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*
branches:
- $scripts_branch
basedir: jenkins-scripts
skip-tag: true
reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git
wipe-workspace: false
clean:
before: true
prune: true
#END: tcwg/default.yaml.inc
- job:
name: tcwg-update-bmk-containers
project-type: freestyle
defaults: global
properties:
- authorization:
anonymous:
- job-read
- job-extended-read
everyone-flat:
- job-build
- job-cancel
- build-discarder:
days-to-keep: 30
num-to-keep: 100
- throttle:
max-per-node: 1
option: project
parameters:
- label:
name: nodes
default: tcwg-bmk-pool
all-nodes: true
matching-label: 'allCases'
description: 'Machines to run on'
- string:
name: distro
default: 'default'
description: 'Distro version to use.'
- bool:
name: force
default: 'false'
description: 'Whether to force update even with no changes in image'
- bool:
name: verbose
default: 'true'
description: 'Whether to be verbose'
- string:
name: scripts_branch
default: master
description: 'Scripts revision to use'
disabled: false
node: tcwg-coordinator
concurrent: true
display-name: 'TCWG CCC Update bmk containers'
# We need to unshare workspace with $NODE_NAME in the path to
# correctly run on tcwg-bmk-* nodes.
workspace: workspace/tcwg-update-bmk-containers_$EXECUTOR_NUMBER/$NODE_NAME
scm:
- jenkins-scripts
triggers:
- timed: '@daily'
wrappers:
- timeout:
timeout: 600
- timestamps
- ssh-agent-credentials:
users:
- 'tcwg-benchmark'
# tcwg-buildslave user id
# ??? Do we need tcwg-buildslave's keys for BMK containers?
- 'e0958a95-204f-4c14-a66c-5e2be6c5d50a'
- build-name:
name: '#${BUILD_NUMBER}-${NODE_NAME}'
builders:
- shell:
command: |
#!/bin/bash
set -ex
# See: LABEL_SED below
set -o pipefail
rm -rf artifacts
mkdir artifacts
echo "$BUILD_URL" > artifacts/mail-body.txt
echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" > artifacts/mail-recipients.txt
declare -A pids
for lock in $(set +f; ls -tr $HOME/boards/$NODE_NAME-*.lock \
$HOME/boards/$NODE_NAME-*.bak*); do
(
flock -e 9
touch $lock
board=$(cat <&9)
# Start the jenkins container so that it can process
# all the queued cleanup and maintenance tasks.
node=$(basename "$board" .tcwglab)
./jenkins-scripts/tcwg-update-bmk-containers.sh \
--board "$board" \
--distro "$distro" \
--force "$force" \
--node "$node" \
--verbose "$verbose"
# Wait for jenkins container to become idle
while sleep 60; do
n_busy=$(source jenkins-scripts/jenkins-helpers.sh
print_number_of_busy_executors "$node")
if [ "$n_busy" = "0" ]; then
break
fi
done
# Now stop the jenkins container so that it's not terminated
# midway some other build by a starting benchmarking job.
ssh -Snone $board docker stop "$node"
) 9<$lock 2>&1 | sed -e "s/^/$(basename $lock): /" &
# LABEL_SED: We need pipefail to get correct result of sub-shell
# "( foo ) | sed" instead of always-succeeding "sed".
pids[$(basename "$lock")]=$!
done
n_good_boards=0
for lock in "${!pids[@]}"; do
res=0 && wait "${pids[$lock]}" || res=$?
case "$res:$lock" in
"0":*".lock")
n_good_boards=$(($n_good_boards + 1))
echo "$lock: SUCCESS" >> artifacts/mail-body.txt
;;
"0":*".bak")
n_good_boards=$(($n_good_boards + 1))
echo "$lock: UNEXPECTED SUCCESS" >> artifacts/mail-body.txt
# Move boards with no STG ticket (e.g., no ".bak.STG-1234")
# back into service.
mv "$HOME/boards/$lock" "$HOME/boards/${lock%.bak}.lock"
;;
"0":*)
# The board appears fine, but it has STG ticket assigned
# to it (e.g., ".bak.STG-1234").
echo "$lock: UNEXPECTED SUCCESS" >> artifacts/mail-body.txt
;;
*:*".lock")
echo "$lock: UNEXPECTED FAILURE" >> artifacts/mail-body.txt
# Move offline boards out of service.
mv "$HOME/boards/$lock" "$HOME/boards/${lock%.lock}.bak"
;;
*:*)
echo "$lock: EXPECTED FAILURE" >> artifacts/mail-body.txt
;;
esac
done
# tcwg-benchmark doesn't have ssh access to ci.linaro.org,
# so use tcwg-buildslave's credentials.
n_executors=$(ssh -p2222 -l tcwg-buildslave@linaro.org \
ci.linaro.org get-node "$NODE_NAME" \
| grep "numExecutors")
n_executors=$(echo "$n_executors" \
| sed -e "s#.*<numExecutors>\([0-9]\+\)</numExecutors>.*#\1#")
if [ x"$n_executors" != x"$n_good_boards" ]; then
echo "UNEXPECTED: Updating number of executors on $NODE_NAME from $n_executors to $n_good_boards" \
>> artifacts/mail-body.txt
if [ "$n_good_boards" = "0" ]; then
# Setting executors to "0" will mightily confuse jenkins;
# the node with 0 executors will be stuck in limbo.
n_good_boards=1
fi
ssh -p2222 -l tcwg-buildslave@linaro.org \
ci.linaro.org get-node "$NODE_NAME" \
| sed -e "s#<numExecutors>\([0-9]\+\)</numExecutors>#<numExecutors>$n_good_boards</numExecutors>#" \
| ssh -p2222 -l tcwg-buildslave@linaro.org \
ci.linaro.org update-node "$NODE_NAME"
fi
if ! grep -q UNEXPECTED artifacts/mail-body.txt; then
exit 0
fi
exit 1
publishers:
- email-ext:
recipients: |
${FILE,path="artifacts/mail-recipients.txt"}
content-type: text
body: |
${FILE,path="artifacts/mail-body.txt"}
failure: true
success: false
aborted: true
send-to:
- recipients
# checksum: 01bb3f2309f5a0ee5b87c5a82836419c
|
