blob: 5c3748b9f0ea8222dde1637acb32fce20e3fbf5f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
- job:
name: jdk8-terasort-benchmark
project-type: matrix
defaults: global
description: |
* Runs the terasort benchmark.
properties:
- authorization:
linaro:
- job-read
- build-discarder:
days-to-keep: 30
num-to-keep: 10
artifact-num-to-keep: 5
disabled: false
node: aarch64-06
display-name: 'OpenJDK 8 - Run terasort benchmark'
axes:
- axis:
type: user-defined
name: JVM_VARIANT
values:
- server
# SRDM: Disable client builds while waiting for LEG-347 to be fixed.
# - client
- axis:
type: user-defined
name: BUILD_TYPE
values:
- release
- axis:
type: slave
name: label
values:
- aarch64-06
execution-strategy:
sequential: true
wrappers:
- workspace-cleanup:
dirmatch: false
- timestamps
- matrix-tie-parent:
node: aarch64-06
builders:
# copyartifact is slow and the file is local
# copy instead of going back and forth between master <-> slave
# - copyartifact:
# project: jdk8-build-image
# filter: 'out/jdk8-${JVM_VARIANT}-${BUILD_TYPE}.tar.gz'
# target: incoming
# flatten: true
- copyartifact:
project: openjdk8-hadoop-LCA14
filter: 'out/openjdk8-hadoop-LCA14.tar.gz'
target: incoming
flatten: true
- shell: |
#!/bin/bash
set -exu
NGIGABYTES=1
## Extract jdk
rm -rf jdk8*
tar xf ~/workspace/jdk8-build-image/BUILD_TYPE/${BUILD_TYPE}/JVM_VARIANT/${JVM_VARIANT}/label/${NODE_NAME}/out/jdk8-${JVM_VARIANT}-${BUILD_TYPE}.tar.gz
export JAVA_HOME=${WORKSPACE}/jdk8-${JVM_VARIANT}-${BUILD_TYPE}
export PATH=${JAVA_HOME}/bin:$PATH
## Extract Hadoop pre-builts
rm -rf openjdk8-hadooop-LCA14
tar xf incoming/openjdk8-hadoop-LCA14.tar.gz
## Benchmark
# FIXME
#(cd incoming; wget --progress=dot -e dotbytes=10M http://openjdk-apm1/openjdk8-build-artefacts/${NGIGABYTES}GB.tar.gz; tar xf ${NGIGABYTES}GB.tar.gz)
(cd incoming; tar xf ~/srv/hadoop-terasort-reference-files/${NGIGABYTES}GB.tar.gz)
TERAGEN_BASELINE_DIR=${WORKSPACE}/incoming
HADOOP_DIR=${WORKSPACE}/openjdk8-hadoop-LCA14
rm -rf out
mkdir out
sed -i '/^export JAVA_HOME=/d' ${HADOOP_DIR}/conf/hadoop-env.sh
echo "export JAVA_HOME=$JAVA_HOME" >> ${HADOOP_DIR}/conf/hadoop-env.sh
sed -i 's|/work/${user.name}/hadoop-tmp|${user.home}/hadoop/tmp|' ${HADOOP_DIR}/conf/core-site.xml
source ${HADOOP_DIR}/env.sh
which hadoop
which java
which hdfs
java -version
stop-dfs.sh
stop-yarn.sh
rm -rf ${HOME}/hadoop/tmp
mkdir -p ${HOME}/hadoop/tmp
hdfs namenode -format -force
start-dfs.sh
start-yarn.sh
# Need time for the datanodes to materialise.
sleep 30
jps
hadoop fs -mkdir -p /user/$USER
hadoop fs -copyFromLocal $TERAGEN_BASELINE_DIR/${NGIGABYTES}GB /user/$USER
trap "stop-dfs.sh; stop-yarn.sh" EXIT
elapsed_time_file=$(mktemp /tmp/benchmark-terasort-XXXXXX.$$)
HADOOP_OPTS=-${JVM_VARIANT} /usr/bin/time -o $elapsed_time_file -f "%e" terasort ${NGIGABYTES}GB ${NGIGABYTES}GB-sorted
hadoop fs -rm -R ${NGIGABYTES}GB-sorted
sed -i 's/\..*//' $elapsed_time_file
elapsed_time=$(cat $elapsed_time_file)
date_as_epoch=$(date --date="$(date +'%Y-%m-%d')" +%s)
echo "$date_as_epoch,$JVM_VARIANT,$NGIGABYTES,$elapsed_time" > out/terasort-results-${JVM_VARIANT}-${BUILD_TYPE}.csv
rm -rf incoming/${NGIGABYTES}*
publishers:
- archive:
artifacts: 'out/terasort-results-*.csv'
|
