aboutsummaryrefslogtreecommitdiff
path: root/jdkX-terasort-benchmark.yaml
blob: 6af837c61236e73e1d9119172143e6cc6a7bea72 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
- job:
    name: jdkX-terasort-benchmark
    project-type: matrix
    defaults: global
    description: |
        * Runs the terasort benchmark.
    properties:
        - authorization:
            linaro:
                - job-read
            openjdk-members:
                - job-extended-read
                - job-build
                - job-cancel
        - build-discarder:
            days-to-keep: 30
            num-to-keep: 10
            artifact-num-to-keep: 5
    disabled: false
    node: j12-qrep-01
    display-name: 'OpenJDK JDK - Run terasort benchmark'
    axes:
        - axis:
            type: user-defined
            name: JVM_VARIANT
            values:
                - server
                - client
        - axis:
            type: user-defined
            name: BUILD_TYPE
            values:
                - release
        - axis:
            type: slave
            name: label
            values:
                - j12-qrep-01
    execution-strategy:
        sequential: true
    wrappers:
        - workspace-cleanup:
            dirmatch: false
        - timestamps
        - matrix-tie-parent:
            node: j12-qrep-01
    builders:
# copyartifact is slow and the file is local
# copy instead of going back and forth between master <-> slave
#        - copyartifact:
#            project: jdkX-build-image
#            filter: 'out/jdkX-${JVM_VARIANT}-${BUILD_TYPE}.tar.gz'
#            target: incoming
#            flatten: true
        - copyartifact:
            project: openjdk8-hadoop-LCA14
            filter: 'out/openjdk8-hadoop-LCA14.tar.gz'
            target: incoming
            flatten: true
        - shell: |
            #!/bin/bash

            set -exu

            NGIGABYTES=1

            # client variant uses server with -XX:TieredStopAtLevel=1
            # zero is a third possible value.
            REAL_VARIANT=${JVM_VARIANT/client/server}

            ## Extract jdk
            rm -rf jdkX*
            tar xf ~/workspace/jdkX-build-image/BUILD_TYPE/${BUILD_TYPE}/JVM_VARIANT/${REAL_VARIANT}/label/${NODE_NAME}/out/jdkX-${REAL_VARIANT}-${BUILD_TYPE}.tar.gz

            export JAVA_HOME=${WORKSPACE}/jdkX-${REAL_VARIANT}-${BUILD_TYPE}
            export PATH=${JAVA_HOME}/bin:$PATH

            ## Extract Hadoop pre-builts
            rm -rf openjdk8-hadooop-LCA14
            tar xf incoming/openjdk8-hadoop-LCA14.tar.gz

            ## Benchmark
            # FIXME
            #(cd incoming; wget --progress=dot -e dotbytes=10M http://openjdk-apm1/openjdk8-build-artefacts/${NGIGABYTES}GB.tar.gz; tar xf ${NGIGABYTES}GB.tar.gz)
            (cd incoming; tar xf ~/srv/hadoop-terasort-reference-files/${NGIGABYTES}GB.tar.gz)

            TERAGEN_BASELINE_DIR=${WORKSPACE}/incoming
            HADOOP_DIR=${WORKSPACE}/openjdk8-hadoop-LCA14

            rm -rf out
            mkdir out

            sed -i '/^export JAVA_HOME=/d' ${HADOOP_DIR}/conf/hadoop-env.sh
            echo "export JAVA_HOME=$JAVA_HOME" >> ${HADOOP_DIR}/conf/hadoop-env.sh
            sed -i 's|/work/${user.name}/hadoop-tmp|${user.home}/hadoop/tmp|' ${HADOOP_DIR}/conf/core-site.xml

            source ${HADOOP_DIR}/env.sh

            which hadoop
            which java
            which hdfs
            java -version

            stop-dfs.sh
            stop-yarn.sh

            rm -rf ${HOME}/hadoop
            mkdir -p ${HOME}/hadoop/tmp
            hdfs namenode -format -force
            start-dfs.sh
            start-yarn.sh

            # Need time for the datanodes to materialise.
            sleep 30
            jps

            hadoop fs -mkdir -p /user/$USER
            hadoop fs -copyFromLocal $TERAGEN_BASELINE_DIR/${NGIGABYTES}GB /user/$USER

            trap "stop-dfs.sh; stop-yarn.sh" EXIT

            # Use only C1 compiler for client runs.
            if [ $JVM_VARIANT = "client" ]; then
                VARIANT_OPT=-XX:TieredStopAtLevel=1
            else
                VARIANT_OPT=-Dharmless.property
            fi

            elapsed_time_file=$(mktemp /tmp/benchmark-terasort-XXXXXX.$$)
            HADOOP_OPTS=${VARIANT_OPT} /usr/bin/time -o $elapsed_time_file -f "%e" terasort ${NGIGABYTES}GB ${NGIGABYTES}GB-sorted

            hadoop fs -rm -R ${NGIGABYTES}GB-sorted
            sed -i 's/\..*//' $elapsed_time_file
            elapsed_time=$(cat $elapsed_time_file)
            date_as_epoch=$(date --date="$(date +'%Y-%m-%d')" +%s)
            echo "$date_as_epoch,$JVM_VARIANT,$NGIGABYTES,$elapsed_time" > out/terasort-results-${JVM_VARIANT}-${BUILD_TYPE}.csv
            rm -rf incoming/${NGIGABYTES}*
    publishers:
        - archive:
            artifacts: 'out/terasort-results-*.csv'