aboutsummaryrefslogtreecommitdiff
path: root/odpi-bigtop-deb.yaml
blob: 7d3a9e088e625b68c741747fde63bd520a5d3a86 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
- job:
    name: odpi-bigtop-deb
    project-type: freestyle
    defaults: global
    logrotate:
        daysToKeep: 30
        numToKeep: 30
        artifactNumToKeep: 1
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            linaro:
                - job-read
                - job-extended-read
                - job-build
                - job-cancel
    parameters:
        - string:
            name: ANT_OPTS
            default: '-Xmx8G'
        - string:
            name: MAVEN_OPTS
            default: '-Xmx8G'
        - string:
            name: JDK_URL
            default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz'
        - string:
            name: MAVEN_URL
            default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz'
        - string:
            name: SCALA_URL
            default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz'
        - string:
            name: NODE_URL
            default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz'
        - string:
            name: PROTOBUF_URL
            default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz'
    disabled: false
    node: docker-jessie-arm64
    display-name: 'ODPi BigTop (deb)'
    wrappers:
        - timestamps
    builders:
        - shell: |
            #!/bin/bash

            set -ex

            # number of CPU_CORES to build with
            export CPU_CORES=$(getconf _NPROCESSORS_ONLN)

            # install pre-requisites
            sudo apt-get update
            sudo apt-get install -y --no-install-recommends \
              ant \
              asciidoc \
              chrpath \
              cmake \
              fuse \
              libcppunit-dev \
              libfuse-dev \
              libjansson-dev \
              libkrb5-dev \
              libldap2-dev \
              liblzo2-dev \
              libmysqlclient-dev \
              libsasl2-dev \
              libsnappy-dev \
              libsqlite3-dev \
              libssl-dev \
              libxml2-dev \
              libxslt1-dev \
              libzip-dev \
              pkg-config \
              python-dev \
              python-setuptools \
              rsync \
              sharutils \
              subversion \
              xmlto

            # download OpenJDK and Maven explicitely
            wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL}
            tar -Jxf jdk*.tar.xz
            tar -zxf apache-maven-*.tar.gz
            tar -zxf scala-*.tgz
            tar -zxf node-*.tar.gz
            tar -zxf protobuf-*.tar.gz

            # replace cacerts with distro-supplied
            cd jdk*/jre/lib/security/
            rm -f cacerts
            ln -s /etc/ssl/certs/java/cacerts

            # set JAVA_HOME
            cd ${WORKSPACE}/jdk*
            export JAVA_HOME=${PWD}

            # set M3_HOME
            cd ${WORKSPACE}/apache-maven-*
            export M3_HOME=${PWD}

            # FIXME switch to nexus.linaro.org
            # hack to use archiva
            #wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml
            #mkdir ~/.m2
            #cp -a conf/settings.xml ~/.m2/settings.xml

            # set SCALA_HOME
            cd ${WORKSPACE}/scala-*
            export SCALA_HOME=${PWD}

            # set PATH
            export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH}
            java -version
            mvn -version

            # build and hookup nodejs
            cd ${WORKSPACE}/node-*
            ./configure --prefix=${WORKSPACE}/node
            make -j${CPU_CORES} install
            export PATH=${WORKSPACE}/node/bin/:${PATH}

            # build and hookup protobuf compiler
            cd ${WORKSPACE}/protobuf-*
            ./configure --prefix=${WORKSPACE}/protobuf
            make -j${CPU_CORES} install
            export PATH=${WORKSPACE}/protobuf/bin:${PATH}
            export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig

            # clone the ODPi BigTop definitions
            git clone --depth 1 https://git.linaro.org/leg/bigdata/bigtop-trunk.git -b erp17.08 ${WORKSPACE}/odpi-bigtop
            cd ${WORKSPACE}/odpi-bigtop

            # FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1.
            git config --global user.name "Linaro CI"
            git config --global user.email "ci_notify@linaro.org"
            git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git
            git fetch scapper
            git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c
            git cherry-pick a4ef371718fc32d25cc01137e559da4079368773

            # okay everything is in place, this build will take a while
            ./gradlew clean hadoop-deb spark-deb hive-deb \
              -Pdist,native-win \
              -DskipTests \
              -Dtar \
              -Dmaven.javadoc.skip=true \
              -PHadoop-2.7 \
              -Pyarn \
              -Phadoop.version=2.7.0 \
              -Dscala-2.11 \
              --debug
    publishers:
        - archive:
            artifacts: 'odpi-bigtop/output/**'
        - email-ext:
            recipients: 'leg-bigdata@linaro.org, fathi.boudra@linaro.org'