aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFathi Boudra <fathi.boudra@linaro.org>2016-12-01 08:12:18 +0200
committerFathi Boudra <fathi.boudra@linaro.org>2016-12-01 08:12:18 +0200
commit8992673d3bc7eda9ac4fc9e333a923dfd7244153 (patch)
tree89fa889c9a4dbfb36db4718d66681a88e534bb4f
parent63fb4334d5806fdd4e89a18dc5cfaa9b499520b7 (diff)
downloadconfigs-8992673d3bc7eda9ac4fc9e333a923dfd7244153.tar.gz
odpi-bigtop-rpm: split the build of hadoop/hive/spark
It allows concurrent builds of each component. Change-Id: Ic0548a926cb024f8a54731bf06e1bce182f99f32 Signed-off-by: Fathi Boudra <fathi.boudra@linaro.org>
-rw-r--r--odpi-bigtop-hadoop-rpm.yaml48
-rw-r--r--odpi-bigtop-hive-rpm.yaml48
-rw-r--r--odpi-bigtop-rpm.yaml155
-rwxr-xr-xodpi-bigtop-rpm/common/builders.sh105
-rwxr-xr-xodpi-bigtop-rpm/hadoop/builders.sh7
-rwxr-xr-xodpi-bigtop-rpm/hive/builders.sh9
-rwxr-xr-xodpi-bigtop-rpm/spark/builders.sh11
-rw-r--r--odpi-bigtop-spark-rpm.yaml48
8 files changed, 283 insertions, 148 deletions
diff --git a/odpi-bigtop-hadoop-rpm.yaml b/odpi-bigtop-hadoop-rpm.yaml
new file mode 100644
index 000000000..d146c0596
--- /dev/null
+++ b/odpi-bigtop-hadoop-rpm.yaml
@@ -0,0 +1,48 @@
+- job:
+ name: odpi-bigtop-hadoop-rpm
+ project-type: freestyle
+ defaults: global
+ properties:
+ - authorization:
+ anonymous:
+ - job-read
+ - job-extended-read
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ artifact-num-to-keep: 1
+ parameters:
+ - string:
+ name: ANT_OPTS
+ default: '-Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: MAVEN_OPTS
+ default: '-B -Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: JDK_URL
+ default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz'
+ - string:
+ name: MAVEN_URL
+ default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz'
+ - string:
+ name: SCALA_URL
+ default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz'
+ - string:
+ name: NODE_URL
+ default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz'
+ - string:
+ name: PROTOBUF_URL
+ default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz'
+ disabled: false
+ node: docker-centos7-aarch64
+ display-name: 'ODPi BigTop Hadoop (rpm)'
+ wrappers:
+ - timestamps
+ builders:
+ - shell:
+ !include-raw:
+ - odpi-bigtop-rpm/common/builders.sh
+ - odpi-bigtop-rpm/hadoop/builders.sh
+ publishers:
+ - archive:
+ artifacts: 'odpi-bigtop/output/**'
diff --git a/odpi-bigtop-hive-rpm.yaml b/odpi-bigtop-hive-rpm.yaml
new file mode 100644
index 000000000..7061f61ec
--- /dev/null
+++ b/odpi-bigtop-hive-rpm.yaml
@@ -0,0 +1,48 @@
+- job:
+ name: odpi-bigtop-hive-rpm
+ project-type: freestyle
+ defaults: global
+ properties:
+ - authorization:
+ anonymous:
+ - job-read
+ - job-extended-read
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ artifact-num-to-keep: 1
+ parameters:
+ - string:
+ name: ANT_OPTS
+ default: '-Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: MAVEN_OPTS
+ default: '-B -Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: JDK_URL
+ default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz'
+ - string:
+ name: MAVEN_URL
+ default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz'
+ - string:
+ name: SCALA_URL
+ default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz'
+ - string:
+ name: NODE_URL
+ default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz'
+ - string:
+ name: PROTOBUF_URL
+ default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz'
+ disabled: false
+ node: docker-centos7-aarch64
+ display-name: 'ODPi BigTop Hive (rpm)'
+ wrappers:
+ - timestamps
+ builders:
+ - shell:
+ !include-raw:
+ - odpi-bigtop-rpm/common/builders.sh
+ - odpi-bigtop-rpm/hive/builders.sh
+ publishers:
+ - archive:
+ artifacts: 'odpi-bigtop/output/**'
diff --git a/odpi-bigtop-rpm.yaml b/odpi-bigtop-rpm.yaml
index 3e9702588..a483632cc 100644
--- a/odpi-bigtop-rpm.yaml
+++ b/odpi-bigtop-rpm.yaml
@@ -5,158 +5,17 @@
logrotate:
daysToKeep: 30
numToKeep: 30
- artifactNumToKeep: 1
properties:
- authorization:
anonymous:
- job-read
- job-extended-read
- parameters:
- - string:
- name: ANT_OPTS
- default: '-Xmx8G -XX:MaxPermSize=1G'
- - string:
- name: MAVEN_OPTS
- default: '-B -Xmx8G -XX:MaxPermSize=1G'
- - string:
- name: JDK_URL
- default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz'
- - string:
- name: MAVEN_URL
- default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz'
- - string:
- name: SCALA_URL
- default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz'
- - string:
- name: NODE_URL
- default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz'
- - string:
- name: PROTOBUF_URL
- default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz'
disabled: false
- node: docker-centos7-aarch64
- display-name: 'ODPi BigTop (rpm)'
- wrappers:
- - timestamps
+ node: master
+ display-name: 'Trigger ODPi BigTop (rpm)'
builders:
- - shell: |
- #!/bin/bash
-
- set -ex
-
- # number of CPU_CORES to build with
- export CPU_CORES=$(getconf _NPROCESSORS_ONLN)
-
- # install pre-requisites
- sudo yum install -y \
- apache-ivy \
- ant \
- asciidoc \
- chrpath \
- cmake \
- cppunit-devel \
- cyrus-sasl-devel \
- fuse \
- fuse-devel \
- gcc-c++ \
- jansson-devel \
- krb5-devel \
- lcms2-devel \
- libtool \
- libxml2-devel \
- libxslt-devel \
- libyaml-devel \
- libzip-devel \
- lzo-devel \
- make \
- mariadb-devel \
- openldap-devel \
- openssl-devel \
- pkgconfig \
- python-devel \
- python-setuptools \
- rpm-build \
- rsync \
- sharutils \
- snappy-devel \
- sqlite-devel \
- subversion \
- unzip \
- wget \
- xmlto
-
- # download OpenJDK and Maven explicitely
- wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL}
- tar -Jxf jdk*.tar.xz
- tar -zxf apache-maven-*.tar.gz
- tar -zxf scala-*.tgz
- tar -zxf node-*.tar.gz
- tar -zxf protobuf-*.tar.gz
-
- # replace cacerts with distro-supplied
- cd jdk*/jre/lib/security/
- rm -f cacerts
- ln -s /etc/pki/java/cacerts
-
- # set JAVA_HOME
- cd ${WORKSPACE}/jdk*
- export JAVA_HOME=${PWD}
-
- # set M3_HOME
- cd ${WORKSPACE}/apache-maven-*
- export M3_HOME=${PWD}
-
- # FIXME switch to nexus.linaro.org
- # hack to use archiva
- #wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml
- #mkdir ~/.m2
- #cp -a conf/settings.xml ~/.m2/settings.xml
-
- # set SCALA_HOME
- cd ${WORKSPACE}/scala-*
- export SCALA_HOME=${PWD}
-
- # set PATH
- export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH}
- java -version
- mvn -version
-
- # build and hookup nodejs
- cd ${WORKSPACE}/node-*
- ./configure --prefix=${WORKSPACE}/node
- make -j${CPU_CORES} install
- export PATH=${WORKSPACE}/node/bin/:${PATH}
-
- # build and hookup protobuf compiler
- cd ${WORKSPACE}/protobuf-*
- ./configure --prefix=${WORKSPACE}/protobuf
- make -j${CPU_CORES} install
- export PATH=${WORKSPACE}/protobuf/bin:${PATH}
- export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig
-
- # clone the ODPi BigTop definitions
- git clone --depth 1 https://git.linaro.org/people/ganesh.raju/bigtop-odpi.git ${WORKSPACE}/odpi-bigtop
- cd ${WORKSPACE}/odpi-bigtop
-
- # FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1.
- git config --global user.name "Linaro CI"
- git config --global user.email "ci_notify@linaro.org"
- git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git
- git fetch scapper
- git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c
- git cherry-pick a4ef371718fc32d25cc01137e559da4079368773
-
- # okay everything is in place, this build will take a while
- ./gradlew clean hadoop-rpm spark-rpm hive-rpm \
- -Pdist,native-win \
- -DskipTests \
- -Dtar \
- -Dmaven.javadoc.skip=true \
- -PHadoop-2.7 \
- -Pyarn \
- -Phadoop.version=2.7.0 \
- -Dscala-2.11 \
- --debug
- publishers:
- - archive:
- artifacts: 'odpi-bigtop/output/**'
+ - trigger-builds:
+ - project:
+ - 'odpi-bigtop-hadoop-rpm'
+ - 'odpi-bigtop-hive-rpm'
+ - 'odpi-bigtop-spark-rpm'
diff --git a/odpi-bigtop-rpm/common/builders.sh b/odpi-bigtop-rpm/common/builders.sh
new file mode 100755
index 000000000..cd5ea8fe7
--- /dev/null
+++ b/odpi-bigtop-rpm/common/builders.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -ex
+
+# number of CPU_CORES to build with
+export CPU_CORES=$(getconf _NPROCESSORS_ONLN)
+
+# install pre-requisites
+sudo yum install -y \
+ apache-ivy \
+ ant \
+ asciidoc \
+ chrpath \
+ cmake \
+ cppunit-devel \
+ cyrus-sasl-devel \
+ fuse \
+ fuse-devel \
+ gcc-c++ \
+ jansson-devel \
+ krb5-devel \
+ lcms2-devel \
+ libtool \
+ libxml2-devel \
+ libxslt-devel \
+ libyaml-devel \
+ libzip-devel \
+ lzo-devel \
+ make \
+ mariadb-devel \
+ openldap-devel \
+ openssl-devel \
+ pkgconfig \
+ python-devel \
+ python-setuptools \
+ rpm-build \
+ rsync \
+ sharutils \
+ snappy-devel \
+ sqlite-devel \
+ subversion \
+ unzip \
+ wget \
+ xmlto
+
+# download OpenJDK and Maven explicitely
+wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL}
+tar -Jxf jdk*.tar.xz
+tar -zxf apache-maven-*.tar.gz
+tar -zxf scala-*.tgz
+tar -zxf node-*.tar.gz
+tar -zxf protobuf-*.tar.gz
+
+# replace cacerts with distro-supplied
+cd jdk*/jre/lib/security/
+rm -f cacerts
+ln -s /etc/pki/java/cacerts
+
+# set JAVA_HOME
+cd ${WORKSPACE}/jdk*
+export JAVA_HOME=${PWD}
+
+# set M3_HOME
+cd ${WORKSPACE}/apache-maven-*
+export M3_HOME=${PWD}
+
+# FIXME switch to nexus.linaro.org
+# hack to use archiva
+#wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml
+#mkdir ~/.m2
+#cp -a conf/settings.xml ~/.m2/settings.xml
+
+# set SCALA_HOME
+cd ${WORKSPACE}/scala-*
+export SCALA_HOME=${PWD}
+
+# set PATH
+export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH}
+java -version
+mvn -version
+
+# build and hookup nodejs
+cd ${WORKSPACE}/node-*
+./configure --prefix=${WORKSPACE}/node
+make -j${CPU_CORES} install
+export PATH=${WORKSPACE}/node/bin/:${PATH}
+
+# build and hookup protobuf compiler
+cd ${WORKSPACE}/protobuf-*
+./configure --prefix=${WORKSPACE}/protobuf
+make -j${CPU_CORES} install
+export PATH=${WORKSPACE}/protobuf/bin:${PATH}
+export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig
+
+# clone the ODPi BigTop definitions
+git clone --depth 1 https://git.linaro.org/people/ganesh.raju/bigtop-odpi.git ${WORKSPACE}/odpi-bigtop
+cd ${WORKSPACE}/odpi-bigtop
+
+# FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1.
+git config --global user.name "Linaro CI"
+git config --global user.email "ci_notify@linaro.org"
+git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git
+git fetch scapper
+git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c
+git cherry-pick a4ef371718fc32d25cc01137e559da4079368773
diff --git a/odpi-bigtop-rpm/hadoop/builders.sh b/odpi-bigtop-rpm/hadoop/builders.sh
new file mode 100755
index 000000000..0628a982a
--- /dev/null
+++ b/odpi-bigtop-rpm/hadoop/builders.sh
@@ -0,0 +1,7 @@
+# okay everything is in place, this build will take a while
+./gradlew clean hadoop-rpm \
+ -Pdist,native-win \
+ -DskipTests \
+ -Dtar \
+ -Dmaven.javadoc.skip=true \
+ --debug
diff --git a/odpi-bigtop-rpm/hive/builders.sh b/odpi-bigtop-rpm/hive/builders.sh
new file mode 100755
index 000000000..cfaceb02a
--- /dev/null
+++ b/odpi-bigtop-rpm/hive/builders.sh
@@ -0,0 +1,9 @@
+# okay everything is in place, this build will take a while
+./gradlew clean hive-rpm \
+ -Pdist,native-win \
+ -DskipTests \
+ -Dtar \
+ -Dmaven.javadoc.skip=true \
+ -PHadoop-2.7 \
+ -Phadoop.version=2.7.2 \
+ --debug
diff --git a/odpi-bigtop-rpm/spark/builders.sh b/odpi-bigtop-rpm/spark/builders.sh
new file mode 100755
index 000000000..660da182e
--- /dev/null
+++ b/odpi-bigtop-rpm/spark/builders.sh
@@ -0,0 +1,11 @@
+# okay everything is in place, this build will take a while
+./gradlew clean spark-rpm \
+ -Pdist,native-win \
+ -DskipTests \
+ -Dtar \
+ -Dmaven.javadoc.skip=true \
+ -PHadoop-2.7 \
+ -Pyarn \
+ -Phadoop.version=2.7.2 \
+ -Dscala-2.11 \
+ --debug
diff --git a/odpi-bigtop-spark-rpm.yaml b/odpi-bigtop-spark-rpm.yaml
new file mode 100644
index 000000000..84292f923
--- /dev/null
+++ b/odpi-bigtop-spark-rpm.yaml
@@ -0,0 +1,48 @@
+- job:
+ name: odpi-bigtop-spark-rpm
+ project-type: freestyle
+ defaults: global
+ properties:
+ - authorization:
+ anonymous:
+ - job-read
+ - job-extended-read
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ artifact-num-to-keep: 1
+ parameters:
+ - string:
+ name: ANT_OPTS
+ default: '-Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: MAVEN_OPTS
+ default: '-B -Xmx8G -XX:MaxPermSize=1G'
+ - string:
+ name: JDK_URL
+ default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz'
+ - string:
+ name: MAVEN_URL
+ default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz'
+ - string:
+ name: SCALA_URL
+ default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz'
+ - string:
+ name: NODE_URL
+ default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz'
+ - string:
+ name: PROTOBUF_URL
+ default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz'
+ disabled: false
+ node: docker-centos7-aarch64
+ display-name: 'ODPi BigTop Spark (rpm)'
+ wrappers:
+ - timestamps
+ builders:
+ - shell:
+ !include-raw:
+ - odpi-bigtop-rpm/common/builders.sh
+ - odpi-bigtop-rpm/spark/builders.sh
+ publishers:
+ - archive:
+ artifacts: 'odpi-bigtop/output/**'