odpi-bigtop-rpm: split the build of hadoop/hive/spark

It allows concurrent builds of each component.

Change-Id: Ic0548a926cb024f8a54731bf06e1bce182f99f32
Signed-off-by: Fathi Boudra <fathi.boudra@linaro.org>

4 files changed, 132 insertions, 0 deletions

diff --git a/odpi-bigtop-rpm/common/builders.sh b/odpi-bigtop-rpm/common/builders.sh
new file mode 100755
index 0000000000..cd5ea8fe7f
--- /dev/null
+++ b/odpi-bigtop-rpm/common/builders.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -ex
+
+# number of CPU_CORES to build with
+export CPU_CORES=$(getconf _NPROCESSORS_ONLN)
+
+# install pre-requisites
+sudo yum install -y \
+  apache-ivy \
+  ant \
+  asciidoc \
+  chrpath \
+  cmake \
+  cppunit-devel \
+  cyrus-sasl-devel \
+  fuse \
+  fuse-devel \
+  gcc-c++ \
+  jansson-devel \
+  krb5-devel \
+  lcms2-devel \
+  libtool \
+  libxml2-devel \
+  libxslt-devel \
+  libyaml-devel \
+  libzip-devel \
+  lzo-devel \
+  make \
+  mariadb-devel \
+  openldap-devel \
+  openssl-devel \
+  pkgconfig \
+  python-devel \
+  python-setuptools \
+  rpm-build \
+  rsync \
+  sharutils \
+  snappy-devel \
+  sqlite-devel \
+  subversion \
+  unzip \
+  wget \
+  xmlto
+
+# download OpenJDK and Maven explicitely
+wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL}
+tar -Jxf jdk*.tar.xz
+tar -zxf apache-maven-*.tar.gz
+tar -zxf scala-*.tgz
+tar -zxf node-*.tar.gz
+tar -zxf protobuf-*.tar.gz
+
+# replace cacerts with distro-supplied
+cd jdk*/jre/lib/security/
+rm -f cacerts
+ln -s /etc/pki/java/cacerts
+
+# set JAVA_HOME
+cd ${WORKSPACE}/jdk*
+export JAVA_HOME=${PWD}
+
+# set M3_HOME
+cd ${WORKSPACE}/apache-maven-*
+export M3_HOME=${PWD}
+
+# FIXME switch to nexus.linaro.org
+# hack to use archiva
+#wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml
+#mkdir ~/.m2
+#cp -a conf/settings.xml ~/.m2/settings.xml
+
+# set SCALA_HOME
+cd ${WORKSPACE}/scala-*
+export SCALA_HOME=${PWD}
+
+# set PATH
+export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH}
+java -version
+mvn -version
+
+# build and hookup nodejs
+cd ${WORKSPACE}/node-*
+./configure --prefix=${WORKSPACE}/node
+make -j${CPU_CORES} install
+export PATH=${WORKSPACE}/node/bin/:${PATH}
+
+# build and hookup protobuf compiler
+cd ${WORKSPACE}/protobuf-*
+./configure --prefix=${WORKSPACE}/protobuf
+make -j${CPU_CORES} install
+export PATH=${WORKSPACE}/protobuf/bin:${PATH}
+export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig
+
+# clone the ODPi BigTop definitions
+git clone --depth 1 https://git.linaro.org/people/ganesh.raju/bigtop-odpi.git ${WORKSPACE}/odpi-bigtop
+cd ${WORKSPACE}/odpi-bigtop
+
+# FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1.
+git config --global user.name "Linaro CI"
+git config --global user.email "ci_notify@linaro.org"
+git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git
+git fetch scapper
+git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c
+git cherry-pick a4ef371718fc32d25cc01137e559da4079368773
diff --git a/odpi-bigtop-rpm/hadoop/builders.sh b/odpi-bigtop-rpm/hadoop/builders.sh
new file mode 100755
index 0000000000..0628a982a9
--- /dev/null
+++ b/odpi-bigtop-rpm/hadoop/builders.sh
@@ -0,0 +1,7 @@
+# okay everything is in place, this build will take a while
+./gradlew clean hadoop-rpm \
+  -Pdist,native-win \
+  -DskipTests \
+  -Dtar \
+  -Dmaven.javadoc.skip=true \
+  --debug
diff --git a/odpi-bigtop-rpm/hive/builders.sh b/odpi-bigtop-rpm/hive/builders.sh
new file mode 100755
index 0000000000..cfaceb02a1
--- /dev/null
+++ b/odpi-bigtop-rpm/hive/builders.sh
@@ -0,0 +1,9 @@
+# okay everything is in place, this build will take a while
+./gradlew clean hive-rpm \
+  -Pdist,native-win \
+  -DskipTests \
+  -Dtar \
+  -Dmaven.javadoc.skip=true \
+  -PHadoop-2.7 \
+  -Phadoop.version=2.7.2 \
+  --debug
diff --git a/odpi-bigtop-rpm/spark/builders.sh b/odpi-bigtop-rpm/spark/builders.sh
new file mode 100755
index 0000000000..660da182ec
--- /dev/null
+++ b/odpi-bigtop-rpm/spark/builders.sh
@@ -0,0 +1,11 @@
+# okay everything is in place, this build will take a while
+./gradlew clean spark-rpm \
+  -Pdist,native-win \
+  -DskipTests \
+  -Dtar \
+  -Dmaven.javadoc.skip=true \
+  -PHadoop-2.7 \
+  -Pyarn \
+  -Phadoop.version=2.7.2 \
+  -Dscala-2.11 \
+  --debug