diff options
-rw-r--r-- | odpi-bigtop-hadoop-rpm.yaml | 48 | ||||
-rw-r--r-- | odpi-bigtop-hive-rpm.yaml | 48 | ||||
-rw-r--r-- | odpi-bigtop-rpm.yaml | 155 | ||||
-rwxr-xr-x | odpi-bigtop-rpm/common/builders.sh | 105 | ||||
-rwxr-xr-x | odpi-bigtop-rpm/hadoop/builders.sh | 7 | ||||
-rwxr-xr-x | odpi-bigtop-rpm/hive/builders.sh | 9 | ||||
-rwxr-xr-x | odpi-bigtop-rpm/spark/builders.sh | 11 | ||||
-rw-r--r-- | odpi-bigtop-spark-rpm.yaml | 48 |
8 files changed, 283 insertions, 148 deletions
diff --git a/odpi-bigtop-hadoop-rpm.yaml b/odpi-bigtop-hadoop-rpm.yaml new file mode 100644 index 0000000000..d146c05963 --- /dev/null +++ b/odpi-bigtop-hadoop-rpm.yaml @@ -0,0 +1,48 @@ +- job: + name: odpi-bigtop-hadoop-rpm + project-type: freestyle + defaults: global + properties: + - authorization: + anonymous: + - job-read + - job-extended-read + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + artifact-num-to-keep: 1 + parameters: + - string: + name: ANT_OPTS + default: '-Xmx8G -XX:MaxPermSize=1G' + - string: + name: MAVEN_OPTS + default: '-B -Xmx8G -XX:MaxPermSize=1G' + - string: + name: JDK_URL + default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz' + - string: + name: MAVEN_URL + default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz' + - string: + name: SCALA_URL + default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz' + - string: + name: NODE_URL + default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz' + - string: + name: PROTOBUF_URL + default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz' + disabled: false + node: docker-centos7-aarch64 + display-name: 'ODPi BigTop Hadoop (rpm)' + wrappers: + - timestamps + builders: + - shell: + !include-raw: + - odpi-bigtop-rpm/common/builders.sh + - odpi-bigtop-rpm/hadoop/builders.sh + publishers: + - archive: + artifacts: 'odpi-bigtop/output/**' diff --git a/odpi-bigtop-hive-rpm.yaml b/odpi-bigtop-hive-rpm.yaml new file mode 100644 index 0000000000..7061f61ec2 --- /dev/null +++ b/odpi-bigtop-hive-rpm.yaml @@ -0,0 +1,48 @@ +- job: + name: odpi-bigtop-hive-rpm + project-type: freestyle + defaults: global + properties: + - authorization: + anonymous: + - job-read + - job-extended-read + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + artifact-num-to-keep: 1 + parameters: + - string: + name: ANT_OPTS + default: '-Xmx8G -XX:MaxPermSize=1G' + - string: + name: MAVEN_OPTS + default: '-B -Xmx8G -XX:MaxPermSize=1G' + - string: + name: JDK_URL + default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz' + - string: + name: MAVEN_URL + default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz' + - string: + name: SCALA_URL + default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz' + - string: + name: NODE_URL + default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz' + - string: + name: PROTOBUF_URL + default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz' + disabled: false + node: docker-centos7-aarch64 + display-name: 'ODPi BigTop Hive (rpm)' + wrappers: + - timestamps + builders: + - shell: + !include-raw: + - odpi-bigtop-rpm/common/builders.sh + - odpi-bigtop-rpm/hive/builders.sh + publishers: + - archive: + artifacts: 'odpi-bigtop/output/**' diff --git a/odpi-bigtop-rpm.yaml b/odpi-bigtop-rpm.yaml index 3e9702588e..a483632ccb 100644 --- a/odpi-bigtop-rpm.yaml +++ b/odpi-bigtop-rpm.yaml @@ -5,158 +5,17 @@ logrotate: daysToKeep: 30 numToKeep: 30 - artifactNumToKeep: 1 properties: - authorization: anonymous: - job-read - job-extended-read - parameters: - - string: - name: ANT_OPTS - default: '-Xmx8G -XX:MaxPermSize=1G' - - string: - name: MAVEN_OPTS - default: '-B -Xmx8G -XX:MaxPermSize=1G' - - string: - name: JDK_URL - default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz' - - string: - name: MAVEN_URL - default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz' - - string: - name: SCALA_URL - default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz' - - string: - name: NODE_URL - default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz' - - string: - name: PROTOBUF_URL - default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz' disabled: false - node: docker-centos7-aarch64 - display-name: 'ODPi BigTop (rpm)' - wrappers: - - timestamps + node: master + display-name: 'Trigger ODPi BigTop (rpm)' builders: - - shell: | - #!/bin/bash - - set -ex - - # number of CPU_CORES to build with - export CPU_CORES=$(getconf _NPROCESSORS_ONLN) - - # install pre-requisites - sudo yum install -y \ - apache-ivy \ - ant \ - asciidoc \ - chrpath \ - cmake \ - cppunit-devel \ - cyrus-sasl-devel \ - fuse \ - fuse-devel \ - gcc-c++ \ - jansson-devel \ - krb5-devel \ - lcms2-devel \ - libtool \ - libxml2-devel \ - libxslt-devel \ - libyaml-devel \ - libzip-devel \ - lzo-devel \ - make \ - mariadb-devel \ - openldap-devel \ - openssl-devel \ - pkgconfig \ - python-devel \ - python-setuptools \ - rpm-build \ - rsync \ - sharutils \ - snappy-devel \ - sqlite-devel \ - subversion \ - unzip \ - wget \ - xmlto - - # download OpenJDK and Maven explicitely - wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL} - tar -Jxf jdk*.tar.xz - tar -zxf apache-maven-*.tar.gz - tar -zxf scala-*.tgz - tar -zxf node-*.tar.gz - tar -zxf protobuf-*.tar.gz - - # replace cacerts with distro-supplied - cd jdk*/jre/lib/security/ - rm -f cacerts - ln -s /etc/pki/java/cacerts - - # set JAVA_HOME - cd ${WORKSPACE}/jdk* - export JAVA_HOME=${PWD} - - # set M3_HOME - cd ${WORKSPACE}/apache-maven-* - export M3_HOME=${PWD} - - # FIXME switch to nexus.linaro.org - # hack to use archiva - #wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml - #mkdir ~/.m2 - #cp -a conf/settings.xml ~/.m2/settings.xml - - # set SCALA_HOME - cd ${WORKSPACE}/scala-* - export SCALA_HOME=${PWD} - - # set PATH - export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH} - java -version - mvn -version - - # build and hookup nodejs - cd ${WORKSPACE}/node-* - ./configure --prefix=${WORKSPACE}/node - make -j${CPU_CORES} install - export PATH=${WORKSPACE}/node/bin/:${PATH} - - # build and hookup protobuf compiler - cd ${WORKSPACE}/protobuf-* - ./configure --prefix=${WORKSPACE}/protobuf - make -j${CPU_CORES} install - export PATH=${WORKSPACE}/protobuf/bin:${PATH} - export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig - - # clone the ODPi BigTop definitions - git clone --depth 1 https://git.linaro.org/people/ganesh.raju/bigtop-odpi.git ${WORKSPACE}/odpi-bigtop - cd ${WORKSPACE}/odpi-bigtop - - # FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1. - git config --global user.name "Linaro CI" - git config --global user.email "ci_notify@linaro.org" - git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git - git fetch scapper - git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c - git cherry-pick a4ef371718fc32d25cc01137e559da4079368773 - - # okay everything is in place, this build will take a while - ./gradlew clean hadoop-rpm spark-rpm hive-rpm \ - -Pdist,native-win \ - -DskipTests \ - -Dtar \ - -Dmaven.javadoc.skip=true \ - -PHadoop-2.7 \ - -Pyarn \ - -Phadoop.version=2.7.0 \ - -Dscala-2.11 \ - --debug - publishers: - - archive: - artifacts: 'odpi-bigtop/output/**' + - trigger-builds: + - project: + - 'odpi-bigtop-hadoop-rpm' + - 'odpi-bigtop-hive-rpm' + - 'odpi-bigtop-spark-rpm' diff --git a/odpi-bigtop-rpm/common/builders.sh b/odpi-bigtop-rpm/common/builders.sh new file mode 100755 index 0000000000..cd5ea8fe7f --- /dev/null +++ b/odpi-bigtop-rpm/common/builders.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +set -ex + +# number of CPU_CORES to build with +export CPU_CORES=$(getconf _NPROCESSORS_ONLN) + +# install pre-requisites +sudo yum install -y \ + apache-ivy \ + ant \ + asciidoc \ + chrpath \ + cmake \ + cppunit-devel \ + cyrus-sasl-devel \ + fuse \ + fuse-devel \ + gcc-c++ \ + jansson-devel \ + krb5-devel \ + lcms2-devel \ + libtool \ + libxml2-devel \ + libxslt-devel \ + libyaml-devel \ + libzip-devel \ + lzo-devel \ + make \ + mariadb-devel \ + openldap-devel \ + openssl-devel \ + pkgconfig \ + python-devel \ + python-setuptools \ + rpm-build \ + rsync \ + sharutils \ + snappy-devel \ + sqlite-devel \ + subversion \ + unzip \ + wget \ + xmlto + +# download OpenJDK and Maven explicitely +wget --progress=dot -e dotbytes=2M ${JDK_URL} ${MAVEN_URL} ${SCALA_URL} ${NODE_URL} ${PROTOBUF_URL} +tar -Jxf jdk*.tar.xz +tar -zxf apache-maven-*.tar.gz +tar -zxf scala-*.tgz +tar -zxf node-*.tar.gz +tar -zxf protobuf-*.tar.gz + +# replace cacerts with distro-supplied +cd jdk*/jre/lib/security/ +rm -f cacerts +ln -s /etc/pki/java/cacerts + +# set JAVA_HOME +cd ${WORKSPACE}/jdk* +export JAVA_HOME=${PWD} + +# set M3_HOME +cd ${WORKSPACE}/apache-maven-* +export M3_HOME=${PWD} + +# FIXME switch to nexus.linaro.org +# hack to use archiva +#wget -q http://people.linaro.org/~fathi.boudra/settings.xml -O conf/settings.xml +#mkdir ~/.m2 +#cp -a conf/settings.xml ~/.m2/settings.xml + +# set SCALA_HOME +cd ${WORKSPACE}/scala-* +export SCALA_HOME=${PWD} + +# set PATH +export PATH=${JAVA_HOME}/bin:${M3_HOME}/bin:${PATH} +java -version +mvn -version + +# build and hookup nodejs +cd ${WORKSPACE}/node-* +./configure --prefix=${WORKSPACE}/node +make -j${CPU_CORES} install +export PATH=${WORKSPACE}/node/bin/:${PATH} + +# build and hookup protobuf compiler +cd ${WORKSPACE}/protobuf-* +./configure --prefix=${WORKSPACE}/protobuf +make -j${CPU_CORES} install +export PATH=${WORKSPACE}/protobuf/bin:${PATH} +export PKG_CONFIG_PATH=${WORKSPACE}/protobuf/lib/pkgconfig + +# clone the ODPi BigTop definitions +git clone --depth 1 https://git.linaro.org/people/ganesh.raju/bigtop-odpi.git ${WORKSPACE}/odpi-bigtop +cd ${WORKSPACE}/odpi-bigtop + +# FIXME Upstream protobuf version 2.5.0 does not support AArch64. Bump up to 2.6.1. +git config --global user.name "Linaro CI" +git config --global user.email "ci_notify@linaro.org" +git remote add scapper https://git.linaro.org/people/steve.capper/odpi-bigtop.git +git fetch scapper +git cherry-pick 3033ede8c0a0ede0323c4e8c946d1293ed64729c +git cherry-pick a4ef371718fc32d25cc01137e559da4079368773 diff --git a/odpi-bigtop-rpm/hadoop/builders.sh b/odpi-bigtop-rpm/hadoop/builders.sh new file mode 100755 index 0000000000..0628a982a9 --- /dev/null +++ b/odpi-bigtop-rpm/hadoop/builders.sh @@ -0,0 +1,7 @@ +# okay everything is in place, this build will take a while +./gradlew clean hadoop-rpm \ + -Pdist,native-win \ + -DskipTests \ + -Dtar \ + -Dmaven.javadoc.skip=true \ + --debug diff --git a/odpi-bigtop-rpm/hive/builders.sh b/odpi-bigtop-rpm/hive/builders.sh new file mode 100755 index 0000000000..cfaceb02a1 --- /dev/null +++ b/odpi-bigtop-rpm/hive/builders.sh @@ -0,0 +1,9 @@ +# okay everything is in place, this build will take a while +./gradlew clean hive-rpm \ + -Pdist,native-win \ + -DskipTests \ + -Dtar \ + -Dmaven.javadoc.skip=true \ + -PHadoop-2.7 \ + -Phadoop.version=2.7.2 \ + --debug diff --git a/odpi-bigtop-rpm/spark/builders.sh b/odpi-bigtop-rpm/spark/builders.sh new file mode 100755 index 0000000000..660da182ec --- /dev/null +++ b/odpi-bigtop-rpm/spark/builders.sh @@ -0,0 +1,11 @@ +# okay everything is in place, this build will take a while +./gradlew clean spark-rpm \ + -Pdist,native-win \ + -DskipTests \ + -Dtar \ + -Dmaven.javadoc.skip=true \ + -PHadoop-2.7 \ + -Pyarn \ + -Phadoop.version=2.7.2 \ + -Dscala-2.11 \ + --debug diff --git a/odpi-bigtop-spark-rpm.yaml b/odpi-bigtop-spark-rpm.yaml new file mode 100644 index 0000000000..84292f9233 --- /dev/null +++ b/odpi-bigtop-spark-rpm.yaml @@ -0,0 +1,48 @@ +- job: + name: odpi-bigtop-spark-rpm + project-type: freestyle + defaults: global + properties: + - authorization: + anonymous: + - job-read + - job-extended-read + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + artifact-num-to-keep: 1 + parameters: + - string: + name: ANT_OPTS + default: '-Xmx8G -XX:MaxPermSize=1G' + - string: + name: MAVEN_OPTS + default: '-B -Xmx8G -XX:MaxPermSize=1G' + - string: + name: JDK_URL + default: 'http://openjdk.linaro.org/releases/jdk8u-server-release-1609.tar.xz' + - string: + name: MAVEN_URL + default: 'http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz' + - string: + name: SCALA_URL + default: 'http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz' + - string: + name: NODE_URL + default: 'https://nodejs.org/dist/v4.2.1/node-v4.2.1.tar.gz' + - string: + name: PROTOBUF_URL + default: 'https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz' + disabled: false + node: docker-centos7-aarch64 + display-name: 'ODPi BigTop Spark (rpm)' + wrappers: + - timestamps + builders: + - shell: + !include-raw: + - odpi-bigtop-rpm/common/builders.sh + - odpi-bigtop-rpm/spark/builders.sh + publishers: + - archive: + artifacts: 'odpi-bigtop/output/**' |