1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
######################################################################
# BIGTOP-2288 Please remove this patch when maven repository is updated
# with version of leveldbjni-all library that is not x86 specific
if [ $HOSTTYPE = "powerpc64le" ] || [ $HOSTTYPE = "aarch64" ] ; then
if [ $HOSTTYPE = "powerpc64le" ] ; then
#download the POWER version of leveldb
git clone git://github.com/ibmsoe/leveldb.git
git clone https://github.com/ibmsoe/leveldbjni.git
else
# download leveldbjni 1.8 from original repo
git clone https://github.com/chirino/leveldb.git
git clone git://github.com/fusesource/leveldbjni.git --branch leveldbjni-1.8 --single-branch
fi
if [ -f /usr/lib/libsnappy.so ]; then
export SNAPPY_HOME=/usr/lib
elif [ -f /usr/lib64/libsnappy.so ]; then
export SNAPPY_HOME=/usr/lib64
fi
export LEVELDB_HOME=`cd leveldb; pwd`
export LEVELDBJNI_HOME=`cd leveldbjni; pwd`
export LIBRARY_PATH=${SNAPPY_HOME}
cd ${LEVELDB_HOME}
export C_INCLUDE_PATH=${LIBRARY_PATH}
export CPLUS_INCLUDE_PATH=${LIBRARY_PATH}
if [ $HOSTTYPE = "aarch64" ] ; then
# apply ARM64 specific patch
patch -p1 -E << 'EOF'
diff a/port/atomic_pointer.h b/port/atomic_pointer.h
--- a/port/atomic_pointer.h
+++ b/port/atomic_pointer.h
@@ -36,6 +36,8 @@
#define ARCH_CPU_X86_FAMILY 1
#elif defined(__ARMEL__)
#define ARCH_CPU_ARM_FAMILY 1
+#elif defined(__aarch64__)
+#define ARCH_CPU_ARM64_FAMILY 1
#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
#define ARCH_CPU_PPC_FAMILY 1
#endif
@@ -93,6 +95,13 @@ inline void MemoryBarrier() {
}
#define LEVELDB_HAVE_MEMORY_BARRIER
+// ARM64
+#elif defined(ARCH_CPU_ARM64_FAMILY) && defined(__linux__)
+inline void MemoryBarrier() {
+ asm volatile("dmb sy" : : : "memory");
+}
+#define LEVELDB_HAVE_MEMORY_BARRIER
+
// PPC
#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
inline void MemoryBarrier() {
@@ -216,6 +225,7 @@ class AtomicPointer {
#undef LEVELDB_HAVE_MEMORY_BARRIER
#undef ARCH_CPU_X86_FAMILY
#undef ARCH_CPU_ARM_FAMILY
+#undef ARCH_CPU_ARM64_FAMILY
#undef ARCH_CPU_PPC_FAMILY
} // namespace port
EOF
fi
git apply ../leveldbjni/leveldb.patch
make libleveldb.a
#Now use maven to build and update the local maven repository with ppc version of
#leveldbjni.
cd ${LEVELDBJNI_HOME}
mvn clean install -DskipTests -P download -Plinux64,all
cd ..
#cleanup
rm -rf ${LEVELDBJNI_HOME}
rm -rf ${LEVELDB_HOME}
fi
## BIGTOP-2288
. `dirname $0`/bigtop.bom
if [ -z "$BUNDLE_SNAPPY" ] ; then
[ -f /usr/lib/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib"
[ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
fi
mkdir build
mkdir build/src
# Variable to define install/deploy the artifacts.
deployarg=${deploy_arg}
[[ $deployarg = "" ]] && deployarg="no"
# Nexus Proxy Urls.
nexus_url="http://nexus.odpi.org:8081/nexus"
nexus_base_url="${nexus_url}/content"
nexus_proxy_url="${nexus_base_url}/groups/public"
# Nexus URL's for artifact deploy.
nexus_repo_url="http://nexus.odpi.org:8081/nexus/content/repositories/ODP/"
repo_id="ODP"
# Build artifacts
MAVEN_OPTS="-Dzookeeper.version=$ZOOKEEPER_VERSION -DdistMgmtStagingId=${repo_id} -DdistMgmtStagingUrl=${nexus_repo_url}"
MAVEN_OPTS="$MAVEN_OPTS -DskipTests -DskipTest -DskipITs"
#mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} -Dtomcat.version=${BIGTOP_TOMCAT_VERSION} -Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz" install "$@"
#mvn site site:stage ${MAVEN_OPTS} $@
# Defining Install and Deploy OPTS for hadoop
HADOOP_INSTALL_OPTS="${MAVEN_OPTS} install"
HADOOP_DEPLOY_OPTS="${MAVEN_OPTS} deploy"
# Based on Environment Flag, setting the right HADOOP_OPTS with install/deploy.
[[ $deployarg = "yes" ]] && HADOOP_OPTS="${HADOOP_DEPLOY_OPTS}" || HADOOP_OPTS="${HADOOP_INSTALL_OPTS}"
#mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar -Dtomcat.version=${BIGTOP_TOMCAT_VERSION} -Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz" ${HADOOP_OPTS} "$@"
mvn clean package -Pdist -Dtar -Dmaven.javadoc.skip=true -DskipTests -fail-at-end -Pnative -Dtomcat.version=${BIGTOP_TOMCAT_VERSION} -Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz" ${HADOOP_OPTS} "$@"
mvn site site:stage ${HADOOP_OPTS} $@
(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
(cd build/src ; tar --strip-components=1 -xzvf ../../hadoop-dist/target/hadoop-${HADOOP_VERSION}-src.tar.gz)
# Copy generated docs
mkdir -p build/share
cp -r target/staging/hadoop-project build/share/doc
# Create a manifest for hadoop client package
(cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list
# Copy fuse output to the build directory
cp hadoop-hdfs-project/hadoop-hdfs/target/native/main/native/fuse-dfs/fuse_dfs build/bin/
|
