aboutsummaryrefslogtreecommitdiff
path: root/build-scripts/helpers
blob: df8264ab6db4967b56edcdc55500f09ef7043287 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
# Set REPO_MIRROR to non-empty value to get around upstream downtimes
#REPO_MIRROR="--repo-url=http://android.git.linaro.org/git-ro/tools/repo"
REPO_MIRROR="--repo-url=git://android.git.linaro.org/tools/repo"

setup-repo-vars () {
    EABI="${EABI-arm-eabi}"
    MANIFEST_REPO="${MANIFEST_REPO-git://android.git.kernel.org/platform/manifest.git}"
    MANIFEST_BRANCH="${MANIFEST_BRANCH-master}"
    MANIFEST_FILENAME="${MANIFEST_FILENAME-default.xml}"
    REPO_GROUPS=${REPO_GROUPS-"all,-notdefault,-eclipse"}
    REPO_QUIET=${REPO_QUIET-false}
    if [ "$REPO_QUIET" == "0" -o "$REPO_QUIET" == "false" ]; then
        REPO_QUIET=""
    else
        REPO_QUIET="--quiet"
    fi
    if [ -n "$REPO_SEED_URL" ]; then
        # If we use seeded builds, use higher value of repo sync -j,
        # as large share of repository content will be local and thus
        # only disk i/o bound.
        SYNC_JOBS=${SYNC_JOBS-5}
        #SYNC_JOBS=5
    else
        SYNC_JOBS=${SYNC_JOBS-3}
        #SYNC_JOBS=3
    fi

}

repo-sync-stubborn () {
    for i in 1 2; do
        time repo sync $REPO_QUIET -j$SYNC_JOBS -f || true
    done
    time repo sync $REPO_QUIET -j$SYNC_JOBS
}

repo-sync-from-mirror () {
    setup-repo-vars

    if [ -d "$REPO_SEED_DIR" ]; then
        repo init $REPO_QUIET -u "${MANIFEST_REPO}" -b "${MANIFEST_BRANCH}" -m "${MANIFEST_FILENAME}" $REPO_MIRROR  -g $REPO_GROUPS --reference=$REPO_SEED
    else
        repo init $REPO_QUIET -u "${MANIFEST_REPO}" -b "${MANIFEST_BRANCH}" -m "${MANIFEST_FILENAME}" $REPO_MIRROR  -g $REPO_GROUPS --depth=1
    fi

    # Save input manifest as build artifact for reference
    mkdir -p out
    cp .repo/manifest.xml out/source-manifest.xml

    if [ -n "$LOCAL_MANIFEST" ]; then
        if [ ! -n "$LOCAL_MANIFEST_BRANCH" ]; then
           LOCAL_MANIFEST_BRANCH="master"
        fi
        cd .repo/
        rm -rf local_manifests
        git clone $LOCAL_MANIFEST -b $LOCAL_MANIFEST_BRANCH local_manifests
        cd -
    fi

    if [ "$REWRITE_MANIFEST" != "0" ]; then
        echo Replace Linaro git URLs with lightweight http git URLs.
        ${BUILD_SCRIPT_ROOT}/rewrite-manifest.py .repo/manifest.xml -o processed-manifest.xml -u "${MANIFEST_REPO}"
        cp processed-manifest.xml .repo/manifest.xml
    fi

    echo ----------------------------
    repo-sync-stubborn
    # Restore source manifest temporarily to create pinned manifest
    cp out/source-manifest.xml .repo/manifest.xml
    repo manifest -r -o out/pinned-manifest.xml
}

repo-sync-from-seed () {
    setup-repo-vars

    local seed_name=$(basename $REPO_SEED_URL .tar.gz)
    mkdir -p $seed_name

    export TIMEFORMAT="TIME: Seed download and uncompress: %lR"
    time curl --silent --show-error "$REPO_SEED_URL" | gzip -d -c | tar --strip-components=1 -C $seed_name -x
    unset TIMEFORMAT

#    # Handle old seed format
#    if [ -d $seed_name/$seed_name ]; then
#        mv $seed_name ${seed_name}_
#        mv ${seed_name}_/$seed_name .
#        rmdir ${seed_name}_
#    fi

    repo init $REPO_QUIET -u "${MANIFEST_REPO}" -b "${MANIFEST_BRANCH}" -m "${MANIFEST_FILENAME}" \
    $REPO_MIRROR --reference=$PWD/$seed_name -g $REPO_GROUPS
    mkdir -p out
    cp .repo/manifest.xml out/source-manifest.xml

    echo Replace Linaro git URLs with lightweight http git URLs.
    ${BUILD_SCRIPT_ROOT}/rewrite-manifest.py .repo/manifest.xml -o processed-manifest.xml -u "${MANIFEST_REPO}"
    cp processed-manifest.xml .repo/manifest.xml

    export TIMEFORMAT="TIME: Repo sync (using seed as reference): %lR"
    repo-sync-stubborn
    unset TIMEFORMAT

    # Restore source manifest temporarily to create pinned manifest
    cp out/source-manifest.xml .repo/manifest.xml
    repo manifest -r -o out/pinned-manifest.xml
}

calc_make_jobs () {
    if [ -z "$MAKE_JOBS" ]; then
        cpuc=`cat /proc/cpuinfo  | grep processor | wc -l`
        MAKE_JOBS=$(($cpuc * 4))
    fi
}

function get_url_basename {
    echo "$1" | sed -e 's/^.*\/\([^/]*\)$/\1/'
}

function get_tarball_version {
#    echo "$1" | sed -e 's/^.*-//' -e 's/\.tar.*$//'
    echo "$1" | sed -r -e 's/\.tar.*$//' -e 's/^[a-z0-9]+-(.*)/\1/'
}
#echo `get_version gcc-linaro-4.5-2011.04-0.tar.bz2`
# linaro-4.5-2011.04-0
#echo `get_version binutils-2.20.1.tar.bz2`
# 2.20.1

#
# Download another builds's artifacts, as if they were
# produced by this build. Used for testing, trigger by BUILD_COPYCAT
# build config setting.
#
download_another_build () {
    local baseurl
    if ! echo "$1" | grep '://' >/dev/null; then
        baseurl="http://snapshots.linaro.org/android/$1"
    else
        baseurl=$1
    fi
    local dir=out/target/product/$TARGET_OUT_DIR
    mkdir -p $dir
    pushd $dir
    wget -nv --no-check-certificate "$baseurl/boot.tar.bz2" || true
    wget -nv --no-check-certificate "$baseurl/boot.img" || true
    wget -nv --no-check-certificate "$baseurl/system.tar.bz2" || true
    wget -nv --no-check-certificate "$baseurl/system.img" || true
    wget -nv --no-check-certificate "$baseurl/userdata.tar.bz2" || true
    wget -nv --no-check-certificate "$baseurl/userdata.img" || true
    popd
    pushd out
    wget -nv --no-check-certificate "$baseurl/source-manifest.xml" || true
    wget -nv --no-check-certificate "$baseurl/pinned-manifest.xml" || true
    popd
    cat >out/BUILD-INFO.txt <<EOF
Format-Version: 0.5

Files-Pattern: *
License-Type: open
EOF

}

flatten_artifacts () {
    if [ "$FLATTEN_ARTIFACTS" != "0" ]; then
        mv out/target/product/$TARGET_OUT_DIR/* out/
        rm -r out/target
    fi
}

# Keep download and unpack overlays split, to download soon after start
# to catch error fast, but preserve ability overwrite something.
download_overlays () {
    local ifs_save="$IFS"
    IFS=";"
    for p in $1; do
        curl --fail --silent --show-error \
            -b license_accepted_$SOURCE_OVERLAY_ACCEPT_LICENSE=yes \
            http://snapshots.linaro.org/android/binaries/$p > $(basename $p)
    done
    IFS="$ifs_save"
}
unpack_overlays () {
    local ifs_save="$IFS"
    IFS=";"
    for p in $1; do
        tar -x -a -f $(basename $p) -C .
    done
    IFS="$ifs_save"
}

unpack_external_tarball () {
    ext_tarball_dir=$BUILD_SCRIPT_ROOT/../../build/external_tarballs

    # Always start clean
    if [[ -d $ext_tarball_dir ]]; then
        rm -rf $ext_tarball_dir
    fi

    mkdir -p $ext_tarball_dir
    pushd $ext_tarball_dir

    local ifs_save="$IFS"
    IFS=";"
    for p in $1; do
        regex_is_http="^https?://"
        regex_file=".*/(.*?)$"

        if [[ $p =~ $regex_is_http ]]; then
            # File is on a remote web server, wget it
            wget -nv --no-check-certificate $p
        fi

        if [[ $p =~ $regex_file ]]; then
            filename=${BASH_REMATCH[1]}
            tar -x -a -f "$filename" -C $ext_tarball_dir
        else
            echo "Fatal Error: $p - file name can not be determined"
            exit 1
        fi
    done
    IFS="$ifs_save"
    popd
}

# Error handler for infrastructure setup (vs compiple) errors
# Wrap infra-setup code in:
# trap infrastructure_error ERR
# trap - ERR
infrastructure_error () {
    echo "Caught infrastructure error - finishing build with 'Not Built' status"
    # Interpreted as NOT_BUILT by Shell Status plugin
    exit 123
}

# Convert Android's TARGET_PRODUCT to linaro-android-media-create --dev option value
product2lamc_dev () {
# Inventory: vexpress,snowball_emmc,mx6qsabrelite,vexpress-a9,panda,iMX53,smdkv310,snowball_sd,beagle,origen,mx53loco
    case "$1" in
        "pandaboard")
            echo -n "panda";;
        "juno")
            echo -n "vexpress";;
        "snowball")
            echo -n "snowball_sd";;
        "full_arndale")
            echo -n "arndale";;
        *)
            echo -n "$1";;
    esac
}

#Check if we are building on VPS
is_on_ec2() {
    [ -e "/etc/cloud/cloud.cfg" ]
}

get_build_config() {
if is_on_ec2; then
    echo "/var/run/"
else
    echo "/tmp/"
fi
}