Staging git reference / snapshots-refgitref

Change-Id: I3334a032fa0b422891b708bb258a0e1adf9d81b2

5 files changed, 49 insertions, 53 deletions

diff --git a/jenkins.sh b/jenkins.sh
index 093b7ad8..feb100e9 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -48,8 +48,8 @@ user_workspace="${WORKSPACE}"
 # The files in this directory are shared across all platforms 
 shared="${HOME}/workspace/shared"
 
-# This is an optional directory for the master copy of the git repositories.
-user_git_repo="${shared}/snapshots"
+# This is an optional directory for the reference copy of the git repositories.
+git_reference="${HOME}/snapshots-ref"
 
 # set default values for options to make life easier
 user_snapshots="${user_workspace}/snapshots"
@@ -189,7 +189,7 @@ fi
 if test x"${abe_dir}" = x; then
     abe_dir=${topdir}
 fi
-$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${user_git_repo} --with-languages=${languages} --enable-schroot-test
+$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${git_reference} --with-languages=${languages} --enable-schroot-test
 
 # Double parallelism for tcwg-ex40-* machines to compensate for really-remote
 # target execution.  GCC testsuites will run with -j 32.
@@ -221,9 +221,16 @@ else
     try_bootstrap=""
 fi
 
+rm -rf ${user_snapshots}
+mkdir -p ${user_snapshots}
+(
+    flock -s 9
+    $CONFIG_SHELL ${abe_dir}/abe.sh ${platform} ${change} --checkout all
+) 9>${git_reference}.lock
+
 # Now we build the cross compiler, for a native compiler this becomes
 # the stage2 bootstrap build.
-$CONFIG_SHELL ${abe_dir}/abe.sh --parallel ${check} ${tars} ${releasestr} ${platform} ${change} ${try_bootstrap} --timeout 100 --build all --disable make_docs > build.out 2> >(tee build.err >&2)
+$CONFIG_SHELL ${abe_dir}/abe.sh --disable update ${check} ${tars} ${releasestr} ${platform} ${change} ${try_bootstrap} --timeout 100 --build all --disable make_docs > build.out 2> >(tee build.err >&2)
 
 # If abe returned an error, make jenkins see this as a build failure
 if test $? -gt 0; then
@@ -340,13 +347,13 @@ sums="`find ${user_workspace} -name \*.sum`"
 # Canadian Crosses are a win32 hosted cross toolchain built on a Linux
 # machine.
 if test x"${canadian}" = x"true"; then
-    $CONFIG_SHELL ${abe_dir}/abe.sh --nodepends --parallel ${change} ${platform} --build all
+    $CONFIG_SHELL ${abe_dir}/abe.sh --disable update --nodepends --parallel ${change} ${platform} --build all
     distro="`lsb_release -sc`"
     # Ubuntu Lucid uses an older version of Mingw32
     if test x"${distro}" = x"lucid"; then
-	$CONFIG_SHELL ${abe_dir}/abe.sh --nodepends --parallel ${change} ${tars} --host=i586-mingw32msvc ${platform} --build all
+	$CONFIG_SHELL ${abe_dir}/abe.sh --disable update --nodepends ${change} ${tars} --host=i586-mingw32msvc ${platform} --build all
     else
-	$CONFIG_SHELL ${abe_dir}/abe.sh --nodepends --parallel ${change} ${tars} --host=i686-w64-mingw32 ${platform} --build all
+	$CONFIG_SHELL ${abe_dir}/abe.sh --disable update --nodepends ${change} ${tars} --host=i686-w64-mingw32 ${platform} --build all
     fi
 fi
 
@@ -382,14 +389,14 @@ if test x"${sums}" != x -o x"${runtests}" != x"true"; then
 	echo "Sent test results"
     fi
     if test x"${tarsrc}" = xtrue -a x"${release}" != x; then
-	allfiles="`ls ${shared}/snapshots/*${release}*.xz`"
+	allfiles="`ls ${user_snapshots}/*${release}*.xz`"
 	srcfiles="`echo ${allfiles} | egrep -v "arm|aarch"`"
 	scp ${srcfiles} ${fileserver}:/home/abe/var/snapshots/ || status=1
 	rm -f ${srcfiles} || status=1
     fi
 
     if test x"${tarbin}" = xtrue -a x"${release}" != x; then
-	allfiles="`ls ${shared}/snapshots/*${release}*.xz`"
+	allfiles="`ls ${user_snapshots}/*${release}*.xz`"
 	binfiles="`echo ${allfiles} | egrep "arm|aarch"`"
 	scp ${binfiles} ${fileserver}:/work/space/binaries/ || status=1
 	rm -f ${binfiles} || status=1
diff --git a/lib/fetch.sh b/lib/fetch.sh
index 5d1771d7..6e8242e8 100755
--- a/lib/fetch.sh
+++ b/lib/fetch.sh
@@ -136,8 +136,11 @@ fetch_http()
     fi
 
     if test ! -e ${local_snapshots}/${getfile} -o x"${force}" = xyes; then
-	notice "Downloading ${getfile} to ${local_snapshots}"
-	if test x"${wget_bin}" != x; then
+	if test -e "${git_reference_dir}/${getfile}"; then
+	    notice "Copying ${getfile} from reference dir to ${local_snapshots}"
+	    dryrun "cp ${git_reference_dir}/${getfile} ${local_snapshots}/${getfile}"
+	elif test x"${wget_bin}" != x; then
+	    notice "Downloading ${getfile} to ${local_snapshots}"
 	    # --continue --progress=bar
 	    # NOTE: the timeout is short, and we only try twice to access the
 	    # remote host. This is to improve performance when offline, or
@@ -147,6 +150,8 @@ fetch_http()
 		warning "downloaded file ${getfile} has zero data!"
 		return 1
 	    fi
+	else
+	    error "Can't find wget to download ${getfile}"
 	fi
     else
 	# We don't want this message for md5sums, since it's so often
diff --git a/scripts/BuildFarm.job b/scripts/BuildFarm.job
index 0a1d1adf..938aabd1 100755
--- a/scripts/BuildFarm.job
+++ b/scripts/BuildFarm.job
@@ -12,18 +12,10 @@ export CONFIG_SHELL="/bin/bash"
 shared="/home/buildslave/workspace/shared"
 abe_dir="${WORKSPACE}/abe"
 user_snapshots="${WORKSPACE}/snapshots"
-snapshots_ref="${shared}/snapshots"
 fileserver="abe.tcwglab.linaro.org"
 
-mkdir -p ${snapshots_ref}/
-rsync -az --update ${fileserver}:snapshots/ ${snapshots_ref}/
-rm -rf ${snapshots_ref}/*~*
-
 rm -rf ${WORKSPACE}/_build ${user_snapshots}
 
-mkdir -p ${user_snapshots}/
-rsync -a ${snapshots_ref}/infrastructure/ ${user_snapshots}/infrastructure/
-
 case ${target} in
     schroot-*)
 	schroot_arch=$(echo ${target} | sed -e "s/^schroot-\([^-]*\)-\(.*\)/\1/")
diff --git a/scripts/MakeRelease.job b/scripts/MakeRelease.job
index c5fd79c8..bdffca5b 100755
--- a/scripts/MakeRelease.job
+++ b/scripts/MakeRelease.job
@@ -60,7 +60,7 @@ cd ${user_workspace}/_build
 
 if ! test -e host.conf; then
   $CONFIG_SHELL ${abe_dir}/configure ${chroot} --with-local-snapshots=${user_snapshots} \
-      --with-git-reference-dir=${shared}/snapshots --with-fileserver=${fileserver}
+      --with-git-reference-dir=${HOME}/snapshots-ref --with-fileserver=${fileserver}
 fi
 
 gcc_source="gcc-linaro-${version}-${release}.tar.xz"
@@ -76,10 +76,17 @@ if test x"${date}" != x; then
     release="${date}"    
 fi
 
+rm -rf ${user_snapshots}
+mkdir -p ${user_snapshots}
+(
+    flock -s 9
+    $CONFIG_SHELL ${abe_dir}/abe.sh gcc=${gcc_source} ${platform} --checkout all
+) 9>${HOME}/snapshots-ref.lock
+
 # Build a toolchain, run the tests, and the create a source release as a tarball
 if test x"${tarsrc}" = x"yes"; then
-    $CONFIG_SHELL ${abe_dir}/abe.sh --release ${release} \
-	--tarsrc ${check} gcc=${gcc_source} ${platform} -build all
+    $CONFIG_SHELL ${abe_dir}/abe.sh --disable update --release ${release} \
+	--tarsrc ${check} gcc=${gcc_source} ${platform} --build all
 fi
 
 # Build a binary release tarball, then run the tests on the installed binaries
diff --git a/test-backport.sh b/test-backport.sh
index 3d2a0541..bcfda019 100755
--- a/test-backport.sh
+++ b/test-backport.sh
@@ -45,11 +45,8 @@ topdir="`dirname ${which_dir}`"
 
 user_snapshots="${WORKSPACE}/snapshots"
 shared="/home/buildslave/workspace/shared/"
-snapshots_ref="${shared}/snapshots"
-
-if test ! -d ${snapshots_ref}/gcc.git; then
-    git clone http://git.linaro.org/git/toolchain/gcc.git ${snapshots_ref}/gcc.git
-fi
+snapshots_ref="${HOME}/snapshots-ref"
+snapshots_ref_lock=${snapshots_ref}.lock
 
 # Configure Abe itself. Force the use of bash instead of the Ubuntu
 # default of dash as some configure scripts go into an infinite loop with
@@ -129,33 +126,21 @@ if test "`echo ${branch} | grep -c gcc.git`" -gt 0; then
     branch="`echo ${branch} | sed -e 's:gcc.git~::'`"
 fi
 
-if test x"${git_reference_dir}" != x; then
-    srcdir="${git_reference_dir}/${branch}"
-else
-    git_reference_dir="${local_snapshots}"
-    srcdir="${local_snapshots}/gcc.git~${branch}"
-fi
+rm -rf ${user_snapshots}
+mkdir -p ${user_snapshots}
+(
+    flock -s 9
+    bash -x ${topdir}/abe.sh ${platform} --checkout all ${user_options}
 
-# Due to update cycles, sometimes the branch isn't in the repository yet.
-exists="`cd ${git_reference_dir}/${repo} && git branch -a | grep -c "${branch}"`"
-if test "${exists}" -eq 0; then
-    pushd ${git_reference_dir}/${repo} && git fetch
-    popd
-fi
-
-rm -fr ${srcdir}
-git-new-workdir ${git_reference_dir}/${repo} ${srcdir} ${branch} || exit 1
-# Make sure we are at the top of ${branch}
-pushd ${srcdir}
-# If in 'detached HEAD' state, don't try to update to the top of the branch
-detached=`git branch | grep detached`
-if test x"${detached}" = x; then
-    git checkout -B ${branch} origin/${branch} || exit 1
-fi
-popd
+    # Workaround broken --checkout all
+    # See https://bugs.linaro.org/show_bug.cgi?id=1338
+    if ! [ -d ${user_snapshots}/gcc.git ]; then
+	git clone --reference=${snapshots_ref}/gcc.git http://git.linaro.org/toolchain/gcc.git ${user_snapshots}/gcc.git
+    fi
+) 9>${snapshots_ref_lock}
 
-# Get the last two revisions
-declare -a revisions=(`cd ${srcdir} && git log -n 2 | grep ^commit | cut -d ' ' -f 2`)
+# Get the last two revisions of ${branch}
+declare -a revisions=(`cd ${user_snapshots}/gcc.git && git rev-parse origin/${branch} origin/${branch}^`)
 
 # Force GCC to not build the docs
 export BUILD_INFO=""
@@ -189,7 +174,7 @@ while test $i -lt ${#revisions[@]}; do
     if test $? -gt 0; then
 	echo "ERROR: Abe failed!"
 	exit 1
-   fi
+    fi
 
     # Compress .sum and .log files
     sums="`find ${local_builds}/${build}/${targetname}/ -name \*.sum`"