Merge branch 'merge'

Conflicts:
	lib/make.sh

27 files changed, 594 insertions, 222 deletions

diff --git a/config/boards/infra-armv8.exp b/config/boards/infra-armv8.exp
new file mode 100644
index 00000000..9eddd7ae
--- /dev/null
+++ b/config/boards/infra-armv8.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname apm-01
diff --git a/config/boards/infra-build.exp b/config/boards/infra-build.exp
new file mode 100644
index 00000000..13aafc15
--- /dev/null
+++ b/config/boards/infra-build.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname build-06
diff --git a/config/boards/infra-buildqemu.exp b/config/boards/infra-buildqemu.exp
new file mode 100644
index 00000000..111052e3
--- /dev/null
+++ b/config/boards/infra-buildqemu.exp
@@ -0,0 +1,4 @@
+load_generic_config tcwg-common
+
+set_board_info hostname build-06
+set_board_info remote_sim "qemu-[lindex [split $target_triplet -] 0]-static -cpu any -L /sysroot"
diff --git a/config/boards/infra-tcwgex40.exp b/config/boards/infra-tcwgex40.exp
new file mode 100644
index 00000000..9e98e8eb
--- /dev/null
+++ b/config/boards/infra-tcwgex40.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname tcwg-ex40-09
diff --git a/config/boards/infra-tcwgex40qemu.exp b/config/boards/infra-tcwgex40qemu.exp
new file mode 100644
index 00000000..c13c0a95
--- /dev/null
+++ b/config/boards/infra-tcwgex40qemu.exp
@@ -0,0 +1,4 @@
+load_generic_config tcwg-common
+
+set_board_info hostname tcwg-ex40-09
+set_board_info remote_sim "qemu-[lindex [split $target_triplet -] 0]-static -cpu any -L /sysroot"
diff --git a/config/boards/tcwg-armv8.exp b/config/boards/tcwg-armv8.exp
new file mode 100644
index 00000000..fde9e6d3
--- /dev/null
+++ b/config/boards/tcwg-armv8.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname juno-01
diff --git a/config/boards/tcwg-build.exp b/config/boards/tcwg-build.exp
new file mode 100644
index 00000000..d81437cc
--- /dev/null
+++ b/config/boards/tcwg-build.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname build-01
diff --git a/config/boards/tcwg-buildqemu.exp b/config/boards/tcwg-buildqemu.exp
new file mode 100644
index 00000000..1ef3881b
--- /dev/null
+++ b/config/boards/tcwg-buildqemu.exp
@@ -0,0 +1,4 @@
+load_generic_config tcwg-common
+
+set_board_info hostname build-01
+set_board_info remote_sim "qemu-[lindex [split $target_triplet -] 0]-static -cpu any -L /sysroot"
diff --git a/config/boards/tcwg-common.exp b/config/boards/tcwg-common.exp
index c3bbe463..e957390a 100644
--- a/config/boards/tcwg-common.exp
+++ b/config/boards/tcwg-common.exp
@@ -3,6 +3,8 @@ load_generic_config schroot-ssh
 
 set_board_info rsh_prog /usr/bin/ssh
 set_board_info rcp_prog /usr/bin/scp
-#set_board_info username buildslave
-set_board_info gcc,timeout 100000
-set_board_info ldflags "-static -lpthread -ldl"
+
+# Compilation timeout 3min
+set_board_info gcc,timeout 180
+# Execution timeout 3min
+set_board_info timeout 180
diff --git a/config/boards/tcwg-tcwgex40.exp b/config/boards/tcwg-tcwgex40.exp
new file mode 100644
index 00000000..b52d1479
--- /dev/null
+++ b/config/boards/tcwg-tcwgex40.exp
@@ -0,0 +1,3 @@
+load_generic_config tcwg-common
+
+set_board_info hostname tcwg-ex40-01
diff --git a/config/boards/tcwg-tcwgex40qemu.exp b/config/boards/tcwg-tcwgex40qemu.exp
new file mode 100644
index 00000000..a3981ad7
--- /dev/null
+++ b/config/boards/tcwg-tcwgex40qemu.exp
@@ -0,0 +1,4 @@
+load_generic_config tcwg-common
+
+set_board_info hostname tcwg-ex40-01
+set_board_info remote_sim "qemu-[lindex [split $target_triplet -] 0]-static -cpu any -L /sysroot"
diff --git a/config/boards/toolchain64/tcwgbuild.exp b/config/boards/toolchain64/tcwgbuild.exp
deleted file mode 100644
index 7364b7cb..00000000
--- a/config/boards/toolchain64/tcwgbuild.exp
+++ /dev/null
@@ -1,3 +0,0 @@
-load_generic_config tcwgcommon
-
-set_board_info hostname tcwgbuild06
diff --git a/config/gcc.conf b/config/gcc.conf
index 888377a0..223102d5 100644
--- a/config/gcc.conf
+++ b/config/gcc.conf
@@ -158,6 +158,12 @@ if test x"${build}" != x"${target}"; then
         stage2_flags="${stage2_flags} --enable-checking=release"
     fi
 
+    if test x"${bootstrap}" = x"yes"; then
+	stage2_flags="${stage2_flags} --enable-bootstrap"
+    else
+	stage2_flags="${stage2_flags} --disable-bootstrap"
+    fi
+
 # native builds are easy, so build almost everything
 else
     if test x"${with_languages}" != x"default"; then
@@ -169,9 +175,9 @@ else
     # Native compilers are bootstrapped by default, but sometimes the extra time isn't
     # desired.
     if test x"${bootstrap}" = x"yes"; then
-	default_configure_flags="${default_configure_flags} --enable-bootstrap" 
+	default_configure_flags="${default_configure_flags} --enable-bootstrap"
     else
-	default_configure_flags="${default_configure_flags} --disable-bootstrap" 
+	default_configure_flags="${default_configure_flags} --disable-bootstrap"
     fi
     case ${target} in
 	arm*-none-linux-gnueabi)
diff --git a/config/linaro.exp b/config/linaro.exp
index 75c06652..8996e4b6 100644
--- a/config/linaro.exp
+++ b/config/linaro.exp
@@ -25,11 +25,29 @@ if {[info exists env(PREFIX_UNDER_TEST)]} {
     set GNAT_UNDER_TEST "[set prefix]gnat $flags"
 }
 
+if {[info exists env(SYSROOT_UNDER_TEST)]} {
+    set native ""
+} else {
+    set native "native"
+}
+
 set myname [get_local_hostname]
 
+case "$myname" in {
+    { localhost } {
+	set target_list { unix }
+    }
+}
+
+if { "$tcl_platform(user)" == "buildslave" } {
+    set machine_prefix "infra"
+} else {
+    set machine_prefix "tcwg"
+}
+
 case "$target_triplet" in { 
-    { "arm-*linux-gnueabi*"} {
-	case "$myname" in {
+    { "arm-*linux-gnueabi*" } {
+	case "$native$myname" in {
 	    { tcwgrob* ripple* darkstar* } {
 #		set target_list { tcwgchromebook }
 #		set target_list { tcwgd01 }
@@ -37,10 +55,10 @@ case "$target_triplet" in {
 #		set target_list { "arm-qemu" } 
 		set target_list { dragon-01 }
 	    }
-	    { build* *ex40-* } {
-		set target_list { d01 }
+	    { build-* tcwg-ex40-* d01-* apm-* juno-* amd-* } {
+		set target_list [list $machine_prefix-armv8]
 	    }
-	    { localhost chromebook* d01* dragon* chrome2-* } {
+	    { natived01-* nativeapm-* nativejuno-* nativeamd-* } {
 		set target_list { unix }
 	    }
 	    default{
@@ -51,28 +69,28 @@ case "$target_triplet" in {
             }
 	}
     }
-    { "arm-*eabi" } {
+    { "arm-*-eabi*" } {
 	global SIM
 	# CPU and sysroot are set through the environment
 	set SIM "qemu-arm"
 	set target_list { "arm-qemu" } 
     }
-    { "armeb-*" } {
+    { "armeb-*-eabi*" } {
 	global SIM
 	# CPU and sysroot are set through the environment
 	set SIM "qemu-armeb"
 	set target_list { "arm-qemu" } 
     }
-    { "aarch64*-none-elf*" } {
+    { "aarch64*-*elf*" } {
 	set target_list { "v8model" } 
 	set env(FOUNDATION_MODEL) "/linaro/foundation-model/Foundation_v8pkg/Foundation_v8"
     }
-    { "aarch64*linux-gnu" } {
-	case "$myname" in {
-	    { tcwgrob build* *ex40* } {
-		set target_list { apm }
+    { "aarch64-*linux*" } {
+	case "$native$myname" in {
+	    { tcwgrob build-* tcwg-ex40-* apm-* juno-* amd-* } {
+		set target_list [list $machine_prefix-armv8]
 	    }
-	    { localhost juno* amd* apm-* } {
+	    { nativeapm-* nativejuno-* nativeamd-* } {
 		set target_list { unix }
 	    }
 	    default {
@@ -82,17 +100,25 @@ case "$target_triplet" in {
 	}
     }
     { "x86_64-*linux-gnu" "i686-*linux-gnu" } {
-	case "$myname" in {
-	    { tcwgrob build* *ex40-* } {
+	case "$native$myname" in {
+	    { build-* } {
+		set target_list [list $machine_prefix-build]
+	    }
+	    { tcwg-ex40-* } {
+		set target_list [list $machine_prefix-tcwgex40]
+	    }
+	    { nativebuild-* nativetcwg-ex40-* } {
 		set target_list { unix }
 	    }
 	}
     }
     { "*linux-gnu*" } {
 	case "$myname" in {
-	    { tcwgrob build* *ex40-* } {
-		set target_list { build  ex40 }
-		set board_info(build,remote_sim) "qemu-[lindex [split $target_triplet -] 0]-static -cpu any -L /sysroot"
+	    { build-* } {
+		set target_list [list $machine_prefix-buildqemu]
+	    }
+	    { tcwg-ex40-* } {
+		set target_list [list $machine_prefix-tcwgex40qemu]
 	    }
 	}
     }
diff --git a/config/schroot-master/chroot.d/.gitignore b/config/schroot-master/chroot.d/.gitignore
index 59913031..bd86dc5f 100644
--- a/config/schroot-master/chroot.d/.gitignore
+++ b/config/schroot-master/chroot.d/.gitignore
@@ -1 +1,2 @@
 tcwg-test-*
+tcwg-build-*
diff --git a/config/schroot-master/tcwg-build/copyfiles b/config/schroot-master/tcwg-build/copyfiles
new file mode 100644
index 00000000..22b29613
--- /dev/null
+++ b/config/schroot-master/tcwg-build/copyfiles
@@ -0,0 +1,4 @@
+# Files to copy into the chroot from the host system.
+#
+# <source and destination>
+/etc/resolv.conf
diff --git a/config/schroot-master/tcwg-build/fstab b/config/schroot-master/tcwg-build/fstab
new file mode 100644
index 00000000..0ea37e1d
--- /dev/null
+++ b/config/schroot-master/tcwg-build/fstab
@@ -0,0 +1,25 @@
+# fstab: static file system information for chroots.
+# Note that the mount point will be prefixed by the chroot path
+# (CHROOT_PATH)
+#
+# <file system>	<mount point>	<type>	<options>	<dump>	<pass>
+/proc		/proc		none    rw,bind        0       0
+/sys		/sys		none    rw,bind        0       0
+
+tmpfs           /run            tmpfs   rw,noexec,nosuid,size=10%,mode=0755    0       0
+none            /run/lock       tmpfs   rw,noexec,nosuid,nodev,size=5242880    0       0
+none            /run/shm        tmpfs   rw,nosuid,nodev 0       0
+
+/dev            /dev            none    rw,bind         0       0
+/dev/pts	/dev/pts	none	rw,bind		0	0
+/home		/home		none	rw,bind		0	0
+#/tmp		/tmp		none	rw,bind		0	0
+
+# It may be desirable to have access to /run, especially if you wish
+# to run additional services in the chroot.  However, note that this
+# may potentially cause undesirable behaviour on upgrades, such as
+# killing services on the host.
+#/run		/run		none	rw,bind		0	0
+#/run/lock	/run/lock	none	rw,bind		0	0
+#/dev/shm	/dev/shm	none	rw,bind		0	0
+#/run/shm	/run/shm	none	rw,bind		0	0
diff --git a/config/schroot-master/tcwg-build/nssdatabases b/config/schroot-master/tcwg-build/nssdatabases
new file mode 100644
index 00000000..722a4516
--- /dev/null
+++ b/config/schroot-master/tcwg-build/nssdatabases
@@ -0,0 +1,11 @@
+# System databases to copy into the chroot from the host system.
+#
+# <database name>
+passwd
+shadow
+group
+gshadow
+services
+protocols
+networks
+hosts
diff --git a/config/schroot-master/tcwg-schroot-farm.sh b/config/schroot-master/tcwg-schroot-farm.sh
deleted file mode 100755
index 5b6541cb..00000000
--- a/config/schroot-master/tcwg-schroot-farm.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-#set -x
-
-master="$(dirname $0)"
-test_schroot="$master/../../scripts/test-schroot.sh"
-
-tcwgbuild_archs=(
-    aarch64-linux-gnu
-    arm-linux-gnueabihf
-    i686-linux-gnu
-    x86_64-linux-gnu
-)
-
-# Generate chroots for all supported targets on tcwgbuild01 ...
-pids=""
-set -e
-for a in "${tcwgbuild_archs[@]}"; do
-    true &
-    #eval $test_schroot -g -c $master -a $a maxim@tcwgbuild01 &
-    pids="$pids $!"
-done
-set +e
-wait $pids
-
-ssh_opts="-o ConnectTimeout=5"
-
-#cmd1="true"
-cmd1="ssh $ssh_opts \$m sudo apt-get install -y schroot"
-#cmd1="ssh $ssh_opts \$m sudo rm -f /etc/schroot/chroot.d/tcwg-test-\\\* /var/chroots/tcwg-test-\\\*"
-#cmd1="echo \$m; ssh $ssh_opts \$m bash -c \\\"schroot -l --all-sessions \| grep -e tcwg-test \\\""
-#cmd1="echo \$m; ssh $ssh_opts \$m bash -c \\\"schroot -l --all-sessions \| grep -e tcwg-test \| xargs -i@ sudo schroot -e -c @\\\""
-
-#cmd2="true"
-cmd2="($test_schroot -c $master -a \$a -o \"$ssh_opts\" \$m && echo \$m:\$a: OK) || echo \$m:\$a: ERROR"
-
-# ... then copy them to all other tcwgbuildXX machines.
-pids=""
-for m in $(for i in `seq 1 6`; do echo maxim@tcwgbuild0$i; done); do
-    eval $cmd1 &
-    pids="$pids $!"
-    for a in i686-linux-gnu x86_64-linux-gnu; do
-	eval $cmd2 &
-	pids="$pids $!"
-    done
-done
-# Copy ARMv7 hard-fp and soft-fp chroots to all D01s and chromebooks.
-for m in $(for i in `seq 1 3`; do echo linaro@tcwg-d01-0$i; done) $(for i in `seq 1 8`; do echo linaro@tcwgchromebook0$i; done); do
-    eval $cmd1 &
-    pids="$pids $!"
-    for a in arm-linux-gnueabihf; do
-	eval $cmd2 &
-	pids="$pids $!"
-    done
-done
-# Copy Aarch64 and ARM 32-bit chroots to APM boards.
-for m in $(for i in `seq 1 3`; do echo linaro@tcwg-apm-0$i; done); do
-    eval $cmd1 &
-    pids="$pids $!"
-    for a in aarch64-linux-gnu arm-linux-gnueabihf; do
-	eval $cmd2 &
-	pids="$pids $!"
-    done
-done
-
-wait $pids
-echo ALL DONE
diff --git a/jenkins.sh b/jenkins.sh
index 2a4c214d..3d311ec2 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -45,19 +45,28 @@ user_workspace="${WORKSPACE}"
 # The files in this directory are shared across all platforms 
 shared="${HOME}/workspace/shared"
 
-# This is where all the git repositories live
-user_git_repo="--with-git-reference-dir=${shared}/snapshots"
+# This is an optional directory for the master copy of the git repositories.
+user_git_repo="${shared}/snapshots"
 
 # set default values for options to make life easier
 user_snapshots="${user_workspace}/snapshots"
 
+# Server to store results on.
+fileserver="abe.tcwglab.linaro.org"
+
+# Compiler languages to build
+languages=default
+
+# Whether attempt bootstrap
+try_bootstrap=false
+
 # The release version string, usually a date
 releasestr=
 
 # This is a string of optional extra arguments to pass to abe at runtime
 user_options=""
 
-OPTS="`getopt -o s:g:c:w:o:f:t:h -l snapshots:gitrepo:abe:workspace:options:fileserver:target:help -- "$@"`"
+OPTS="`getopt -o s:g:c:w:o:f:l:rt:b:h -l snapshots:,gitrepo:,abe:,workspace:,options:,fileserver:,languages:,runtests,target:,bootstrap,help -- "$@"`"
 while test $# -gt 0; do
     echo 1 = "$1"
     case $1 in
@@ -68,7 +77,9 @@ while test $# -gt 0; do
         -w|--workspace) user_workspace=$2 ;;
         -o|--options) user_options=$2 ;;
         -f|--fileserver) fileserver=$2 ;;
+        -l|--languages) languages=$2 ;;
         -r|--runtests) runtests="true" ;;
+        -b|--bootstrap) try_bootstrap="true" ;;
 	-h|--help) usage ;;
     esac
     shift
@@ -95,11 +106,6 @@ if test "`echo $user_options | grep -c -- --release`" -gt 0; then
     releasestr="--release ${release}"
 fi
 
-# This is an optional directory for the master copy of the git repositories.
-if test x"${user_git_repo}" = x; then
-    user_git_repo="--with-git-reference-dir=${shared}/snapshots"
-fi
-
 # Get the versions of dependant components to use
 changes=""
 if test x"${gmp_snapshot}" != x"latest" -a x"${gmp_snapshot}" != x; then
@@ -173,7 +179,13 @@ fi
 if test x"${abe_dir}" = x; then
     abe_dir=${topdir}
 fi
-$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${shared}/snapshots
+$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${user_git_repo} --with-languages=${languages} --enable-schroot-test
+
+# Double parallelism for tcwg-ex40-* machines to compensate for really-remote
+# target execution.  GCC testsuites will run with -j 32.
+case "$(hostname)" in
+    "tcwg-ex40-"*) sed -i -e "s/cpus=8/cpus=16/" host.conf ;;
+esac
 
 # load commonly used varibles set by configure
 if test -e "${PWD}/host.conf"; then
@@ -186,19 +198,28 @@ fi
 # Delete the previous test result files to avoid problems.
 find ${user_workspace} -name \*.sum -exec rm {} \;  2>&1 > /dev/null
 
-# For cross build. For cross builds we build a native GCC, and then use
-# that to compile the cross compiler to bootstrap. Since it's just
-# used to build the cross compiler, we don't bother to run 'make check'.
-if test x"${bootstrap}" = xtrue; then
-    $CONFIG_SHELL ${abe_dir}/abe.sh --parallel ${change} --bootstrap --build all
+if test x"${try_bootstrap}" = xtrue; then
+    # Attempt to bootstrap GCC is build and target are compatible
+    build1="$(grep "^build=" host.conf | sed -e "s/build=\(.*\)-\(.*\)-\(.*\)-\(.*\)/\1-\3-\4/")"
+    target1="$(echo ${target} | sed -e "s/\(.*\)-\(.*\)-\(.*\)-\(.*\)/\1-\3-\4/")"
+    if test x"${build1}" = x"${target1}" -o x"${platform}" = x""; then
+	try_bootstrap="--enable bootstrap"
+    else
+	try_bootstrap="--disable bootstrap"
+    fi
+else
+    try_bootstrap=""
 fi
 
 # Now we build the cross compiler, for a native compiler this becomes
 # the stage2 bootstrap build.
-$CONFIG_SHELL ${abe_dir}/abe.sh --parallel ${check} ${tars} ${releasestr} ${platform} ${change} --timeout 100 --build all
+$CONFIG_SHELL ${abe_dir}/abe.sh --parallel ${check} ${tars} ${releasestr} ${platform} ${change} ${try_bootstrap} --timeout 100 --build all --disable make_docs > build.out 2> >(tee build.err >&2)
 
 # If abe returned an error, make jenkins see this as a build failure
 if test $? -gt 0; then
+    echo "================= TAIL OF LOG: BEGIN ================="
+    tail -n 50 build.out
+    echo "================= TAIL OF LOG: FINISH ================="
     exit 1
 fi
 
@@ -323,16 +344,24 @@ if test x"${sums}" != x -o x"${runtests}" != x"true"; then
 	for s in ${sums}; do
 	    test_logs="$test_logs ${s%.sum}.log"
 	done
-	scp ${sums} $test_logs ${fileserver}:${basedir}/${dir}/
+
+	logs_dir=$(mktemp -d)
+	cp ${sums} ${test_logs} ${logs_dir}/
 	
 	# Copy over the logs from make check, which we need to find testcase errors.
 	checks="`find ${user_workspace} -name check\*.log`"
-	scp ${checks} ${fileserver}:${basedir}/${dir}/
+	cp ${checks} ${logs_dir}/
 	
 	# Copy over the build logs
 	logs="`find ${user_workspace} -name make\*.log`"
-	scp ${logs} ${fileserver}:${basedir}/${dir}/
-	ssh ${fileserver} xz ${basedir}/${dir}/\*.sum ${basedir}/${dir}/\*.log
+	cp ${logs} ${logs_dir}/
+
+	# Copy stdout and stderr output from abe.
+	cp build.out build.err ${logs_dir}/
+
+	xz ${logs_dir}/*
+	scp ${logs_dir}/* ${fileserver}:${basedir}/${dir}/
+	rm -rf ${logs_dir}
 	scp ${abe_dir}/tcwgweb.sh ${fileserver}:/tmp/tcwgweb$$.sh
 	ssh ${fileserver} /tmp/tcwgweb$$.sh --email --base ${basedir}/${dir}
 	ssh ${fileserver} rm -f /tmp/tcwgweb$$.sh
diff --git a/lib/checkout.sh b/lib/checkout.sh
index 907520a6..2459c987 100644
--- a/lib/checkout.sh
+++ b/lib/checkout.sh
@@ -254,6 +254,7 @@ checkout()
 		notice "Cloning $1 in ${srcdir}"
 		dryrun "git_robust clone $git_reference_opt ${url} ${repodir}"
 	    fi
+
 	    if test ! -d ${srcdir}; then
 		# By definition a git commit resides on a branch.  Therefore specifying a
 		# branch AND a commit is redundant and potentially contradictory.  For this
@@ -270,23 +271,6 @@ checkout()
 		    # it doesn't exist already.
 		    dryrun "(cd ${srcdir} && git checkout -B local_${revision})"
 	        else
-		    if test x"${branch}" != x""; then
-			# Sometimes, after removing a srcdir and re-running, the branch
-			# you're trying to checkout will already be a named branch in the
-			# repodir, so we have to delete it so that the new checkout will
-			# get the latest, updated branch source.
-			notice "Checking for existing named branch ${branch} in ${repodir}"
-
-			# Don't test this for dryrun because repodir probably won't exist.
-			if test x"${dryrun}" = no; then
-			    local existing_branch=`(cd ${repodir} && git branch -a | grep -c "^.*[[:space:]]\{1,\}${branch}")`
-			    if test ${existing_branch} -gt 0; then
-				notice "Removing previously named branch ${branch} from ${repodir}"
-				dryrun "(cd ${repodir} && git branch -D ${branch})"
-			    fi
-			fi
-		    fi
-
 		    notice "Checking out ${branch:+branch ${branch}}${branch-master branch} for ${tool} in ${srcdir}"
 		    dryrun "${NEWWORKDIR} ${local_snapshots}/${repo} ${srcdir} ${branch}"
 		    if test $? -gt 0; then
@@ -295,7 +279,7 @@ checkout()
 		    fi
 		fi
 		# dryrun "git_robust clone --local ${local_snapshots}/${repo} ${srcdir}"
-		# dryrun "(cd ${srcdir} && git checkout ${branch})"
+		# dryrun "(cd ${srcdir} && git checkout -B ${branch})"
 	    elif test x"${supdate}" = xyes; then
 		# Some packages allow the build to modify the source directory and
 		# that might screw up abe's state so we restore a pristine branch.
@@ -310,12 +294,12 @@ checkout()
 		if test x"${revision}" != x""; then
 		    # No need to pull.  A commit is a single moment in time
 		    # and doesn't change.
-		    dryrun "(cd ${srcdir} && git checkout local_${revision})"
+		    dryrun "(cd ${srcdir} && git checkout -B local_${revision})"
 		else
 		    # Make sure we are on the correct branch.
 		    # This is a no-op if $branch is empty and it
 		    # just gets master.
-		    dryrun "(cd ${srcdir} && git checkout ${branch})"
+		    dryrun "(cd ${srcdir} && git checkout -B ${branch} origin/${branch})"
 		    dryrun "(cd ${srcdir} && git_robust pull)"
 		fi
 	    fi
diff --git a/lib/common.sh b/lib/common.sh
index e23f3ecc..996e1e89 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -31,6 +31,7 @@ set -o pipefail
 . "${topdir}/lib/testcode.sh" || exit 1
 . "${topdir}/lib/git-parser.sh" || exit 1
 . "${topdir}/lib/stamp.sh" || exit 1
+. "${topdir}/lib/schroot.sh" || exit 1
 . "${topdir}/lib/gerrit.sh" || exit 1
 
 #
diff --git a/lib/make.sh b/lib/make.sh
index 9c6d0928..33287684 100755
--- a/lib/make.sh
+++ b/lib/make.sh
@@ -455,16 +455,20 @@ make_all()
 
 # Print path to dynamic linker in sysroot
 # $1 -- sysroot path
+# $2 -- whether dynamic linker is expected to exist
 find_dynamic_linker()
 {
     local sysroots="$1"
+    local strict="$2"
     local dynamic_linker c_library_version
 
     # Programmatically determine the embedded glibc version number for
     # this version of the clibrary.
-    c_library_version="`${sysroots}/usr/bin/ldd --version | head -n 1 | sed -e "s/.* //"`"
-    dynamic_linker="`find ${sysroots} -type f -name ld-${c_library_version}.so`"
-    if [ -z "$dynamic_linker" ]; then
+    if test -x "${sysroots}/usr/bin/ldd"; then
+	c_library_version="`${sysroots}/usr/bin/ldd --version | head -n 1 | sed -e "s/.* //"`"
+	dynamic_linker="`find ${sysroots} -type f -name ld-${c_library_version}.so`"
+    fi
+    if $strict && [ -z "$dynamic_linker" ]; then
         error "Couldn't find dynamic linker ld-${c_library_version}.so in ${sysroots}"
         exit 1
     fi
@@ -567,22 +571,12 @@ make_install()
     fi
 
     if test x"${tool}" = x"gcc"; then
-        local libs="`find ${builddir}/${target} -name \*.so\* -o -name \*.a`"
-#        if test "`echo ${target} | egrep -c 'aarch64|x86_64'`" -gt 0 ; then
-	if test x"${build_arch}" = x"aarch64" -o x"${build_arch}" = x"x86_64"; then
-            gcc_libpath=${sysroots}/usr/lib64/
-        else
-            gcc_libpath=${sysroots}/usr/lib/
-        fi
-        if test ! -e ${gcc_libpath}; then
-            dryrun "mkdir -p ${gcc_libpath}"
-	fi
-        dryrun "rsync -av ${libs} ${gcc_libpath}"
+	dryrun "copy_gcc_libs_to_sysroot \"${local_builds}/destdir/${host}/bin/${target}-gcc --sysroot=${sysroots}\""
     fi
 
     if test "`echo ${tool} | grep -c glibc`" -gt 0 -a "`echo ${target} | grep -c aarch64`" -gt 0; then
         local dynamic_linker
-        dynamic_linker="$(find_dynamic_linker "$sysroots")"
+        dynamic_linker="$(find_dynamic_linker "$sysroots" true)"
         local dynamic_linker_name="`basename ${dynamic_linker}`"
 
         # 64 bit architectures don't populate sysroot/lib, which unfortunately other
@@ -701,16 +695,54 @@ make_check()
     fi
 
     if test x"${parallel}" = x"yes"; then
-        local make_flags="${make_flags} -j ${cpus}"
+	local make_flags
+	case "${target}" in
+	    "$build"|*"-elf"*) make_flags="${make_flags} -j ${cpus}" ;;
+	    # Double parallelization when running tests on remote boards
+	    # to avoid host idling when waiting for the board.
+	    *) make_flags="${make_flags} -j $((2*${cpus}))" ;;
+	esac
     fi
 
     # load the config file for Linaro build farms
     export DEJAGNU=${topdir}/config/linaro.exp
 
+    # Run tests
     local checklog="${builddir}/check-${tool}.log"
     if test x"${build}" = x"${target}" -a x"${tarbin}" != x"yes"; then
         dryrun "make check RUNTESTFLAGS=\"${runtest_flags} --xml=${tool}.xml \" ${make_flags} -w -i -k -C ${builddir} 2>&1 | tee ${checklog}"
     else
+	local exec_tests
+	exec_tests=false
+	case "$tool" in
+	    gcc) exec_tests=true ;;
+	    binutils)
+		if [ x"$2" = x"gdb" ]; then
+		    exec_tests=true
+		fi
+		;;
+	esac
+
+	eval "schroot_make_opts="
+	eval "schroot_boards="
+
+	local schroot_port
+	if $exec_tests && [ x"$schroot_test" = x"yes" ]; then
+	    # Start schroot sessions on target boards that support it
+	    schroot_port="$(print_schroot_port)"
+	    local schroot_sysroot
+	    case "${target}" in
+		*"-elf"*) schroot_sysroot="" ;;
+		*) schroot_sysroot="$(make_target_sysroot)" ;;
+	    esac
+	    start_schroot_sessions "${target}" "${schroot_port}" "${schroot_sysroot}" "${builddir}"
+	    if test "$?" != "0"; then
+		stop_schroot_sessions "${schroot_port}" ${schroot_boards}
+		return 1
+	    fi
+	    rm -rf "$schroot_sysroot"
+	fi
+
 	case ${tool} in
 	    binutils)
 		local dirs="/binutils /ld /gas"
@@ -720,10 +752,6 @@ make_check()
 		local dirs="/gdb"
 		local check_targets="check-gdb"
 		;;
-#	    gcc)
-#		local dirs="/gcc"
-#		local check_targets="check-gcc check-c++ check-gfortran check-go check-target-libstdc++-v3 check-target-libgomp"
-#		;;
 	    *)
 		local dirs="/"
 		local check_targets="check"
@@ -735,14 +763,17 @@ make_check()
 	fi
 
 	for i in ${dirs}; do
-            dryrun "make ${check_targets} SYSROOT_UNDER_TEST=${sysroots} PREFIX_UNDER_TEST=\"${local_builds}/destdir/${host}/bin/${target}-\" RUNTESTFLAGS=\"${runtest_flags}\" ${make_flags} -w -i -k -C ${builddir}$i 2>&1 | tee ${checklog}"
+            dryrun "make ${check_targets} SYSROOT_UNDER_TEST=${sysroots} FLAGS_UNDER_TEST=\"\" PREFIX_UNDER_TEST=\"${local_builds}/destdir/${host}/bin/${target}-\" RUNTESTFLAGS=\"${runtest_flags}\" ${schroot_make_opts} ${make_flags} -w -i -k -C ${builddir}$i 2>&1 | tee ${checklog}"
 	done
+
+	# Stop schroot sessions
+	stop_schroot_sessions "$schroot_port" ${schroot_boards}
        
         if test x"${tool}" = x"gcc"; then
             rm -rf ${sysroots}/etc/ld.so.cache
 	fi
     fi
-    
+
     return 0
 }
 
@@ -843,3 +874,48 @@ EOF
     return 0
 }
 
+# Make a single-use target sysroot with all shared libraries for testing.
+# NOTE: It is responsibility of the caller to "rm -rf" the sysroot.
+# $1 - compiler (and any compiler flags) to query multilib information
+make_target_sysroot()
+{
+    trace "$*"
+
+    local sysroot
+    sysroot=/tmp/sysroot.$$
+    rsync -a $sysroots/ $sysroot/
+
+    if test "`echo ${target} | grep -c aarch64`" -gt 0; then
+	# Remove symlink lib64 -> lib to make sysroot debian-compatible.
+	rm $sysroot/lib
+    fi
+
+    echo $sysroot
+}
+
+# $1 - compiler (and any compiler flags) to query multilib information
+copy_gcc_libs_to_sysroot()
+{
+    local libgcc
+    local ldso
+    local gcc_lib_path
+    local sysroot_lib_dir
+
+    ldso="$(find_dynamic_linker "${sysroots}" false)"
+    if ! test -z "${ldso}"; then
+	libgcc="libgcc_s.so"
+    else
+	libgcc="libgcc.a"
+    fi
+
+    libgcc="$($@ -print-file-name=${libgcc})"
+    gcc_lib_path="$(dirname "${libgcc}")"
+
+    if ! test -z "${ldso}"; then
+	sysroot_lib_dir="$(dirname ${ldso})"
+    else
+	sysroot_lib_dir="${sysroots}/usr/lib"
+    fi
+
+    rsync -a ${gcc_lib_path}/ ${sysroot_lib_dir}/
+}
diff --git a/lib/schroot.sh b/lib/schroot.sh
new file mode 100755
index 00000000..ffb27912
--- /dev/null
+++ b/lib/schroot.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Print DJ board files used for testing
+# $1 - [optional] target triplet
+print_schroot_board_files()
+{
+    trace "$*"
+
+    local target="$1"
+
+    if ! [ -z "$target" ]; then
+	target="--target=$target"
+    fi
+
+    # Run dummy runtest to figure out which target boards load schroot-ssh.exp
+    runtest $target --tool none 2>&1 \
+	| grep -B5 "^Using .*/schroot-ssh.exp as generic interface file for target\.\$" \
+	| grep "^Using .* as board description file for target\.\$" \
+	| sed -e "s/^Using \(.*\) as board description file for target\.\$/\1/"
+    rm -f none.log none.sum
+}
+
+# Print unique port to be used for ssh server on the boards
+print_schroot_port()
+{
+    trace "$*"
+
+    # Set build_n to the last digit appearing in hostname.
+    # E.g., tcwgbuild04 becomes "4" (and so does x86_64).
+    # This is purely for convenience when determining which hosts use
+    # a particular target board.
+    # If the port turns out to be already in use on one of the boards then
+    # test-schroot.sh will gracefully fail.
+    local build_n
+    build_n="$(hostname | sed -e "s/.*\([0-9]\).*/\1/" | grep "[0-9]")"
+    test -z "$build_n" && build_n="0"
+    port=$((50000+1000*$build_n+$RANDOM%1000))
+    # We return ports between 50000 and 59999
+    echo $port
+}
+
+# Start schroot sessions on boards.
+# $1 - target triplet
+# $2 - port on which to start ssh server in schroot session
+# $3 - [optional] target sysroot
+# $4 - [optional] directory on host that should be mounted on target
+start_schroot_sessions()
+{
+    trace "$*"
+
+    local target="$1"
+    local port="$2"
+    local sysroot="$3"
+    local shared_dir="$4"
+
+    local target_opt
+    if ! [ -z "$target" ]; then
+	target_opt="-a $target"
+    fi
+
+    local sysroot_opt
+    local sysroot_env
+    if ! [ -z "$sysroot" ]; then
+	local multilib_dir
+	multilib_dir="$(find_dynamic_linker "$sysroot" true)"
+	multilib_dir="$(basename $(dirname $multilib_dir))"
+	sysroot_opt="-l $sysroot -h $multilib_dir"
+	sysroot_env="SYSROOT_UNDER_TEST=$sysroot"
+    fi
+
+    local shared_dir_opt
+    local shared_dir_ok=false
+    if ! [ -z "$shared_dir" ]; then
+	shared_dir_opt="-d $shared_dir"
+	shared_dir_ok=true
+    fi
+
+    local -a board_exps
+    board_exps=($(eval $sysroot_env print_schroot_board_files "$target"))
+    for board_exp in "${board_exps[@]}"; do
+	local hostname sysroot lib_path multilib_dir
+
+	# Get board hostname
+	hostname="$(grep "^set_board_info hostname " $board_exp | sed -e "s/^set_board_info hostname //")"
+	if [ -z "$hostname" ]; then
+	    error "Board file $board_exp uses schroot testing, but does not define \"set_board_info hostname\""
+	    continue
+	fi
+
+	local log=$(mktemp)
+	local result
+	# This command is very handy to have in logs to reproduce test
+	# environment.
+	set -x
+	# Start testing schroot session.
+	dryrun "$topdir/scripts/test-schroot.sh -v -b $target_opt -m -e $board_exp $sysroot_opt $shared_dir_opt $hostname:$port" > $log 2>&1; result="$?"
+	set +x
+
+	cat $log >&2
+
+	if ! grep -q "shared directory .*: SUCCESS" $log; then
+	    shared_dir_ok=false
+	fi
+
+	rm -f $log
+
+	# Print the hostname to the caller
+	schroot_boards="$schroot_boards $hostname"
+
+	if test "$result" != "0"; then
+	    return 1
+	fi
+    done
+
+    if [ -z "$schroot_boards" ]; then
+	return 0
+    fi
+
+    schroot_make_opts="SCHROOT_PORT=$port"
+    if $shared_dir_ok; then
+	schroot_make_opts="$schroot_make_opts SCHROOT_SHARED_DIR=$shared_dir"
+    fi
+}
+
+# Stop schroot sessions on given boards/ports
+# $1 - port on which ssh server for schroot session was started
+# $2, $3, ... - hostnames of boards as printed by start_schroot_sessions.
+stop_schroot_sessions()
+{
+    trace "$*"
+
+    local port="$1"
+    shift 1
+
+    for hostname in "$@"; do
+	dryrun "${topdir}/scripts/test-schroot.sh -v -f $hostname:$port"
+    done
+}
diff --git a/scripts/BuildFarm.job b/scripts/BuildFarm.job
index b378cea5..ffc79362 100755
--- a/scripts/BuildFarm.job
+++ b/scripts/BuildFarm.job
@@ -7,22 +7,50 @@ Files-Pattern: *
 License-Type: open
 EOF
 
-export runtests=true
-
 export CONFIG_SHELL="/bin/bash"
-shared="/home/buildslave/workspace/shared/"
+shared="/home/buildslave/workspace/shared"
 abe_dir="${WORKSPACE}/abe"
 user_snapshots="${WORKSPACE}/snapshots"
 snapshots_ref="${shared}/snapshots"
+fileserver="abe.tcwglab.linaro.org"
+
+mkdir -p ${snapshots_ref}/
+rsync -az --update ${fileserver}:snapshots/ ${snapshots_ref}/
+rm -rf ${snapshots_ref}/*~*
+
+rm -rf ${WORKSPACE}/_build ${user_snapshots}
+
+mkdir -p ${user_snapshots}/
+rsync -a ${snapshots_ref}/infrastructure/ ${user_snapshots}/infrastructure/
+
+case ${target} in
+    schroot-*)
+	schroot_arch=$(echo ${target} | sed -e "s/^schroot-\([^-]*\)-\(.*\)/\1/")
+	target=$(echo ${target} | sed -e "s/^schroot-\([^-]*\)-\(.*\)/\2/")
+	CONFIG_SHELL="schroot -c tcwg-build-${schroot_arch}-trusty --preserve-environment -- ${CONFIG_SHELL}"
+	;;
+esac
+
+if test x"${bootstrap}" = x"true"; then
+    bootstrap="--bootstrap"
+else
+    bootstrap=""
+fi
 
 rm -fr ${WORKSPACE}/_build
 mkdir -p ${WORKSPACE}/_build
 
 cd ${WORKSPACE}/_build
-$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${shared}/snapshots
-$CONFIG_SHELL -x ${abe_dir}/jenkins.sh ${release}
+$CONFIG_SHELL -x ${abe_dir}/jenkins.sh --runtests -l ${languages} -f ${fileserver} ${release} ${bootstrap}
 
 # force a failure of abe has build problems.
 if test $? -gt 0; then
   exit 1
 fi
+
+# Leave source and build trees for *Farm jobs; delete for all other jobs
+echo "JOB_NAME = ${JOB_NAME}"
+case "${JOB_NAME}" in
+    *"Farm"*) ;;
+    *) rm -rf ${WORKSPACE}/_build ${user_snapshots} ;;
+esac
diff --git a/scripts/test-schroot.sh b/scripts/test-schroot.sh
index b2608566..c7221447 100755
--- a/scripts/test-schroot.sh
+++ b/scripts/test-schroot.sh
@@ -13,9 +13,10 @@ sysroot=""
 ssh_master=false
 target_ssh_opts=""
 host_ssh_opts=""
+profile="tcwg-test"
 multilib_path="lib"
 
-while getopts "a:bc:d:e:fgh:l:mo:p:qv" OPTION; do
+while getopts "a:bc:d:e:fgh:l:mo:p:P:qv" OPTION; do
     case $OPTION in
 	a) arch=$OPTARG ;;
 	b) begin_session=true ;;
@@ -29,6 +30,7 @@ while getopts "a:bc:d:e:fgh:l:mo:p:qv" OPTION; do
 	m) ssh_master=true ;;
 	o) target_ssh_opts="$OPTARG" ;;
 	p) host_ssh_opts="$OPTARG" ;;
+	P) profile="$OPTARG" ;;
 	q) exec > /dev/null ;;
 	v) set -x ;;
     esac
@@ -77,21 +79,36 @@ if ! triplet_to_deb_arch "$arch" >/dev/null 2>&1; then
     arch="native"
 fi
 
-if [ "x$arch" = "xnative" ]; then
-    cpu="$(ssh $target uname -m)"
-    case "$cpu" in
-	aarch64) arch=aarch64-linux-gnu ;;
-	armv7l) arch=arm-linux-gnueabihf ;;
-	armv7*) arch=arm-linux-gnueabi ;;
-	i686) arch=i686-linux-gnu ;;
-	x86_64) arch=x86_64-linux-gnu ;;
-	*)
-	    echo "ERROR: unrecognized native target $cpu"
-	    exit 1
-	    ;;
-    esac
+cpu="$(ssh $target uname -m)"
+case "$cpu" in
+    aarch64) native_arch=aarch64-linux-gnu ;;
+    armv7l) native_arch=arm-linux-gnueabihf ;;
+    armv7*) native_arch=arm-linux-gnueabi ;;
+    i686) native_arch=i686-linux-gnu ;;
+    x86_64) native_arch=x86_64-linux-gnu ;;
+    *)
+	echo "ERROR: unrecognized native target $cpu"
+	exit 1
+	;;
+esac
+
+if [ x"$arch" = x"native" ]; then
+    arch="$native_arch"
 fi
 
+deb_arch="$(triplet_to_deb_arch $arch)"
+
+case "$cpu:$deb_arch" in
+    "x86_64:amd64"|"x86_64:i386") ;;
+    "x86_64:"*)
+	use_qemu=true
+	arch=$native_arch
+	;;
+esac
+
+deb_arch="$(triplet_to_deb_arch $arch)"
+deb_dist="$(triplet_to_deb_dist $arch)"
+
 if [ "x$board_exp" != "x" ] ; then
     lava_json="$(grep "^set_board_info lava_json " $board_exp | sed -e "s/^set_board_info lava_json //")"
     if [ "x$lava_json" != "x" ] && $begin_session; then
@@ -112,12 +129,12 @@ if [ "x$board_exp" != "x" ] ; then
     fi
 fi
 
-deb_arch="$(triplet_to_deb_arch $arch)"
-deb_dist="$(triplet_to_deb_dist $arch)"
+orig_target_ssh_opts="$target_ssh_opts"
+target_ssh_opts="$target_ssh_opts -o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/tmp/ssh-$profile-$port-%u-%r@%h:%p"
 
-schroot_id=tcwg-test-$deb_arch-$deb_dist
+schroot_id=$profile-$deb_arch-$deb_dist
 
-schroot="ssh $target_ssh_opts $target schroot -r -c session:tcwg-test-$port -d / -u root --"
+schroot="ssh $target_ssh_opts $target schroot -r -c session:$profile-$port -d / -u root --"
 rsh_opts="$target_ssh_opts -o Port=$port -o StrictHostKeyChecking=no"
 rsh="ssh $rsh_opts"
 user="$(ssh $target_ssh_opts $target echo \$USER)"
@@ -152,18 +169,24 @@ if $gen_schroot; then
 	fi
     done
 
+    # Configure APT sources.
     case "$deb_arch" in
-	amd64) extra_packages="qemu-user-static" ;;
-	*) extra_packages="" ;;
+	amd64|i386) deb_mirror="http://archive.ubuntu.com/ubuntu/" ;;
+	*) deb_mirror="http://ports.ubuntu.com/ubuntu-ports/" ;;
+    esac
+    ssh $target_ssh_opts $target \
+	sudo chroot $chroot bash -c "\"for i in '' -updates -security -backports; do for j in '' -src; do echo deb\\\$j $deb_mirror $deb_dist\\\$i main restricted universe multiverse >> /etc/apt/sources.list; done; done\""
+
+    case "$deb_arch" in
+	amd64) extra_packages="qemu-user-static gdb gdbserver" ;;
+	*) extra_packages="gdb gdbserver" ;;
+    esac
+
+    case "$profile" in
+	"tcwg-build") extra_packages="autoconf autogen automake bash bison build-essential ccrypt dejagnu flex gawk git g++ gcc libncurses5-dev libtool make texinfo wget xz-utils" ;;
     esac
 
     if ! [ -z "$extra_packages" ]; then
-	case "$deb_arch" in
-	    amd64) deb_mirror="http://archive.ubuntu.com/ubuntu/" ;;
-	    *) deb_mirror="http://ports.ubuntu.com/ubuntu-ports/" ;;
-	esac
-	ssh $target_ssh_opts $target \
-	    sudo chroot $chroot bash -c "\"for i in '' -updates -security -backports; do for j in '' -src; do echo deb\\\$j $deb_mirror $deb_dist\\\$i main restricted universe multiverse >> /etc/apt/sources.list; done; done\""
 	ssh $target_ssh_opts $target \
 	    sudo chroot $chroot apt-get update
 	ssh $target_ssh_opts $target \
@@ -175,6 +198,21 @@ if $gen_schroot; then
 	    sudo rm -f $chroot/usr/bin/qemu-\*-static
     fi
 
+    # Install foundation model in x86_64 chroots for bare-metal testing
+    if [ x"$deb_arch" = x"amd64" ]; then
+	ssh $target_ssh_opts $target \
+	    sudo mkdir -p $chroot/linaro/foundation-model/Foundation_v8pkg
+	ssh $target_ssh_opts $target \
+	    sudo rsync -a /linaro/foundation-model/Foundation_v8pkg/ $chroot/linaro/foundation-model/Foundation_v8pkg/
+    fi
+
+    if echo "$extra_packages" | grep -q "git"; then
+	ssh $target_ssh_opts $target \
+	    sudo ln -s /usr/share/doc/git/contrib/workdir/git-new-workdir $chroot/usr/local/bin/
+	ssh $target_ssh_opts $target \
+	    sudo chmod a+x $chroot/usr/share/doc/git/contrib/workdir/git-new-workdir
+    fi
+
     ssh $target_ssh_opts $target \
 	sudo mkdir -p /var/chroots/
     ssh $target_ssh_opts $target \
@@ -183,14 +221,20 @@ if $gen_schroot; then
     ssh $target_ssh_opts $target \
 	sudo rm -rf $chroot
 
+    case "$deb_arch" in
+	armhf|i386) personality="personality=linux32" ;;
+	*) personality="" ;;
+    esac
+
     ssh $target_ssh_opts $target \
 	sudo bash -c "\"cat > /etc/schroot/chroot.d/$schroot_id\"" <<EOF
 [$schroot_id]
 type=file
 file=/var/chroots/$schroot_id.tgz
-groups=buildslave,users
-root-groups=buildslave,users
-profile=tcwg-test
+groups=users
+root-groups=users
+profile=$profile
+$personality
 EOF
 
     if ! [ -z "$schroot_master" ]; then
@@ -213,12 +257,14 @@ if ! [ -z "$schroot_master" ]; then
     cat $schroot_master/chroot.d/$schroot_id | ssh $target_ssh_opts $target \
 	sudo bash -c "\"cat > /etc/schroot/chroot.d/$schroot_id\""
 
-    (cd $schroot_master && tar -c tcwg-test/ | ssh $target_ssh_opts $target \
-	sudo bash -c "\"cd /etc/schroot && rm -rf tcwg-test && tar -x && chown -R root:root tcwg-test/\"")
+    (cd $schroot_master && tar -c $profile/ | ssh $target_ssh_opts $target \
+	sudo bash -c "\"cd /etc/schroot && rm -rf $profile && tar -x && chown -R root:root $profile/\"")
 fi
 
 if $begin_session; then
-    ssh $target_ssh_opts $target schroot -b -c chroot:$schroot_id -n tcwg-test-$port -d /
+    ssh $target_ssh_opts $target schroot -b -c chroot:$schroot_id -n $profile-$port -d /
+    $schroot sh -c "\"echo $user - data $((1024*1024)) >> /etc/security/limits.conf\""
+    $schroot sh -c "\"echo $user - nproc 1000 >> /etc/security/limits.conf\""
     # Set ssh port
     $schroot sed -i -e "\"s/^Port 22/Port $port/\"" /etc/ssh/sshd_config
     # Run as root
@@ -241,6 +287,9 @@ if $begin_session; then
     $rsh root@$target rsync -a /root/ $home/
     $rsh root@$target chown -R $user $home/
 
+    $rsh root@$target "echo 1 > /dont_keep_session"
+    $rsh root@$target chmod 0666 /dont_keep_session
+
     echo $target:$port started schroot: $rsh $target
 fi
 
@@ -261,13 +310,29 @@ if ! [ -z "$shared_dir" ]; then
     test -z "$host_ssh_port" && host_ssh_port="22"
     # Establish port forwarding
     $rsh -fN -S none -R $tmp_ssh_port:127.0.0.1:$host_ssh_port $target
-    $rsh $target sshfs -o ssh_command="ssh -o Port=$tmp_ssh_port -o IdentityFile=$home/.ssh/id_rsa-test-schroot.$$ -o StrictHostKeyChecking=no $host_ssh_opts" "$USER@127.0.0.1:$shared_dir" "$shared_dir"
+    # Recent versions of sshfs fail if ssh_command has more than a single
+    # white spaces between options or ends with a space; filter ssh_command.
+    ssh_command="$(echo "ssh -o Port=$tmp_ssh_port -o IdentityFile=$home/.ssh/id_rsa-test-schroot.$$ -o StrictHostKeyChecking=no $host_ssh_opts" | sed -e "s/ \+/ /g" -e "s/ \$//")"
+    try="0"
+    while [ "$try" -lt "3" ]; do
+	$rsh $target sshfs -C -o ssh_command="\"$ssh_command\"" "$USER@127.0.0.1:$shared_dir" "$shared_dir" | true
+	if [ x"${PIPESTATUS[0]}" != x"0" ]; then
+	    try=$(($try + 1))
+	    sleep 1
+	    continue
+	fi
+	break
+    done
 
     # Remove temporary key and delete extra empty lines at the end of file.
     sed -i -e "/.*test-schroot\.$$\$/d" -e '/^$/N;/\n$/D' ~/.ssh/authorized_keys
     rm ~/.ssh/id_rsa-test-schroot.$$*
 
-    echo $target:$port shared directory $shared_dir
+    if [ "$try" != "3" ]; then
+	echo "$target:$port shared directory $shared_dir: SUCCESS"
+    else
+	echo "$target:$port shared directory $shared_dir: FAIL"
+    fi
 fi
 
 if ! [ -z "$sysroot" ]; then
@@ -285,20 +350,38 @@ EOF
 	# Remove /etc/ld.so.cache to workaround QEMU problem for targets with
 	# different endianness (i.e., /etc/ld.so.cache is endian-dependent).
 	$rsh root@$target "rm /etc/ld.so.cache"
+	if [ -e $sysroot/lib64/ld-linux-aarch64.so.1 ]; then
+	    # Our aarch64 sysroot has everything in /lib64, but executables
+	    # still expect to find dynamic linker under
+	    # /lib/ld-linux-aarch64.so.1
+	    $rsh root@$target "ln -s /sysroot/lib64 /sysroot/lib"
+	fi
 	# Cleanup runaway QEMU processes that ran for more than 2 minutes.
-	$rsh -f $target bash -c "\"while sleep 30; do ps uxf | sed -e \\\"s/ \+/ /g\\\" | cut -d\\\" \\\" -f 2,10- | grep \\\"^[0-9]\+ [0-9]*2:[0-9]\+ ._ qemu-\\\" | cut -d\\\" \\\" -f 1 | xargs -r kill -9; done\""
+	# Note the "-S none" option -- ssh does not always detach from process
+	# when multiplexing is used.  I think this is a bug in ssh.
+	# We calculate delay in this fashion to avoid multi-thread tests
+	# getting through a minute of usertime in 60/#_of_cpus seconds.
+	delay=$((60 / $($rsh $target getconf _NPROCESSORS_ONLN)))
+	$rsh -f -S none $target bash -c "\"while sleep $delay; do ps uxf | sed -e \\\"s/ \+/ /g\\\" | cut -d\\\" \\\" -f 2,10- | grep \\\"^[0-9]\+ [0-9]*2:[0-9]\+ ._ qemu-\\\" | cut -d\\\" \\\" -f 1 | xargs -r kill -9; done\""
     fi
     echo $target:$port installed sysroot $sysroot
 fi
 
 if $ssh_master; then
-    $rsh -fMN $target
+    ssh $orig_target_ssh_opts -o Port=$port -o StrictHostKeyChecking=no -fMN $target
 fi
 
-if $finish_session; then
+# Keep the session alive when file /dont_kill_me is present
+if $finish_session && [ x`$schroot cat /dont_keep_session` = x"1" ]; then
     $schroot iptables -I INPUT -p tcp --dport $port -j REJECT || true
-    $schroot /etc/init.d/ssh stop || true
-    ssh $target_ssh_opts $target schroot -e -c session:tcwg-test-$port
+    ssh $target_ssh_opts $target schroot -f -e -c session:$profile-$port | true
+    if [ x"${PIPESTATUS[0]}" != x"0" ]; then
+	# tcwgbuildXX machines have a kernel problem that a bind mount will be
+	# forever busy if it had an sshfs under it.  Seems like fuse is not
+	# cleaning up somethings.  The workaround is to lazy unmount the bind.
+	$schroot umount -l /
+	ssh $target_ssh_opts $target schroot -f -e -c session:$profile-$port
+    fi
     echo $target:$port finished session
 fi
 
diff --git a/tcwgweb.sh b/tcwgweb.sh
index 6f4eca38..1708942a 100755
--- a/tcwgweb.sh
+++ b/tcwgweb.sh
@@ -114,11 +114,19 @@ difftwodirs ()
     fi
 
     # Don't diff it's already been done
-    if test -e $next/testsuite-diff.txt; then
+    if test -e $next/testsuite-diff-$(basename ${prev}).txt; then
 	return 0
     fi
+
+    local prev0=${prev}
+    local next0=${next}
+    tmpdir=$(mktemp -d)
+    prev=${tmpdir}/prev
+    next=${tmpdir}/next
+    rsync -a ${prev0}/ ${prev}/
+    rsync -a ${next0}/ ${next}/
     
-    echo "Diffing: ${prev} against ${next}..."
+    echo "Diffing: ${prev0} against ${next0}..."
 
     local gcc_version="`grep 'gcc_version=' ${next}/manifest.txt | cut -d '=' -f 2`"
     if test x"${gcc_version}" = x"gcc.git"; then
@@ -134,10 +142,8 @@ difftwodirs ()
     else
 	local pversion=${cversion}
     fi
-    local toplevel="`dirname ${prev}`"
 
-#    diffdir="/tmp/diffof-${gcc_version}"
-    diffdir="/tmp/diffof-${pversion}-${cversion}"
+    diffdir="${tmpdir}/diffof-${pversion}-${cversion}"
     mkdir -p ${diffdir}
     local files="`ls ${prev}/*.sum.xz | wc -l`"
     if test ${files} -gt 0; then
@@ -150,14 +156,13 @@ difftwodirs ()
     local regressions=0
     touch ${resultsfile}
     echo "Comparison of ${gcc_branch} between:" >> ${resultsfile}
-    echo "	${prev} and" >> ${resultsfile}
-    echo "	${next}" >> ${resultsfile}
-    echo "	" >> ${resultsfile}
-    for i in gcc g\+\+ libstdc++ gas gdb glibc egibc newlib binutils libatomic libgomp libitm; do
+    echo "	${prev0} and" >> ${resultsfile}
+    echo "	${next0}" >> ${resultsfile}
+    for i in gcc g\+\+ libstdc++ ld gas gdb glibc egibc newlib binutils libatomic libgomp libitm; do
 	if test -e ${prev}/$i.sum -a -e ${next}/$i.sum; then
            sort ${prev}/$i.sum -o ${prev}/$i-sort.sum
            sort ${next}/$i.sum -o ${next}/$i-sort.sum
-           diff -U 0 ${prev}/$i-sort.sum ${next}/$i-sort.sum 2>&1 | egrep '^[+-]PASS|^[-]FAIL|^[+-]XPASS|^[+-]XFAIL' 2>&1 | sort -k 2 2>&1 > ${diffdir}/diff-$i.txt
+           diff -U 0 ${prev}/$i-sort.sum ${next}/$i-sort.sum 2>&1 | egrep '^[+-]PASS|^[+-]FAIL|^[+-]XPASS|^[+-]XFAIL' 2>&1 | sort -k 2 2>&1 > ${diffdir}/diff-$i.txt
             rm ${prev}/$i-sort.sum ${next}/$i-sort.sum
 	    if test -s ${diffdir}/diff-$i.txt; then
 		if test `grep -c ^\+PASS ${diffdir}/diff-$i.txt` -gt 0; then
@@ -218,18 +223,8 @@ difftwodirs ()
 
     mailto "Test results for ${gcc_branch}" ${resultsfile} ${userid}
 
-    rm -fr ${diffdir}
-    local incr=`expr ${incr} + 1`
+    rm -fr ${tmpdir}
 
-    # Not all subdirectories have uncompressed sum files
-    local files="`ls ${prev}/*.sum | wc -l`"
-    if test ${files} -gt 0; then
-	xz ${prev}/*.sum
-    fi
-    local files="`ls ${next}/*.sum | wc -l`"
-    if test ${files} -gt 0; then
-	xz ${next}/*.sum
-    fi
     echo ${returnstr}
 }
 
@@ -271,7 +266,7 @@ testfile()
     origdir="`basename $1`"
     nextdir="`basename $2`"
  
-    cat <<EOF > ${diffdir}/testsuite-diff.txt
+    cat <<EOF > ${diffdir}/testsuite-diff-${origdir}.txt
 Difference in testsuite results between:
  ${orig} build ${origdir}
 and the one before it:
@@ -280,8 +275,7 @@ and the one before it:
 ------
 EOF
     
-    cat ${diffdir}/diff.txt  >> ${diffdir}/testsuite-diff.txt
-    cp  ${diffdir}/testsuite-diff.txt  $1
+    cat ${diffdir}/diff.txt  >> ${diffdir}/testsuite-diff-${origdir}.txt
     cp  ${diffdir}/testsuite-diff.txt  $2
 }