Merge remote-tracking branch 'origin/bernie/sysroot' into will/ilp32_sysrootwill/ilp32_sysroot

Conflicts:
	lib/globals.sh

35 files changed, 2725 insertions, 410 deletions

diff --git a/config/CPU2000.conf b/config/CPU2000.conf
new file mode 100644
index 00000000..0a3ba2a1
--- /dev/null
+++ b/config/CPU2000.conf
@@ -0,0 +1,19 @@
+# This is a list of packages that must be installed on the build machine
+depends=""
+
+# spec does not have a configure script
+configure="no"
+
+#Note that the makefile will cross-compile the benchmark, but only ever natively compiles the tools
+#--no-print-directory is important, as we otherwise catch paths as part of the checksum that identifies
+#whether the build options have changed. This is a problem when cross-building. Or even when native
+#building and passing the results around, which appears to be an expected (supported?) use case, see
+#http://www.spec.org/cpu2006/docs/runspec.html#section3.1.1
+default_makeflags="BUILD=${build} --no-print-directory"
+if test x"${build}" = x"${target}"; then
+  default_makeflags="${default_makeflags} CROSS_COMPILE="
+else
+  default_makeflags="${default_makeflags} CROSS_COMPILE=${target}-"
+fi
+
+benchlog=result
diff --git a/config/CPU2006.conf b/config/CPU2006.conf
new file mode 100644
index 00000000..0a3ba2a1
--- /dev/null
+++ b/config/CPU2006.conf
@@ -0,0 +1,19 @@
+# This is a list of packages that must be installed on the build machine
+depends=""
+
+# spec does not have a configure script
+configure="no"
+
+#Note that the makefile will cross-compile the benchmark, but only ever natively compiles the tools
+#--no-print-directory is important, as we otherwise catch paths as part of the checksum that identifies
+#whether the build options have changed. This is a problem when cross-building. Or even when native
+#building and passing the results around, which appears to be an expected (supported?) use case, see
+#http://www.spec.org/cpu2006/docs/runspec.html#section3.1.1
+default_makeflags="BUILD=${build} --no-print-directory"
+if test x"${build}" = x"${target}"; then
+  default_makeflags="${default_makeflags} CROSS_COMPILE="
+else
+  default_makeflags="${default_makeflags} CROSS_COMPILE=${target}-"
+fi
+
+benchlog=result
diff --git a/config/eembc.conf b/config/EEMBC.conf
index 83f8062a..f9baf7a8 100644
--- a/config/eembc.conf
+++ b/config/EEMBC.conf
@@ -9,12 +9,3 @@ default_makeflags="build -j 1 "
 if test x"${build}" != x"${target}"; then
   default_makeflags="${default_makeflags} CROSS_COMPILE=${target}-"
 fi
-
-#command to run the benchmark
-benchcmd="make -C ${builddir} -s rerun COMPILER_FLAGS='-O3 -mfpu=neon -mcpu=native -DNDEBUG -DHOST_EXAMPLE_CODE=1'"
-
-#number of times to run the benchmark
-benchcount="5"
-
-#pattern to find all relative logs, relative to build directory
-benchlog="*/gcc_{size,time}.log"
diff --git a/config/boards/bench/arndale.conf b/config/boards/bench/arndale.conf
new file mode 100644
index 00000000..c8cc37c9
--- /dev/null
+++ b/config/boards/bench/arndale.conf
@@ -0,0 +1,11 @@
+servicectl=yes
+netctl=yes
+freq=900MHz
+benchcore=0
+othercore=1
+ip=arndale.json
+#ip=linaro@10.2.201.83
+boot_timeout=90
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=Arndale HW_CPU=Cortex-A15 HW_FPU=TODO"
diff --git a/config/boards/bench/arndale.json b/config/boards/bench/arndale.json
new file mode 100644
index 00000000..2c9886be
--- /dev/null
+++ b/config/boards/bench/arndale.json
@@ -0,0 +1,41 @@
+{
+  "job_name": "bench-arndale",
+  "device_type": "arndale",
+  "timeout": 1800,
+  "actions": [
+    {
+      "command": "deploy_linaro_image",
+      "parameters": {
+           "hwpack": "http://releases.linaro.org/14.04/ubuntu/arndale/hwpack_linaro-arndale_20140417-630_armhf_supported.tar.gz",
+           "rootfs": "http://releases.linaro.org/14.04/ubuntu/arndale/linaro-saucy-server-20140410-652.tar.gz"
+        }
+    },
+    {
+      "command": "lava_test_shell",
+      "parameters": {
+          "testdef_repos": [
+            {
+                "git-repo": "http://git.linaro.org/toolchain/lavabench.git",
+                "testdef": "bench-session-debian.yaml",
+                "parameters": {
+                    "GATEWAY": "10.0.0.1",
+                    "PUB_KEY": "",
+                    "LISTENER_ADDR": "",
+                    "LISTENER_PORT": ""
+                }
+            }
+        ],
+	"timeout": 691200
+      }
+    },
+    {
+      "command": "submit_results",
+      "parameters":
+        {
+          "server": "",
+          "stream": ""
+        }
+    } 
+
+  ]
+}
diff --git a/config/boards/bench/arndale.services b/config/boards/bench/arndale.services
new file mode 100644
index 00000000..0f26e6f6
--- /dev/null
+++ b/config/boards/bench/arndale.services
@@ -0,0 +1,87 @@
+mountall-net
+mountnfs-bootclean.sh
+passwd
+rc
+rsyslog
+#startpar-bridge
+#tty4
+udev
+upstart-udev-bridge
+ureadahead-other
+console-setup
+hwclock-save
+irqbalance
+plymouth-log
+#systemd-logind KEEP (can't bring down)
+#tty5
+failsafe
+mountall.sh
+atd
+dbus
+mounted-var
+plymouth
+resolvconf
+udev-fallback-graphics
+ssh
+checkroot.sh
+control-alt-delete
+gator-daemon
+hwclock
+mounted-proc
+#auto-serial-console
+setvtrgb
+shutdown
+cron
+mountall
+mounted-debugfs
+mountkernfs.sh
+console
+mounted-run
+checkfs.sh
+checkroot-bootclean.sh
+kmod
+mountnfs.sh
+openvt
+plymouth-stop
+#rcS
+ufw
+#wait-for-state
+bootmisc.sh
+flush-early-job-log
+friendly-recovery
+rc-sysinit
+upstart-socket-bridge
+mountdevsubfs.sh
+tty2
+udevtrigger
+upstart-file-bridge
+container-detect
+mounted-dev
+mtab.sh
+tty3
+udev-finish
+hostname
+mountall-reboot
+mysql
+mountall-shell
+mounted-tmp
+#network-interface
+#network-interface
+#network-interface
+#plymouth-ready KEEP - we break if we stop this
+plymouth-splash
+plymouth-upstart-bridge
+tty1
+udevmonitor
+dmesg
+mountall-bootclean.sh
+#network-interface-security
+#network-interface-security
+#network-interface-security
+#network-interface-security
+#networking
+procps
+#tty6
+console-font
+#network-interface-container
+ureadahead
diff --git a/config/boards/bench/juno-a53.conf b/config/boards/bench/juno-a53.conf
new file mode 100644
index 00000000..4a8dfcbf
--- /dev/null
+++ b/config/boards/bench/juno-a53.conf
@@ -0,0 +1,10 @@
+servicectl=no
+netctl=yes
+freq=
+benchcore=1
+othercore=0
+ip=juno-a53.json
+boot_timeout=90
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=Juno HW_CPU=Cortex-A57 HW_FPU=fp_asimd_evtstrm_aes_pmull_sha1_sha2_crc32"
diff --git a/config/boards/bench/juno-a53.json b/config/boards/bench/juno-a53.json
new file mode 100644
index 00000000..95a74789
--- /dev/null
+++ b/config/boards/bench/juno-a53.json
@@ -0,0 +1,41 @@
+{
+    "job_name": "bench-juno-a53",
+    "device_type": "juno",
+    "tags": [ "serial-console-server" ],
+    "timeout": 691200,
+    "actions": [
+        {
+            "command": "deploy_linaro_image",
+            "parameters": {
+                "hwpack": "http://people.linaro.org/~bernie.ogden/hwpack_linaro-lt-vexpress64-rtsm_20150114-706_arm64_supported.tar.gz",
+                "rootfs": "http://people.linaro.org/~bernie.ogden/linaro-utopic-developer-20150114-87.tar.gz"
+            }
+        },
+        {
+            "command": "lava_test_shell",
+            "parameters": {
+                "testdef_repos": [
+                    {
+                        "git-repo": "git://git.linaro.org/toolchain/lavabench.git",
+                        "testdef": "bench-session-debian.yaml",
+                        "parameters": {
+                           "GATEWAY": "10.0.0.1",
+                           "PUB_KEY": "",
+                           "LISTENER_ADDR": "",
+                           "LISTENER_PORT": ""
+                        }
+                    }
+                ],
+                "timeout": 691200
+            }
+        },
+        {
+            "command": "submit_results",
+            "parameters": {
+                "server": "",
+                "stream": ""
+            }
+        }
+    ]
+}
+
diff --git a/config/boards/bench/juno-a53.services b/config/boards/bench/juno-a53.services
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/config/boards/bench/juno-a53.services
diff --git a/config/boards/bench/juno-a57.conf b/config/boards/bench/juno-a57.conf
new file mode 100644
index 00000000..59b9c133
--- /dev/null
+++ b/config/boards/bench/juno-a57.conf
@@ -0,0 +1,10 @@
+servicectl=no
+netctl=yes
+freq=
+benchcore=5
+othercore=0
+ip=juno-a57.json
+boot_timeout=90
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=Juno HW_CPU=Cortex-A57 HW_FPU=fp_asimd_evtstrm_aes_pmull_sha1_sha2_crc32"
diff --git a/config/boards/bench/juno-a57.json b/config/boards/bench/juno-a57.json
new file mode 100644
index 00000000..db5a49dc
--- /dev/null
+++ b/config/boards/bench/juno-a57.json
@@ -0,0 +1,41 @@
+{
+    "job_name": "bench-juno-a57",
+    "device_type": "juno",
+    "tags": [ "serial-console-server" ],
+    "timeout": 691200,
+    "actions": [
+        {
+            "command": "deploy_linaro_image",
+            "parameters": {
+                "hwpack": "http://people.linaro.org/~bernie.ogden/hwpack_linaro-lt-vexpress64-rtsm_20150114-706_arm64_supported.tar.gz",
+                "rootfs": "http://people.linaro.org/~bernie.ogden/linaro-utopic-developer-20150114-87.tar.gz"
+            }
+        },
+        {
+            "command": "lava_test_shell",
+            "parameters": {
+                "testdef_repos": [
+                    {
+                        "git-repo": "git://git.linaro.org/toolchain/lavabench.git",
+                        "testdef": "bench-session-debian.yaml",
+                        "parameters": {
+                           "GATEWAY": "10.0.0.1",
+                           "PUB_KEY": "",
+                           "LISTENER_ADDR": "",
+                           "LISTENER_PORT": ""
+                        }
+                    }
+                ],
+                "timeout": 691200
+            }
+        },
+        {
+            "command": "submit_results",
+            "parameters": {
+                "server": "",
+                "stream": ""
+            }
+        }
+    ]
+}
+
diff --git a/config/boards/bench/juno-a57.services b/config/boards/bench/juno-a57.services
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/config/boards/bench/juno-a57.services
diff --git a/config/boards/bench/kvm.conf b/config/boards/bench/kvm.conf
new file mode 100644
index 00000000..96f63379
--- /dev/null
+++ b/config/boards/bench/kvm.conf
@@ -0,0 +1,14 @@
+#NB: You must run with '-c' (don't be cautious) for a kvm. This is because
+#the scripts assume a minimum of 2 cores on the target and the kvm only has
+#one. (The symptom is that taskset will fail.)
+
+servicectl=no
+netctl=no
+freq=
+benchcore=
+othercore=
+ip=kvm.json
+boot_timeout=60
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=localhost-unknown HW_CPU=localhost-unknown HW_FPU=localhost-unknown"
diff --git a/config/boards/bench/kvm.json b/config/boards/bench/kvm.json
new file mode 100644
index 00000000..9041241e
--- /dev/null
+++ b/config/boards/bench/kvm.json
@@ -0,0 +1,39 @@
+{
+    "job_name": "bench-kvm",
+    "device_type": "kvm",
+    "timeout": 1800,
+    "actions": [
+        {
+            "command": "deploy_linaro_image",
+            "parameters": {
+                "image": "http://images.validation.linaro.org/ubuntu-14-04-server-base.img.gz"
+            }
+        },
+        {
+            "command": "lava_test_shell",
+            "parameters": {
+                "testdef_repos": [
+                    {
+                        "git-repo": "git://git.linaro.org/toolchain/lavabench.git",
+                        "testdef": "bench-session-debian.yaml",
+                        "parameters": {
+                           "GATEWAY": "10.0.0.1",
+                           "PUB_KEY": "",
+                           "LISTENER_ADDR": "",
+                           "LISTENER_PORT": ""
+                        }
+                    }
+                ],
+                "timeout": 691200
+            }
+        },
+        {
+            "command": "submit_results",
+            "parameters": {
+                "server": "",
+                "stream": ""
+            }
+        }
+    ]
+}
+
diff --git a/config/boards/bench/kvm.services b/config/boards/bench/kvm.services
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/config/boards/bench/kvm.services
diff --git a/config/boards/bench/localhost.conf b/config/boards/bench/localhost.conf
new file mode 100644
index 00000000..674a13e5
--- /dev/null
+++ b/config/boards/bench/localhost.conf
@@ -0,0 +1,10 @@
+servicectl=no
+netctl=no
+freq=
+benchcore=0
+othercore=1
+ip=localhost
+uncontrolled=yes
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=localhost-unknown HW_CPU=localhost-unknown HW_FPU=localhost-unknown"
diff --git a/config/boards/bench/localhost.services b/config/boards/bench/localhost.services
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/config/boards/bench/localhost.services
diff --git a/config/boards/bench/panda-es.conf b/config/boards/bench/panda-es.conf
new file mode 100644
index 00000000..d4002dc5
--- /dev/null
+++ b/config/boards/bench/panda-es.conf
@@ -0,0 +1,11 @@
+servicectl=yes
+netctl=yes
+freq=
+benchcore=0
+othercore=1
+ip=panda-es.json
+#ip=linaro@10.2.201.83
+boot_timeout=90
+
+#Avoid spaces within parameters, they aren't worth the heartache
+board_benchargs="HW_MODEL=Panda-ES HW_CPU=Cortex-A9 HW_FPU=swp_half_thumb_fastmult_vfp_edsp_thumbee_neon_vfpv3_tls_vfpd3"
diff --git a/config/boards/bench/panda-es.json b/config/boards/bench/panda-es.json
new file mode 100644
index 00000000..630acb0d
--- /dev/null
+++ b/config/boards/bench/panda-es.json
@@ -0,0 +1,41 @@
+{
+  "job_name": "bench-panda-es",
+  "device_type": "panda-es",
+  "timeout": 1800,
+  "actions": [
+    {
+      "command": "deploy_linaro_image",
+      "parameters": {
+           "hwpack": "http://releases.linaro.org/14.05/ubuntu/panda/hwpack_linaro-panda_20140525-654_armhf_supported.tar.gz",
+           "rootfs": "http://releases.linaro.org/14.05/ubuntu/panda/linaro-trusty-developer-20140522-661.tar.gz"
+        }
+    },
+    {
+      "command": "lava_test_shell",
+      "parameters": {
+          "testdef_repos": [
+            {
+                "git-repo": "http://git.linaro.org/toolchain/lavabench.git",
+                "testdef": "bench-session-debian.yaml",
+                "parameters": {
+                    "GATEWAY": "10.0.0.1",
+                    "PUB_KEY": "",
+                    "LISTENER_ADDR": "",
+                    "LISTENER_PORT": ""
+                }
+            }
+        ],
+	"timeout": 691200
+      }
+    },
+    {
+      "command": "submit_results",
+      "parameters":
+        {
+          "server": "",
+          "stream": ""
+        }
+    } 
+
+  ]
+}
diff --git a/config/boards/bench/panda-es.services b/config/boards/bench/panda-es.services
new file mode 100644
index 00000000..5ee3f7e9
--- /dev/null
+++ b/config/boards/bench/panda-es.services
@@ -0,0 +1,81 @@
+mountall-net
+mountnfs-bootclean.sh
+passwd
+rc
+rsyslog
+#screen-cleanup
+#tty4 KEEP
+udev
+upstart-udev-bridge
+ureadahead-other
+console-setup
+hwclock-save
+plymouth-log
+#tty5 KEEP
+failsafe
+mountall.sh
+mounted-var
+plymouth
+resolvconf
+ssh
+udev-fallback-graphics
+checkroot.sh
+control-alt-delete
+gator-daemon
+hwclock
+mounted-proc
+alsa-store
+#auto-serial-console KEEP
+setvtrgb
+shutdown
+alsa-restore
+cron
+mountall
+mounted-debugfs
+mountkernfs.sh
+console
+mounted-run
+checkfs.sh
+checkroot-bootclean.sh
+kmod
+mountnfs.sh
+openvt
+plymouth-stop
+#rcS KEEP
+#wait-for-state DON'T EXPECT THIS TO BE RUNNING, IT DOESN'T BEHAVE NICELY IN RESPONSE TO SERVICE COMMANDS
+bootmisc.sh
+flush-early-job-log
+rc-sysinit
+upstart-socket-bridge
+mountdevsubfs.sh
+#tty2 KEEP
+udevtrigger
+upstart-file-bridge
+container-detect
+mounted-dev
+mtab.sh
+#tty3 KEEP
+udev-finish
+hostname
+mountall-reboot
+mountall-shell
+mounted-tmp
+#network-interface KEEP
+#network-interface KEEP
+#network-interface KEEP
+plymouth-splash
+plymouth-upstart-bridge
+#tty1 KEEP
+udevmonitor
+dmesg
+mountall-bootclean.sh
+#network-interface-security KEEP
+#network-interface-security KEEP
+#network-interface-security KEEP
+#network-interface-security KEEP
+#networking KEEP
+procps
+#tty6 KEEP
+console-font
+#network-interface-container KEEP
+ureadahead
diff --git a/config/fakebench.conf b/config/fakebench.conf
new file mode 100644
index 00000000..95932346
--- /dev/null
+++ b/config/fakebench.conf
@@ -0,0 +1,13 @@
+# This is a list of packages that must be installed on the build machine
+depends=""
+
+# eembc does not have a configure script
+configure="no"
+
+#eembc build has a race condition, don't try to parallelize
+default_makeflags=
+
+#command to run the benchmark
+benchcmd="echo fake; true"
+
+benchlog="fake/fakeresults fake/extralog"
diff --git a/config/sources.conf b/config/sources.conf
index 3adfaa7a..7b8d9caa 100644
--- a/config/sources.conf
+++ b/config/sources.conf
@@ -55,4 +55,7 @@ qemu.git        git://git.qemu.org/qemu.git
 qemu-linaro.git http://git.linaro.org/qemu/qemu-linaro.git
 
 #Benchmarks
-eembc.git	ssh://somehost/somepath/eembc.git
+eembc.git	git://somehost/somepath/eembc.git
+CPU2000.git	git://somehost/somepath/CPU2000.git
+CPU2006.git	git://somehost/somepath/CPU2006.git
+fakebench.git	git://git.linaro.org/toolchain/fakebench.git
diff --git a/lib/benchmark.sh b/lib/benchmark.sh
deleted file mode 100644
index 5f0625e9..00000000
--- a/lib/benchmark.sh
+++ /dev/null
@@ -1,115 +0,0 @@
-bench_run ()
-{
-  local builddir="`get_builddir $1`"
-
-  local tool="`get_toolname $1`"
-  local runlog="${builddir}/run-${tool}.log"
-  local cmd="`grep ^benchcmd= ${topdir}/config/${tool}.conf | cut -d '\"' -f 2`"
-  local count="`grep ^benchcount= ${topdir}/config/${tool}.conf | cut -d '\"' -f 2`"
-
-  if test x"${cmd}" = x; then
-    error "No benchcmd for ${tool}"
-    return 1
-  fi
-  if test x"${count}" = x; then
-    warning "No benchcount for ${tool}, defaulting to 5"
-    count=5
-  fi
-
-  dryrun "rm -f ${runlog}"
-  if test $? -gt 0; then
-    error "Failed to delete old runlog ${runlog}"
-    return 1
-  fi
-
-  for i in `seq 1 "${count}"`; do
-    dryrun "eval \"${cmd}\" 2>&1 | tee -a ${runlog}"
-    if test $? -gt 0; then
-      error "${cmd} failed"
-      return 1
-    fi
-    dryrun "echo -e \"\nRun $i::\" | tee -a ${runlog}"
-    bench_log "${tool}" "${runlog}" "${builddir}"
-    if test $? -gt 0; then
-      error "Logging failed for ${tool}"
-      return 1
-    fi
-    dryrun "echo -e \"\n\" | tee -a ${runlog}"
-  done
-
-  return 0
-}
-
-bench_log ()
-{
-  local tool="$1"
-  local out_log="$2"
-  local builddir="$3"
-
-  local in_log="`grep ^benchlog= ${topdir}/config/${tool}.conf | cut -d '\"' -f 2`"
-  if test x"${in_log}" = x; then
-    error "No benchlog in ${1}.conf"
-    return 1
-  fi
-
-  for log in ${in_log}; do
-    dryrun "cat ${builddir}/${log} | tee -a ${out_log}"
-    if test $? -gt 0; then
-      error "Could not tee log ${log} to ${out_log}"
-      return 1
-    fi
-  done
-
-  return 0
-}
-
-dump_host_info ()
-{
-  echo "GCCVERSION=`${CROSS_COMPILE}gcc --version | head -n1`"
-  echo "GXXVERSION=`${CROSS_COMPILE}g++ --version | head -n1`"
-  echo "DATE=`date +%Y-%m-%d`"
-  echo "ARCH=`uname -m`"
-  echo "CPU=`grep -E "^(model name|Processor)" /proc/cpuinfo | head -n1 | tr -s [:space:] | awk -F: '{print $2;}'`"
-  echo "OS=`lsb_release -sd`"
-  echo "TOPDIR=`pwd`"
-  echo "date:`date --rfc-3339=seconds -u`"
-  echo
-  echo "uname:`uname -a`"
-  echo
-  echo lsb_release:
-  lsb_release -a
-  echo
-  echo /proc/version:
-  cat /proc/version
-  echo
-  echo "gcc: `dpkg -s gcc | grep ^Version`"
-  gcc --version
-  echo "as: `dpkg -s binutils | grep ^Version`"
-  as --version
-  echo
-  echo ldd:
-  ldd --version
-  echo
-  echo free:
-  free
-  echo
-  echo ulimit:
-  bash -c "ulimit -a"
-  echo
-  echo cpuinfo:
-  cat /proc/cpuinfo
-  echo gdb:
-  dpkg -s gdb | grep ^Version
-  gdb --version
-  echo gcc-binary:
-  #$(PWD)/$(@D)/gcc-binary/bin/gcc --version || true
-  echo
-  echo libc6:
-  dpkg -s libc6 | grep ^Version
-  echo PATH:
-  echo $PATH
-  echo
-  echo cpufreq-info:
-  echo `cpufreq-info`
-  echo
- }
diff --git a/lib/common.sh b/lib/common.sh
index 96dd3d52..67c984c2 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -33,6 +33,7 @@ set -o pipefail
 . "${topdir}/lib/stamp.sh" || exit 1
 . "${topdir}/lib/schroot.sh" || exit 1
 . "${topdir}/lib/gerrit.sh" || exit 1
+. "${topdir}/lib/remote.sh" || exit 1
 
 #
 # All the set* functions set global variables used by the other functions.
@@ -59,6 +60,9 @@ set_dbpasswd()
 
 # if --dryrun is passed to abe.sh, then commands are echoed instead of
 # of executed.
+# NOTE: This function must not run any commands in the background between
+#       'eval' and 'return'. This is so that runes such as
+#       'dryrun "foo&"; wait $!' will work.
 dryrun()
 {
     if test x"${dryrun}" = xyes; then
@@ -70,8 +74,8 @@ dryrun()
 	    read answer
 	    return $?
 	fi
-        echo "RUN: $1"
-	eval $1
+        echo "RUN: $1" 1>&2
+	eval "$1"
 	return $?
     fi
 
@@ -273,7 +277,7 @@ get_builddir()
     # as well because we might be passed a tar file.
     local dir="`normalize_path $1`"
 
-    if test x"$2" = x"libgloss"; then
+    if test x"${2:+$2}" = x"libgloss"; then
      	echo "${local_builds}/${host}/${target}/${dir}/${target}/libgloss"
     else
 	echo "${local_builds}/${host}/${target}/${dir}${ABI:+-$ABI}${2:+-$2}"
diff --git a/lib/configure.sh b/lib/configure.sh
index e21e7e9d..c87394b5 100755
--- a/lib/configure.sh
+++ b/lib/configure.sh
@@ -283,9 +283,9 @@ configure_build()
 	        return $?
 	    fi
         else
-            dryrun "rsync -a --exclude=.git/ ${srcdir}/ ${builddir}"
+            dryrun "rsync -a --delete --exclude=.git/ ${srcdir}/ ${builddir}"
 	    if test $? -gt 0; then
-	        error "Copy of $1 failed (rsync -a ${srcdir} ${builddir})"
+	        error "Copy of $1 failed (rsync -a --delete ${srcdir}/ ${builddir})"
 	        return $?
 	    fi
         fi
diff --git a/lib/globals.sh b/lib/globals.sh
index 84827e14..f5caf8c8 100644
--- a/lib/globals.sh
+++ b/lib/globals.sh
@@ -68,13 +68,13 @@ distribution=
 make_flags=
 
 # These can be changed by environment variables
-if test x"${SNAPSHOTS_URL}" != x -o x"${ABE_SNAPSHOTS}" != x; then
+if test x"${SNAPSHOTS_URL:-}" != x -o x"${ABE_SNAPSHOTS:-}" != x; then
     snapshots="${SNAPSHOTS_URL}"
 fi
-if test x"${ABE_DBUSER}" != x; then
+if test x"${ABE_DBUSER:-}" != x; then
     dbuser="${ABE_DBUSER}"
 fi
-if test x"${ABE_DBPASSWD}" != x; then
+if test x"${ABE_DBPASSWD:-}" != x; then
     dbpasswd="${ABE_DBPASSWD}"
 fi
 
@@ -98,10 +98,15 @@ override_runtestflags=
 
 aarch64_abilist=lp64
 
-if test x"${BUILD_NUMBER}" = x; then
+if test x"${BUILD_NUMBER:-}" = x; then
     export BUILD_NUMBER=${RANDOM}
 fi
 
+export CBUILD2_SHA="`cd ${topdir} && git rev-parse HEAD`"
+if test $? -ne 0; then
+    CBUILD2_SHA=0
+fi
+
 gerrit_host="review.linaro.org"
 gerrit_port="29418"
 gerrit_username=""
diff --git a/lib/remote.sh b/lib/remote.sh
new file mode 100644
index 00000000..f8e5d4de
--- /dev/null
+++ b/lib/remote.sh
@@ -0,0 +1,164 @@
+#TODO: Check that remote_{down,up)load behave as one might think
+#TODO: Copy more of the dejagnu contract? e.g. return target-side name of copied file
+#TODO: Tests?
+
+#ON USING remote_exec*
+#---------------------
+#Quotes in command can be tricky. It's a good idea to use single-quotes when
+#you can get away with it, because these are easier to think about. Remember:
+#1) It isn't legal to escape a single-quote within single-quotes
+#2) \ is only removed on expansion if it actualy escaped something
+#3) \ only escapes $,`,\ when in `` (critically, it does not escape either kind of quote) - but if you push \ through to the execution of the contents of the `` (by typing \\ everywhere you want \) then will of course behave as normal when shoved through the shell commands within  the ``.
+#Because of (1), it's sensible to use "" rather than '' when we're expecting input that
+#may in turn contain quotes. And actually more robust to do that in general.
+#I may never fathom the laws on behaviour of \, so further dragons may lurk here.
+#Some examples of how to call:
+#remote_exec2 localhost "grep 'foo bar' ~/file"
+##Becomes ssh localhost "grep 'foo bar' ~/file"
+##Becomes grep 'foo bar' ~/file
+#
+#x="`remote_exec2 localhost \"grep 'foo bar' ~/file\"`"
+##Becomes ssh localhost "grep 'foo bar' ~/file"
+##Becomes grep 'foo bar' ~/file
+##So x ends up as the non-expanded, non-globbed output of that grep command
+#
+#Four ways to grep for a single-quote - the first 2 are vim-highlighting-friendly
+#x="$(remote_exec2 localhost "grep \"'\" ~/file")"
+#x="`remote_exec2 localhost "grep \\\"'\\\" ~/file"`"
+#x="`remote_exec2 localhost \"grep \\\"'\\\" ~/file\"`"
+#x="`remote_exec2 localhost "grep \\"'\\" ~/file"`"
+#These (all?) effectively expand to:
+##ssh localhost "grep \"'\" ~/file"
+##grep "'" ~/file
+##So x ends up as the non-expanded, non-globbed output of that grep command
+#
+#One way to grep for a double-quote.
+#x="`remote_exec localhost "grep \\\"\\\\\\\\\\"\\\" ~/file"`"
+#My personal favourite. Using single-quotes would be much simpler, but there
+#are real cases where we can't do that. Expands to:
+##ssh localhost "grep \"\\\"\" ~/file"
+##grep "\"" ~/file
+##So x ends up as the non-expanded, non-globbed output of that grep command
+
+remote_download()
+{
+  OPTIND=1
+  local retries=0
+  while getopts r: flag; do
+    case "${flag}" in
+      r) retries="${OPTARG}";;
+      *)
+         echo "Bad arg" 1>&2
+         return 1
+      ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+
+  local target="${1:-}"
+  local sourcefile="${2:-}"
+  local destfile="${3:-}"
+  if test x"${target}" = x; then
+    error "target not specified"
+    return 1
+  fi
+  if test x"${sourcefile}" = x; then
+    error "file/dir to copy not specified"
+    return 1
+  fi
+  if test x"${destfile}" = x; then
+    error "file/dir to copy to not specified"
+    return 1
+  fi
+  shift 3
+
+  local c
+  for ((c = ${retries}; c >= 0; c--)); do
+    dryrun "rsync -e \"ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR $*\" -avzx '${target}:${sourcefile}' '${destfile}' > /dev/null"
+    if test $? -eq 0; then
+      return 0
+    elif test $c -gt 0; then
+      warning "Download of '${target}:${sourcefile}' to '${destfile}' failed: will try $c more times"
+      sleep 3
+    fi
+  done
+  error "Download of '${target}:${sourcefile}' to '${destfile}' failed"
+  return 1
+}
+
+remote_upload()
+{
+  OPTIND=1
+  local retries=0
+  while getopts r: flag; do
+    case "${flag}" in
+      r) retries="${OPTARG}";;
+      *)
+         echo "Bad arg" 1>&2
+         return 1
+      ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+
+  local target="${1:-}"
+  local sourcefile="${2:-}"
+  local destfile="${3:-}"
+  if test x"${target}" = x; then
+    error "target not specified"
+    return 1
+  fi
+  if test x"${sourcefile}" = x; then
+    error "file/dir to copy from not specified"
+    return 1
+  fi
+  if test x"${destfile}" = x; then
+    error "file/dir to copy to not specified"
+    return 1
+  fi
+  shift 3
+
+  local c
+  for ((c = ${retries}; c >= 0; c--)); do
+    dryrun "rsync -e \"ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR $*\" -avzx '${sourcefile}' '${target}:${destfile}' > /dev/null"
+    if test $? -eq 0; then
+      return 0
+    elif test $c -gt 0; then
+      warning "Upload of '${sourcefile}' to '${target}:${destfile}' failed: will try $c more times"
+      sleep 3
+    fi
+  done
+  error "Upload of '${sourcefile}' to '${target}:${destfile}' failed"
+  return 1
+}
+
+remote_exec()
+{
+  local target="${1//\"/\\\"}"
+  local cmd="${2//\"/\\\"}"
+  if test $# -lt 2; then
+    error "Target and/or command not specified"
+    return 1
+  fi
+  shift 2
+  dryrun "ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR $* \"${target}\" \"${cmd}\""
+  return $?
+}
+
+remote_exec_async()
+{
+  local target="${1//\"/\\\"}"
+  local cmd="${2//\"/\\\"}"
+  local stdoutfile="${3}"
+  local stderrfile="${4}"
+  if test $# -lt 4; then
+    error "Target and/or command not specified"
+    return 1
+  fi
+  shift 4
+
+  dryrun "ssh -n -o PasswordAuthentication=no -o PubkeyAuthentication=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR $* ${target} -- \"exec 1>${stdoutfile}; exec 2>${stderrfile}; ${cmd}; echo EXIT CODE: \\\$?\" &"
+
+  #Backgrounded command won't give a meaningful error code
+  return 0
+}
diff --git a/runbenchmark.sh b/runbenchmark.sh
deleted file mode 100755
index 228bee0c..00000000
--- a/runbenchmark.sh
+++ /dev/null
@@ -1,276 +0,0 @@
-#!/bin/bash
-
-# load the configure file produced by configure
-if test -e "`dirname $0`/host.conf"; then
-    . "`dirname $0`/host.conf"
-else
-    echo "WARNING: no host.conf file!  Did you run configure?" 1>&2
-fi
-
-# load commonly used functions
-script="`which $0`"
-topdir="`dirname ${script}`"
-app="`basename $0`"
-
-. "${topdir}/lib/common.sh" || exit 1
-. "${topdir}/lib/benchmark.sh" || exit 1
-
-usage()
-{
-  # Format this section with 75 columns.
-  cat << EOF
-  ${app}	[-b, --build]
-  [-r, --run]
-  [-c, --clean]
-  [-x, --extract]
-  [-l, --list-of-benchmarks=benchmark1,benchmark2..]
-  [-t, --gcc-binary-tarball=tarball]
-  [-d, --dir-to-build=directory]
-  [    --extract-dir=directory]
-  [-C, --additional-cflags]
-  [-L, --aditional-lflags]
-  [    --controlled]
-EOF
-  return 0
-}
-
-setup_cpu()
-{
-  sudo cpufreq-set -g performance
-  for p in `(ps ax --format='%p' | tail -n +2)`; do
-    sudo taskset -a -p 0x1 $p 2>&1;
-  done
-}
-
-restore_cpu()
-{
-  for p in `(ps ax --format='%p' | tail -n +2)`; do
-    sudo taskset -a -p 0xFFFFFFFF $p 2>&1;
-  done
-  sudo cpufreq-set -g  conservative
-}
-
-help()
-{
-  # Format this section with 75 columns.
-  cat << EOF
-  NAME
-
-  ${app} - the Linaro Toolchain Becnhmarking Framework.
-
-  SYNOPSIS
-
-EOF
-    usage
-    cat << EOF
-
-DESCRIPTION
-
-  ${app} is a toolchain benchmarking framework.
-
-PRECONDITIONS
-
-  FIXME
-
-OPTIONS
-
-  -b, --build
-	       Just build the benchmarks. Default behavior is
-	       to extract, clean, build and run the benchmark list.
-
-  -r, --run
-	       Just run the benchmarks. Default behavior is
-	       to extract, clean, build and run the benchmark list.
-
-  -c, --clean
-	       Just clean the benchmarks. Default behavior is
-	       to extract, clean, build and run the benchmark list.
-
-  -x, --extract
-               Just extract the benchmarks. Default behavior is
-	       to extract, clean, build and run the benchmark list.
-
-  -l, --list-of-benchmarks=benchmark1,benchmark2..
-	       Specify the list of benchmarks to run,
-
-  -p, --gcc-binary-path=path
-               Spevify the precompiled gcc path to use for
-	       compiling benchmarks.
-
-  -d, --dir-to-build=directory
-	       Specify the path of directory in which bennchrks
-	       should be placed and executed.
-
-  --extract-dir=directory
-               Specify the path of directory from which benchmarks
-               should be extracted.
-
-  -C, --additional-cflags
-	       Additional CFLAGS for compiling benchmarks.
-
-  -L, --aditional-lflags
-	       Additional LFLAGS for linking benchmarks.
-
-  --controlled
-               Minimise sources of noise during run
-               E.G. Shuts down unneeded services, shunts remaining
-               services to a second CPU, shuts down network.
-
-EXAMPLES
-
-  Clean, build and run eembc without extracting:
-
-    runbenchmark.sh -l=eembc -c -b -r
-
-  Run spec2k in a controlled way:
-
-    runbenchmark.sh -l=spec2k --controlled
-
-
-PRECONDITION FILES
-
-AUTHOR
-  Kugan Vivekanandarajah <kuganv@linaro.org>
-  Bernard Ogden <bernie.ogden@linaro.org>
-
-EOF
-    return 0
-}
-
-extract=false
-clean=false
-build=false
-run=false
-build_pgo=false
-controlled=false
-
-while :
-do
-  case $1 in
-    -h | --help | -\?)
-      help
-      exit 0
-      ;;
-    --controlled)
-      controlled=true
-      shift
-      ;;
-    -x | --extract)
-      extract=true
-      shift
-      ;;
-    -b | --build)
-      clean=true
-      extract=true
-      build=true
-      shift
-      ;;
-    -c | --clean)
-      extract=true
-      clean=true
-      shift
-      ;;
-    -r | --run)
-      extract=true
-      clean=true
-      build=true
-      run=true
-      shift
-      ;;
-    -l=* | --list=*)
-      list=${1#*=}
-      shift
-      ;;
-    -C=* | --additional-cflags=*)
-      XCFLAGS=${1#*=}
-      shift
-      ;;
-    -L=* | --aditional-lflags=*)
-      XLFLAGS=${1#*=}
-      shift
-      ;;
-    --pgo)
-      build_pgo=true
-      shift
-      ;;
-    --file=*)
-      file=${1#*=}        # Delete everything up till "="
-      shift
-      ;;
-    -x=* | --extract-dir==*)
-      SRC_PATH=${1#*=}
-      shift
-      ;;
-    --) # End of all options
-      shift
-      break
-      ;;
-    -*)
-      warning "Unknown option (ignored): $1" >&2
-      shift
-      ;;
-    *)  # no more options. Stop while loop
-      break
-      ;;
-  esac
-done
-
-if test x"$list" = x; then
-  error "Benchmark list is empty"
-  exit 1
-fi
-if test x"$list" = xall; then
-  list=coremark,gmpbench,gnugo,skiabench,denbench,eembc,spec2k,libavbench,eembc_office,nbench
-fi
-
-$extract || $clean || $build || $run || { extract=true;clean=true;build=true;run=true; }
-
-#Cribbed from abe.sh
-#TODO: Push this change back to last merge? Not really needed if I submit generated patches.
-#      But may still be worth it for my sanity.
-make_docs=no
-install=no
-
-#TODO How does primary abe do this?
-dump_host_info  > host.txt
-#fetch md5sums
-
-for b in ${list//,/ };
-do
-  if $extract; then
-    echo "Extract benchmark $b"
-    url="`get_source $b`"
-    if test $? -gt 0; then
-      error "Couldn't find the source for ${do_checkout}"
-      build_failure
-    fi
-    checkout ${url}
-    if test $? -gt 0; then
-      error "--checkout ${url} failed."
-      build_failure
-    fi
-  fi
-
-#TODO is this actually doing anything when we started from fresh checkout?
-#TODO clean seems to need to run as part of 'build', fix that up so that I can run it (if I can't already)
-  #if $clean; then
-  #  echo "Clean benchmark $b"
-  #  make_clean $b || exit
-  #fi
-
-  #TODO Delete the PGO part unless we find a benchmark that actually uses it
-  if $build; then
-    if $build_pgo; then
-      echo "Build benchmark $b with pgo"
-      build_with_pgo $ctx || exit
-    else
-      echo "Build benchmark $b"
-      build $b || exit
-    fi
-  fi
-
-  if $run; then
-    echo "Run benchmark $b"
-    bench_run $b $controlled || exit
-  fi
-done
diff --git a/scripts/Benchmark.job b/scripts/Benchmark.job
new file mode 100644
index 00000000..faad8b85
--- /dev/null
+++ b/scripts/Benchmark.job
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+whoami
+#if test x"`whoami`" != benchmark; then
+#  ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no benchmark@localhost whoami
+#  exit 1
+#fi
+rm -rf ${WORKSPACE}/bin
+mkdir ${WORKSPACE}/bin
+cd ${WORKSPACE}/bin
+wget ${aarch64_gcc}
+tar xf gcc-linaro-aarch64-linux-gnu-4.9-2014.09_linux.tar.xz
+export PATH=${WORKSPACE}/bin/gcc-linaro-aarch64-linux-gnu-4.9-2014.09_linux/bin:$PATH
+cd ${WORKSPACE}/abe
+./configure
+LAVA_SERVER=bogden:q6ndgs71bgocf7lc3q4s2gpqal0shnfy6h32x4i3oy4q9u8qos0su3zjy6qhco7akc6tddujs4dmyt2osojfx9fq500fci95sh20bzxoolhhq2u9sqyyif8qk4d68kaw@validation.linaro.org/RPC2/ ./scripts/benchmark.sh -t aarch64-linux-gnu -b fakebench kvm
+scp -r ./fakebench-log abe:/work/benchmarking
diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh
new file mode 100755
index 00000000..a8e60afc
--- /dev/null
+++ b/scripts/benchmark.sh
@@ -0,0 +1,259 @@
+#!/bin/bash
+#This script is an ad-hoc way of doing things pending a DejaGNU
+#implementation that will avoid wheel re-invention. Let's not
+#sink too much time into making this script beautiful.
+
+#TODO Convert as much as possible into a function, so that we don't share global namespace with abe except where we mean to
+#     Better - confine abe to a subshell
+
+set -o pipefail
+set -o nounset
+
+#Make sure that subscripts clean up - we must not leave benchmark sources or data lying around,
+#we should not leave lava targets reserved
+trap clean_top EXIT
+trap 'exit ${error}' TERM INT HUP QUIT
+
+error=1
+declare -A runpids
+
+clean_top()
+{
+  for runpid in "${!runpids[@]}"; do
+    if kill -0 "${runpid}" 2>/dev/null; then
+      kill "${runpid}" 2>/dev/null
+      wait "${runpid}"
+      if test $? -ne 0; then
+        error=1
+      fi
+    fi
+  done
+  if test x"${cmpbuild:-}" != x; then
+    rm -f "${cmpbuild}"
+    if test $? -ne 0; then
+      echo "Failed to delete compressed benchmark output ${cmpbuild}" 1>&2
+      error=1
+    fi
+  fi
+  exit ${error}
+}
+
+function usage
+{
+  cat << EOF
+$0 [-tckhl] -b <benchmark> <board...>
+
+  -b   Identify the benchmark to build, e.g. fakebench, eembc. Compulsory.
+  -c   Cautious. If this is set, failure in any stage of target setup will
+       be treated as an error, even if recoverable. On by default.
+  -h   Show this help.
+  -k   Keep. If this is set, benchmark sources and results will be left on
+       target. LAVA targets will not be released.
+  -t   Target triple to build benchmark for e.g. arm-linux-gnueabihf. Used
+       only for building. Blank implies native build.
+  -l   Sysroot to install on target. Blank uses native libraries. This option
+       can only be used for LAVA targets.
+
+  <board...> may be anything that has a file in config/boards/bench, e.g. the
+  existence of arndale.conf means that you can put arndale here. At least one
+  target may be specified. ssh targets must only be specified once. LAVA
+  targets can be specified as many times as you like.
+
+  If building natively, board is optional. If not given, the benchmark will
+  run on localhost.
+EOF
+}
+
+set_toolchain()
+{
+  local target_gcc="${target:+${target}-}gcc"
+  if test x"${toolchain_path:-}" = x; then
+    which "${target_gcc}" > /dev/null 2>&1
+    if test $? -ne 0; then
+      echo "No toolchain specified and unable to find a suitable gcc on the path" 1>&2
+      echo "Looked for ${target:+${target}-}gcc" 1>&2
+      exit 1
+    else
+      echo "No toolchain specified, using `which ${target_gcc}`, found on PATH" 1>&2
+    fi
+  else
+    if test -f "${toolchain_path}/bin/${target_gcc}"; then
+      PATH="${toolchain_path}/bin:$PATH"
+    else
+      echo "Toolchain directory ${toolchain_path} does not contain bin/${target_gcc}" 1>&2
+      exit 1
+    fi
+  fi
+}
+
+# load the configure file produced by configure
+if test -e "${PWD}/host.conf"; then
+    . "${PWD}/host.conf"
+else
+    echo "ERROR: no host.conf file!  Did you run configure?" 1>&2
+    exit 1
+fi
+topdir="${abe_path}" #abe global, but this should be the right value for abe
+
+
+#Sanity Checks
+if test "$((`umask` & 077))" -ne 63; then
+  echo "umask grants permissions to group and world, will remove those permissions" 1>&2
+  if ! umask g-rwx,o-rwx; then
+    echo "umask failed, wibble, aborting" 1>&2
+    exit 1
+  fi
+fi
+
+#TODO: Really, this check should operate on the route from the git server to localhost
+. "${topdir}/scripts/benchutil.sh"
+if ! check_private_route localhost; then
+  echo "Do not appear to be on private network, conservatively aborting" 1>&2
+  exit 1
+fi
+#End sanity checks
+
+
+run_benchargs=""
+skip_build=
+toolchain_path=
+cautious='-c'
+keep= #if set, don't clean up benchmark output on target, don't kill lava targets
+target=
+sysroot_path=
+while getopts a:i:t:b:l:kchs flag; do
+  case "${flag}" in
+    a) run_benchargs="${OPTARG}";;
+    s) skip_build=1;;
+    i) toolchain_path="${OPTARG}";;
+    l) sysroot_path="${OPTARG}";;
+    t) target="${OPTARG}";; #have to be careful with this one, it is meaningful to sourced abe files in subshells below
+    b) benchmark="${OPTARG}";;
+    c) cautious=;;
+    k)
+       keep='-k'
+       echo 'Keep (-k) set: possibly sensitive benchmark data will be left on target'
+       echo 'Continue? (y/N)'
+       read answer
+       if ! echo "${answer}" | egrep -i '^(y|yes)[[:blank:]]*$' > /dev/null; then
+         exit 0
+       fi
+    ;;
+    h)
+       usage
+       exit 0
+    ;;
+    *)
+       echo "Bad arg" 1>&2
+       exit 1
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+devices=("$@") #Duplicate targets are fine for lava, they will resolve to different instances of the same machine.
+               #Duplicate targets not fine for ssh access, where they will just resolve to the same machine every time.
+               #TODO: Check for multiple instances of a given non-lava target
+set_toolchain
+
+if test x"${benchmark:-}" = x; then
+  echo "No benchmark given (-b)" 1>&2
+  echo "Sensible values might be eembc, spec2000, spec2006" 1>&2
+  exit 1
+fi
+if test x"${sysroot_path:-}" != x; then
+  tmp_sysroot_lib="${sysroot_path#*:}"
+  if test x"${tmp_sysroot_lib}" = x"${sysroot_path}"; then
+    tmp_sysroot_lib=lib
+  fi
+  tmp_sysroot_path="${sysroot_path%:*}"
+  if ! test -d "${tmp_sysroot_path}"; then
+    echo "Sysroot path '${tmp_sysroot_path}' does not exist" 1>&2
+    exit 1
+  else
+    if ! test -d "${tmp_sysroot_path}/${tmp_sysroot_lib}"; then
+      echo "Sysroot path '${sysroot_path}' does not look like a sysroot: lib dir ${tmp_sysroot_lib} does not exist" 1>&2
+      exit 1
+    fi
+    if ! test -d "${tmp_sysroot_path}/usr/${tmp_sysroot_lib}"; then
+      echo "Sysroot path '${sysroot_path}' does not look like a sysroot: lib dir usr/${tmp_sysroot_lib} does not exist" 1>&2
+      exit 1
+    fi
+  fi
+fi
+if test x"${target:-}" = x; then #native build
+  if test ${#devices[@]} -eq 0; then
+    devices=("localhost") #Note that we still need passwordless ssh to
+                          #localhost. This could be fixed if anyone _really_
+                          #needs it, but DejaGNU will presumably fix for free.
+  #else - we're doing a native build and giving devices other than localhost
+  #       for measurement, that's fine. But giving both localhost and other
+  #       devices is unlikely to work, given that we'll be both shutting down
+  #       localhost and using it to dispatch benchmark jobs. Therefore TODO:
+  #       check for a device list composed of localhost plus other targets
+  fi
+else #cross-build, implies we need remote devices
+  if test ${#devices[@]} -eq 0; then
+    echo "--target implies cross-compilation, but no devices given for run" 1>&2
+    exit 1
+  fi
+fi
+
+if test x"${skip_build:-}" = x; then
+  #abe can build the benchmarks just fine
+  ("${topdir}"/abe.sh --build "${benchmark}.git" ${target:+--target "${target}"})
+  if test $? -ne 0; then
+    echo "Error while building benchmark ${benchmark}" 1>&2
+    exit 1
+  fi
+fi
+
+builddir="`target2="${target}"; . ${abe_top}/host.conf && . ${topdir}/lib/common.sh && if test x"${target2}" != x; then target="${target2}"; fi && get_builddir $(get_URL ${benchmark}.git)`"
+if test $? -ne 0; then
+  echo "Unable to get builddir" 1>&2
+  exit 1
+fi
+
+#Compress build to a tmpfile in our top-level working directory
+#This should be good for bandwidth
+#By keeping file at top level, we make sure that everything sensitive is in one place
+cmpbuild="`mktemp -p ${abe_top} -t ${benchmark}_XXXXXXX.tar.bz2`"
+if test $? -ne 0; then
+  echo "Unable to create temporary file for compressed build output" 1>&2
+  exit 1
+fi
+if ! tar cjf "${cmpbuild}" -C "${builddir}/.." "`basename ${builddir}`"; then
+  echo "Unable to compress ${builddir} to ${cmpbuild}" 1>&2
+  exit 1
+fi
+for device in "${devices[@]}"; do
+  "${topdir}"/scripts/runbenchmark.sh -b "${benchmark}" -d "${device}" -t "${cmpbuild}" -a "${run_benchargs}" ${sysroot_path:+-l "${sysroot_path}"} ${keep} ${cautious} &
+  runpids[$!]=''
+done
+
+running_pids=("${!runpids[@]}")
+while true; do
+  for running_pid in "${running_pids[@]}"; do
+    kill -0 "${running_pid}" 2>/dev/null
+    if test $? -ne 0; then #Process cannot be signalled, reap it
+      wait "${running_pid}"
+      if test $? -ne 0; then
+        error=1
+      fi
+      unset runpids["${running_pid}"]
+    fi
+  done
+  running_pids=("${!runpids[@]}")
+  if test ${#running_pids[@]} -eq 0; then
+    break
+  else
+    sleep 60&
+    wait $!
+  fi
+done
+
+echo
+echo "All runs completed"
+exit ${error}
+
+#TODO: I suppose I might want a 'delete local copies of source/built benchmark'
+
diff --git a/scripts/benchutil.sh b/scripts/benchutil.sh
new file mode 100644
index 00000000..25e29681
--- /dev/null
+++ b/scripts/benchutil.sh
@@ -0,0 +1,214 @@
+set -o pipefail
+set -o nounset
+
+function get_addr
+{
+  local hostname="${1:-localhost}"
+  local listener_addr
+  if test x"${hostname}" = xlocalhost; then
+    listener_addr="`hostname -I`"
+  else
+    listener_addr="`ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes ${hostname} 'hostname -I'`"
+  fi
+  if test $? -ne 0; then
+    echo "Failed to run 'hostname -I' on ${hostname}" 1>&2
+    return 1
+  fi
+
+  #We only try to support IPv4 for now
+  listener_addr="`echo ${listener_addr} | tr ' ' '\n' | grep '^\([[:digit:]]\{1,3\}\.\)\{3\}[[:digit:]]\{1,3\}$'`"
+  if test $? -ne 0; then
+    echo "No IPv4 address found, aborting" 1>&2
+    return 1
+  fi
+
+  #We don't try to figure out what'll happen if we've got multiple IPv4 interfaces
+  if test "`echo ${listener_addr} | wc -w`" -ne 1; then
+    echo "Multiple IPv4 addresses found, aborting" 1>&2
+    return 1
+  fi
+
+  echo "${listener_addr}" | sed 's/^[[:blank:]]*//' | sed 's/[[:blank:]]*$//'
+}
+
+#Return 0 if we're inside the lava network
+#Return 1 if we're outside the lava network
+#Return 2 if we don't know where we are
+#Typically, 2 will be returned if we can't ssh into the hackbox. This tends
+#to mean we're not configured to do so and could happen both inside and outside
+#the LAVA network.
+function lava_network
+{
+  local hackbox_mac
+
+  hackbox_mac="`ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes ${1:+${1}@}lab.validation.linaro.org 'cat /sys/class/net/eth0/address'`"
+  if test $? -ne 0; then
+    echo "Failed to get hackbox mac" 1>&2
+    echo "Tried: ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes ${1:+${1}@}lab.validation.linaro.org 'cat /sys/class/net/eth0/address'" >&2
+    return 2 #We couldn't get the mac, stop trying to figure out where we are
+  fi
+  arp 10.0.0.10 | grep -q "${hackbox_mac}";
+  if test $? -eq 0; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+function lava_user
+{
+  local usr="${USER}"
+  if echo "$1" | grep -q '^http'; then
+    echo "LAVA URL must exclude protocol (e.g. http://, https://)" 1>&2
+    return 1
+  fi
+  if echo "$1" | grep -Eq '^.+@'; then
+    usr="${1/@*}"
+    usr="${usr/:*}"
+  fi
+  echo "${usr}"
+}
+
+function lava_server
+{
+  if echo "$1" | grep -q '^http'; then
+    echo "LAVA URL must exclude protocol (e.g. http://, https://)" 1>&2
+    return 1
+  fi
+  if echo "$1" | grep -Eq '^.+@'; then
+    echo "$1" | sed 's/[^@]*@//'
+  else
+    echo "$1"
+  fi
+}
+
+#Attempt to use read to discover whether there is a record to read from the producer
+#If we time out, check to see whether the producer still seems to be alive.
+#We can check more than one pid, if we have visibility of some other process(s) that we
+#would also like to monitor while we wait to read something. If any of these
+#processes seem dead then return 2, otherwise keep trying to read.
+#Once we've established that there seems to be a record, try to read it with a
+#read timeout. If we fail to read within read_timeout, return 1 to indicate
+#read failure - but the producer may still be alive in this case.
+#Can also set a deadline - bgread will return 3 if it hasn't seen any new output
+#before deadline expires. deadline is only checked at read_timeout intervals.
+#Typical invocation: foo="`bgread ${child_pid} <&3`"
+#Invocation with read checks every 5 seconds, failure after 2 minutes and two
+#pids to monitor:
+#foo="`bgread -T 120 -t 5 ${child_pid} ${other_pid} <&3`"
+function bgread
+{
+  OPTIND=1
+  local read_timeout=60
+  local deadline=
+  local pid
+
+  while getopts T:t: flag; do
+    case "${flag}" in
+      t) read_timeout="${OPTARG}";;
+      T) deadline="$((${OPTARG} + `date +%s`))";;
+      *)
+         echo "Bad arg" 1>&2
+         return 1
+      ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+
+  if test $# -eq 0; then
+    echo "No pid(s) to poll!" 1>&2
+    return 1
+  fi
+  local buffer=''
+  local line=''
+
+  #We have to be careful here. If the read times out when there was a partial
+  #record on the fifo then the part that has been read just gets discarded. We
+  #can avoid this problem by using -N to ensure that we read the minimal amount
+  #and DO NOT discard it. -N 0 might be intuited to do the right thing, but is
+  #arguably undefined behaviour and empirically doesn't work.
+  #Read 1 char then timeout if it isn't a delimiter: buffer is the char, exit code 0 OR
+  #Read the delimiter, don't timeout: buffer is empty, exit code 0 OR
+  #Fail to read any chars coz there aren't any, then timeout: buffer is empty, exit code 1
+  while ! read -N 1 -t "${read_timeout}" buffer; do
+    for pid in "$@"; do
+      kill -0 "${pid}" > /dev/null 2>&1 || return 2
+    done
+    if test x"${deadline:-}" != x; then
+      if test `date +%s` -ge ${deadline}; then
+        echo "bgread timed out" 1>&2
+        return 3
+      fi
+    fi
+  done
+
+  #If we get here, we managed to read 1 char. If we have a null string just
+  #return it (the record was empty). Otherwise, assume the rest of the record is ready to be read,
+  #especially within the generous timeout that we allow.
+  if test x"${buffer:-}" != x; then
+    read -t 60 line
+    if test $? -ne 0; then
+      echo "Record did not complete" 1>&2
+      return 1
+    fi
+  fi
+  echo "${buffer}${line}"
+  return 0
+}
+
+#Use ping to perform a traceroute-like check of route to some target
+#It's probably not guaranteed that other protocols (or even future pings) will
+#take the same route, this is just a conservative sanity check.
+function check_private_route
+{
+  local myaddr
+  local pingout
+  local ttl
+  local retry
+
+  if test x"${1:-}" = x; then
+    echo "check_private_route requires a parameter" 1>&2
+    return 1
+  fi
+
+  #Extended regexps (use grep -E)
+  local block24='10\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
+  local block20='172\.(1[6-9]|2[0-9]|3[0-1])\.[[:digit:]]+\.[[:digit:]]+'
+  local block16='192\.168\.[[:digit:]]+\.[[:digit:]]+'
+
+  myaddr="`get_addr`"
+  if test $? -ne 0; then
+    echo "Cannot get a usable IP address to check route" 1>&2
+    return 1
+  fi
+
+  #Check we're on something in a private network to start with
+  if ! echo "${myaddr}" | grep -Eq "^(${block24}|${block20}|${block16})$"; then
+    echo "Own IP address ${myaddr} does not match any known private network range" 1>&2
+    return 1
+  fi
+
+  #Check every stop along the way to target. DO NOT check target itself - assume
+  #that we don't hop off our network even if its IP appears to be non-private.
+  #This is a crude, but generic and unprivileged, way of doing traceroute - what
+  #we really want is the routing tables, I think.
+  for ttl in {1..10}; do
+    #This thing sometimes fails spuriously - I can't fathom why, so we retry
+    #a few times. Sigh.
+    for retry in {1..5}; do
+      pingout="`ping -n -t ${ttl} -c 1 $1`"
+      if test $? -eq 0; then
+        return 0 #We've reached the target
+      fi
+      echo "${pingout}" | grep -Eq "^From (${block24}|${block20}|${block16}) icmp_seq=1 Time to live exceeded$"
+      if test $? -eq 0; then
+        continue 2
+      fi
+    done
+    echo "Surprising stop on hop ${ttl} on route to benchmark target: '${pingout}'" 1>&2
+    return 1
+  done
+
+  echo "Failed to reach benchmark target within ${ttl} hops" 1>&2
+  return 1
+}
diff --git a/scripts/controlledrun.sh b/scripts/controlledrun.sh
new file mode 100755
index 00000000..0218d78f
--- /dev/null
+++ b/scripts/controlledrun.sh
@@ -0,0 +1,595 @@
+#!/bin/bash
+
+#TODO: Test/finish crouton support
+
+#Assumptions:
+#1) We have are root, or a non-root user with passwordless sudo
+#2) We don't care about cleanup. We have a cleanup handler for exit, but not
+#   for signals. If you kill it, the target could be in a messy state.
+#3) Target system is running Crouton or Ubuntu. It might well work with other
+#   distributions, though, especially the parts that don't assume upstart.
+
+#Error returns from the 'teardown' functions are ignored by default - we can
+#still run the benchmark, just with less faith in repeatability. Specifying -c
+#(cautiousness) will cause error on exit from a teardown function.
+
+#Rebuild functions just return 0 if there is nothing to do
+
+set -o pipefail
+set -o nounset
+declare -a stopped_services
+declare -a bound_processes
+declare -a downed_interfaces
+old_policy=
+
+#The chroot part is often a nop, but will get us out of chroot if necessary
+if test "${USER}" = root; then
+  sudo="chroot /proc/1/root"
+else
+  sudo="sudo chroot /proc/1/root"
+fi
+
+cleanup()
+{
+  local ret=$?
+  local tmp
+
+  if test x"${rva_setting:-}" != x; then
+    ${sudo} bash -c "echo ${rva_setting} > /proc/sys/kernel/randomize_va_space"
+    if test $? -ne 0; then
+      echo "Failed to restore ASLR setting" | tee -a /dev/stderr "${log}"
+      ret=1
+    fi
+  fi
+
+  start_services
+  tmp=$?
+  if test ${tmp} -gt ${ret}; then
+    ret=${tmp}
+  fi
+  restore_policy
+  tmp=$?
+  if test ${tmp} -gt ${ret}; then
+    ret=${tmp}
+  fi
+  unbind_processes
+  tmp=$?
+  if test ${tmp} -gt ${ret}; then
+    ret=${tmp}
+  fi
+  start_network
+  tmp=$?
+  if test ${tmp} -gt ${ret}; then
+    ret=${tmp}
+  fi
+
+  if test ${ret} -gt 0; then
+    echo "Problem restoring target system" | tee -a /dev/stderr "${log}" > /dev/null
+    if test ${cautiousness} -eq 1; then
+      exit 1
+    fi
+  fi
+}
+
+#Service control is a hairy land. We limit this function to handling upstart
+#services for now - experiments so far seem to show this gives enough
+#repeatability.  A little research indicates that:
+# * The service utility will let us query both upstart and SysV-style init services
+# * The service utility's output cannot necessarily be trusted
+# * systemd is coming and might change the story further
+#So we might want to improve this function in the future, or might be forced
+#to change it.
+
+#(Upstart) services can have dependencies that are hard to determine. Therefore
+#we pass an ordered list of services to stop. We assume that:
+#1) Target is in a known state
+#2) List of services to stop is in a dependency-friendly order - though we may
+#   often get away with ignoring this one for simple systems
+#3) Network servies must be left alone as we handle them separately, if only
+#   for convenience in interactive use
+
+#In the past we've kept services matching these patterns:
+#For crouton - 
+#  "dbus|boot-services|shill|wpasupplicant|tty"
+#For Linux -
+#  keep="dbus|network|tty|rcS|auto-serial-console"
+#If running on an Ubuntu desktop, keeping lightdm is sensible
+#If running in a non-native chroot, keep binfmt-support
+stop_services()
+{
+  local service
+  local ret=0
+  local service_status
+  for service in `cat $1 | grep -v '^[[:blank:]]*#'`; do
+    service_status=`${sudo} status "${service}"`
+    if test $? -ne 0; then
+      echo "Service '${service}' does not exist" | tee -a /dev/stderr "${log}" > /dev/null
+      ret=1
+      continue
+    fi
+    if echo "${service_status}" | grep ' stop/waiting$' > /dev/null; then
+      echo "Service '${service}' already stopped" | tee -a /dev/stderr "${log}" > /dev/null
+      continue
+    fi
+    if echo "${service_status}" | grep -v ' start/running$' > /dev/null; then
+      echo "Service '${service}' does not appear to be running. Will try to stop it anyway. Are you specifying services in the right order?" | tee -a /dev/stderr "${log}" > /dev/null
+    fi
+    ${sudo} stop "${service}" > /dev/null
+    if test $? -eq 0; then
+      stopped_services+=("${service}")
+    else
+      echo "Service '${service}' could not be stopped" | tee -a /dev/stderr "${log}" > /dev/null
+      ret=1
+    fi
+  done
+
+  return ${ret}
+}
+
+#Start services in reverse order, to get the dependencies right
+start_services()
+{
+  ret=0
+  if test x"${stopped_services-}" = x; then
+    return 0
+  fi
+  for ((i=${#stopped_services[@]}-1; i>=0; i--)); do
+    ${sudo} start "${stopped_services[$i]}" > /dev/null
+    if test $? -ne 0; then
+      echo "Failed to restart service '${stopped_service[$i]}'" | tee -a /dev/stderr "${log}" > /dev/null
+      ret=1
+    fi
+  done
+  return ${ret}
+}
+
+set_policy()
+{
+  old_policy=(`cpufreq-info -p`) #0 = min freq, 1 = max freq, 2 = governor
+  if test $? -ne 0 || \
+     test x"${old_policy[0]:-}" = x || \
+     test x"${old_policy[1]:-}" = x || \
+     test x"${old_policy[2]:-}" = x; then
+    echo "Frequency scaling not supported" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+  ${sudo} cpufreq-set -g userspace -d "${freq}" -u "${freq}"
+  if test $? -ne 0; then
+      old_policy=
+      echo "Frequency scaling not supported" | tee -a /dev/stderr "${log}" > /dev/null
+      return 1
+  fi
+}
+
+restore_policy()
+{
+  #If freq scaling was unsupported then there is nothing to do
+  if test x"${old_policy:-}" = x; then
+    return 0
+  fi
+  ${sudo} cpufreq-set -g "${old_policy[2]}" -d "${old_policy[0]}" -u "${old_policy[1]}"
+  if test $? -ne 0; then
+    echo "Unable to restore policy '${old_policy}'" | tee -a /dev/stderr "${log}" > /dev/null
+  fi
+}
+
+#Bind all existing processes to CPU $1.  We then run benchmarks on CPU #1.
+#Note that some processes cannot be bound, for example per-cpu kernel threads.
+bind_processes()
+{
+  ${sudo} taskset -a -p -c $1 1 > /dev/null
+  if test $? -ne 0; then
+    echo "CPU bind not supported" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+
+  bound_processes=(1)
+  local all_processes
+  all_processes=`ps ax --format='%p' | tail -n +3`
+  if test $? -ne 0; then
+    echo "Unable to list processes" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+
+  local p
+  local ppid
+  local ppcmd
+  local output
+  local ret=0
+  for p in ${all_processes}; do
+    ppid="`ps -p $p -o ppid=`"
+    if test $? -ne 0; then
+      continue #Probably some process completed since we made the list
+    fi
+    if test ${ppid} -ne 0; then
+      ppcmd="`ps -p ${ppid} -o cmd=`" 
+      if test $? -ne 0; then
+        echo "Failed to get cmd for pid $ppid (parent of $pid)" 1>&2
+      fi
+      if [[ "${ppcmd}" = *kthreadd* ]]; then
+        continue #don't try to change the affinity of kernel procs
+      fi
+    fi
+    output="`${sudo} taskset -a -p -c $1 ${p} 2>&1`"
+    if test $? -eq 0; then
+      bound_processes+=("${p}")
+    else
+      local name="`grep Name: /proc/$p/status | cut -f 2`"
+      echo "Failed to bind $name to CPU $1: $output" | tee -a /dev/stderr "${log}" > /dev/null
+      ret=1
+    fi
+  done
+
+  return ${ret}
+}
+
+#TODO: Strictly we shoud rebind to the same mask as before. That would be
+#      very easy to do with an associative array, but I don't fancy putting
+#      a bash 4 dependency here just yet
+unbind_processes()
+{
+  if test x"${bound_processes-}" = x; then
+    #Either taskset isn't working, or we didn't change any affinities
+    return 0
+  fi
+
+  local p
+  local ret=0
+  for p in "${bound_processes[@]}"; do
+    local output
+    output="`${sudo} taskset -a -p 0xFFFFFFFF ${p} 2>&1`"
+    if test $? -ne 0; then
+      local name="`grep Name: /proc/$p/status | cut -f 2`"
+      echo "Failed to unbind $name: $output" | tee -a /dev/stderr "${log}" > /dev/null
+      ret=1
+    fi
+  done
+
+  return ${ret}
+}
+
+#It would be more consistent to get the user to tell us how to manipulate the
+#network, but this should work fine and it is convenient.
+#We don't stop loopback, that would be madness
+stop_network()
+{
+  #Stop network on crouton (untested)
+  if croutonversion > /dev/null 2>&1; then
+    #TODO: Rather than sleep 2, we should spin until we see that those services are stopped
+    #      Although perhaps we can count on the stop command not exiting until the service is really stopped
+    ${sudo} /bin/bash -c 'stop shill && stop wpasupplicant' && sleep 2
+    if test $? -ne 0; then
+      echo "Failed to stop network" | tee -a /dev/stderr "${log}" > /dev/null
+      return 1
+    fi
+    downed_interfaces+=("crouton")
+    return 0
+  fi
+
+  #Stop network on not-crouton
+
+  #Get interfaces
+  local -a interfaces
+  #TODO: Remote corner case - this'll break on interface names with a space in
+  interfaces=(`ifconfig | cut -f 1 -d ' ' | sed '/^$/d' | grep -v '^lo$'`)
+  if test $? -ne 0; then
+    echo "Failed to read network interfaces" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+  if test x"${interfaces-}" = x; then
+    echo "No interfaces found. Wibble." | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+
+  #Work out how to stop interfaces by stopping one of them
+  local netcmd
+  if ${sudo} stop network-interface INTERFACE="${interfaces[0]}" >/dev/null 2>&1; then
+    netcmd="${sudo} stop network-interface INTERFACE="
+  elif ${sudo} ifdown "${interfaces[0]}"; then #don't redirect stderr as this is our last try and failure information would be helpful
+    netcmd="${sudo} ifdown "
+  else
+    echo "Cannot bring down network interfaces" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+  downed_interfaces+=("${interfaces[0]}")
+
+  local ret=0
+
+  #Stop any remaining interfaces
+  if test ${#interfaces[@]} -gt 1; then
+    local interface
+    interfaces=("${interfaces[@]:1}")
+    for interface in "${interfaces[@]}"; do
+      bash -c "${netcmd}${interface}"
+      if test $? -eq 0; then
+        downed_interfaces+=("${interface}")
+      else
+        echo "Failed to bring down network interface '${interface}'" | tee -a /dev/stderr "${log}" > /dev/null
+        ret=1
+      fi
+    done
+  fi
+
+  #Ensure that interfaces are have finished going down
+  #TODO: A little manpage scanning suggests that this isn't needed at least the upstart case
+  for i in {0..4}; do
+    if test x"`ifconfig | cut -f 1 -d ' ' | sed '/^$/d' | grep -v '^lo$'`" = x; then
+      break
+    fi
+    sleep 2
+    echo "Brought-down network interface(s) "`ifconfig | cut -f 1 -d ' ' | sed '/^$/d' | grep -v '^lo$'`" still up after >10s" | tee -a /dev/stderr "${log}" > /dev/null
+    echo "Will continue and hope for the best unless we're being cautious (-c)" | tee -a /dev/stderr "${log}" > /dev/null
+    ret=1
+  done
+    
+  return ${ret}
+} 
+
+start_network()
+{
+  if test x"${downed_interfaces-}" = x; then
+    return 0
+  fi
+
+  if croutonversion > /dev/null 2>&1; then
+    ${sudo} /bin/bash -c 'start wpasupplicant && start shill' && ${sudo} /sbin/iptables -P INPUT ACCEPT
+    if test $? -ne 0; then
+      echo "Failed to restart network" | tee -a /dev/stderr "${log}" > /dev/null
+      return 1
+    fi
+    return 0
+  fi
+
+  local netcmd
+  local i
+  if ${sudo} /bin/bash -c "start network-interface INTERFACE=${downed_interfaces[0]}" >/dev/null 2>&1; then
+    netcmd="${sudo} start network-interface INTERFACE="
+  elif ${sudo} /bin/bash -c "ifup ${downed_interfaces[0]}"; then #don't redirect stderr as this is our last try and failure information would be helpful
+    netcmd="${sudo} ifup "
+  else
+    echo "Cannot bring up network interfaces" | tee -a /dev/stderr "${log}" > /dev/null
+    return 1
+  fi
+  for i in {1..60}; do
+    echo "Ping ${downed_interfaces[0]}: $i" | tee -a /dev/stderr "${log}" > /dev/null
+    ping -c 1 "`ip -f inet -o addr show ${downed_interfaces[0]} | awk '{print $4}' | sed 's#/.*##'`" > /dev/null
+    if test $? -eq 0; then
+      break
+    fi
+    sleep 1
+    false
+  done
+  if test $? -ne 0; then
+    echo "Restored interface ${downed_interfaces[0]} not answering pings after ${i} seconds" | tee -a /dev/stderr "${log}" > /dev/null
+    echo "Ping rune: ping -c 1 \"`ip -f inet -o addr show ${downed_interfaces[0]} | awk '{print $4}' | sed 's#/.*##'`\" > /dev/null" | tee -a /dev/stderr "${log}" > /dev/null
+  fi
+  downed_interfaces=("${downed_interfaces[@]:1}")
+
+  local ret=0
+  if test ${#downed_interfaces[@]} -gt 1; then
+    local interface
+    for interface in "${downed_interfaces[@]}"; do
+      bash -c "${netcmd}${interface}"
+      if test $? -ne 0; then
+        echo "Failed to bring up network interface '${interface}'" | tee -a /dev/stderr "${log}" > /dev/null
+        ret=1
+      fi
+      for i in {1..60}; do
+        ping -c 1 "`ip -f inet -o addr show ${interface} | awk '{print $4}' | sed 's#/.*##'`" > /dev/null
+        if test $? -eq 0; then
+	  break
+        fi
+        sleep 1
+        false
+      done
+      if test $? -ne 0; then
+        echo "Restored interface ${interface} not answering pings after ${i} seconds" | tee -a /dev/stderr "${log}" > /dev/null
+        echo "Ping rune: ping -c 1 \"`ip -f inet -o addr show ${downed_interfaces[0]} | awk '{print $4}' | sed 's#/.*##'`\" > /dev/null" | tee -a /dev/stderr "${log}" > /dev/null
+      fi
+    done
+  fi
+  return ${ret}
+} 
+
+services_file=''
+log=/dev/null
+freq=''
+bench_cpu=0
+non_bench_cpu=''
+cautiousness=0
+do_network=0
+do_aslr=1 #Enabled by default
+do_renice=1 #Enabled by default
+while getopts s:f:b:p:cnul: flag; do
+  case $flag in
+    s)  services_file="${OPTARG}";;
+    f)  freq="${OPTARG}";;
+    b)  bench_cpu="${OPTARG}";;
+    p)  non_bench_cpu="${OPTARG}";;
+    c)  cautiousness=1;;
+    n)  do_network=1;;
+    l)  log="${OPTARG}";;
+    u)  #Set everything to 'uncontrolled', even the controls that default on
+        sudo=''
+        services_file=''
+        freq=''
+        bench_cpu=''
+        non_bench_cpu=''
+        do_network=0
+        do_aslr=0
+        do_renice=0
+        echo "Uncontrolled (-u) set, no controls enabled" 1>&2
+        echo "Individual control flags set after -u will still be respected" 1>&2
+    ;;
+    *)
+        echo 'Unknown option' | tee -a /dev/stderr "${log}" > /dev/null
+        exit 1
+    ;;
+  esac
+done
+
+echo "$@" | tee -a "${log}"
+echo | tee -a "${log}"
+
+shift $((OPTIND - 1))
+
+#Cheap sanity checks, before we start tearing the target down
+if test x"${services_file:-}" != x; then
+  if test \! -f "${services_file}"; then
+    echo "Services file '${services_file}' missing" | tee -a /dev/stderr "${log}" > /dev/null
+    exit 1
+  fi
+  if test x"`cat ${services_file:-}`" = x; then
+    echo "Services file '${services_file}' is empty" | tee -a /dev/stderr "${log}" > /dev/null
+    exit 1
+  fi
+fi
+
+echo "$bench_cpu" | grep '^[[:digit:]]*$' > /dev/null
+if test $? -ne 0; then
+  echo "Benchmark CPU (-b) must be null or a decimal number" | tee -a /dev/stderr "${log}" > /dev/null
+  exit 1
+fi
+echo "$non_bench_cpu" | grep '^[[:digit:]]*$' > /dev/null
+if test $? -ne 0; then
+  echo "Non-benchmark CPU (-p) must be null or a decimal number" | tee -a /dev/stderr "${log}" > /dev/null
+  exit 1
+fi
+if test x"${bench_cpu:-}" != x && test x"${non_bench_cpu:-}" != x && test x"${bench_cpu:-}" = x"${non_bench_cpu:-}"; then
+  echo "If set, benchmark CPU (-b) and non-benchmark CPU (-p) must be different" | tee -a /dev/stderr "${log}" > /dev/null
+  exit 1
+fi
+
+cmd="$@"
+
+if test x"${bench_cpu:-}" != x; then
+  taskset -c ${bench_cpu} true
+  if test $? -ne 0; then
+    echo "Could not bind benchmark to CPU ${bench_cpu}" | tee -a /dev/stderr "${log}" > /dev/null
+    exit 1
+  fi
+fi
+
+#Put the target back in order before we quit
+trap cleanup EXIT
+trap 'exit 1' TERM INT HUP QUIT
+
+if test x"${services_file:-}" != x; then
+  stop_services "${services_file}"
+  if test $? -ne 0 -a ${cautiousness} -eq 1; then
+    exit 1
+  fi
+fi
+if test x"${freq:-}" != x; then
+  set_policy "${freq}"
+  if test $? -ne 0 -a ${cautiousness} -eq 1; then
+    exit 1
+  fi
+fi
+if test x"${non_bench_cpu:-}" != x; then
+  bind_processes ${non_bench_cpu}
+  if test $? -ne 0 -a ${cautiousness} -eq 1; then
+    exit 1
+  fi
+fi
+if test ${do_network} -eq 1; then
+  stop_network
+  if test $? -ne 0 -a ${cautiousness} -eq 1; then
+    exit 1
+  fi
+fi
+
+#Report status of the target
+echo | tee -a "${log}"
+echo "** Target Status **" | tee -a "${log}"
+echo "===================" | tee -a "${log}"
+echo "General Information:" | tee -a "${log}"
+uname -a | tee -a "${log}"
+if test -f /etc/lsb-release; then
+  cat /etc/lsb-release | tee -a "${log}"
+fi
+echo | tee -a "${log}"
+#A little research shows that it is unclear how
+#reliable or complete the information from either
+#initctl or service is. So we make a best effort.
+echo "** (Possibly) Running Services:" | tee -a "${log}"
+echo "According to initctl:" | tee -a "${log}"
+${sudo} initctl list | grep running | tee -a "${log}"
+if test $? -ne 0; then
+  echo "*** initctl unable to list running services" | tee -a "${log}"
+fi
+echo "According to service:" | tee -a "${log}"
+${sudo} service --status-all 2>&1 | grep -v '^...-' | tee -a "${log}"
+if test $? -ne 0; then
+  echo "*** service unable to list (possibly) running services" | tee -a "${log}"
+fi
+echo | tee -a "${log}"
+echo "** CPUFreq:" | tee -a "${log}"
+${sudo} cpufreq-info -p | tee -a "${log}"
+if test $? -ne 0; then
+  echo "*** Unable to get CPUFreq info" | tee -a "${log}"
+fi
+echo | tee -a "${log}"
+echo "** Affinity Masks:" | tee -a "${log}"
+all_processes=`ps ax --format='%p' | tail -n +2`
+if test $? -eq 0; then
+  for p in ${all_processes}; do
+    ${sudo} taskset -a -p ${p} | tee -a "${log}"
+    if test $? -ne 0; then
+      echo "*** Unable to get affinity mask for process ${p}" | tee -a "${log}"
+    fi
+  done
+else
+  echo "*** Unable to get affinity mask info" | tee -a "${log}"
+fi
+echo | tee -a "${log}"
+echo "** Live Network Interfaces:" | tee -a "${log}"
+${sudo} ifconfig | tee -a "${log}"
+if test $? -ne 0; then
+  echo "*** Unable to get network info" | tee -a "${log}"
+fi
+echo "===================" | tee -a "${log}"
+echo | tee -a "${log}"
+
+#"setarch `uname -m` -R" would be a tidier way to run our benchmark without ASLR,
+#but doesn't work on our machines (setarch rejects the value of uname -m, and some
+#obvious alternatives, as invalid).
+if test ${do_aslr} -eq 1; then
+  rva_setting="`cat /proc/sys/kernel/randomize_va_space`"
+  ${sudo} bash -c 'echo 0 > /proc/sys/kernel/randomize_va_space'
+  if test $? -ne 0; then
+    echo "Error when disabling ASLR" | tee -a /dev/stderr "${log}"
+    if test "${cautiousness}" -eq 1; then
+      exit 1
+    fi
+  fi
+fi
+
+#By setting our own niceness, we don't force the benchmark to run as root
+if test ${do_renice} -eq 1; then
+  #Don't use $sudo, we don't want to break out of chroot here
+  if test "${USER}" = root; then
+         renice -19 $$
+  else
+    sudo renice -19 $$
+  fi
+  if test $? -ne 0; then
+    echo "Failed to set niceness to -19" 1>&2
+  fi
+fi
+
+#Finally, run the command!
+#We don't tee it, just in case it contains any sensitive output
+#TODO We expect to be running with stdout & stderr redirected, insert a test for this
+if test x"${bench_cpu:-}" != x; then
+  cmd="taskset -c ${bench_cpu} ${cmd}"
+fi
+echo "Running ${cmd}"
+${cmd}
+if test $? -eq 0; then
+  echo "Run of ${cmd} complete" | tee -a "${log}"
+  exit 0
+else
+  echo "Run of ${cmd} failed" | tee -a /dev/stderr "${log}" > /dev/null
+  exit 1
+fi
diff --git a/scripts/establish_listener.sh b/scripts/establish_listener.sh
new file mode 100755
index 00000000..d21f4a57
--- /dev/null
+++ b/scripts/establish_listener.sh
@@ -0,0 +1,174 @@
+#!/bin/bash
+set -o pipefail
+set -o nounset
+
+# load the configure file produced by configure
+if test -e "${PWD}/host.conf"; then
+    . "${PWD}/host.conf"
+else
+    echo "ERROR: no host.conf file!  Did you run configure?" 1>&2
+    exit 1
+fi
+topdir="${abe_path}" #abe global, but this should be the right value for abe
+. "${topdir}"/scripts/benchutil.sh
+if test $? -ne 0; then
+  echo "Unable to source ${topdir}/benchutil.sh" 1>&2
+  exit 1
+fi
+
+trap cleanup EXIT
+trap 'exit ${error}' TERM INT HUP QUIT
+
+error=1
+listener_pid=
+forward_pid=
+pseudofifo_pid=
+temps="`mktemp -dt XXXXXXXXX`" || exit 1
+listener_file="${temps}/listener_file"
+gateway=
+
+function cleanup
+{
+  error=$?
+  if test x"${listener_pid:-}" != x; then
+    kill "${listener_pid}" 2>/dev/null
+    wait "${listener_pid}"
+  fi
+  if test x"${pseudofifo_pid:-}" != x; then
+    kill "${pseudofifo_pid}" 2>/dev/null
+    #Substituted process is not our child and cannot be waited on. Fortunately,
+    #it doesn't matter too much when it dies.
+  fi
+  if test x"${forward_pid:-}" != x; then
+    kill "${forward_pid}" 2>/dev/null
+    wait "${forward_pid}"
+  fi
+  if test -d "${temps}"; then
+    exec 3>&-
+    rm -rf ${temps}
+    if test $? -ne 0; then
+      echo "Failed to delete temporary file store ${temps}" 1>&2
+    fi
+    error=1
+  fi
+  exit "${error}"
+}
+
+#A fifo would make much more sense, but nc doesn't like it
+touch "${listener_file}"
+if test $? -ne 0; then
+  echo "Failed to create listener file '${listener_file}'" 1>&2
+  exit 1
+fi
+
+#The trap is just to suppress the 'Terminated' message
+exec 3>&-
+exec 3< <(trap 'exit' TERM; tail -f "${listener_file}"& echo $! >> "${listener_file}"; wait)
+read -t 60 pseudofifo_pid <&3
+if test $? -ne 0; then
+  echo "Failed to read pseudofifo pid" 1>&2
+  exit 1
+fi
+
+forward_fifo="${temps}/forward_fifo"
+mkfifo "${forward_fifo}" || exit 1
+
+while getopts f: flag; do
+  case "${flag}" in
+    f) gateway="${OPTARG}" ;;
+    *)
+       echo "Bad arg" 1>&2
+       exit 1
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+if test $# -ne 3; then
+  echo "establish_listener needs 3 args, got $#" 1>&2
+  for arg in "$@"; do echo "${arg}" 1>&2; done
+  exit 1
+fi
+if test x"${gateway:-}" != x; then
+  if ! echo "${gateway}" | grep -q '.\+:.\+'; then
+    echo "If specifying a gateway to forward through, must be in format 'internal_interface:external_interface'" 1>&2
+    echo "Got: ${gateway}" 1>&2
+    exit 1
+  fi
+fi
+
+listener_addr="$1"
+ping -c 1 "${listener_addr}" > /dev/null
+if test $? -ne 0; then
+  echo "Unable to ping host ${listener_addr}" 1>&2
+  exit 1
+fi
+start_port="$2"
+end_port="$3"
+
+for ((listener_port=${start_port}; listener_port < ${end_port}; listener_port++)); do
+  #Try to listen on the port. nc will fail if something has snatched it.
+  echo "Attempting to establish listener on ${listener_addr}:${listener_port}" 1>&2
+  nc -kl "${listener_addr}" "${listener_port}" >> "${listener_file}"&
+  listener_pid=$!
+
+  #nc doesn't confirm that it's got the port, so we spin until either:
+  #1) We see that the port has been taken by our process
+  #2) We see our process exit (implying that the port was taken)
+  #3) We've waited long enough
+  #(listener_pid can't be reused until we wait on it)
+  for j in {1..5}; do
+    if test x"`lsof -i tcp@${listener_addr}:${listener_port} | sed 1d | awk '{print $2}'`" = x"${listener_pid}"; then
+      break 2; #success, exit outer loop
+    elif ! ps -p "${listener_pid}" > /dev/null; then
+      #listener has exited, reap it and go back to start of outer loop
+      wait "${listener_pid}"
+      listener_pid=
+      continue 2
+    else
+      sleep 1
+    fi
+  done
+
+  #We gave up waiting, kill and reap the nc process
+  kill "${listener_pid}"
+  wait "${listener_pid}"
+  listener_pid=
+done
+
+if test x"${listener_pid:-}" = x; then
+  echo "Failed to find a free port in range ${start_port}-${end_port}" 1>&2
+  exit 1
+fi
+
+if test x"${gateway:-}" != x; then
+  internal_interface="${gateway/%:*}"
+  external_interface="${gateway/#*:}"
+  ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes -NR ${internal_interface/%:*}:0:${listener_addr}:${listener_port} ${external_interface} >"${forward_fifo}" 2>&1 &
+  forward_pid=$!
+  read -t 30 line < "${forward_fifo}"
+  if test $? -ne 0; then
+    echo "Timeout while establishing port forward" 1>&2
+    exit 1
+  fi
+  if echo ${line} | grep -q "^Allocated port [[:digit:]]\\+ for remote forward to ${listener_addr}:${listener_port}"; then
+    listener_port="`echo ${line} | cut -d ' ' -f 3`"
+  else
+    echo "Unable to get port forwarded for listener" 1>&2
+    echo "Tried: ssh -o PasswordAuthentication=no -o PubkeyAuthentication=yes -NR ${internal_interface/%:*}:0:${listener_addr}:${listener_port} ${external_interface}" 1>&2
+    echo "Got: $line" 1>&2
+    exit 1
+  fi
+  listener_addr="`get_addr ${external_interface}`" || exit 1
+fi
+
+echo "${listener_addr}"
+echo "${listener_port}"
+
+while true; do
+  line="`bgread ${pseudofifo_pid} <&3`"
+  if test $? -ne 0; then
+    echo "Failed to read pseudofifo pid" 1>&2
+    exit 1
+  fi
+  echo $line
+done
diff --git a/scripts/lava.sh b/scripts/lava.sh
new file mode 100755
index 00000000..f558877f
--- /dev/null
+++ b/scripts/lava.sh
@@ -0,0 +1,349 @@
+#!/bin/bash
+
+#Deps: lava-tool, auth token for lava-tool
+set -o pipefail
+set -o nounset
+# load the configure file produced by configure
+if test -e "${PWD}/host.conf"; then
+    . "${PWD}/host.conf"
+else
+    echo "ERROR: no host.conf file!  Did you run configure?" 1>&2
+    exit 1
+fi
+topdir="${abe_path}" #abe global, but this should be the right value for abe
+. "${topdir}"/scripts/benchutil.sh
+if test $? -ne 0; then
+  echo "Unable to source ${topdir}/benchutil.sh" 1>&2
+  exit 1
+fi
+listener_pid=
+waiter=
+keep=1
+error=1
+trap 'keep=0; exit ${error}' USR1
+trap release EXIT
+trap 'exit ${error}' TERM INT HUP QUIT
+
+function retrying_lava_tool
+{
+  local c
+  for c in {4..0}; do
+    lava-tool "$@" && return 0
+    if test $c -eq 0; then
+      return 1
+    else
+      echo "lava-tool $@ failed" 1>&2
+      echo "May be spurious: ${c} retries remaining" 1>&2
+      sleep 5
+    fi
+  done
+  return 1
+}
+
+function job_status
+{
+  if test x"${id:-}" = x -o x"${lava_url:-}" = x; then
+    echo "Job ID or LAVA URL not defined" 1>&2
+    return 1
+  fi
+
+  local jobstatus
+  jobstatus="`retrying_lava_tool job-status https://${lava_url} ${id}`"
+  if test $? -ne 0; then
+    echo "Job ${id} disappeared!" 1>&2
+    return 1
+  fi
+  echo "${jobstatus}" | grep "Job Status: $1" > /dev/null
+  if test $? -ne 0; then
+    echo "Job ${id} has surprising status" 1>&2
+    echo -e "${jobstatus}" 1>&2
+    return 1
+  fi
+
+  return 0
+}
+
+release()
+{
+  if test x"${waiter:-}" != x; then
+    kill "${waiter}" 2>/dev/null
+    wait "${waiter}"
+  fi
+  if test x"${listener_pid:-}" != x; then
+    kill "${listener_pid}" 2>/dev/null
+    wait "${listener_pid}"
+  fi
+  if test x"${temps:-}" != x; then
+    if test -d "${temps}"; then
+      exec 3>&-
+      rm -rf "${temps}"
+      if test $? -ne 0; then
+        echo "Failed to delete temporary file store ${temps}" 1>&2
+      fi
+    fi
+  fi
+  if test x"${id:-}" != x; then
+    if test ${keep} -eq 0; then
+      retrying_lava_tool cancel-job https://"${lava_url}" "${id}"
+      if test $? -eq 0; then
+        echo "Cancelled job ${id}"
+        error=0
+      else
+        echo "Failed to cancel job ${id}" 1>&2
+        echo "Run 'lava-tool cancel-job https://"${lava_url}" "${id}"' to cancel" 1>&2
+        error=1
+      fi
+    else
+      echo "Did not cancel job ${id} - superior did not request cancellation."
+      echo "You probably have some cleanup to do."
+      echo "When you've finished, cancel by running:"
+      echo "lava-tool cancel-job https://${lava_url} ${id}"
+      error=0
+    fi
+    echo "Getting present job status:"
+    retrying_lava_tool job-status "https://${lava_url}" "${id}"
+    if test $? -ne 0; then
+      echo "Was unable to get job status"
+    fi
+    lava_url=${lava_url%/RPC2/}
+    echo "Log should be at: https://${lava_url#*@}/scheduler/job/${id}/log_file#bottom"
+  fi
+  exit "${error}"
+}
+
+lava_url="${LAVA_SERVER:-}"
+lava_server=
+lava_user=
+lava_json=
+boot_timeout="$((120*60))" #2 hours
+key=${LAVA_SSH_KEYFILE:-}
+while getopts s:j:b:p: flag; do
+  case "${flag}" in
+    s) lava_url="${OPTARG}";;
+    j) lava_json="${OPTARG}";;
+    b) boot_timeout="$((OPTARG*60))";;
+    p) key="${OPTARG}";;
+    *)
+       echo 'Unknown option' 1>&2
+       exit 1
+    ;;
+  esac
+done
+
+expr ${lava_url:?'Must give a LAVA URL (-s) or set $LAVA_SERVER'} > /dev/null
+lava_user="`lava_user ${lava_url}`"
+if test $? -ne 0; then
+  echo "Unable to find username from ${lavaserver}" 1>&2
+  exit 1
+fi
+lava_server="`lava_server ${lava_url}`"
+if test $? -ne 0; then
+  echo "Unable to find username from ${lavaserver}" 1>&2
+  exit 1
+fi
+
+if ! test -f ${lava_json:?'Must give a json file (-j)'}; then
+  echo "JSON file ${lava_json} not a file"
+  exit 1
+fi
+
+shift $((OPTIND - 1))
+if test ${#@} -ne 0; then
+  echo -n "Unknown option(s): " 1>&2
+  for opt in "$@"; do
+    echo -n " '${opt}'" 1>&2
+  done
+  echo 1>&2
+  exit 1
+fi
+
+#Store the public key in key - if no public key file given on CLI, try a few
+#sensible defaults. (We only ever end up sharing a public key, so there's no
+#security issue here.)
+if test x"${key:-}" = x; then
+  if ssh-add -l; then
+    key="`ssh-add -L | head -n1 2>/dev/null`"
+    if test $? -ne 0; then
+      echo "Failed to get public key from ssh-agent" 1>&2
+      exit 1
+    fi
+  elif test -f ~/.ssh/id_rsa; then
+    key="`ssh-keygen -y -f ~/.ssh/id_rsa`"
+    if test $? -ne 0; then
+      echo "Failed to get public key from private key file" 1>&2
+      exit 1
+    fi
+  else
+    echo "Could not find a key to authenticate with target (tried ssh-agent, ~/.ssh/id_rsa)" 1>&2
+    exit 1
+  fi
+else
+  if test -f "${key}"; then
+    #Build in a little protection against accidental private key publication
+    if head -n 1 "${key}" | grep PRIVATE > /dev/null; then
+      echo "Given key file appears to be a private key, will generate public key" 1>&2
+      key="`ssh-keygen -y -f ${key}`"
+      if test $? -ne 0; then
+        echo "Failed to get public key from private key file" 1>&2
+        exit 1
+      fi
+    elif head -n 1 "${key}" | grep -v ^ssh-; then
+      echo "Given key file does not look like an ssh public key" 1>&2
+      exit 1
+    else
+      key="`cat ${key}`"
+    fi
+  else
+    echo "Public key file ${key} does not exist or is not a file" 1>&2
+    exit 1
+  fi
+fi
+
+#Convert key into a sed-friendly replacement string (i.e. escape chars that are special on the RHS of s//)
+#Danger - Don't change the sed runes to backticks, they resolve differently and render all the matches as ampersands
+key="$(set -f; echo ${key} | sed 's/[\/&]/\\&/g')"
+
+temps="`mktemp -dt XXXXXXXXX`" || exit 1
+listener_fifo="${temps}/listener_fifo"
+mkfifo "${listener_fifo}" || exit 1
+exec 3>&-
+exec 3<> "${listener_fifo}"
+json_copy="${temps}/job.json"
+cp "${lava_json}" "${json_copy}"
+sed -i "s/^\(.*\"PUB_KEY\":\)[^\"]*\".*\"[^,]*\(,\?\)[[:blank:]]*$/\1 \"${key}\"\2/" "${json_copy}"
+if test $? -ne 0; then
+  echo "Failed to populate json file with public key" 1>&2
+  exit 1
+fi
+sed -i "s+^\(.*\"server\":\)[^\"]*\".*\"[^,]*\(,\?\)[[:blank:]]*\$+\1 \"https://${lava_user}@${lava_server}\"\2+" "${json_copy}"
+sed -i "s+^\(.*\"stream\":\)[^\"]*\".*\"[^,]*\(,\?\)[[:blank:]]*\$+\1 \"/private/personal/${lava_user}/\"\2+" "${json_copy}"
+
+lava_network "${lava_user}"
+in_lab=$?
+if test ${in_lab} -eq 2; then
+  echo "Unable to determine whether I am inside the LAVA lab, assuming that I am not" 1>&2
+fi
+
+listener_addr="`get_addr`"
+if test $? -ne 0; then
+  echo "Unable to get IP for listener" 1>&2
+  exit 1
+fi
+if test ${in_lab} -eq 0; then
+  "${topdir}"/scripts/establish_listener.sh ${listener_addr} 4200 5200 >&3 &
+else
+  "${topdir}"/scripts/establish_listener.sh -f 10.0.0.10:${lava_user}@lab.validation.linaro.org ${listener_addr} 4200 5200 >&3 &
+fi
+listener_pid=$!
+listener_addr="`bgread -T 60 ${listener_pid} <&3`"
+if test $? -ne 0; then
+  echo "Failed to read listener address" 1>&2
+  exit 1
+fi
+listener_port="`bgread -T 60 ${listener_pid} <&3`"
+if test $? -ne 0; then
+  echo "Failed to read listener port" 1>&2
+  exit 1
+fi
+echo "Listener ${listener_addr}:${listener_port}"
+
+sed -i "s/^\(.*\"LISTENER_ADDR\":\)[^\"]*\".*\"[^,]*\(,\?\)[[:blank:]]*$/\1 \"${listener_addr}\"\2/" "${json_copy}"
+if test $? -ne 0; then
+  echo "Failed to populate json file with listener ip" 1>&2
+  exit 1
+fi
+sed -i "s/^\(.*\"LISTENER_PORT\":\)[^\"]*\".*\"[^,]*\(,\?\)[[:blank:]]*$/\1 \"${listener_port}\"\2/" "${json_copy}"
+if test $? -ne 0; then
+  echo "Failed to populate json file with listener port" 1>&2
+  exit 1
+fi
+
+id="`retrying_lava_tool submit-job https://${lava_url} ${json_copy}`"
+if test $? -ne 0; then
+  echo "Failed to submit job" 1>&2
+  echo "URL: https://${lava_url}" 1>&2
+  echo "JSON: " 1>&2
+  cat "${json_copy}" 1>&2
+  exit 1
+fi
+
+#TODO: Should be able to use cut at the end of this pipe, but when lava-tool
+#      is invoked through expect wrapper this line ends up with a carriage return 
+#      at the end. Should be fixed on the expect side, or expect script should
+#      be discarded, but hack it here for now.
+id="`echo ${id} | grep 'submitted as job id: [[:digit:]]\+' | grep -o '[[:digit:]]\+'`"
+if test $? -ne 0; then
+  echo "Failed to read job id" 1>&2
+  echo "Input string was: ${id}" 1>&2
+  echo "JSON: " 1>&2
+  cat "${json_copy}" 1>&2
+  exit 1
+fi
+echo "Dispatched LAVA job ${id}"
+
+sleep 15& #A short delay here is handy when debugging (if the LAVA queues are empty then we'll dispatch fast, but not instantly)
+wait $!
+
+#Monitor job status until it starts running or fails
+#TODO: This block assumes that lava_tool doesn't return until the job is in 'Submitted' state, which I haven't checked
+#TODO: In principle we want a timeout here, but we could be queued for a very long time, and that could be fine
+while true; do
+  jobstatus="`retrying_lava_tool job-status https://${lava_url} ${id}`"
+  if test $? -ne 0; then
+    echo "Job ${id} disappeared!" 1>&2
+    exit 1
+  fi
+  echo "${jobstatus}" | grep 'Job Status: Running' > /dev/null
+  if test $? -eq 0; then
+    break
+  fi
+  echo "${jobstatus}" | grep 'Job Status: Submitted' > /dev/null
+  if test $? -ne 0; then
+    echo "Job ${id} has surprising status, giving up" 1>&2
+    echo -e "${jobstatus}" 1>&2
+    exit 1
+  fi
+  sleep 60&
+  wait $!
+done
+
+echo "Job ${id} is running, waiting for boot"
+
+deadline=$((`date +%s` + boot_timeout))
+while test `date +%s` -lt ${deadline}; do
+  read -t 60 user_ip <&3
+  err=$?
+  if test ${err} -eq 0; then
+    break
+  elif test ${err} -le 128; then
+    echo "Non-timeout error code ${err} while trying to read user_ip" 1>&2
+    exit 1
+  fi
+  if ! job_status 'Running'; then
+    echo "Bad status, giving up" 1>&2
+    exit 1
+  fi
+done
+
+#We've finished with the listener
+kill ${listener_pid} 2>/dev/null
+#Don't wait on it - that way, when we kill and wait on exit, we know we're
+#still dealing with the same process. (Killing it more than once is harmless.)
+
+if test x"${user_ip:-}" = x; then
+  echo "LAVA boot abandoned after $((boot_timeout/60)) minutes" 1>&2
+  exit 1
+fi
+
+echo "LAVA target ready at ${user_ip}"
+
+#Wait to be killed, at which point we cancel the job
+while true; do
+  sleep 600 &
+  waiter=$!
+  wait ${waiter}
+  if ! job_status Running; then
+    echo "Job ${id} has bad status, giving up" 1>&2
+    exit 1
+  fi
+done
diff --git a/scripts/runbenchmark.sh b/scripts/runbenchmark.sh
new file mode 100755
index 00000000..9d66ed5b
--- /dev/null
+++ b/scripts/runbenchmark.sh
@@ -0,0 +1,443 @@
+#!/bin/bash
+#This script takes a build of a benchmark and transfers it to, and runs it on
+#a single target. The target may be LAVA or non-LAVA. The script is mostly
+#LAVA-agnostic - apart from the section that initiates the LAVA session, there
+#is some awareness of it in the exit handler, and that's all.
+set -o pipefail
+set -o nounset
+
+trap clean_benchmark EXIT
+trap 'exit ${error}' TERM INT HUP QUIT
+
+session_pid=
+lava_pid=
+listener_pid=
+benchmark=
+device=
+keep=
+cautious=''
+build_dir=
+lava_target=
+run_benchargs=
+sysroot_path=
+while getopts b:d:t:a:l:kc flag; do
+  case "${flag}" in
+    k) keep='-k';;
+    c) cautious='-c';;
+    b) benchmark="${OPTARG}";;
+    d) device="${OPTARG}";;
+    t) buildtar="${OPTARG}";;
+    a) run_benchargs="${OPTARG}";;
+    l) sysroot_path="${OPTARG}";;
+    *)
+       echo "Bad arg" 1>&2
+       exit 1
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+if test $# -ne 0; then
+  echo "Surplus arguments: $@" 1>&2
+  exit 1
+fi
+
+error=1
+tee_output=/dev/null
+
+# load the configure file produced by configure
+if test -e "${PWD}/host.conf"; then
+    . "${PWD}/host.conf"
+else
+    echo "ERROR: no host.conf file!  Did you run configure?" 1>&2
+    exit 1
+fi
+topdir="${abe_path}" #abe global, but this should be the right value for abe
+confdir="${topdir}/config/boards/bench"
+if test x"${LAVA_SERVER:-}" != x; then
+  lava_url="${LAVA_SERVER}"
+else
+  lava_url="${USER}@validation.linaro.org/RPC2/"
+  echo "Environment variable LAVA_SERVER not set, defaulted to ${lava_url}" 1>&2
+fi
+
+benchlog="`. ${abe_top}/host.conf && . ${topdir}/lib/common.sh && read_config ${benchmark}.git benchlog`"
+if test $? -ne 0; then
+  echo "Unable to read benchmark config file for ${benchmark}" 1>&2
+  exit 1
+fi
+
+. "${topdir}"/scripts/benchutil.sh
+if test $? -ne 0; then
+  echo "+++ Unable to source ${topdir}/benchutil.sh" 1>&2
+  exit 1
+fi
+. "${confdir}/${device}.conf" #We can't use abe's source_config here as it requires us to have something get_toolname can parse
+if test $? -ne 0; then
+  echo "+++ Failed to source ${confdir}/${device}.conf" 1>&2
+  exit 1
+fi
+
+temps="`mktemp -dt XXXXXXXXX`" || exit 1
+listener_file="${temps}/listener_file"
+listener_fifo="${temps}/listener_fifo"
+lava_fifo="${temps}/lava_fifo"
+mkfifo "${listener_fifo}" || exit 1
+exec 3>&-
+exec 3<> "${listener_fifo}"
+mkfifo "${lava_fifo}" || exit 1
+exec 4>&-
+exec 4<> "${lava_fifo}"
+
+#Make sure that subscripts clean up - we must not leave benchmark sources or data lying around,
+#we should not leave lava targets reserved
+clean_benchmark()
+{
+  error=$?
+
+  if test x"${ip:-}" != x; then
+    if test x"${target_dir:-}" = x; then
+      echo "No directory to remove from ${ip}"
+    elif test x"${keep}" = 'x-k'; then
+      echo "Not removing ${target_dir} from ${ip} as -k was given. You might want to go in and clean up."
+    elif ! expr "${target_dir}" : '\(/tmp\)' > /dev/null; then
+      echo "Cowardly refusing to delete ${target_dir} from ${ip}. Not rooted at /tmp. You might want to go in and clean up." 1>&2
+      error=1
+    else
+      (. "${topdir}"/lib/common.sh; remote_exec "${ip}" "rm -rf ${target_dir}" ${ssh_opts})
+      if test $? -eq 0; then
+        echo "Removed ${target_dir} from ${ip}"
+      else
+        echo "Failed to remove ${target_dir} from ${ip}. You might want to go in and clean up." 1>&2
+        error=1
+      fi
+    fi
+  else
+    echo "Target post-boot initialisation did not happen, thus nothing to clean up."
+  fi
+
+  if test x"${session_pid:-}" != x; then
+    kill "${session_pid}" 2>/dev/null
+    wait "${session_pid}"
+  fi
+
+  if test x"${listener_pid:-}" != x; then
+    kill "${listener_pid}" 2>/dev/null
+    wait "${listener_pid}"
+  fi
+
+  if test x"${lava_pid:-}" != x; then
+    if test ${error} -ne 0 || test x"${keep}" = 'x-k'; then
+      echo "Not killing lava session, to ensure session remains open for investigation/cleanup."
+      kill "${lava_pid}" 2>/dev/null
+      wait "${lava_pid}"
+    else
+      kill -USR1 "${lava_pid}" 2>/dev/null
+      wait "${lava_pid}"
+    fi
+
+    #Make sure we see any messages from the lava.sh handlers
+    dd iflag=nonblock <&4 2>/dev/null | awk "{print \"${lava_target}: \" \$0}"
+  fi
+
+  #Delete these last so that we can still get messages through the lava fifo
+  if test -d "${temps}"; then
+    exec 3>&-
+    exec 4>&-
+    rm -rf "${temps}"
+    if test $? -ne 0; then
+      echo "Failed to delete ${temps}" 1>&2
+      error=1
+    fi
+  fi
+
+  exit "${error}"
+}
+
+#Handle LAVA case
+echo "${ip}" | grep '\.json$' > /dev/null
+if test $? -eq 0; then
+  lava_user="`lava_user ${lava_url}`"
+  if test $? -ne 0; then
+    echo "Unable to find username from ${lava_url}" 1>&2
+    exit 1
+  fi
+  lava_network "${lava_user}"
+  case $? in
+    2) echo "Unable to determing location w.r.t. lava lab: assuming outside" 1>&2 ;;
+    1)
+      gateway=lab.validation.linaro.org
+      ssh_opts="${LAVA_SSH_KEYFILE:+-o IdentityFile=${LAVA_SSH_KEYFILE}} -o ProxyCommand='ssh ${lava_user}@${gateway} nc -q0 %h %p'"
+      establish_listener_opts="-f 10.0.0.10:${lava_user}@${gateway}"
+
+      #LAVA targets need to boot - do an early check that the route to the gateway is private, so that we can fail fast
+      if ! check_private_route "${gateway}"; then
+        echo "Failed to confirm that route to target is private, conservatively aborting" 1>&2
+        exit 1
+      fi
+  esac
+
+  lava_target="${ip}"
+  ip=''
+  tee_output=/dev/console
+  echo "Acquiring LAVA target ${lava_target}"
+  echo "${topdir}/scripts/lava.sh -s ${lava_url} -j ${confdir}/${lava_target} -b ${boot_timeout:-30}"
+
+  ${topdir}/scripts/lava.sh -s "${lava_url}" -j "${confdir}/${lava_target}" -b "${boot_timeout-:30}" >&4 2>&1 &
+  if test $? -ne 0; then
+    echo "+++ Failed to acquire LAVA target ${lava_target}" 1>&2
+    exit 1
+  fi
+  lava_pid=$!
+  while true; do
+    line="`bgread -t 5 ${lava_pid} <&4`"
+    if test $? -ne 0; then
+      echo "${lava_target}: Failed to read lava output" 1>&2
+      exit 1
+    fi
+    echo "${lava_target}: $line"
+    if echo "${line}" | grep '^LAVA target ready at ' > /dev/null; then
+      ip="`echo ${line} | cut -d ' ' -f 5 | sed 's/\s*$//'`"
+      break
+    fi
+  done
+
+  if test x"${sysroot_path:-}" != x; then
+    sysroot_lib="${sysroot_path#*:}"
+    if test x"${sysroot_lib}" = x"${sysroot_path}"; then
+      sysroot_lib=lib
+    else
+      sysroot_path="${sysroot_path%:*}"
+    fi
+    if ! (. "${topdir}"/lib/common.sh; remote_exec "${ip}" true ${ssh_opts}) > /dev/null 2>&1; then
+      echo "Unable to connect to target ${ip:+(unknown)} after boot" 1>&2
+      exit 1
+    fi
+    tmpsysdir="`(. ${topdir}/lib/common.sh; remote_exec ${ip} "mktemp -dt sysroot_XXXXX" ${ssh_opts})`"
+    (. ${topdir}/lib/common.sh; remote_upload -r 3 "${ip}" "${sysroot_path}/" "${tmpsysdir}"/sysroot ${ssh_opts})
+    if test $? -ne 0; then
+      echo "Failed to upload sysroot to target" 1>&2
+      exit 1
+    fi
+    #TODO We're relying on a symlink from lib to lib64 being present, where relevant
+    #TODO Would really be better to do this with a (s)chroot, to allow use on non-LAVA
+    #     targets. But after ~1 day of experimenting with test-schroot.sh, opted to do
+    #     this to unblock this use case for LAVA targets.
+    (. ${topdir}/lib/common.sh; remote_exec "${ip}" "echo -e '/lib\n/usr/lib\n > ld.so.conf.new' && \
+                         cat /etc/ld.so.conf >> /etc/ld.so.conf.new && \
+                         mv /etc/ld.so.conf.new /etc/ld.so.conf && \
+                         rsync -a ${tmpsysdir}/sysroot/ / && \
+                         ldconfig" ${ssh_opts})
+    if test $? -ne 0; then
+      echo "Failed to install sysroot on target" 1>&2
+      exit 1
+    fi
+    if ! (. ${topdir}/lib/common.sh; remote_exec "${ip}" true ${ssh_opts}) > /dev/null 2>&1; then
+      echo "Unable to run simple command on target after sysroot installation" 1>&2
+      exit 1
+    fi
+  fi
+
+  #After this point, lava.sh should produce no output until we reach the exit handlers.
+  #Our exit handler checks the pipe from lava.sh before closing down.
+
+  if test x"${ip:-}" = x; then
+    echo "+++ Failed to acquire LAVA target ${lava_target}" 1>&2
+    exit 1
+  fi
+else
+  if test x"${sysroot_path:-}" != x; then
+    echo "Cannot install a sysroot on non-LAVA targets" 1>&2
+    exit 1
+  fi
+  gateway="${ip/*@}"
+  ssh_opts=
+  establish_listener_opts=
+fi
+#LAVA-agnostic from here, apart from a section in the exit handler, and bgread
+#monitoring of the LAVA process while we're waiting for the benchmark to end
+
+#Set up our listener
+listener_addr="`get_addr`"
+if test $? -ne 0; then
+  echo "Unable to get IP for listener" 1>&2
+  exit 1
+fi
+"${topdir}"/scripts/establish_listener.sh ${establish_listener_opts} "${listener_addr}" 4200 5200 >&3 &
+listener_pid=$!
+listener_addr="`bgread -T 60 ${listener_pid} <&3`"
+if test $? -ne 0; then
+  echo "Failed to read listener address" 1>&2
+  exit 1
+fi
+listener_port="`bgread -T 60 ${listener_pid} <&3`"
+if test $? -ne 0; then
+  echo "Failed to read listener port" 1>&2
+  exit 1
+fi
+echo "Listener ${listener_addr}:${listener_port}"
+
+if ! (. "${topdir}"/lib/common.sh; remote_exec "${ip}" true ${ssh_opts}) > /dev/null 2>&1; then
+  echo "Unable to connect to target ${ip:+(unknown)} after boot" 1>&2
+  exit 1
+fi
+
+#Should be a sufficient UID, as we wouldn't want to run multiple benchmarks on the same target at the same time
+logdir="${abe_top}/${benchmark}-log/${device}_${ip}_`date -u +%F_%T.%N`"
+if test -e "${logdir}"; then
+  echo "Log output directory ${logdir} already exists" 1>&2
+fi
+mkdir -p "${logdir}/${benchmark}.git"
+if test $? -ne 0; then
+  echo "Failed to create dir ${logdir}" 1>&2
+  exit 1
+fi
+
+#Create and populate working dir on target
+target_dir="`. ${topdir}/lib/common.sh; remote_exec ${ip} 'mktemp -dt XXXXXXX' ${ssh_opts}`"
+if test $? -ne 0; then
+  echo "Unable to get tmpdir on target" 1>&2
+  exit 1
+fi
+if ! check_private_route "${gateway}"; then
+  echo "Failed to confirm that route to target is private, conservatively aborting" 1>&2
+  exit 1
+fi
+for thing in "${buildtar}" "${topdir}/scripts/controlledrun.sh" "${confdir}/${device}.services"; do
+  (. "${topdir}"/lib/common.sh; remote_upload -r 3 "${ip}" "${thing}" "${target_dir}/`basename ${thing}`" ${ssh_opts})
+  if test $? -ne 0; then
+    echo "Unable to copy ${thing}" to "${ip}:${target_dir}/${thing}" 1>&2
+    exit 1
+  fi
+done
+
+#Compose and run the ssh command.
+#We have to run the ssh command asynchronously, because having the network down during a long-running benchmark will result in ssh
+#death sooner or later - we can stop ssh client and ssh server from killing the connection, but the TCP layer will get it eventually.
+
+#These parameters sourced from the conf file at beginning of this function
+flags="${benchcore:+-b ${benchcore}} ${othercore:+-p ${othercore}}"
+if test x"${netctl:-}" = xyes; then
+  flags+=" -n"
+fi
+if test x"${servicectl:-}" = xyes; then
+  flags+=" -s ${target_dir}/${device}.services"
+fi
+if test x"${freqctl:-}" = xyes; then
+  flags+=" -f"
+fi
+
+#But, if uncontrolled is set, override all other flags
+if test x"${uncontrolled:-}" = xyes; then
+  echo "Running without any target controls or special (sudo) privileges, due to 'uncontrolled=yes' in target config file"
+  flags="-u"
+fi
+
+#TODO: Repetition of hostname echoing is ugly, but seems to be needed -
+#      perhaps there is some delay after the interface comes up
+(
+   pids=()
+   cleanup()
+   {
+     local pid
+     for pid in "${pids[@]}"; do
+       if test x"${pid:-}" != x; then
+         kill ${pid} 2>/dev/null
+         wait ${pid} 2>/dev/null
+       fi
+     done
+     exit
+   }
+   trap cleanup EXIT
+
+   . "${topdir}"/lib/common.sh
+   remote_exec_async \
+     "${ip}" \
+     "echo 'STARTED' | nc ${listener_addr} ${listener_port} && \
+      cd ${target_dir} && \
+      tar xjf `basename ${buildtar}` && \
+      cd `tar tjf ${buildtar} | head -n1` && \
+     ../controlledrun.sh ${cautious} ${flags} -l ${tee_output} -- ./linarobench.sh ${board_benchargs:-} -- ${run_benchargs:-}; \
+     ret=\\\$?; \
+     for i in {1..10}; do \
+       echo \"\\\${USER}@\\\`ifconfig eth0 | grep 'inet addr' | sed 's/[^:]*://' | cut -d ' ' -f 1\\\`:\\\${ret}\" | nc ${listener_addr} ${listener_port}; \
+     done; \
+     if test \\\${ret} -eq 0; then \
+       true; \
+     else \
+       false; \
+     fi" \
+     "${target_dir}/stdout" "${target_dir}/stderr" \
+     ${ssh_opts}
+   if test $? -ne 0; then
+     echo "Something went wrong when we tried to dispatch job" 1>&2
+     exit 1
+   fi
+   pids+=($!)
+   sleep infinity&
+   waiter=$!
+   pids+=(${waiter})
+   wait ${waiter}
+)&
+session_pid=$!
+
+#lava_pid will expand to empty if we're not using lava
+handshake="`bgread -T 300 ${listener_pid} ${lava_pid} <&3`"
+if test $? -ne 0 -o x"${handshake:-}" != 'xSTARTED'; then
+  echo "Did not get handshake from target, giving up" 1>&2
+  exit 1
+fi
+
+#lava_pid will expand to empty if we're not using lava
+#No sense in setting a deadline on this one, it's order of days for many cases
+ip="`bgread ${listener_pid} ${lava_pid} <&3`"
+if test $? -ne 0; then
+  if test x"${lava_pid:-}" = x; then
+    echo "Failed to read post-benchmark-run IP" 1>&2
+  else
+    echo "LAVA process died, or otherwise failed while waiting to read post-benchmark-run IP" 1>&2
+  fi
+  exit 1
+fi
+
+error="`echo ${ip} | sed 's/.*://'`"
+if test $? -ne 0; then
+  echo "Unable to determine exit code, assuming the worst." 1>&2
+  error=1
+fi
+ip="`echo ${ip} | sed 's/:.*//' | sed 's/\s*$//'`"
+if test $? -ne 0; then
+  echo "Unable to determine IP, giving up." 1>&2
+  exit 1
+fi
+
+if ! (. "${topdir}"/lib/common.sh; remote_exec "${ip}" true ${ssh_opts}) > /dev/null 2>&1; then
+  echo "Unable to connect to target after ${ip:+(unknown)} benchmark run" 1>&2
+  exit 1
+fi
+
+if test ${error} -ne 0; then
+  echo "Command failed: will try to get logs" 1>&2
+  echo "Target: ${ip}:${target_dir}" 1>&2
+  error=1
+fi
+
+#Several days might have passed, re-check the route
+if ! check_private_route "${gateway}"; then
+  echo "Failed to confirm that route to target is private, conservatively aborting" 1>&2
+  exit 1
+fi
+for log in ../stdout ../stderr linarobenchlog ${benchlog}; do
+  mkdir -p "${logdir}/${benchmark}.git/`dirname ${log}`"
+  (. "${topdir}"/lib/common.sh; remote_download -r 3 "${ip}" "${target_dir}/${benchmark}.git/${log}" "${logdir}/${benchmark}.git/${log}" ${ssh_opts})
+  if test $? -ne 0; then
+    echo "Error while getting log ${log}: will try to get others" 1>&2
+    error=1
+  fi
+done
+
+if test ${error} -eq 0; then
+  echo "+++ Run of ${benchmark} on ${device} succeeded"
+else
+  echo "+++ Run of ${benchmark} on ${device} failed"
+fi
+exit ${error}