Retry creation of schroot testing sessions

This patch fixes BZ1302, which is about a clash between a randomly choosen
name for a new schroot session and an existing schroot session.  We randomly
choose port number, on which ssh server is started inside schroot session,
among 1000 ports, which sometimes can cause a clash (port number is embedded
in schroot session's name).
The patch refactors and cleans up lib/schroot.sh to gracefully retry with
a new schroot_port number.
Tested in both normal and "clash" cases.
Additionally the patch cleans up schroot handling in lib/make.sh and moves
schroot-internal stuff to lib/schroot.sh.

Change-Id: I9842ec8602e5859854eb16b1af6b509df5bf3463

2 files changed, 104 insertions, 65 deletions

diff --git a/lib/make.sh b/lib/make.sh
index 95ba9be4..87a73661 100755
--- a/lib/make.sh
+++ b/lib/make.sh
@@ -799,28 +799,27 @@ make_check()
 		;;
 	esac
 
+	# Declare schroot_make_opts.  Its value will be set in
+	# start_schroot_sessions depending on features that target board[s]
+	# support.
 	eval "schroot_make_opts="
-	eval "schroot_boards="
 
 	# Export SCHROOT_TEST so that we can choose correct boards
 	# in config/linaro.exp
 	export SCHROOT_TEST="$schroot_test"
 
-	local schroot_port
 	if $exec_tests && [ x"$schroot_test" = x"yes" ]; then
 	    # Start schroot sessions on target boards that support it
-	    schroot_port="$(print_schroot_port)"
 	    local schroot_sysroot
 	    case "${target}" in
-		*"-elf"*) schroot_sysroot="" ;;
+		*"-elf"*) schroot_sysroot="$(mktemp -d)" ;;
 		*) schroot_sysroot="$(make_target_sysroot)" ;;
 	    esac
-	    start_schroot_sessions "${target}" "${schroot_port}" "${schroot_sysroot}" "${builddir}"
+	    start_schroot_sessions "${target}" "${schroot_sysroot}" "${builddir}"
+	    rm -rf "$schroot_sysroot"
 	    if test "$?" != "0"; then
-		stop_schroot_sessions "${schroot_port}" ${schroot_boards}
 		return 1
 	    fi
-	    rm -rf "$schroot_sysroot"
 	fi
 
 	case ${tool} in
@@ -830,7 +829,7 @@ make_check()
 		;;
 	    gdb)
 	        # Stop schroot sessions
-	        stop_schroot_sessions "$schroot_port" ${schroot_boards}
+	        stop_schroot_sessions
 	        unset SCHROOT_TEST
 		#local dirs="/gdb"
 		#local check_targets="check-gdb"
@@ -851,7 +850,7 @@ make_check()
 	done
 
 	# Stop schroot sessions
-	stop_schroot_sessions "$schroot_port" ${schroot_boards}
+	stop_schroot_sessions
 	unset SCHROOT_TEST
        
         if test x"${tool}" = x"gcc"; then
diff --git a/lib/schroot.sh b/lib/schroot.sh
index ffb27912..8bd5a97b 100755
--- a/lib/schroot.sh
+++ b/lib/schroot.sh
@@ -39,19 +39,27 @@ print_schroot_port()
     echo $port
 }
 
-# Start schroot sessions on boards.
-# $1 - target triplet
-# $2 - port on which to start ssh server in schroot session
-# $3 - [optional] target sysroot
-# $4 - [optional] directory on host that should be mounted on target
-start_schroot_sessions()
+# Start schroot session on a single board.
+# $1 - target triplet (can be empty, which means "native")
+# $2 - sysroot to copy to target (can be empty)
+# $3 - directory on host that should be mounted on target via sshfs
+# $4 - dejagnu board file
+# $5 - filename for log
+start_schroot_session()
 {
-    trace "$*"
-
     local target="$1"
-    local port="$2"
-    local sysroot="$3"
-    local shared_dir="$4"
+    local sysroot="$2"
+    local shared_dir="$3"
+    local board_exp="$4"
+    local log="$5"
+
+    # Get board hostname
+    local hostname
+    hostname="$(grep "^set_board_info hostname " $board_exp | sed -e "s/^set_board_info hostname //")"
+    if [ -z "$hostname" ]; then
+	error "Board file $board_exp uses schroot testing, but does not define \"set_board_info hostname\""
+	continue
+    fi
 
     local target_opt
     if ! [ -z "$target" ]; then
@@ -59,80 +67,112 @@ start_schroot_sessions()
     fi
 
     local sysroot_opt
-    local sysroot_env
     if ! [ -z "$sysroot" ]; then
 	local multilib_dir
 	multilib_dir="$(find_dynamic_linker "$sysroot" true)"
 	multilib_dir="$(basename $(dirname $multilib_dir))"
 	sysroot_opt="-l $sysroot -h $multilib_dir"
-	sysroot_env="SYSROOT_UNDER_TEST=$sysroot"
     fi
 
-    local shared_dir_opt
-    local shared_dir_ok=false
-    if ! [ -z "$shared_dir" ]; then
-	shared_dir_opt="-d $shared_dir"
-	shared_dir_ok=true
+    local result
+    # This command is very handy to have in logs to reproduce test
+    # environment.
+    local was_verbose="${-//[^x]/}"
+    if [ x"$was_verbose" != x"x" ]; then set -x; fi
+    # Start testing schroot session.
+    dryrun "$topdir/scripts/test-schroot.sh -v -b $target_opt -m -e $board_exp $sysroot_opt -d $shared_dir $hostname:$schroot_port" > $log 2>&1; result="$?"
+    if [ x"$was_verbose" != x"x" ]; then set +x; fi
+
+    cat $log >&2
+
+    if grep -q "Failed to write session file: File exists" $log; then
+	result="2"
+    else
+	# Add $hostname to the list of boards to cleanup in
+	# stop_schroot_sessions
+	schroot_boards="$schroot_boards $hostname"
+
+	if test x"$result" != x"0"; then
+	    result="1"
+	fi
+    fi
+
+    return "$result"
+}
+
+# Start schroot sessions on all relevant boards.
+# This routine sets global $schroot_make_opts to pass info upstream.
+# Local state is maintained in $schroot_boards and $schroot_port variables.
+# $1 - target triplet
+# $2 - target sysroot
+# $3 - directory on host that should be mounted on target
+start_schroot_sessions()
+{
+    trace "$*"
+
+    local target="$1"
+    local sysroot="$2"
+    local shared_dir="$3"
+
+    local sysroot_env
+    if ! [ -z "$sysroot" ]; then
+	sysroot_env="SYSROOT_UNDER_TEST=$sysroot"
     fi
 
     local -a board_exps
     board_exps=($(eval $sysroot_env print_schroot_board_files "$target"))
-    for board_exp in "${board_exps[@]}"; do
-	local hostname sysroot lib_path multilib_dir
-
-	# Get board hostname
-	hostname="$(grep "^set_board_info hostname " $board_exp | sed -e "s/^set_board_info hostname //")"
-	if [ -z "$hostname" ]; then
-	    error "Board file $board_exp uses schroot testing, but does not define \"set_board_info hostname\""
-	    continue
-	fi
 
-	local log=$(mktemp)
-	local result
-	# This command is very handy to have in logs to reproduce test
-	# environment.
-	set -x
-	# Start testing schroot session.
-	dryrun "$topdir/scripts/test-schroot.sh -v -b $target_opt -m -e $board_exp $sysroot_opt $shared_dir_opt $hostname:$port" > $log 2>&1; result="$?"
-	set +x
+    if [ -z "${board_exps[@]}" ]; then
+	return 0
+    fi
+
+    # Result of "0" means OK; "1" means FAIL; "2" means RETRY.
+    local result="2"
 
-	cat $log >&2
+    while test x"$result" = x"2"; do
+	eval "schroot_boards="
+	eval "schroot_port=$(print_schroot_port)"
 
-	if ! grep -q "shared directory .*: SUCCESS" $log; then
-	    shared_dir_ok=false
-	fi
+	local shared_dir_ok=true
+	local board_exp
 
-	rm -f $log
+	for board_exp in "${board_exps[@]}"; do
+	    local log=$(mktemp)
 
-	# Print the hostname to the caller
-	schroot_boards="$schroot_boards $hostname"
+	    start_schroot_session "$1" "$2" "$3" "$board_exp" "$log"
+	    result="$?"
 
-	if test "$result" != "0"; then
-	    return 1
-	fi
+	    if ! grep -q "shared directory .*: SUCCESS" $log; then
+		shared_dir_ok=false
+	    fi
+
+	    rm -f $log
+
+	    if test x"$result" != x"0"; then
+		stop_schroot_sessions
+		break
+	    fi
+	done
     done
 
-    if [ -z "$schroot_boards" ]; then
-	return 0
+    if test x"$result" != x"0"; then
+	return 1
     fi
 
-    schroot_make_opts="SCHROOT_PORT=$port"
+    schroot_make_opts="SCHROOT_PORT=$schroot_port"
     if $shared_dir_ok; then
 	schroot_make_opts="$schroot_make_opts SCHROOT_SHARED_DIR=$shared_dir"
     fi
 }
 
-# Stop schroot sessions on given boards/ports
-# $1 - port on which ssh server for schroot session was started
-# $2, $3, ... - hostnames of boards as printed by start_schroot_sessions.
+# Stop schroot sessions on $schroot_boards
 stop_schroot_sessions()
 {
     trace "$*"
 
-    local port="$1"
-    shift 1
-
-    for hostname in "$@"; do
-	dryrun "${topdir}/scripts/test-schroot.sh -v -f $hostname:$port"
+    for hostname in $schroot_boards; do
+	dryrun "${topdir}/scripts/test-schroot.sh -v -f $hostname:$schroot_port"
     done
+
+    schroot_boards=""
 }