tcwg/round-robin.yaml.inc: Use long-lived containers instead of docker-run.sh

... to preserve state of precommit builds between "build/test" and
"notify" stages.  In precommit builds we use a scratch volume for
$WORKSPACE, so, as soon as docker-run.sh completes, scratch volume
is destroyed.  For normal builds using several containers is not
a problem since $WORKSPACE is bind-mounted from the host machine.

This change makes the job control container runtime explicitly
by starting containers using start-container-docker.sh and
cleaning them up using "trap cleanup_all_containers EXIT", which
is embedded into artifacts/jenkins/run-build.env.  We need to add
"trap '' EXIT" to the [successful] end of every step to extend
container lifetime.  The new step "run-build-fini" cleans up
containers of a successful build.

Change-Id: I8fd652b06431167b3dab295b3d5d1d995580d754

1 files changed, 51 insertions, 14 deletions

diff --git a/tcwg_gnu_native.yaml b/tcwg_gnu_native.yaml
index 2c4e30cc58..116ded6ee7 100644
--- a/tcwg_gnu_native.yaml
+++ b/tcwg_gnu_native.yaml
@@ -237,6 +237,8 @@
             fi
           done
 
+          trap "" EXIT
+
 - builder:
     name: trigger-followup-builds-2
     builders:
@@ -270,8 +272,7 @@
           # Note that we add --notify to the manifest -- we will need this
           # later in round-robin-baseline.sh when re-writing history
           # to update notifications files in interesting-commits and jira.
-          "${{docker_run[@]}}" \
-            -- \
+          build_container_exec \
             ./jenkins-scripts/round-robin-notify.sh \
             @@rr[top_artifacts] artifacts --notify "$notify" \
             __stage init __verbose true \
@@ -287,8 +288,7 @@
 
           # Full notification
           echo "NOTE: Notify developers about this build"
-          "${{docker_run[@]}}" \
-            -- \
+          build_container_exec \
             ./jenkins-scripts/round-robin-notify.sh \
             @@rr[top_artifacts] artifacts __TCWG_JIRA_TOKEN "$TCWG_JIRA_TOKEN" \
             __stage full __verbose true \
@@ -338,6 +338,8 @@
             artifacts/jenkins/mail-recipients.bak
           fi
 
+          trap "" EXIT
+
 - builder:
     name: run-exit-on-failure
     builders:
@@ -352,6 +354,8 @@
           fi
           # Only successful builds run beyond this point.
 
+          trap "" EXIT
+
 - builder:
     name: run-push-baseline
     builders:
@@ -367,8 +371,7 @@
             exit 0
           fi
 
-          "${{docker_run[@]}}" \
-            -- \
+          build_container_exec \
             ./jenkins-scripts/round-robin-baseline.sh \
             @@rr[top_artifacts] artifacts \
             __build_script {build_script} \
@@ -387,6 +390,8 @@
             exit 1
           fi
 
+          trap "" EXIT
+
 
 - builder:
     name: run-build-init
@@ -438,6 +443,8 @@
                 fi
               done
             fi
+
+            trap "" EXIT
           unstable-return: 125
 
 - builder:
@@ -449,13 +456,17 @@
             set -ex
             source artifacts/jenkins/run-build.env
 
-            docker_run=(
-              ./jenkins-scripts/docker-run.sh
+            docker_opts=(
               --distro $distro
               --arch {docker_arch}
               --node $NODE_NAME
             )
 
+            ./jenkins-scripts/start-container-docker.sh --prefix build_ \
+              "${{docker_opts[@]}}" > build_container.sh
+            source build_container.sh
+            trap "cleanup_all_containers" EXIT
+
             # Run the build script to fetch base-artifacts/, which are
             # required to fetch baseline sources to apply patches to in
             # pw-apply.sh below.  This also initializes the build manifest
@@ -465,8 +476,7 @@
             # after fetching the baseline and "__" avoids saving this parameter
             # in the manifest, so that we don't see this option in the main
             # run of the build script below.
-            "${{docker_run[@]}}" \
-              -- \
+            build_container_exec \
               ./jenkins-scripts/{build_script} \
               %%rr[top_artifacts] artifacts \
               ==rr[ci_project] $ci_project \
@@ -480,8 +490,15 @@
               __finish_at reset_artifacts
 
             cat >> artifacts/jenkins/run-build.env <<EOF
-            docker_run=(${{docker_run[@]+$(printf "%q$IFS" "${{docker_run[@]}}")}})
+            docker_opts=(${{docker_opts[@]+$(printf "%q$IFS" "${{docker_opts[@]}}")}})
+            source build_container.sh
+            trap "cleanup_all_containers" EXIT
             EOF
+
+            # Now that we have "trap cleanup_all_containers EXIT" in
+            # run-build.env we need to reset the trap at the end of every
+            # step to preserve container[s] for the next step.
+            trap "" EXIT
           unstable-return: 125
 
 - builder:
@@ -524,6 +541,8 @@
             build_name="$build_name"
             branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}})
             EOF
+
+            trap "" EXIT
           unstable-return: 125
 
 - builder:
@@ -541,8 +560,7 @@
             # where the above run finished.  This allows us to use read-only
             # bind-mount for base-artifacts/, which, otherwise, may require
             # multi-gig rsync.
-            "${{docker_run[@]}}" \
-              -- \
+            build_container_exec \
               ./jenkins-scripts/{build_script} \
               @@rr[top_artifacts] artifacts \
               "${{branch_opt[@]}}" \
@@ -596,9 +614,23 @@
             cat >> artifacts/jenkins/run-build.env <<EOF
             build_name="$build_name"
             EOF
+
+            trap "" EXIT
           unstable-return: 125
 
 - builder:
+    name: run-build-fini
+    builders:
+      - shell:
+          command: |
+            #!/bin/bash
+            set -ex
+            source artifacts/jenkins/run-build.env
+            # The "trap cleanup_all_containers EXIT" statement in
+            # artifacts/jenkins/run-build.env will remove build and precommit
+            # containers.
+
+- builder:
     name: run-build
     builders:
       - conditional-step:
@@ -677,6 +709,11 @@
       - build-name-setter:
           name: 'artifacts/jenkins/build-name'
           file: true
+      - conditional-step:
+          condition-kind: current-status
+          steps:
+            - run-build-fini:
+                dummy: dummy
 
 
 - publisher:
@@ -902,4 +939,4 @@
           docker_arch: '{docker_arch}'
     publishers:
       - bisect-publishers
-# checksum: 564cd23665764084c233887988ff6ba8
+# checksum: 82a75cc1560e2a86b39c10da3d4429eb