Add parsing of ping test results for better automation

Rather than just dumping a set of ping results for the user to track,
add extra functions in the test-common wrapper.

For each test area in each test, mark the test area between

 log "CHECK $AREA START"

and

 log "CHECK $AREA END"

These will mark the interesting test areas, i.e. after VLAN changes
etc. have settled. Also, just after the START tag include some
statements like

 log "CHECK $AREA CHECK FOO:machine1:machine2:machine3 BAR:machine4:machine5"

to define the expected connectivity results. FOO and BAR are machine
groups that are expected to see each other *only*, i.e. they've been
isolated using VLANs. At the very end of the test, add a call to

  check_test_steps

to use this information and give a summary of what has happened in
each of the test areas. $AREA is a free-form string, just don't use
START, END, CHECK or whitespace within the name.

I've updated complex-4switches-2vlans-1 to use this new code, and it's
found another bug for me!

Change-Id: Ib41c7f1cc2fa8096c5579ab56d4199386abe6843

2 files changed, 115 insertions, 2 deletions

diff --git a/test/complex-4switches-2vlans-1 b/test/complex-4switches-2vlans-1
index 2d19e8b..2644738 100644
--- a/test/complex-4switches-2vlans-1
+++ b/test/complex-4switches-2vlans-1
@@ -40,7 +40,10 @@ clear_logs
 # Start all the test machines logging, then wait 60s to let all of
 # them show baseline results before we start testing
 start_logging
+log "CHECK INIT START"
+log "CHECK INIT CHECK VLAN_BASE:imx5301:panda01:arndale02:arndale03:imx5302:arndale01:panda02:panda03"
 pause 60
+log "CHECK INIT END"
 
 # Create 2 VLANs: tag 30, named "test30" and tag 31, named "test31"
 log "Creating new VLAN tag 30"
@@ -61,19 +64,36 @@ pause 10
 log "Moving imx5301 to VLAN ID $VLAN_ID1"
 run_admin_command --set_port_current_vlan ${imx5301_PORT_ID} $VLAN_ID1
 pause 60
+log "CHECK STEP1 START"
+log "CHECK STEP1 CHECK VLAN_BASE:panda01:arndale02:arndale03:imx5302:arndale01:panda02:panda03"
+pause 60
+log "CHECK STEP1 END"
+
 log "Moving panda01 to VLAN ID $VLAN_ID1"
 run_admin_command --set_port_current_vlan ${panda01_PORT_ID} $VLAN_ID1
 log "Done moving ports to VLAN ID $VLAN_ID1"
+pause 60
+log "CHECK STEP2 START"
+log "CHECK STEP2 CHECK VLAN_30:imx5301:panda01 VLAN_BASE:arndale02:arndale03:imx5302:arndale01:panda02:panda03"
+pause 60
+log "CHECK STEP2 END"
 
 log "Moving arndale02 to VLAN ID $VLAN_ID2"
 run_admin_command --set_port_current_vlan ${arndale02_PORT_ID} $VLAN_ID2
 pause 60
+log "CHECK STEP3 START"
+log "CHECK STEP3 CHECK VLAN_30:imx5301:panda01 VLAN_BASE:arndale03:imx5302:arndale01:panda02:panda03"
+pause 60
+log "CHECK STEP3 END"
+
 log "Moving arndale03 to VLAN ID $VLAN_ID2"
 run_admin_command --set_port_current_vlan ${arndale03_PORT_ID} $VLAN_ID2
 log "Done moving ports to VLAN ID $VLAN_ID2"
-
-# Wait 60s for everything to settle and logs to show it again
 pause 60
+log "CHECK STEP4 START"
+log "CHECK STEP4 CHECK VLAN_30:imx5301:panda01 VLAN_31:arndale02:arndale03 VLAN_BASE:imx5302:arndale01:panda02:panda03"
+pause 60
+log "CHECK STEP4 END"
 
 # Move test machines back to their base VLANs
 log "Moving ports back to base"
@@ -86,6 +106,11 @@ log "Done moving ports back to base"
 # Wait 60s for everything to settle
 pause 60
 
+log "CHECK FINI START"
+log "CHECK FINI CHECK VLAN_BASE:imx5301:panda01:arndale02:arndale03:imx5302:arndale01:panda02:panda03"
+pause 60
+log "CHECK FINI END"
+
 # Check that they're all back on their base VLANs
 log "Checking base VLANs after the test"
 verify_all_hosts_are_base
@@ -108,6 +133,9 @@ clear_logs
 # And now sort the logs so we have clean output
 sort -u $LOGFILE
 
+# How did the test do?
+check_test_steps
+
 echo "Full details are in $LOGFILE, use \"sort -u $LOGFILE\" to read them in order."
 
 # DONE!
diff --git a/test/test-common b/test/test-common
index 22d9130..8776489 100755
--- a/test/test-common
+++ b/test/test-common
@@ -3,6 +3,9 @@
 #
 # Common set of definitions and functions to help with driving VLANd
 # in testing
+#
+# This specifically depends on *gawk* for the 2d arrays used in
+# check_test_step() at the bottom
 
 TOPDIR=$(dirname $0)/..
 
@@ -161,4 +164,86 @@ cleanup () {
     fi
 }
 
+list_test_steps () {
+    sort -u ${LOGFILE} | awk '
+        /CHECK.*START/  {
+            gsub("^.*CHECK ","")
+            gsub(" START.*$","")
+            start[$0]=1
+        }
+        /CHECK.*END/    {
+            gsub("^.*CHECK ","")
+            gsub(" END.*$","")
+            if(start[$0] == 1) {
+                print $0
+                start[$0] = 0
+            }
+        }'
+}
+
+check_test_steps () {
+    for STEP in $(list_test_steps); do
+        echo "Checking connectivity results for ${STEP}: "
+        check_test_step ${STEP}
+    done
+}
+
+check_test_step () {
+    STEP=$1
+    sort -u ${LOGFILE} | awk -v STEP=$1 "
+        /CHECK $STEP START/  { running=1 }
+        /CHECK $STEP END/    { running=0 }
+        /CHECK $STEP CHECK/  {
+            for( i = 4; i <= NF; i++) {
+                num_mach = split(\$i, mach_list, \":\")
+                for (j = 2; j <= num_mach; j++) {
+                    test[mach_list[1]][mach_list[j]] = 1
+                }
+            }
+            if (0 == 1) {
+                for (group in test) {
+                    printf(\"  $STEP test group %s contains: \", group)
+                    for (host in test[group]) {
+                        printf(\"%s \", host)
+                    }
+                    printf(\"\n\")
+                }
+            }
+        }
+        /UP/                 {
+            if(running) {
+                OK=0
+                for (group in test) {
+                    if (test[group][\$2] && test[group][\$4]) {
+                        #printf(\"GOOD UP: %s to %s (%s)\n\",\$2,\$4,group)
+                        OK=1
+                        success++
+                    }
+                }
+                if (OK == 0) {
+                    printf(\"    BAD UP: %s to %s\n\",\$2,\$4)
+                    fail++
+                }
+            }
+        }
+        /DOWN/               {
+            if(running) {
+                OK=1
+                for (group in test) {
+                    if (test[group][\$2] && test[group][\$4]) {
+                        printf(\"    BAD DOWN: %s to %s (%s)\n\",\$2,\$4,group)
+                        OK=0
+                        fail++
+                    }
+                }
+                if (OK == 1) {
+                    # printf(\"GOOD DOWN: %s to %s\n\",\$2,\$4)
+                    success++
+                }
+            }
+        }
+        END { printf(\"  success: %d, fail: %d\n\", success, fail) }
+"
+}
+
 trap cleanup 0