Add reference output with -l option.

Change-Id: I3518a9d866770af2996c338309c5c4a9abc5e3a4

2 files changed, 338 insertions, 0 deletions

diff --git a/testsuite/Makefile b/testsuite/Makefile
index cabb3b6..8925ba8 100644
--- a/testsuite/Makefile
+++ b/testsuite/Makefile
@@ -1,5 +1,6 @@
 check:
 	LC_ALL=C ../compare_dg_tests.pl sum-1.txt sum-2.txt > cmp-1-2.txt ; diff cmp-1-2.txt expected-1-2.txt
+	LC_ALL=C ../compare_dg_tests.pl -l sum-1.txt sum-2.txt > cmp-1-2-long.txt ; diff cmp-1-2-long.txt expected-1-2-long.txt
 
 clean:
 	rm -f cmp-1-2.txt
diff --git a/testsuite/expected-1-2-long.txt b/testsuite/expected-1-2-long.txt
new file mode 100644
index 0000000..83ecb5c
--- /dev/null
+++ b/testsuite/expected-1-2-long.txt
@@ -0,0 +1,337 @@
+Comparing:
+REFERENCE:sum-1.txt
+CURRENT:  sum-2.txt
+
+              `                              +---------+---------+
+o  RUN STATUS:                               |   REF   |   RES   |
+  +------------------------------------------+---------+---------+
+  | Passes                            [PASS] |       9 |      11 |
+  | Unexpected fails                  [FAIL] |      10 |      10 |
+  | Errors                           [ERROR] |       1 |       1 |
+  | Unexpected passes                [XPASS] |       8 |       9 |
+  | Expected fails                   [XFAIL] |       8 |       9 |
+  | Unresolved                  [UNRESOLVED] |       7 |       7 |
+  | Unsupported                [UNSUPPORTED] |       7 |       7 |
+  | Untested                      [UNTESTED] |       7 |       7 |
+  +------------------------------------------+---------+---------+
+
+    REF PASS ratio: 0.303571
+    RES PASS ratio: 0.333333
+    ***** ERROR: REF PASS ratio is abnormally low *****
+    ***** ERROR: RES PASS ratio is abnormally low *****
+    ***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****
+    ***** WARNING: No RES execution test FAILed. Check execution engine configuration. *****
+
+o  REGRESSIONS:
+  +------------------------------------------+---------+
+  | PASS now XPASS            [PASS =>XPASS] |       1 |
+  | PASS now FAIL             [PASS => FAIL] |       1 |
+  | PASS now UNRESOLVED       [PASS =>UNRES] |       1 |
+  | ERROR appears             [     =>ERROR] |       1 |
+  | XPASS now FAIL            [XPASS=> FAIL] |       1 |
+  | XPASS appears             [     =>XPASS] |       1 |
+  | XPASS now UNRESOLVED      [XPASS=>UNRES] |       1 |
+  | FAIL appears              [     => FAIL] |       1 |
+  | FAIL now UNRESOLVED       [FAIL =>UNRES] |       1 |
+  | XFAIL now FAIL            [XFAIL=> FAIL] |       1 |
+  | XFAIL disappears          [XFAIL=>     ] |       1 |
+  | XFAIL now UNRESOLVED      [XFAIL=>UNRES] |       1 |
+  | UNSUPPORTED now FAIL      [UNSUP=> FAIL] |       1 |
+  | UNSUPPORTED now XPASS     [UNSUP=>XPASS] |       1 |
+  | UNSUPP now UNRESOLVED     [UNSUP=>UNRES] |       1 |
+  | UNTESTED now FAIL         [UNTES=> FAIL] |       1 |
+  | UNTESTED now XPASS        [UNTES=>XPASS] |       1 |
+  | UNTESTED now UNRESOLVED   [UNTES=>UNRES] |       1 |
+  | UNRESOLVED now FAIL       [UNRES=> FAIL] |       1 |
+  | UNRESOLVED now XPASS      [UNRES=>XPASS] |       1 |
+  | UNRESOLVED appears        [     =>UNRES] |       1 |
+  +------------------------------------------+---------+
+  | TOTAL_REGRESSIONS                        |      21 |
+  +------------------------------------------+---------+
+
+  - PASS now XPASS            [PASS =>XPASS]:
+
+  test2-pass-xpass
+
+  - PASS now FAIL             [PASS => FAIL]:
+
+  test3-pass-fail
+
+  - PASS now UNRESOLVED       [PASS =>UNRES]:
+
+  test8-pass-unresolved
+
+  - ERROR appears             [     =>ERROR]:
+
+  (DejaGnu) proc "scan-dump-tree-not fail_test optimized 3" does not exist.
+
+  - XPASS now FAIL            [XPASS=> FAIL]:
+
+  test3-xpass-fail
+
+  - XPASS appears             [     =>XPASS]:
+
+  test2-xpass-appears
+
+  - XPASS now UNRESOLVED      [XPASS=>UNRES]:
+
+  test8-xpass-unresolved
+
+  - FAIL appears              [     => FAIL]:
+
+  test3-fail-appears
+
+  - FAIL now UNRESOLVED       [FAIL =>UNRES]:
+
+  test8-fail-unresolved
+
+  - XFAIL now FAIL            [XFAIL=> FAIL]:
+
+  test3-xfail-fail
+
+  - XFAIL disappears          [XFAIL=>     ]:
+
+  test5-xfail-disappears
+
+  - XFAIL now UNRESOLVED      [XFAIL=>UNRES]:
+
+  test8-xfail-unresolved
+
+  - UNSUPPORTED now FAIL      [UNSUP=> FAIL]:
+
+  test3-unsupported-fail
+
+  - UNSUPPORTED now XPASS     [UNSUP=>XPASS]:
+
+  test2-unsupported-xpass
+
+  - UNSUPP now UNRESOLVED     [UNSUP=>UNRES]:
+
+  test7-unsupported-unresolved
+
+  - UNTESTED now FAIL         [UNTES=> FAIL]:
+
+  test3-untested-fail
+
+  - UNTESTED now XPASS        [UNTES=>XPASS]:
+
+  test2-untested-xpass
+
+  - UNTESTED now UNRESOLVED   [UNTES=>UNRES]:
+
+  test7-untested-unresolved
+
+  - UNRESOLVED now FAIL       [UNRES=> FAIL]:
+
+  test3-unresolved-fail
+
+  - UNRESOLVED now XPASS      [UNRES=>XPASS]:
+
+  test2-unresolved-xpass
+
+  - UNRESOLVED appears        [     =>UNRES]:
+
+  test4-unresolved-appears
+
+
+o  MINOR TO BE CHECKED:
+  +------------------------------------------+---------+
+  | PASS disappears           [PASS =>     ] |       1 |
+  | PASS now XFAIL            [PASS =>XFAIL] |       1 |
+  | New PASS                  [     => PASS] |       1 |
+  | PASS now UNSUPPORTED      [PASS =>UNSUP] |       1 |
+  | PASS now UNTESTED         [PASS =>UNTES] |       1 |
+  | XPASS now PASS            [XPASS=> PASS] |       1 |
+  | XPASS now XFAIL           [XPASS=>XFAIL] |       1 |
+  | XPASS now UNSUPPORTED     [XPASS=>UNSUP] |       1 |
+  | XPASS now UNTESTED        [XPASS=>UNTES] |       1 |
+  | XPASS disappears          [XPASS=>     ] |       1 |
+  | FAIL now PASS             [FAIL => PASS] |       2 |
+  | FAIL now XPASS            [FAIL =>XPASS] |       1 |
+  | FAIL now XFAIL            [FAIL =>XFAIL] |       1 |
+  | FAIL now UNSUPPORTED      [FAIL =>UNSUP] |       1 |
+  | FAIL now UNTESTED         [FAIL =>UNTES] |       1 |
+  | FAIL disappears           [FAIL =>     ] |       1 |
+  | XFAIL now PASS            [XFAIL=> PASS] |       1 |
+  | XFAIL now XPASS           [XFAIL=>XPASS] |       1 |
+  | XFAIL now UNSUPPORTED     [XFAIL=>UNSUP] |       1 |
+  | XFAIL now UNTESTED        [XFAIL=>UNTES] |       1 |
+  | XFAIL appears             [     =>XFAIL] |       1 |
+  | UNSUPPORTED now PASS      [UNSUP=> PASS] |       1 |
+  | UNSUPPORTED now XFAIL     [UNSUP=>XFAIL] |       1 |
+  | UNSUPP now UNTESTED       [UNSUP=>UNTES] |       1 |
+  | UNSUPPORTED disappears    [UNSUP=>     ] |       1 |
+  | UNSUPPORTED appears       [     =>UNSUP] |       1 |
+  | UNTESTED now PASS         [UNTES=> PASS] |       1 |
+  | UNTESTED now XFAIL        [UNTES=>XFAIL] |       1 |
+  | UNTESTED now UNSUPP       [UNTES=>UNSUP] |       1 |
+  | UNTESTED disappears       [UNTES=>     ] |       1 |
+  | UNTESTED appears          [     =>UNTES] |       1 |
+  | UNRESOLVED now PASS       [UNRES=> PASS] |       1 |
+  | UNRESOLVED now XFAIL      [UNRES=>XFAIL] |       1 |
+  | UNRESOLVED now UNSUPP     [UNRES=>UNSUP] |       1 |
+  | UNRESOLVED now UNTESTED   [UNRES=>UNTES] |       1 |
+  | UNRESOLVED disappears     [UNRES=>     ] |       1 |
+  | ERROR disappears          [ERROR=>     ] |       1 |
+  +------------------------------------------+---------+
+  | TOTAL_MINOR_TO_BE_CHECKED                |      38 |
+  +------------------------------------------+---------+
+
+  - PASS disappears           [PASS =>     ]:
+
+  test5-pass-disappears
+
+  - PASS now XFAIL            [PASS =>XFAIL]:
+
+  test4-pass-xfail
+
+  - New PASS                  [     => PASS]:
+
+  test1-pass-appears
+
+  - PASS now UNSUPPORTED      [PASS =>UNSUP]:
+
+  test6-pass-unsupported
+
+  - PASS now UNTESTED         [PASS =>UNTES]:
+
+  test7-pass-untested
+
+  - XPASS now PASS            [XPASS=> PASS]:
+
+  test1-xpass-pass
+
+  - XPASS now XFAIL           [XPASS=>XFAIL]:
+
+  test4-xpass-xfail
+
+  - XPASS now UNSUPPORTED     [XPASS=>UNSUP]:
+
+  test6-xpass-unsupported
+
+  - XPASS now UNTESTED        [XPASS=>UNTES]:
+
+  test7-xpass-untested
+
+  - XPASS disappears          [XPASS=>     ]:
+
+  test5-xpass-disappears
+
+  - XFAIL now PASS            [XFAIL=> PASS]:
+
+  test1-xfail-pass
+
+  - XFAIL now XPASS           [XFAIL=>XPASS]:
+
+  test2-xfail-xpass
+
+  - FAIL now PASS             [FAIL => PASS]:
+
+  test1-fail-pass
+  test1-fail-pass execution
+
+  - FAIL now XPASS            [FAIL =>XPASS]:
+
+  test2-fail-xpass
+
+  - FAIL now XFAIL            [FAIL =>XFAIL]:
+
+  test4-fail-xfail
+
+  - FAIL now UNSUPPORTED      [FAIL =>UNSUP]:
+
+  test6-fail-unsupported
+
+  - FAIL now UNTESTED         [FAIL =>UNTES]:
+
+  test7-fail-untested
+
+  - FAIL disappears           [FAIL =>     ]:
+
+  test5-fail-disappears
+
+  - XFAIL now PASS            [XFAIL=> PASS]:
+
+  test1-xfail-pass
+
+  - XFAIL now XPASS           [XFAIL=>XPASS]:
+
+  test2-xfail-xpass
+
+  - XFAIL now UNSUPPORTED     [XFAIL=>UNSUP]:
+
+  test6-xfail-unsupported
+
+  - XFAIL now UNTESTED        [XFAIL=>UNTES]:
+
+  test7-xfail-untested
+
+  - XFAIL appears             [     =>XFAIL]:
+
+  test4-xfail-appears
+
+  - UNSUPPORTED now PASS      [UNSUP=> PASS]:
+
+  test1-unsupported-pass
+
+  - UNSUPPORTED now XFAIL     [UNSUP=>XFAIL]:
+
+  test4-unsupported-xfail
+
+  - UNSUPP now UNTESTED       [UNSUP=>UNTES]:
+
+  test6-unsupported-untested
+
+  - UNSUPPORTED disappears    [UNSUP=>     ]:
+
+  test5-unsupported-disappears
+
+  - UNSUPPORTED appears       [     =>UNSUP]:
+
+  test4-unsupported-appears
+
+  - UNTESTED now PASS         [UNTES=> PASS]:
+
+  test1-untested-pass
+
+  - UNTESTED now XFAIL        [UNTES=>XFAIL]:
+
+  test4-untested-xfail
+
+  - UNTESTED now UNSUPP       [UNTES=>UNSUP]:
+
+  test6-untested-unsupported
+
+  - UNTESTED disappears       [UNTES=>     ]:
+
+  test5-untested-disappears
+
+  - UNTESTED appears          [     =>UNTES]:
+
+  test4-untested-appears
+
+  - UNRESOLVED now PASS       [UNRES=> PASS]:
+
+  test1-unresolved-pass
+
+  - UNRESOLVED now XFAIL      [UNRES=>XFAIL]:
+
+  test4-unresolved-xfail
+
+  - UNRESOLVED now UNSUPP     [UNRES=>UNSUP]:
+
+  test6-unresolved-unsupported
+
+  - UNRESOLVED now UNTESTED   [UNRES=>UNTES]:
+
+  test7-unresolved-untested
+
+  - UNRESOLVED disappears     [UNRES=>     ]:
+
+  test5-unresolved-disappears
+
+  - ERROR disappears          [ERROR=>     ]:
+
+  (DejaGnu) proc "scan-dump-tree-not fail_test optimized 1" does not exist.
+
+