diff options
| -rw-r--r-- | tcwg-benchmark-results.broken-list | 97 | ||||
| -rwxr-xr-x | tcwg_bmk-build.sh | 23 |
2 files changed, 119 insertions, 1 deletions
diff --git a/tcwg-benchmark-results.broken-list b/tcwg-benchmark-results.broken-list new file mode 100644 index 00000000..85d65264 --- /dev/null +++ b/tcwg-benchmark-results.broken-list @@ -0,0 +1,97 @@ +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2/2458 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2/2578 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2/2824 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2/3089 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2_LTO/2416 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2_LTO/2550 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2_LTO/2707 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2_LTO/2852 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O2_LTO/2981 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3/2511 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3/2632 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3/2831 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3/3056 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3_LTO/2798 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3_LTO/2947 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-O3_LTO/3142 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os/2356 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os/2567 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os/2791 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os/2962 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os/3162 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os_LTO/2379 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os_LTO/2488 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os_LTO/2642 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os_LTO/2848 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-master-arm-spec2k6-Os_LTO/3073 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2/2750 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2/2967 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2/3135 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2_LTO/2470 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2_LTO/2661 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2_LTO/2857 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O2_LTO/3187 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3/2335 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3/2539 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3/2742 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3/2936 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3/3172 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3_LTO/2323 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3_LTO/2532 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3_LTO/2720 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3_LTO/2921 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-O3_LTO/3165 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os/2423 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os/2554 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os/2679 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os/3123 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os_LTO/2496 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os_LTO/2670 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os_LTO/2883 +tk1_32/tcwg_bmk_tk1/jenkins-full-gnu-release-arm-spec2k6-Os_LTO/3151 +tk1/tcwg_bmk/gnu-master-arm-spec2k6-O3-4374 +tk1/tcwg_bmk/gnu-master-arm-spec2k6-O3_LTO-4704 +tk1/tcwg_bmk/gnu-master-arm-spec2k6-O3_LTO-4788 +tk1/tcwg_bmk/llvm-master-arm-spec2k6-O2-5917 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2143 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2182 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2252 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2393 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2542 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2/2732 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2_LTO/2248 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2_LTO/2380 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2_LTO/2543 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O2_LTO/2727 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O3/2563 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O3/2769 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-O3_LTO/2745 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os/2213 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os/2316 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os/2458 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os/2621 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os/2798 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os_LTO/2442 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os_LTO/2593 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-master-aarch64-spec2k6-Os_LTO/2823 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O2/2763 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O2_LTO/2267 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O2_LTO/2451 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O2_LTO/2683 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3/2349 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3/2528 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3/2692 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3_LTO/2151 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3_LTO/2259 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3_LTO/2447 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-O3_LTO/2681 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os/2318 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os/2392 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os/2564 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os/2829 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os_LTO/2298 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os_LTO/2403 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os_LTO/2594 +tx1_64/tcwg_bmk_tx1/jenkins-full-gnu-release-aarch64-spec2k6-Os_LTO/2847 +tx1/tcwg_bmk/gnu-master-aarch64-spec2k6-O3-4348 +tx1/tcwg_bmk/llvm-master-aarch64-spec2k6-O3-4358 diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh index fc416314..9ce91d99 100755 --- a/tcwg_bmk-build.sh +++ b/tcwg_bmk-build.sh @@ -389,6 +389,16 @@ no_regression_vs_p () if [ ! -f "$ref_artifacts/results_id-1" ] || [ ! -f "$ref_artifacts/results_id-2" ]; then return 0 fi + # <Workaround> missing reference results, which we have listed in + # tcwg-benchmark-results.broken-list. Once all entries referencing missing + # results are discarded, we'll remove this workaround. + # Otherwise compare_results will fail while fetching baseline results, + # and we'll consider this failure as a regression. + if cat "$scripts/tcwg-benchmark-results.broken-list" \ + | grep -q "^$(cat $ref_artifacts/results_id-1)\$\|^$(cat $ref_artifacts/results_id-2)\$"; then + return 0 + fi + # </Workaround> compare_results "$ref_artifacts/results_id-1" "$ref_artifacts/results_id-2" "--num_dsos 1 --num_symbols 0" while IFS= read -r -d '' i do @@ -466,7 +476,18 @@ no_regression_to_base_p () if ! [ -f "$ref_artifacts/results_id" ]; then return 0 - elif ! [ -f "$new_artifacts/results_id" ]; then + fi + # <Workaround> missing reference results, which we have listed in + # tcwg-benchmark-results.broken-list. Once all entries referencing missing + # results are discarded, we'll remove this workaround. + # Otherwise compare_results will fail while fetching baseline results, + # and we'll consider this failure as a regression. + if cat "$scripts/tcwg-benchmark-results.broken-list" \ + | grep -q "^$(cat $ref_artifacts/results_id)\$"; then + return 0 + fi + # </Workaround> + if ! [ -f "$new_artifacts/results_id" ]; then return 1 fi |