testsuite: Produce JUnit output

Test Plan: Validate, try ingesting into Jenkins.

Reviewers: austin

Subscribers: rwbarton, thomie

GHC Trac Issues: #13716

Differential Revision: https://phabricator.haskell.org/D3796

7 files changed, 52 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 16071f6..073c5a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -171,6 +171,7 @@ _darcs/
 /rts/package.conf.install.raw
 /stage3.package.conf
 /testsuite_summary*.txt
+/testsuite*.xml
 /testlog*
 /utils/mkUserGuidePart/mkUserGuidePart.cabal
 /utils/runghc/runghc.cabal
diff --git a/testsuite/driver/junit.py b/testsuite/driver/junit.py
new file mode 100644
index 0000000..01a5f47
--- /dev/null
+++ b/testsuite/driver/junit.py
@@ -0,0 +1,38 @@
+from datetime import datetime
+import xml.etree.ElementTree as ET
+
+def junit(t):
+    testsuites = ET.Element('testsuites')
+    testsuite = ET.SubElement(testsuites, 'testsuite',
+                              id = "0",
+                              package = 'ghc',
+                              tests = str(t.total_tests),
+                              failures = str(len(t.unexpected_failures) + len(t.unexpected_stat_failures)),
+                              errors = str(len(t.framework_failures)),
+                              timestamp = datetime.now().isoformat())
+
+    for result, group in [('stat failure', t.unexpected_stat_failures),
+                          ('unexpected failure', t.unexpected_failures)]:
+        for (directory, testname, reason, way) in group:
+            testcase = ET.SubElement(testsuite, 'testcase',
+                                     classname = testname,
+                                     name = way)
+            result = ET.SubElement(testcase, 'failure',
+                                   type = result,
+                                   message = reason)
+
+    for (directory, testname, reason, way) in t.framework_failures:
+        testcase = ET.SubElement(testsuite, 'testcase',
+                                 classname = testname,
+                                 name = way)
+        result = ET.SubElement(testcase, 'error',
+                               type = "framework failure",
+                               message = reason)
+
+    for (directory, testname, way) in t.expected_passes:
+        testcase = ET.SubElement(testsuite, 'testcase',
+                                 classname = testname,
+                                 name = way)
+
+    return ET.ElementTree(testsuites)
+
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index f7064a5..f0c635f 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -26,6 +26,7 @@ import subprocess
 
 from testutil import *
 from testglobals import *
+from junit import junit
 
 # Readline sometimes spews out ANSI escapes for some values of TERM,
 # which result in test failures. Thus set TERM to a nice, simple, safe
@@ -57,6 +58,7 @@ parser.add_argument("--threads", type=int, help="threads to run simultaneously")
 parser.add_argument("--check-files-written", help="check files aren't written by multiple tests") # NOTE: This doesn't seem to exist?
 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
 parser.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
+parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
 
 args = parser.parse_args()
 
@@ -310,6 +312,9 @@ else:
         with open(config.summary_file, 'w') as file:
             summary(t, file)
 
+    if args.junit:
+        junit(t).write(args.junit)
+
 cleanup_and_exit(0)
 
 # Note [Running tests in /tmp]
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index fc050e6..5e7142d 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -140,6 +140,7 @@ class TestRun:
        self.framework_failures = []
        self.framework_warnings = []
 
+       self.expected_passes = []
        self.unexpected_passes = []
        self.unexpected_failures = []
        self.unexpected_stat_failures = []
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 26e3d17..15c773e 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -863,6 +863,7 @@ def do_test(name, way, func, args, files):
 
     if passFail == 'pass':
         if _expect_pass(way):
+            t.expected_passes.append((directory, name, way))
             t.n_expected_passes += 1
         else:
             if_verbose(1, '*** unexpected pass for %s' % full_name)
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index 6c39636..a21c4bb 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -246,6 +246,10 @@ RUNTEST_OPTS +=  \
 
 RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
 
+ifneq "$(JUNIT_FILE)" ""
+RUNTEST_OPTS +=  \
+  --junit "$(JUNIT_FILE)"
+endif
 ifneq "$(SUMMARY_FILE)" ""
 RUNTEST_OPTS +=  \
 	--summary-file "$(SUMMARY_FILE)"
diff --git a/validate b/validate
index 09f4fd2..d885bd7 100755
--- a/validate
+++ b/validate
@@ -296,6 +296,7 @@ rm -f testsuite_summary.txt testsuite_summary_stage1.txt
 $make -C testsuite/tests $BINDIST $PYTHON_ARG \
       $MAKE_TEST_TARGET stage=2 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
       NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../testsuite_summary.txt \
+      JUNIT_FILE=../../testsuite.xml \
       2>&1 | tee testlog
 
 # Run a few tests using the stage1 compiler.
@@ -304,6 +305,7 @@ $make -C testsuite/tests $BINDIST $PYTHON_ARG \
 $make -C testsuite/tests/stage1 $PYTHON_ARG \
       $MAKE_TEST_TARGET stage=1 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
       NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../../testsuite_summary_stage1.txt \
+      JUNIT_FILE=../../../testsuite_stage1.xml \
       2>&1 | tee testlog-stage1
 
 echo