From 222319081515f9f6a4c699bd3a406f181a6ddc3b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Jan 2019 06:19:27 -0800 Subject: tests: optionally write results as JUnit-style .xml This will come in handy when publishing the results of Git's test suite during an automated Azure DevOps run. Note: we need to make extra sure that invalid UTF-8 encoding is turned into valid UTF-8 (using the Replacement Character, \uFFFD) because t9902's trace contains such invalid byte sequences, and the task in the Azure Pipeline that uploads the test results would refuse to do anything if it was asked to parse an .xml file with invalid UTF-8 in it. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index 1a44c81..044b4f7 100644 --- a/Makefile +++ b/Makefile @@ -754,6 +754,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o TEST_BUILTINS_OBJS += test-subprocess.o TEST_BUILTINS_OBJS += test-urlmatch-normalization.o +TEST_BUILTINS_OBJS += test-xml-encode.o TEST_BUILTINS_OBJS += test-wildmatch.o TEST_BUILTINS_OBJS += test-windows-named-pipe.o TEST_BUILTINS_OBJS += test-write-cache.o diff --git a/t/.gitignore b/t/.gitignore index 348715f..91cf577 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -2,3 +2,4 @@ /test-results /.prove /chainlinttmp +/out/ diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index bfb195b..4b4b397 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -49,6 +49,7 @@ static struct test_cmd cmds[] = { { "submodule-nested-repo-config", cmd__submodule_nested_repo_config }, { "subprocess", cmd__subprocess }, { "urlmatch-normalization", cmd__urlmatch_normalization }, + { "xml-encode", cmd__xml_encode }, { "wildmatch", cmd__wildmatch }, #ifdef GIT_WINDOWS_NATIVE { "windows-named-pipe", cmd__windows_named_pipe }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 042f124..c0ab65e 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -45,6 +45,7 @@ int cmd__submodule_config(int argc, const char **argv); int cmd__submodule_nested_repo_config(int argc, const char **argv); int cmd__subprocess(int argc, const char **argv); int cmd__urlmatch_normalization(int argc, const char **argv); +int cmd__xml_encode(int argc, const char **argv); int cmd__wildmatch(int argc, const char **argv); #ifdef GIT_WINDOWS_NATIVE int cmd__windows_named_pipe(int argc, const char **argv); diff --git a/t/helper/test-xml-encode.c b/t/helper/test-xml-encode.c new file mode 100644 index 0000000..a648bbd --- /dev/null +++ b/t/helper/test-xml-encode.c @@ -0,0 +1,80 @@ +#include "test-tool.h" + +static const char *utf8_replace_character = "�"; + +/* + * Encodes (possibly incorrect) UTF-8 on to , to be embedded + * in an XML file. + */ +int cmd__xml_encode(int argc, const char **argv) +{ + unsigned char buf[1024], tmp[4], *tmp2 = NULL; + ssize_t cur = 0, len = 1, remaining = 0; + unsigned char ch; + + for (;;) { + if (++cur == len) { + len = xread(0, buf, sizeof(buf)); + if (!len) + return 0; + if (len < 0) + die_errno("Could not read "); + cur = 0; + } + ch = buf[cur]; + + if (tmp2) { + if ((ch & 0xc0) != 0x80) { + fputs(utf8_replace_character, stdout); + tmp2 = NULL; + cur--; + continue; + } + *tmp2 = ch; + tmp2++; + if (--remaining == 0) { + fwrite(tmp, tmp2 - tmp, 1, stdout); + tmp2 = NULL; + } + continue; + } + + if (!(ch & 0x80)) { + /* 0xxxxxxx */ + if (ch == '&') + fputs("&", stdout); + else if (ch == '\'') + fputs("'", stdout); + else if (ch == '"') + fputs(""", stdout); + else if (ch == '<') + fputs("<", stdout); + else if (ch == '>') + fputs(">", stdout); + else if (ch >= 0x20) + fputc(ch, stdout); + else if (ch == 0x09 || ch == 0x0a || ch == 0x0d) + fprintf(stdout, "&#x%02x;", ch); + else + fputs(utf8_replace_character, stdout); + } else if ((ch & 0xe0) == 0xc0) { + /* 110XXXXx 10xxxxxx */ + tmp[0] = ch; + remaining = 1; + tmp2 = tmp + 1; + } else if ((ch & 0xf0) == 0xe0) { + /* 1110XXXX 10Xxxxxx 10xxxxxx */ + tmp[0] = ch; + remaining = 2; + tmp2 = tmp + 1; + } else if ((ch & 0xf8) == 0xf0) { + /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + tmp[0] = ch; + remaining = 3; + tmp2 = tmp + 1; + } else + fputs(utf8_replace_character, stdout); + } + + return 0; +} diff --git a/t/test-lib.sh b/t/test-lib.sh index a1abb11..a3b2166 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -139,6 +139,9 @@ do verbose_log=t tee=t ;; + --write-junit-xml) + write_junit_xml=t + ;; --stress) stress=t ;; --stress=*) @@ -622,11 +625,24 @@ trap 'exit $?' INT TERM HUP # the test_expect_* functions instead. test_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$*" + fi test_success=$(($test_success + 1)) say_color "" "ok $test_count - $@" } test_failure_ () { + if test -n "$write_junit_xml" + then + junit_insert="" + junit_insert="$junit_insert $(xml_attr_encode \ + "$(printf '%s\n' "$@" | sed 1d)")" + junit_insert="$junit_insert" + write_junit_xml_testcase "$1" " $junit_insert" + fi test_failure=$(($test_failure + 1)) say_color error "not ok $test_count - $1" shift @@ -635,11 +651,19 @@ test_failure_ () { } test_known_broken_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (breakage fixed)" + fi test_fixed=$(($test_fixed+1)) say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (known breakage)" + fi test_broken=$(($test_broken+1)) say_color warn "not ok $test_count - $@ # TODO known breakage" } @@ -897,6 +921,10 @@ test_start_ () { test_count=$(($test_count+1)) maybe_setup_verbose maybe_setup_valgrind + if test -n "$write_junit_xml" + then + junit_start=$(test-tool date getnanos) + fi } test_finish_ () { @@ -934,6 +962,13 @@ test_skip () { case "$to_skip" in t) + if test -n "$write_junit_xml" + then + message="$(xml_attr_encode "$skipped_reason")" + write_junit_xml_testcase "$1" \ + " " + fi + say_color skip >&3 "skipping test: $@" say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true @@ -949,9 +984,51 @@ test_at_end_hook_ () { : } +write_junit_xml () { + case "$1" in + --truncate) + >"$junit_xml_path" + junit_have_testcase= + shift + ;; + esac + printf '%s\n' "$@" >>"$junit_xml_path" +} + +xml_attr_encode () { + printf '%s\n' "$@" | test-tool xml-encode +} + +write_junit_xml_testcase () { + junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" + shift + junit_attrs="$junit_attrs classname=\"$this_test\"" + junit_attrs="$junit_attrs time=\"$(test-tool \ + date getnanos $junit_start)\"" + write_junit_xml "$(printf '%s\n' \ + " " "$@" " ")" + junit_have_testcase=t +} + test_done () { GIT_EXIT_OK=t + if test -n "$write_junit_xml" && test -n "$junit_xml_path" + then + test -n "$junit_have_testcase" || { + junit_start=$(test-tool date getnanos) + write_junit_xml_testcase "all tests skipped" + } + + # adjust the overall time + junit_time=$(test-tool date getnanos $junit_suite_start) + sed "s/]*/& time=\"$junit_time\"/" \ + <"$junit_xml_path" >"$junit_xml_path.new" + mv "$junit_xml_path.new" "$junit_xml_path" + + write_junit_xml " " "" + fi + if test -z "$HARNESS_ACTIVE" then mkdir -p "$TEST_RESULTS_DIR" @@ -1178,6 +1255,7 @@ then else mkdir -p "$TRASH_DIRECTORY" fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 @@ -1191,6 +1269,19 @@ then test_done fi +if test -n "$write_junit_xml" +then + junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" + mkdir -p "$junit_xml_dir" + junit_xml_base=${0##*/} + junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" + junit_attrs="name=\"${junit_xml_base%.sh}\"" + junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ + date +%Y-%m-%dT%H:%M:%S)\"" + write_junit_xml --truncate "" " " + junit_suite_start=$(test-tool date getnanos) +fi + # Provide an implementation of the 'yes' utility yes () { if test $# = 0 -- cgit v0.10.2-6-g49f6