For Gitlab to detect and report the failing tests directly on the MR front page. This has the disadvantage of making the test job output more silent, but the results are still available as artifacts (in JUnit XML format), which hopefully should be enough if we need to have a better look at it. The tests standard output is included in the reports too.
I believe it would also show some failure statistics that could help deciding on whether a test is regularly failing or not, but I wasn't able to really test that it does. In order to help it doing it, it also implements hashing of test sources to generate test IDs that are more stable than `source:line`, although it could be done later.
All the succeeding (and todo) tests are grouped together and reported as once, as they would otherwise be split into individual success lines in the Gitlab UI, which would make it unnecessary large. Only failures and skipped tests are reported as individual entries.
-- v2: gitlab: Use winetest JUnit output mode. winetest: Implement JUnit report output mode. winetest: Introduce some test report helpers. winetest: Pass output file handle to xprintf. winetest: Always use a temporary file for test output. winetest: Add printf attributes to xprintf. winetest: Add printf attributes to strmake.
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/send.c | 6 +++--- programs/winetest/util.c | 12 +++++++----- programs/winetest/winetest.h | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/programs/winetest/send.c b/programs/winetest/send.c index 257d9b75f1d..8c07f3b8ae7 100644 --- a/programs/winetest/send.c +++ b/programs/winetest/send.c @@ -30,7 +30,7 @@ #define URL_PATH "/submit" #define SEP "--8<--cut-here--8<--" #define CONTENT_HEADERS "Content-Type: multipart/form-data; boundary="" SEP ""\r\n" \ - "Content-Length: %u\r\n\r\n" + "Content-Length: %lu\r\n\r\n" static const char body1[] = "--" SEP "\r\n" "Content-Disposition: form-data; name="reportfile"; filename="%s"\r\n" "Content-Type: application/octet-stream\r\n\r\n"; @@ -218,7 +218,7 @@ send_file_direct (const char * url, const char *name) return 1; }
- str = strmake (&count, "Received %s (%d bytes).\n", + str = strmake (&count, "Received %s (%ld bytes).\n", name, filesize); ret = total < count || memcmp (str, buffer + total - count, count) != 0; free(str); @@ -412,7 +412,7 @@ send_file_wininet (const char *url, const char *name) while (bytes_read != 0);
free(str); - str = strmake (&count, "Received %s (%d bytes).\n", + str = strmake (&count, "Received %s (%ld bytes).\n", name, filesize); if (total < count || memcmp (str, buffer + total - count, count) != 0) { buffer[total] = 0; diff --git a/programs/winetest/util.c b/programs/winetest/util.c index bd5b584c3c4..f1f887020a1 100644 --- a/programs/winetest/util.c +++ b/programs/winetest/util.c @@ -50,7 +50,8 @@ char *xstrdup( const char *str ) return res; }
-static char *vstrfmtmake (size_t *lenp, const char *fmt, va_list ap) +static char *vstrfmtmake( size_t *lenp, const char *fmt, va_list ap ) __WINE_PRINTF_ATTR(2,0); +static char *vstrfmtmake( size_t *lenp, const char *fmt, va_list ap ) { size_t size = 1000; char *p; @@ -77,14 +78,15 @@ char *vstrmake (size_t *lenp, va_list ap) return vstrfmtmake (lenp, fmt, ap); }
-char * WINAPIV strmake (size_t *lenp, ...) +char *WINAPIV strmake( size_t *len, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3); +char *WINAPIV strmake( size_t *len, const char *fmt, ... ) { va_list ap; char *p;
- va_start (ap, lenp); - p = vstrmake (lenp, ap); - va_end (ap); + va_start( ap, fmt ); + p = vstrfmtmake( len, fmt, ap ); + va_end( ap ); return p; }
diff --git a/programs/winetest/winetest.h b/programs/winetest/winetest.h index 60b661fd3b1..ab8898e7834 100644 --- a/programs/winetest/winetest.h +++ b/programs/winetest/winetest.h @@ -25,12 +25,14 @@ #include <stdio.h> #include <stdlib.h> #include <stdarg.h> +#include "wine/debug.h" + +extern char *WINAPIV strmake( size_t *len, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3);
void fatal (const char* msg); void warning (const char* msg); void WINAPIV xprintf (const char *fmt, ...); char *vstrmake (size_t *lenp, va_list ap); -char * WINAPIV strmake (size_t *lenp, ...); int goodtagchar (char c); const char *findbadtagchar (const char *tag);
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 22 +++++++++++----------- programs/winetest/util.c | 3 ++- programs/winetest/winetest.h | 5 ++--- 3 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index dc4fc4b3581..28f48b28cb2 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -516,8 +516,8 @@ static void print_version (void) xprintf (" Description=%s\n", description ); if (url) xprintf (" URL=%s\n", url ); - xprintf (" dwMajorVersion=%u\n dwMinorVersion=%u\n" - " dwBuildNumber=%u\n PlatformId=%u\n szCSDVersion=%s\n", + xprintf (" dwMajorVersion=%lu\n dwMinorVersion=%lu\n" + " dwBuildNumber=%lu\n PlatformId=%lu\n szCSDVersion=%s\n", ver.dwMajorVersion, ver.dwMinorVersion, ver.dwBuildNumber, ver.dwPlatformId, ver.szCSDVersion);
@@ -547,7 +547,7 @@ static void print_version (void) DWORD prodtype = 0;
pGetProductInfo(ver.dwMajorVersion, ver.dwMinorVersion, ver.wServicePackMajor, ver.wServicePackMinor, &prodtype); - xprintf(" dwProductInfo=%u\n", prodtype); + xprintf(" dwProductInfo=%lu\n", prodtype); } }
@@ -558,9 +558,9 @@ static void print_language(void) LANGID (WINAPI *pGetUserDefaultUILanguage)(void); LANGID (WINAPI *pGetThreadUILanguage)(void);
- xprintf (" SystemDefaultLCID=%04x\n", GetSystemDefaultLCID()); - xprintf (" UserDefaultLCID=%04x\n", GetUserDefaultLCID()); - xprintf (" ThreadLocale=%04x\n", GetThreadLocale()); + xprintf (" SystemDefaultLCID=%04lx\n", GetSystemDefaultLCID()); + xprintf (" UserDefaultLCID=%04lx\n", GetUserDefaultLCID()); + xprintf (" ThreadLocale=%04lx\n", GetThreadLocale());
hkernel32 = GetModuleHandleA("kernel32.dll"); pGetSystemPreferredUILanguages = (void*)GetProcAddress(hkernel32, "GetSystemPreferredUILanguages"); @@ -584,7 +584,7 @@ static void print_language(void) if (pGetThreadUILanguage) xprintf (" ThreadUILanguage=%04x\n", pGetThreadUILanguage()); xprintf (" KeyboardLayout=%p\n", GetKeyboardLayout(0)); - xprintf (" Country=%d\n", GetUserGeoID(GEOCLASS_NATION)); + xprintf (" Country=%ld\n", GetUserGeoID(GEOCLASS_NATION)); xprintf (" ACP=%d\n", GetACP()); }
@@ -907,10 +907,10 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch } if (status == -2) status = -GetLastError(); free(cmd); - xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size); + xprintf ("%s:%s:%04lx done (%d) in %lds %luB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size); if (size > MAX_OUTPUT_SIZE) { - xprintf ("%s:%s:%04x The test prints too much data (%u bytes)\n", test->name, subtest, pid, size); + xprintf ("%s:%s:%04lx The test prints too much data (%lu bytes)\n", test->name, subtest, pid, size); failures++; } else if (status) failures++; @@ -1102,7 +1102,7 @@ extract_test_proc (HMODULE hModule, LPCSTR lpszType, LPSTR lpszName, LONG_PTR lP xprintf (" %s=dll is missing the requested side-by-side version\n", dllname); break; default: - xprintf (" %s=load error %u\n", dllname, err); + xprintf (" %s=load error %lu\n", dllname, err); break; } } @@ -1191,7 +1191,7 @@ run_tests (char *logname, char *outdir) nextline = strsize?eol+1:NULL; if (eol > strres && *(eol-1) == '\r') eol--; } - xprintf (" %.*s\n", eol-strres, strres); + xprintf (" %.*s\n", (int)(eol-strres), strres); strres = nextline; } xprintf ("Operating system version:\n"); diff --git a/programs/winetest/util.c b/programs/winetest/util.c index f1f887020a1..07ab74d7996 100644 --- a/programs/winetest/util.c +++ b/programs/winetest/util.c @@ -90,7 +90,8 @@ char *WINAPIV strmake( size_t *len, const char *fmt, ... ) return p; }
-void WINAPIV xprintf (const char *fmt, ...) +void WINAPIV xprintf( const char *fmt, ... ) __WINE_PRINTF_ATTR(1,2); +void WINAPIV xprintf( const char *fmt, ... ) { va_list ap; size_t size; diff --git a/programs/winetest/winetest.h b/programs/winetest/winetest.h index ab8898e7834..ca7337ea3e0 100644 --- a/programs/winetest/winetest.h +++ b/programs/winetest/winetest.h @@ -27,19 +27,18 @@ #include <stdarg.h> #include "wine/debug.h"
+extern HANDLE logfile; +extern void WINAPIV xprintf( const char *fmt, ... ) __WINE_PRINTF_ATTR(1,2); extern char *WINAPIV strmake( size_t *len, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3);
void fatal (const char* msg); void warning (const char* msg); -void WINAPIV xprintf (const char *fmt, ...); char *vstrmake (size_t *lenp, va_list ap); int goodtagchar (char c); const char *findbadtagchar (const char *tag);
int send_file (const char *url, const char *name);
-extern HANDLE logfile; - /* GUI definitions */
#include <windows.h>
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index 28f48b28cb2..997f35b8fa0 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -883,28 +883,22 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch } else { + char *data, tmpname[MAX_PATH]; + HANDLE tmpfile = create_temp_file( tmpname ); int status; DWORD pid, size, start = GetTickCount(); char *cmd = strmake (NULL, "%s %s", test->exename, subtest); + report (R_STEP, "Running: %s:%s", test->name, subtest); xprintf ("%s:%s start %s\n", test->name, subtest, file); /* Flush to disk so we know which test caused Windows to crash if it does */ FlushFileBuffers(out_file); - if (quiet_mode > 1) - { - char *data, tmpname[MAX_PATH]; - HANDLE tmpfile = create_temp_file( tmpname ); - status = run_ex (cmd, tmpfile, tempdir, 120000, FALSE, &pid); - data = flush_temp_file( tmpname, tmpfile, &size ); - if (status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); - free( data ); - } - else - { - DWORD start_size = GetFileSize( out_file, NULL ); - status = run_ex (cmd, out_file, tempdir, 120000, FALSE, &pid); - size = GetFileSize( out_file, NULL ) - start_size; - } + + status = run_ex( cmd, tmpfile, tempdir, 120000, FALSE, &pid ); + data = flush_temp_file( tmpname, tmpfile, &size ); + if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); + free( data ); + if (status == -2) status = -GetLastError(); free(cmd); xprintf ("%s:%s:%04lx done (%d) in %lds %luB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size);
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 1 + programs/winetest/util.c | 8 +++----- programs/winetest/winetest.h | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index 997f35b8fa0..bd4744557dc 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -68,6 +68,7 @@ static char build_id[64]; static BOOL is_wow64; static int failures; static int quiet_mode; +static HANDLE logfile;
/* filters for running only specific tests */ static char **filters; diff --git a/programs/winetest/util.c b/programs/winetest/util.c index 07ab74d7996..17313c4115f 100644 --- a/programs/winetest/util.c +++ b/programs/winetest/util.c @@ -25,8 +25,6 @@
#include "winetest.h"
-HANDLE logfile = 0; - void *xalloc (size_t len) { void *p = malloc( len ); @@ -90,8 +88,8 @@ char *WINAPIV strmake( size_t *len, const char *fmt, ... ) return p; }
-void WINAPIV xprintf( const char *fmt, ... ) __WINE_PRINTF_ATTR(1,2); -void WINAPIV xprintf( const char *fmt, ... ) +void WINAPIV output( HANDLE file, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3); +void WINAPIV output( HANDLE file, const char *fmt, ... ) { va_list ap; size_t size; @@ -103,7 +101,7 @@ void WINAPIV xprintf( const char *fmt, ... ) head = buffer; va_end (ap); while (size) { - if (!WriteFile( logfile, head, size, &written, NULL )) + if (!WriteFile( file, head, size, &written, NULL )) report (R_FATAL, "Can't write logs: %u", GetLastError()); head += written; size -= written; diff --git a/programs/winetest/winetest.h b/programs/winetest/winetest.h index ca7337ea3e0..8fb93e93dbb 100644 --- a/programs/winetest/winetest.h +++ b/programs/winetest/winetest.h @@ -27,8 +27,8 @@ #include <stdarg.h> #include "wine/debug.h"
-extern HANDLE logfile; -extern void WINAPIV xprintf( const char *fmt, ... ) __WINE_PRINTF_ATTR(1,2); +#define xprintf( fmt, ... ) output( logfile, fmt, ## __VA_ARGS__ ) +extern void WINAPIV output( HANDLE file, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3); extern char *WINAPIV strmake( size_t *len, const char *fmt, ... ) __WINE_PRINTF_ATTR(2,3);
void fatal (const char* msg);
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index bd4744557dc..375e5a5daed 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -870,6 +870,26 @@ get_subtests (const char *tempdir, struct wine_test *test, LPSTR res_name) return 0; }
+static void report_test_start( struct wine_test *test, const char *subtest, const char *file ) +{ + report( R_STEP, "Running: %s:%s", test->name, subtest ); + xprintf( "%s:%s start %s\n", test->name, subtest, file ); +} + +static void report_test_done( struct wine_test *test, const char *subtest, const char *file, DWORD pid, DWORD ticks, + HANDLE out_file, UINT status, const char *data, DWORD size ) +{ + if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); + xprintf( "%s:%s:%04lx done (%d) in %lds %luB\n", test->name, subtest, pid, status, ticks / 1000, size ); + if (size > MAX_OUTPUT_SIZE) xprintf( "%s:%s:%04lx The test prints too much data (%lu bytes)\n", test->name, subtest, pid, size ); +} + +static void report_test_skip( struct wine_test *test, const char *subtest, const char *file ) +{ + report( R_STEP, "Skipping: %s:%s", test->name, subtest ); + xprintf( "%s:%s skipped %s\n", test->name, subtest, file ); +} + static void run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const char *tempdir) { @@ -878,8 +898,7 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch
if (test_filtered_out( test->name, subtest )) { - report (R_STEP, "Skipping: %s:%s", test->name, subtest); - xprintf ("%s:%s skipped %s\n", test->name, subtest, file); + report_test_skip( test, subtest, file ); nr_of_skips++; } else @@ -890,25 +909,19 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch DWORD pid, size, start = GetTickCount(); char *cmd = strmake (NULL, "%s %s", test->exename, subtest);
- report (R_STEP, "Running: %s:%s", test->name, subtest); - xprintf ("%s:%s start %s\n", test->name, subtest, file); + report_test_start( test, subtest, file ); /* Flush to disk so we know which test caused Windows to crash if it does */ FlushFileBuffers(out_file);
status = run_ex( cmd, tmpfile, tempdir, 120000, FALSE, &pid ); + if (status == -2 && GetLastError()) status = -GetLastError(); + free(cmd); + data = flush_temp_file( tmpname, tmpfile, &size ); - if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); + report_test_done( test, subtest, file, pid, GetTickCount() - start, out_file, status, data, size ); free( data );
- if (status == -2) status = -GetLastError(); - free(cmd); - xprintf ("%s:%s:%04lx done (%d) in %lds %luB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size); - if (size > MAX_OUTPUT_SIZE) - { - xprintf ("%s:%s:%04lx The test prints too much data (%lu bytes)\n", test->name, subtest, pid, size); - failures++; - } - else if (status) failures++; + if (status || size > MAX_OUTPUT_SIZE) failures++; } if (failures) report (R_STATUS, "Running tests - %u failures", failures); }
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 154 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index 375e5a5daed..0bf6fea2204 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -69,6 +69,7 @@ static BOOL is_wow64; static int failures; static int quiet_mode; static HANDLE logfile; +static HANDLE junit;
/* filters for running only specific tests */ static char **filters; @@ -870,6 +871,31 @@ get_subtests (const char *tempdir, struct wine_test *test, LPSTR res_name) return 0; }
+static char *xmlescape( const char *src, const char *end, int comment ) +{ + char *dst = calloc( 1, (end - src) * 6 + 1 ), *tmp; + + while (end > src && (end[-1] == '\r' || end[-1] == '\n')) end--; + for (tmp = dst; tmp && src < end; src++) + { + if (comment && *src == '-') tmp += sprintf( tmp, "-" ); + else if (comment) *tmp++ = *src; + else if (*src == '&') tmp += sprintf( tmp, "&" ); + else if (*src == '"') tmp += sprintf( tmp, """ ); + else if (*src == '<') tmp += sprintf( tmp, "<" ); + else if (*src == '>') tmp += sprintf( tmp, ">" ); + else if (*src < ' ' && *src != '\t' && *src != '\r' && *src != '\n') + { + char *esc = strmake( NULL, "\x%02x", *src ); + tmp += sprintf( tmp, "%s", esc ); + free( esc ); + } + else *tmp++ = *src; + } + + return dst; +} + static void report_test_start( struct wine_test *test, const char *subtest, const char *file ) { report( R_STEP, "Running: %s:%s", test->name, subtest ); @@ -882,6 +908,108 @@ static void report_test_done( struct wine_test *test, const char *subtest, const if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); xprintf( "%s:%s:%04lx done (%d) in %lds %luB\n", test->name, subtest, pid, status, ticks / 1000, size ); if (size > MAX_OUTPUT_SIZE) xprintf( "%s:%s:%04lx The test prints too much data (%lu bytes)\n", test->name, subtest, pid, size ); + + if (junit) + { + int total = 0, fail_total = 0, skip_total = 0, failures = 0; + const char *next, *line, *ptr, *dir = strrchr( file, '/' ); + float time, last_time = 0; + + for (line = next = data; *line; line = next) + { + int count, todo_count, flaky_count, fail_count, skip_count; + + if ((next = strchr( next, '\n' ))) next += 1; + else next = line + strlen( line ); + if (!(ptr = strchr( line, ' ' ))) continue; + + if (sscanf( ptr, " %d tests executed (%d marked as todo, %d as flaky, %d failur%*[^)]), %d skipped.", + &count, &todo_count, &flaky_count, &fail_count, &skip_count ) == 5) + { + total += count; + fail_total += fail_count; + skip_total += skip_count; + } + } + + output( junit, " <testsuite name="%s:%s" file="%s" time="%f" tests="%d" failures="%d" skipped="%d">\n", + test->name, subtest, file, ticks / 1000.0, total, fail_total, skip_total ); + + for (line = next = data; *line; line = next) + { + struct { const char *pattern; int length; int error; } patterns[] = + { + #define X(x, y) {.pattern = x, .length = strlen(x), .error = y} + X("Tests skipped: ", -1), + X("Test failed: ", 1), + X("Test succeeded inside todo block: ", 1), + X("Test succeeded inside flaky todo block: ", 1), + #undef X + }; + char src[256], *name, *message; + int i, n, len; + + if ((next = strchr( next, '\n' ))) next += 1; + else next = line + strlen( line ); + + if ((len = sscanf( line, "%255[^:\n]:%d:%f", src, &n, &time )) != 2 && len != 3) continue; + if (len == 2) time = last_time; + + if (!(ptr = strchr( line, ' ' ))) continue; + ptr++; + + message = xmlescape( ptr, next, 0 ); + name = strmake( NULL, "%s:%d %s", src, n, message ); + for (i = 0; i < ARRAY_SIZE(patterns); i++) + { + if (!strncmp( ptr, patterns[i].pattern, patterns[i].length )) + { + output( junit, " <testcase classname="%s:%s" name="%s:%s %s" file="%.*s%s#L%d" assertions="1" time="%f">", + test->name, subtest, test->name, subtest, name, (int)(dir - file + 1), file, src, n, time - last_time ); + if (patterns[i].error == -1) output( junit, "<system-out>%s</system-out><skipped/>", message ); + else if (patterns[i].error == 1) output( junit, "<system-out>%s</system-out><failure/>", message ); + output( junit, "</testcase>\n" ); + + if (patterns[i].error == 1) failures++; + last_time = time; + break; + } + } + free( name ); + free( message ); + } + + if (total > fail_total + skip_total) + { + output( junit, " <testcase classname="%s:%s" name="%d succeeding tests" file="%s" assertions="%d" time="%f"/>\n", + test->name, subtest, total - fail_total - skip_total, file, total - fail_total - skip_total, ticks / 1000.0 - last_time ); + } + + if (status == WAIT_TIMEOUT) + { + output( junit, " <testcase classname="%s:%s" name="%s:%s timeout" file="%s" assertions="%d" time="%f">", + test->name, subtest, test->name, subtest, file, total, ticks / 1000.0 ); + output( junit, "<system-out>Test timeout after %f seconds</system-out><failure/>", ticks / 1000.0 ); + output( junit, "</testcase>\n" ); + } + else if (status != failures) + { + output( junit, " <testcase classname="%s:%s" name="%s:%s status %d" file="%s" assertions="%d" time="%f">", + test->name, subtest, test->name, subtest, status, file, total, ticks / 1000.0 ); + output( junit, "<system-out>Test exited with status %d</system-out><failure/>", status ); + output( junit, "</testcase>\n" ); + } + if (size > MAX_OUTPUT_SIZE) + { + output( junit, " <testcase classname="%s:%s" name="%s:%s output overflow" file="%s" assertions="%d" time="%f">", + test->name, subtest, test->name, subtest, file, total, ticks / 1000.0 ); + output( junit, "<system-out>Test prints too much data (%lukB > %ukB)</system-out><failure/>", + size / 1024, MAX_OUTPUT_SIZE / 1024 ); + output( junit, "</testcase>\n" ); + } + + output( junit, " </testsuite>\n" ); + } }
static void report_test_skip( struct wine_test *test, const char *subtest, const char *file ) @@ -1183,6 +1311,11 @@ run_tests (char *logname, char *outdir)
report (R_DIR, "%s", tempdir);
+ if (junit) + { + output( junit, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ); + output( junit, "<testsuites>\n" ); + } xprintf ("Version 4\n"); xprintf ("Tests from build %s\n", build_id[0] ? build_id : "-" ); xprintf ("Archive: -\n"); /* no longer used */ @@ -1268,6 +1401,11 @@ run_tests (char *logname, char *outdir) } } report (R_DELTA, 0, "Running: Done"); + if (junit) + { + output( junit, "</testsuites>\n" ); + CloseHandle( junit ); + }
report (R_STATUS, "Cleaning up - %u failures", failures); if (strcmp(logname, "-") != 0) CloseHandle( logfile ); @@ -1348,6 +1486,7 @@ usage (void) " -n exclude the specified tests\n" " -p shutdown when the tests are done\n" " -q quiet mode, no output at all\n" +" -J FILE output junit XML report into FILE\n" " -o FILE put report into FILE, do not submit\n" " -s FILE submit FILE, do not run tests\n" " -S URL URL to submit the results to\n" @@ -1359,7 +1498,7 @@ usage (void) int __cdecl main( int argc, char *argv[] ) { BOOL (WINAPI *pIsWow64Process)(HANDLE hProcess, PBOOL Wow64Process); - char *logname = NULL, *outdir = NULL; + char *logname = NULL, *outdir = NULL, *path = NULL; const char *extract = NULL; const char *cp, *submit = NULL, *submiturl = NULL; int reset_env = 1; @@ -1407,6 +1546,14 @@ int __cdecl main( int argc, char *argv[] ) exit( 2 ); } break; + case 'J': + if (!(path = argv[++i])) + { + usage(); + exit( 2 ); + } + junit = create_output_file( path ); + break; case 'm': if (!(email = argv[++i])) { @@ -1524,6 +1671,11 @@ int __cdecl main( int argc, char *argv[] ) SetEnvironmentVariableA( "WINETEST_INTERACTIVE", "0" ); SetEnvironmentVariableA( "WINETEST_REPORT_SUCCESS", "0" ); } + if (junit) + { + SetEnvironmentVariableA( "WINETEST_COLOR", "0" ); + SetEnvironmentVariableA( "WINETEST_TIME", "1" ); + }
if (nb_filters && !exclude_tests) {
From: Rémi Bernon rbernon@codeweavers.com
--- tools/gitlab/test.yml | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-)
diff --git a/tools/gitlab/test.yml b/tools/gitlab/test.yml index ad2258ce3dd..e5d0602884b 100644 --- a/tools/gitlab/test.yml +++ b/tools/gitlab/test.yml @@ -57,7 +57,13 @@ test-linux-64: - job: build-linux script: - export WINETEST_COLOR=1 - - wine usr/local/lib/wine/x86_64-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL $INCLUDE_TESTS + - wine usr/local/lib/wine/x86_64-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL -J winetest.xml $INCLUDE_TESTS + artifacts: + when: always + paths: + - winetest.xml + reports: + junit: winetest.xml
test-linux-32: extends: .wine-test @@ -69,7 +75,13 @@ test-linux-32: - job: build-linux script: - export WINETEST_COLOR=1 - - wine usr/local/lib/wine/i386-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL -n $EXCLUDE_TESTS + - wine usr/local/lib/wine/i386-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL -J winetest.xml -n $EXCLUDE_TESTS + artifacts: + when: always + paths: + - winetest.xml + reports: + junit: winetest.xml
test-win10-21h2-32: stage: test @@ -84,13 +96,23 @@ test-win10-21h2-32: - win10-21h2 script: - $WINETEST_ARGS = @(Get-Content ./winetest.args) - - if ($WINETEST_ARGS.count -gt 0) { ./winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } else { echo $null >winetest.log } + - | + if ($WINETEST_ARGS.count -gt 0) { + ./winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL -J winetest.tmp @WINETEST_ARGS >winetest.log + } else { + echo '<?xml version="1.0" encoding="UTF-8"?><testsuites/>' >winetest.tmp + echo $null >winetest.log + } after_script: + - Get-Content ./winetest.tmp | Set-Content -Encoding utf8 winetest.xml - Get-Content ./winetest.log artifacts: when: always paths: + - winetest.xml - winetest.log + reports: + junit: winetest.xml
test-win10-21h2-64: stage: test @@ -105,13 +127,23 @@ test-win10-21h2-64: - win10-21h2 script: - $WINETEST_ARGS = @(Get-Content ./winetest.args) - - if ($WINETEST_ARGS.count -gt 0) { ./winetest64.exe -q -q -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } else { echo $null >winetest.log } + - | + if ($WINETEST_ARGS.count -gt 0) { + ./winetest64.exe -q -q -o - -t gitlab -u $CI_JOB_URL -J winetest.tmp @WINETEST_ARGS >winetest.log + } else { + echo '<?xml version="1.0" encoding="UTF-8"?><testsuites/>' >winetest.tmp + echo $null >winetest.log + } after_script: + - Get-Content ./winetest.tmp | Set-Content -Encoding utf8 winetest.xml - Get-Content ./winetest.log artifacts: when: always paths: + - winetest.xml - winetest.log + reports: + junit: winetest.xml
debian-32: extends: .wine-test
v2: Drop the test ids computation for now, output the JUnit report to a separate file instead of replacing the classic output, add printf attributes to some functions to check arguments.
-
Rémi Bernon -
Rémi Bernon (＠rbernon)