For Gitlab to detect and report the failing tests directly on the MR front page. This has the disadvantage of making the test job output more silent, but the results are still available as artifacts (in JUnit XML format), which hopefully should be enough if we need to have a better look at it. The tests standard output is included in the reports too.
I believe it would also show some failure statistics that could help deciding on whether a test is regularly failing or not, but I wasn't able to really test that it does. In order to help it doing it, it also implements hashing of test sources to generate test IDs that are more stable than `source:line`, although it could be done later.
All the succeeding (and todo) tests are grouped together and reported as once, as they would otherwise be split into individual success lines in the Gitlab UI, which would make it unnecessary large. Only failures and skipped tests are reported as individual entries.
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index dc4fc4b3581..f28f907dc0d 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -883,28 +883,22 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch } else { + char *data, tmpname[MAX_PATH]; + HANDLE tmpfile = create_temp_file( tmpname ); int status; DWORD pid, size, start = GetTickCount(); char *cmd = strmake (NULL, "%s %s", test->exename, subtest); + report (R_STEP, "Running: %s:%s", test->name, subtest); xprintf ("%s:%s start %s\n", test->name, subtest, file); /* Flush to disk so we know which test caused Windows to crash if it does */ FlushFileBuffers(out_file); - if (quiet_mode > 1) - { - char *data, tmpname[MAX_PATH]; - HANDLE tmpfile = create_temp_file( tmpname ); - status = run_ex (cmd, tmpfile, tempdir, 120000, FALSE, &pid); - data = flush_temp_file( tmpname, tmpfile, &size ); - if (status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); - free( data ); - } - else - { - DWORD start_size = GetFileSize( out_file, NULL ); - status = run_ex (cmd, out_file, tempdir, 120000, FALSE, &pid); - size = GetFileSize( out_file, NULL ) - start_size; - } + + status = run_ex( cmd, tmpfile, tempdir, 120000, FALSE, &pid ); + data = flush_temp_file( tmpname, tmpfile, &size ); + if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); + free( data ); + if (status == -2) status = -GetLastError(); free(cmd); xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size);
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index f28f907dc0d..4764075d166 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -869,6 +869,26 @@ get_subtests (const char *tempdir, struct wine_test *test, LPSTR res_name) return 0; }
+static void report_test_start( struct wine_test *test, const char *subtest, const char *file ) +{ + report( R_STEP, "Running: %s:%s", test->name, subtest ); + xprintf( "%s:%s start %s\n", test->name, subtest, file ); +} + +static void report_test_done( struct wine_test *test, const char *subtest, const char *file, UINT pid, UINT ticks, + HANDLE out_file, UINT status, const char *output, DWORD size ) +{ + if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, output, size, &size, NULL ); + xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, ticks / 1000, size); + if (size > MAX_OUTPUT_SIZE) xprintf ("%s:%s:%04x The test prints too much data (%u bytes)\n", test->name, subtest, pid, size); +} + +static void report_test_skip( struct wine_test *test, const char *subtest, const char *file ) +{ + report( R_STEP, "Skipping: %s:%s", test->name, subtest ); + xprintf( "%s:%s skipped %s\n", test->name, subtest, file ); +} + static void run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const char *tempdir) { @@ -877,8 +897,7 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch
if (test_filtered_out( test->name, subtest )) { - report (R_STEP, "Skipping: %s:%s", test->name, subtest); - xprintf ("%s:%s skipped %s\n", test->name, subtest, file); + report_test_skip( test, subtest, file ); nr_of_skips++; } else @@ -889,25 +908,19 @@ run_test (struct wine_test* test, const char* subtest, HANDLE out_file, const ch DWORD pid, size, start = GetTickCount(); char *cmd = strmake (NULL, "%s %s", test->exename, subtest);
- report (R_STEP, "Running: %s:%s", test->name, subtest); - xprintf ("%s:%s start %s\n", test->name, subtest, file); + report_test_start( test, subtest, file ); /* Flush to disk so we know which test caused Windows to crash if it does */ FlushFileBuffers(out_file);
status = run_ex( cmd, tmpfile, tempdir, 120000, FALSE, &pid ); + if (status == -2 && GetLastError()) status = -GetLastError(); + free(cmd); + data = flush_temp_file( tmpname, tmpfile, &size ); - if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, data, size, &size, NULL ); + report_test_done( test, subtest, file, pid, GetTickCount() - start, out_file, status, data, size ); free( data );
- if (status == -2) status = -GetLastError(); - free(cmd); - xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, (GetTickCount()-start)/1000, size); - if (size > MAX_OUTPUT_SIZE) - { - xprintf ("%s:%s:%04x The test prints too much data (%u bytes)\n", test->name, subtest, pid, size); - failures++; - } - else if (status) failures++; + if (status || size > MAX_OUTPUT_SIZE) failures++; } if (failures) report (R_STATUS, "Running tests - %u failures", failures); }
From: Rémi Bernon rbernon@codeweavers.com
--- programs/winetest/main.c | 161 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 155 insertions(+), 6 deletions(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index 4764075d166..90e9bbad2eb 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -68,6 +68,7 @@ static char build_id[64]; static BOOL is_wow64; static int failures; static int quiet_mode; +static int junit_mode;
/* filters for running only specific tests */ static char **filters; @@ -869,24 +870,152 @@ get_subtests (const char *tempdir, struct wine_test *test, LPSTR res_name) return 0; }
+static char *strescape( const char *src, const char *end, int comment ) +{ + char *dst = calloc( 1, (end - src) * 6 + 1 ), *tmp; + + while (end > src && (end[-1] == '\r' || end[-1] == '\n')) end--; + for (tmp = dst; tmp && src < end; src++) + { + if (!junit_mode) *tmp++ = *src; + else if (comment && *src == '-') tmp += sprintf( tmp, "-" ); + else if (comment) *tmp++ = *src; + else if (*src == '&') tmp += sprintf( tmp, "&" ); + else if (*src == '"') tmp += sprintf( tmp, """ ); + else if (*src == '<') tmp += sprintf( tmp, "<" ); + else if (*src == '>') tmp += sprintf( tmp, ">" ); + else if (*src < ' ' && *src != '\t' && *src != '\r' && *src != '\n') + { + char *esc = strmake( NULL, "\x%02x", *src ); + tmp += sprintf( tmp, "%s", esc ); + free( esc ); + } + else *tmp++ = *src; + } + + return dst; +} + static void report_test_start( struct wine_test *test, const char *subtest, const char *file ) { report( R_STEP, "Running: %s:%s", test->name, subtest ); - xprintf( "%s:%s start %s\n", test->name, subtest, file ); + if (!junit_mode) xprintf( "%s:%s start %s\n", test->name, subtest, file ); }
static void report_test_done( struct wine_test *test, const char *subtest, const char *file, UINT pid, UINT ticks, HANDLE out_file, UINT status, const char *output, DWORD size ) { - if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, output, size, &size, NULL ); - xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, ticks / 1000, size); - if (size > MAX_OUTPUT_SIZE) xprintf ("%s:%s:%04x The test prints too much data (%u bytes)\n", test->name, subtest, pid, size); + if (junit_mode) + { + int total = 0, fail_total = 0, skip_total = 0, failures = 0; + const char *next, *line, *ptr, *dir = strrchr( file, '/' ); + float time, last_time = 0; + char *tmp; + + for (line = next = output; *line; line = next) + { + int count, todo_count, flaky_count, fail_count, skip_count; + + if ((next = strchr( next, '\n' ))) next += 1; + else next = line + strlen( line ); + if (!(ptr = strchr( line, ' ' ))) continue; + + if (sscanf( ptr, " %d tests executed (%d marked as todo, %d as flaky, %d failur%*[^)]), %d skipped.", + &count, &todo_count, &flaky_count, &fail_count, &skip_count ) == 5) + { + total += count; + fail_total += fail_count; + skip_total += skip_count; + } + } + + xprintf( " <testsuite name="%s:%s" file="%s" time="%f" tests="%d" failures="%d" skipped="%d">\n", test->name, subtest, + file, ticks / 1000.0, total, fail_total, skip_total ); + xprintf( " <system-out>" ); + tmp = strescape( output, output + size, 0 ); + WriteFile( out_file, tmp, strlen( tmp ), &size, NULL ); + free( tmp ); + xprintf( "</system-out>\n" ); + + for (line = next = output; *line; line = next) + { + struct { const char *pattern; int length; int error; } patterns[] = + { + #define X(x, y) {.pattern = x, .length = strlen(x), .error = y} + X("Tests skipped: ", -1), + X("Test failed: ", 1), + X("Test succeeded inside todo block: ", 1), + X("Test succeeded inside flaky todo block: ", 1), + #undef X + }; + char src[256], *name; + int i, n, len; + + if ((next = strchr( next, '\n' ))) next += 1; + else next = line + strlen( line ); + + if ((len = sscanf( line, "%255[^:\n]:%d:%f", src, &n, &time )) != 2 && len != 3) continue; + if (len == 2) time = last_time; + + if (!(ptr = strchr( line, ' ' ))) continue; + ptr++; + + name = strmake( NULL, "%s:%d", src, n ); + for (i = 0; i < ARRAY_SIZE(patterns); i++) + { + if (!strncmp( ptr, patterns[i].pattern, patterns[i].length )) + { + char *message = strescape( ptr, next, 0 ); + xprintf( " <testcase classname="%s:%s" name="%s:%s %s" file="%.*s%s#L%d" assertions="1" time="%f">", + test->name, subtest, test->name, subtest, name, dir - file + 1, file, src, n, time - last_time ); + if (patterns[i].error == -1) xprintf( "<system-out>%s</system-out><skipped/>", message ); + else if (patterns[i].error == 1) xprintf( "<system-out>%s</system-out><failure/>", message ); + xprintf( "</testcase>\n" ); + free( message ); + + if (patterns[i].error == 1) failures++; + last_time = time; + break; + } + } + free( name ); + } + + if (total > fail_total + skip_total) + { + xprintf( " <testcase classname="%s:%s" name="%d succeeding tests" file="%s" assertions="%d" time="%f"/>\n", + test->name, subtest, total - fail_total - skip_total, file, total - fail_total - skip_total, ticks / 1000.0 - last_time ); + } + + if (status == WAIT_TIMEOUT) + { + xprintf( " <testcase classname="%s:%s" name="%s:%s timeout" file="%s" assertions="%d" time="%f">", + test->name, subtest, test->name, subtest, file, total, ticks / 1000.0 ); + xprintf( "<system-out>Test timeout after %f seconds</system-out><failure/>", ticks / 1000.0 ); + xprintf( "</testcase>\n" ); + } + else if (status != failures) + { + xprintf( " <testcase classname="%s:%s" name="%s:%s status %d" file="%s" assertions="%d" time="%f">", + test->name, subtest, test->name, subtest, status, file, total, ticks / 1000.0 ); + xprintf( "<system-out>Test exited with status %d</system-out><failure/>", status ); + xprintf( "</testcase>\n" ); + } + + xprintf( " </testsuite>\n" ); + } + else + { + if (quiet_mode <= 1 || status || size > MAX_OUTPUT_SIZE) WriteFile( out_file, output, size, &size, NULL ); + xprintf ("%s:%s:%04x done (%d) in %ds %uB\n", test->name, subtest, pid, status, ticks / 1000, size); + if (size > MAX_OUTPUT_SIZE) xprintf ("%s:%s:%04x The test prints too much data (%u bytes)\n", test->name, subtest, pid, size); + } }
static void report_test_skip( struct wine_test *test, const char *subtest, const char *file ) { report( R_STEP, "Skipping: %s:%s", test->name, subtest ); - xprintf( "%s:%s skipped %s\n", test->name, subtest, file ); + if (!junit_mode) xprintf( "%s:%s skipped %s\n", test->name, subtest, file ); }
static void @@ -1182,6 +1311,11 @@ run_tests (char *logname, char *outdir)
report (R_DIR, "%s", tempdir);
+ if (junit_mode) + { + xprintf( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ); + xprintf( "<testsuites>\n<!--\n" ); + } xprintf ("Version 4\n"); xprintf ("Tests from build %s\n", build_id[0] ? build_id : "-" ); xprintf ("Archive: -\n"); /* no longer used */ @@ -1189,6 +1323,7 @@ run_tests (char *logname, char *outdir) xprintf ("Build info:\n"); strres = extract_rcdata ("BUILD_INFO", "STRINGRES", &strsize); while (strres) { + char *tmp; eol = memchr (strres, '\n', strsize); if (!eol) { nextline = NULL; @@ -1198,7 +1333,10 @@ run_tests (char *logname, char *outdir) nextline = strsize?eol+1:NULL; if (eol > strres && *(eol-1) == '\r') eol--; } - xprintf (" %.*s\n", eol-strres, strres); + + tmp = strescape( strres, eol, 1 ); + xprintf (" %s\n", tmp ); + free( tmp ); strres = nextline; } xprintf ("Operating system version:\n"); @@ -1237,6 +1375,7 @@ run_tests (char *logname, char *outdir) if (aborting) return logname;
xprintf ("Test output:\n" ); + if (junit_mode) xprintf ("-->\n" );
report (R_DELTA, 0, "Extracting: Done");
@@ -1267,6 +1406,7 @@ run_tests (char *logname, char *outdir) } } report (R_DELTA, 0, "Running: Done"); + if (junit_mode) xprintf( "</testsuites>\n" );
report (R_STATUS, "Cleaning up - %u failures", failures); if (strcmp(logname, "-") != 0) CloseHandle( logfile ); @@ -1347,6 +1487,7 @@ usage (void) " -n exclude the specified tests\n" " -p shutdown when the tests are done\n" " -q quiet mode, no output at all\n" +" -J junit mode, output XML report\n" " -o FILE put report into FILE, do not submit\n" " -s FILE submit FILE, do not run tests\n" " -S URL URL to submit the results to\n" @@ -1406,6 +1547,9 @@ int __cdecl main( int argc, char *argv[] ) exit( 2 ); } break; + case 'J': + junit_mode = 1; + break; case 'm': if (!(email = argv[++i])) { @@ -1523,6 +1667,11 @@ int __cdecl main( int argc, char *argv[] ) SetEnvironmentVariableA( "WINETEST_INTERACTIVE", "0" ); SetEnvironmentVariableA( "WINETEST_REPORT_SUCCESS", "0" ); } + if (junit_mode) + { + SetEnvironmentVariableA( "WINETEST_COLOR", "0" ); + SetEnvironmentVariableA( "WINETEST_TIME", "1" ); + }
if (nb_filters && !exclude_tests) {
From: Rémi Bernon rbernon@codeweavers.com
--- tools/gitlab/test.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-)
diff --git a/tools/gitlab/test.yml b/tools/gitlab/test.yml index ad2258ce3dd..ade56a609d5 100644 --- a/tools/gitlab/test.yml +++ b/tools/gitlab/test.yml @@ -57,7 +57,13 @@ test-linux-64: - job: build-linux script: - export WINETEST_COLOR=1 - - wine usr/local/lib/wine/x86_64-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL $INCLUDE_TESTS + - wine usr/local/lib/wine/x86_64-windows/winetest.exe -q -q -J -o winetest.xml -t gitlab -u $CI_JOB_URL $INCLUDE_TESTS + artifacts: + when: always + paths: + - winetest.xml + reports: + junit: winetest.xml
test-linux-32: extends: .wine-test @@ -69,7 +75,13 @@ test-linux-32: - job: build-linux script: - export WINETEST_COLOR=1 - - wine usr/local/lib/wine/i386-windows/winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL -n $EXCLUDE_TESTS + - wine usr/local/lib/wine/i386-windows/winetest.exe -q -q -J -o winetest.xml -t gitlab -u $CI_JOB_URL -n $EXCLUDE_TESTS + artifacts: + when: always + paths: + - winetest.xml + reports: + junit: winetest.xml
test-win10-21h2-32: stage: test @@ -84,13 +96,17 @@ test-win10-21h2-32: - win10-21h2 script: - $WINETEST_ARGS = @(Get-Content ./winetest.args) - - if ($WINETEST_ARGS.count -gt 0) { ./winetest.exe -q -q -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } else { echo $null >winetest.log } + - | + if ($WINETEST_ARGS.count -gt 0) { ./winetest.exe -q -q -J -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } + else { echo '<?xml version="1.0" encoding="UTF-8"?><testsuites/>' >winetest.log } after_script: - - Get-Content ./winetest.log + - Get-Content ./winetest.log | Set-Content -Encoding utf8 winetest.xml artifacts: when: always paths: - - winetest.log + - winetest.xml + reports: + junit: winetest.xml
test-win10-21h2-64: stage: test @@ -105,13 +121,17 @@ test-win10-21h2-64: - win10-21h2 script: - $WINETEST_ARGS = @(Get-Content ./winetest.args) - - if ($WINETEST_ARGS.count -gt 0) { ./winetest64.exe -q -q -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } else { echo $null >winetest.log } + - | + if ($WINETEST_ARGS.count -gt 0) { ./winetest64.exe -q -q -J -o - -t gitlab -u $CI_JOB_URL @WINETEST_ARGS >winetest.log } + else { echo '<?xml version="1.0" encoding="UTF-8"?><testsuites/>' >winetest.log } after_script: - - Get-Content ./winetest.log + - Get-Content ./winetest.log | Set-Content -Encoding utf8 winetest.xml artifacts: when: always paths: - - winetest.log + - winetest.xml + reports: + junit: winetest.xml
debian-32: extends: .wine-test
From: Rémi Bernon rbernon@codeweavers.com
--- tools/makedep.c | 37 +++++++++++++++++++++++++++---------- tools/wrc/parser.y | 4 +++- 2 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/tools/makedep.c b/tools/makedep.c index e8ffd2c8395..b564b53ea30 100644 --- a/tools/makedep.c +++ b/tools/makedep.c @@ -347,6 +347,16 @@ static void output( const char *format, ... ) }
+/******************************************************************* + * cmd_prefix + */ +static const char *cmd_prefix( const char *cmd ) +{ + if (!silent_rules) return ""; + return strmake( "$(quiet_%s)", cmd ); +} + + /******************************************************************* * strarray_get_value * @@ -2423,16 +2433,6 @@ static const char *get_debug_file( struct makefile *make, const char *name, unsi }
-/******************************************************************* - * cmd_prefix - */ -static const char *cmd_prefix( const char *cmd ) -{ - if (!silent_rules) return ""; - return strmake( "$(quiet_%s)", cmd ); -} - - /******************************************************************* * output_winegcc_command */ @@ -3627,11 +3627,28 @@ static void output_test_module( struct makefile *make, unsigned int arch ) struct strarray dep_libs = empty_strarray; struct strarray all_libs = empty_strarray; struct makefile *parent = get_parent_makefile( make ); + struct incl_file *source; const char *debug_file; unsigned int link_arch;
if (!get_link_arch( make, arch, &link_arch )) return;
+ LIST_FOR_EACH_ENTRY( source, &make->sources, struct incl_file, entry ) + { + const char *output_rsrc; + if (source->file->flags & FLAG_GENERATED) continue; + + output_rsrc = strmake( "%s%s.res", arch_dirs[arch], source->name ); + strarray_add( &make->res_files[arch], output_rsrc ); + output( "%s:", obj_dir_path( make, output_rsrc ) ); + output_filename( source->filename ); + output_filename( tools_path( make, "wrc" )); + output( "\n" ); + + output( "\t%secho "%s TESTSRC \"%s\"" | %s -u -o $@\n", cmd_prefix( "WRC" ), source->name, + source->filename, tools_path( make, "wrc" ) ); + } + strarray_addall( &all_libs, add_import_libs( make, &dep_libs, make->imports, IMPORT_TYPE_DIRECT, arch ) ); strarray_addall( &all_libs, add_import_libs( make, &dep_libs, default_imports, IMPORT_TYPE_DEFAULT, arch ) );
diff --git a/tools/wrc/parser.y b/tools/wrc/parser.y index 6b362b06b6b..94986435bee 100644 --- a/tools/wrc/parser.y +++ b/tools/wrc/parser.y @@ -2152,7 +2152,9 @@ static raw_data_t *load_file(string_t *filename, language_t lang) name = convert_string_utf8( filename, 0 ); else name = xstrdup( filename->str.cstr ); - if (!(path = wpp_find_include(name, input_name))) + if (*name == '/') + path = xstrdup( name ); + else if (!(path = wpp_find_include(name, input_name))) yyerror("Cannot open file %s", name); if (!(fp = fopen( path, "rb" ))) yyerror("Cannot open file %s", name);
From: Rémi Bernon rbernon@codeweavers.com
This hashes 5 lines of source code before and after the check, to create unique identifier for each test that is more robust to moving code and modified line numbers. --- programs/winetest/main.c | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-)
diff --git a/programs/winetest/main.c b/programs/winetest/main.c index 90e9bbad2eb..00a5e715c5d 100644 --- a/programs/winetest/main.c +++ b/programs/winetest/main.c @@ -36,6 +36,18 @@ #include "winetest.h" #include "resource.h"
+typedef struct +{ + ULONG Unknown[6]; + ULONG State[5]; + ULONG Count[2]; + UCHAR Buffer[64]; +} SHA_CTX; + +VOID WINAPI A_SHAInit(SHA_CTX *ctx); +VOID WINAPI A_SHAUpdate(SHA_CTX *ctx, const UCHAR *buffer, UINT size); +VOID WINAPI A_SHAFinal(SHA_CTX *ctx, PULONG result); + /* Don't submit the results if more than SKIP_LIMIT tests have been skipped */ #define SKIP_LIMIT 10
@@ -69,6 +81,7 @@ static BOOL is_wow64; static int failures; static int quiet_mode; static int junit_mode; +static int context_lines = 5;
/* filters for running only specific tests */ static char **filters; @@ -896,6 +909,36 @@ static char *strescape( const char *src, const char *end, int comment ) return dst; }
+static char *get_test_name( HMODULE module, const char *src, int num, int *size ) +{ + const char *source, *next, *line; + ULONG sha1_hash[5]; + HGLOBAL handle; + SHA_CTX ctx; + HRSRC rsrc; + + if (!(rsrc = FindResourceA( module, src, "TESTSRC" ))) return NULL; + if (!(*size = SizeofResource( module, rsrc ))) return NULL; + if (!(handle = LoadResource( module, rsrc))) return NULL; + if (!(source = LockResource( handle ))) return NULL; + + A_SHAInit( &ctx ); + for (line = next = source; line < source + *size; line = next) + { + if ((next = memchr( next, '\n', source + *size - next ))) next += 1; + else next = source + *size; + + if (--num > context_lines) continue; + A_SHAUpdate( &ctx, (BYTE *)line, next - line ); + while (line < next && (*line == ' ' || *line == '\t')) line++; + if (num <= -context_lines) break; + } + A_SHAFinal( &ctx, sha1_hash ); + + if (num > context_lines) return NULL; + return strmake( NULL, "%08x%08x%08x%08x%08x", sha1_hash[0], sha1_hash[1], sha1_hash[2], sha1_hash[3], sha1_hash[4] ); +} + static void report_test_start( struct wine_test *test, const char *subtest, const char *file ) { report( R_STEP, "Running: %s:%s", test->name, subtest ); @@ -907,6 +950,7 @@ static void report_test_done( struct wine_test *test, const char *subtest, const { if (junit_mode) { + HMODULE module = LoadLibraryExA( test->exename, NULL, LOAD_LIBRARY_AS_DATAFILE ); int total = 0, fail_total = 0, skip_total = 0, failures = 0; const char *next, *line, *ptr, *dir = strrchr( file, '/' ); float time, last_time = 0; @@ -960,7 +1004,7 @@ static void report_test_done( struct wine_test *test, const char *subtest, const if (!(ptr = strchr( line, ' ' ))) continue; ptr++;
- name = strmake( NULL, "%s:%d", src, n ); + if (!(name = get_test_name( module, src, n, &len ))) name = strmake( NULL, "%s:%d", src, n ); for (i = 0; i < ARRAY_SIZE(patterns); i++) { if (!strncmp( ptr, patterns[i].pattern, patterns[i].length )) @@ -1003,6 +1047,7 @@ static void report_test_done( struct wine_test *test, const char *subtest, const }
xprintf( " </testsuite>\n" ); + FreeLibrary( module ); } else {
Alternatively, it could perhaps output the JUnit report to a separate file *in addition* to the classic report format, so we can output both and upload the JUnit report to Gitlab and the classic one for Marvin.
For Gitlab to detect and report the failing tests directly on the MR front page. This has the disadvantage of making the test job output more silent, but the results are still available as artifacts (in JUnit XML format), which hopefully should be enough if we need to have a better look at it. The tests standard output is included in the reports too.
Very nice!
I believe it would also show some failure statistics that could help deciding on whether a test is regularly failing or not, but I wasn't able to really test that it does. In order to help it doing it, it also implements hashing of test sources to generate test IDs that are more stable than `source:line`, although it could be done later.
Including the entire source doubles the size of winetest.exe, it would be nice to find a better way.
We could probably use upx for the Gitlab CI, but I still think it would be nice to find a solution that doesn't require shipping the entire source.
Okay, I don't have any (other) good idea tbh, I'll probably leave it for later if it's an issue and hopefully using the test source + line + message would be enough for Gitlab to not mess the results up.
I've considered doing some hashing in the test code, but the only context we can get in `winetest_ok` (and others similar) calls is the test expression, possibly with a rolling buffer of previously executed tests. That doesn't feel like a very useful or unique information about the test being executed. Capturing the test full expressions would also increase the exe size.
Also considered other things like function-relative (or at least relative to the first test of a function) line numbers, which doesn't feel very robust to code addition/removal.
Overall nothing felt as satisfying as being able to hash the surrounding source lines (actually the preceding lines are the most meaningful, but including a few lines after can be useful to capture ok calls on multiple lines).
-
Alexandre Julliard (＠julliard) -
Rémi Bernon
-
Rémi Bernon (＠rbernon)