This pattern shows the number of failed test units for each test configuration. This allows detecting when results get worse after a VM update, or when a test configuration gets worse results than its peers.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- winetest/build-patterns | 121 +++++++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 38 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns index f6704bd00..c9fcb6638 100755 --- a/winetest/build-patterns +++ b/winetest/build-patterns @@ -31,12 +31,19 @@ sub BEGIN } unshift @INC, $1 if ($0 =~ m=^(/.*)/[^/]+$=); } -use vars qw/$workdir $gitdir $gitweb $patternbuilds $fixed_threshold/; +use vars qw/$workdir $gitdir $gitweb @groups $patternbuilds $fixed_threshold/; require "winetest.conf";
my $name0=$0; $name0 =~ s+^.*/++;
+# Map dissect's IDs to the platform groups +my %idmap; +foreach my $group (@groups) +{ + map { $idmap{$_} = $group } @{$group->{ids}}; +} +
# # Common helpers @@ -249,6 +256,16 @@ my %reports; # - name # The uniquely identifying test name in the form 'module:unit'. # +# - id +# The HTML id for linking to this test. +# +# - title +# The test's HTML title. +# +# - desc +# An optional chunk of HTML shown above the test pattern which can be used +# to provide additional information about the test. +# # - source # The source file for that test unit. # @@ -310,7 +327,7 @@ my %reports; # - commits # A hashtable of the commit objects, indexed first buy build name and then # by commit id. -my %tests; +my %tests = ( 0 => { name => "0", id => "summary", newmodes => {} } );
# A set containing the test units source files. my %testsources; @@ -360,7 +377,9 @@ foreach my $build (@sortedbuilds) if (!$test) { $tests{$testname} = $test = { + id => $testname, name => $testname, + title => "<a href='tests/$testname.html'>$testname</a>", source => $source, }; $testsources{$source} = 1; @@ -377,6 +396,10 @@ foreach my $build (@sortedbuilds) next; } $test->{testreports}->{$reportdir}->{status}->{$build->{name}} = $status; + + # Count failed test units + next if ($status =~ /^(?:missing.|skipped|stub)/); + $tests{0}->{testreports}->{$reportdir}->{status}->{$build->{name}}++; } } } @@ -882,7 +905,15 @@ sub write_pattern_line($$$) else { ($symbol, $class, $title, my $link, $attrs) = get_status_html($status, $test->{colors}); - if ($link eq "t") + if ($test->{name} eq "0") + { + $tag = "a"; + my $platform = $reports{$reportdir}->{platform}; + my $group = $idmap{$platform} || $idmap{unknown}; + $attrs .= sprintf " href='%s/index_%s.html'", + $build->{name}, $group->{name}; + } + elsif ($link eq "t") { $tag = "a"; $attrs .= sprintf " href='%s/%s/%s.html'", @@ -1042,8 +1073,11 @@ sub write_pattern($$$) my ($html, $test, $pagereports) = @_;
compute_set_colors($test->{colors}); - write_newmodes_line($html, $test) if (%{$test->{newmodes}});
+ print $html $test->{desc} if (defined $test->{desc}); + + print $html "<div class='test'>\n"; + write_newmodes_line($html, $test) if (%{$test->{newmodes}}); foreach my $reportdir (@sortedreports) { next if (!$pagereports->{$reportdir}); @@ -1051,6 +1085,7 @@ sub write_pattern($$$) write_pattern_line($html, $test, $reportdir); } write_commits($html, $test) if (%{$test->{commits}}); + print $html "</div>\n"; }
sub write_patterns_list($$$$) @@ -1063,8 +1098,8 @@ sub write_patterns_list($$$$) my $testname = $testnames->[$i]; my $test = $tests{$testname};
- print $html "<div class='testfile' id='$testname'>\n"; - print $html "<div class='updownbar'><a href='tests/$testname.html'>$testname</a>"; + print $html "<div class='testfile' id='$test->{id}'>\n"; + print $html "<div class='updownbar'>$test->{title}"; print $html "<div class='ralign'>"; if ($mainpage) { @@ -1078,9 +1113,8 @@ sub write_patterns_list($$$$)
print $html "</div></div>\n";
- print $html "<div class='test'>\n"; write_pattern($html, $test, $pagereports); - print $html "</div></div>\n"; + print $html "</div>\n"; } }
@@ -1114,30 +1148,12 @@ sub write_patterns_page($$$) </head> <body> <div class="navbar"><a href="..">index</a> | Failure patterns: <a href="patterns.html">all</a> | <a href="patterns-tb-win.html">testbot windows</a> | <a href="patterns-tb-wine.html">testbot wine</a></div> - -<div class='testfile' id='summary'> -<div class='updownbar'>$title</div> -<p>The purpose of this page is to:</p> -<ul> - <li>Help detect new test failures, that is test / Wine regressions.</li> - <li>Help diagnose failures by showing when they are specific to a set of Windows versions, a locale, a day of the week, etc.</li> -</ul> -<p>The patterns on this page read as follows:</p> -<ul> - <li>Each line corresponds to a test configuration sorted by platform and tag name.</li> - <li>Test configurations with no failure for a given test are omitted from that test's pattern.</li> - <li>Each column corresponds to a WineTest build from oldest to newest.</li> - <li>Lines with a _N suffix are when WineTest was run more than once for a test configuration + build combination. They are rare and thus hang at the bottom of the pattern to not disrupt it.</li> - <li>Each cell is a single letter describing the result for that test. See the tooltips for a description. Click on the letter to go to that result page.</li> - <li>Each result is also color coded to make the patterns pop out. There is one color per type of result (crash , timeout, etc.), and one per number of failures (to help detect changes).</li> - <li>Failure modes that have not been seen before (test that newly crashes, new failure count, etc.) are highlighted, as well as the configuration names in which it appears.</li> - <li>Below the patterns is a line showing which builds have potentially related commits if any. -</ul> EOF
# Show the test units with recent failures first. # Also use these lists to link the test units to each other. my %lists = ( + summary => {testnames => [ 0 ]}, recent => {title => "Recent failures", desc => "recently started failing (the $patternbuilds oldest builds had no failure).", testnames => []}, @@ -1194,6 +1210,7 @@ EOF } } next if ($last == -1); # no report had a pattern of interest + next if ($testname eq "0");
$test->{newmodes} = {}; foreach my $status (keys %$newfailures) @@ -1221,23 +1238,51 @@ EOF }
# Generate the lists index (and up test unit links) - print $html "<p>The patterns are grouped according to the likelyhood that they contain new failures:</p>\n"; - print $html "<ul>\n"; - my @listids = ("recent", "newmode", "regular", "old"); + $tests{0}->{title} = $title; + $tests{0}->{desc} = " +<p>The purpose of this page is to:</p> +<ul> + <li>Help detect new test failures, that is test / Wine regressions.</li> + <li>Help diagnose failures by showing when they are specific to a set of Windows versions, a locale, a day of the week, etc.</li> + <li>Help detect test configuration changes that increase the number of failed test units.</li> + <li>Help identify test configurations that get worse results than their peers.</li> +</ul> + +<p>The patterns on this page read as follows:</p> +<ul> + <li>Each line corresponds to a test configuration sorted by platform and tag name.</li> + <li>Test configurations with no failure for a given test are omitted from that test's pattern.</li> + <li>Each column corresponds to a WineTest build from oldest to newest.</li> + <li>Lines with a _N suffix are when WineTest was run more than once for a test configuration + build combination. They are rare and thus hang at the bottom of the pattern to not disrupt it.</li> + <li>Each cell is a single letter describing the result for that test. See the tooltips for a description. Click on the letter to go to that result page.</li> + <li>Each result is also color coded to make the patterns pop out. There is one color per type of result (crash , timeout, etc.), and one per number of failures (to help detect changes).</li> + <li>Failure modes that have not been seen before (test that newly crashes, new failure count, etc.) are highlighted, as well as the configuration names in which it appears.</li> + <li>Below the patterns is a line showing which builds have potentially related commits if any. +</ul> + +<p>The patterns are grouped according to the likelyhood that they contain new failures:</p> +<ul>"; + + my @listids = ("summary", "recent", "newmode", "regular", "old"); my ($prevunit, $nolist) = ("", 1); foreach my $listid (@listids) { my $list = $lists{$listid}; my $count = @{$list->{testnames}}; next if (!$count); - print $html "<li><a href='#$listid'>$count tests</a> $list->{desc}</li>\n"; - $nolist = 0; $list->{prevunit} = $prevunit; - $prevunit = $list->{testnames}->[-1]; + $prevunit = $tests{$list->{testnames}->[-1]}->{id}; + + next if (!$list->{desc}); + $tests{0}->{desc} .= "<li><a href='#$listid'>$count tests</a> $list->{desc}</li>\n"; + $nolist = 0; + } + $tests{0}->{desc} .= "<p>No failures. Don't change anything!</p>\n" if ($nolist); + $tests{0}->{desc} .= "</ul>"; + if (!$nolist) + { + $tests{0}->{desc} .= "<p>Unlike the other patterns on this page, the one below shows the number of failed test units for each configuration.</p>"; } - print $html "<p>No failures. Don't change anything!</p>\n" if ($nolist); - print $html "</ul>\n"; - print $html "</div>\n";
# Link the last test unit of each list to the first one of the next list my $nextunit; @@ -1247,7 +1292,7 @@ EOF my $count = @{$list->{testnames}}; next if (!$count); $list->{nextunit} = $nextunit; - $nextunit = $list->{testnames}->[0]; + $nextunit = $tests{$list->{testnames}->[0]}->{id}; }
# Generate the test unit lists themselves @@ -1257,7 +1302,7 @@ EOF my $list = $lists{$listid}; my $count = @{$list->{testnames}}; next if (!$count); - print $html "<h2 id='$listid'>$list->{title}</h2>\n"; + print $html "<h2 id='$listid'>$list->{title}</h2>\n" if ($list->{title}); write_patterns_list($html, $pagereports, $mainpage, $list); }