This makes it possible to identify when the failure count changes, such as when a patch adds two new failures to a test that already had five, or when the failure count depending on the test configuration.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- The pattern page will automatically be updated when the next report arrives.
Maybe there's a way to tweak the gradients to get more contrasting colors. In particular the blue / green yellow are sometimes hard to distinguish. But I'm not much of a color guy so I'll leave that to someone else. Of course another approach is to have fewer failure modes ;-) --- winetest/build-patterns | 116 +++++++++++++++++++++++++++++++--------- winetest/report.css | 4 +- 2 files changed, 93 insertions(+), 27 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns index f470c266f..9dc25816f 100755 --- a/winetest/build-patterns +++ b/winetest/build-patterns @@ -247,6 +247,9 @@ my %reports; # - name # The uniquely identifying test name in the form 'module:unit'. # +# - colors +# A hashtable of colors indexed by failure count. +# # - testreports # A hashtable mapping report directory names to objects storing the results # for that test and report combination. Each testreport object has the @@ -421,44 +424,101 @@ foreach my $testname (keys %tests) }
+# +# Compute color gradients +# + +sub color2html($) +{ + my ($c) = @_; + return sprintf "#%02x%02x%02x", $c->[0], $c->[1], $c->[2]; +} + +sub blend($$$) +{ + my ($p, $start, $end) = @_; + my $r = [int($start->[0] * (1 - $p) + $end->[0] * $p + 0.5), + int($start->[1] * (1 - $p) + $end->[1] * $p + 0.5), + int($start->[2] * (1 - $p) + $end->[2] * $p + 0.5)]; + return $r; +} + +my @keycolors = ([0, 255, 255], # cyan + [0, 255, 0], # green + [255, 255, 0], # yellow + [255, 0, 0], # red +); + +# Use colors to differentiate the set values. Each unique value is assigned +# a color (in HTML format) picked along a series of gradients passing by the +# colors defined in @keycolors. +sub compute_set_colors($) +{ + my ($set) = @_; + my @values = sort { $a <=> $b } keys %$set; + my $count = @values; + if ($count == 1) + { + $set->{$values[0]} = color2html($keycolors[0]); + } + else + { + my $k = 0; + my ($start, $end) = (-1, 0); + for (0..$count-1) + { + while (!$end or $_ > $end) + { + $k++; + $start = $end; + $end = ($count-1) * $k / (@keycolors-1); + } + $set->{$values[$_]} = color2html(blend(($_-$start)/($end-$start), + $keycolors[$k-1], + $keycolors[$k])); + } + } +} + + # # Write the failure patterns page for the given set of reports #
my %status2html = ( # Dll information status values - # <status> => [ <symbol>, <class-char>, <title>, <link> ] - "missing" => ["n", "n", "not run for an unknown reason", "report"], - "missingdll" => ["m", "m", "missing dll", "version"], - "missingentrypoint" => ["e", "e", "missing entry point", "version"], - "missingordinal" => ["o", "o", "missing ordinal", "version"], - "missingsxs" => ["v", "v", "missing side-by-side dll version", "version"], - "stub" => ["u", "u", "stub Windows dll", "version"], - "native" => ["N", "N", "native Windows dll", "version"], - "loaderror258" => ["I", "I", "timed out while getting the test list", "version"], + # <status> => [ <symbol>, <class-char>, <title>, <link>, <attrs> ] + "missing" => ["n", "n", "not run for an unknown reason", "report", ""], + "missingdll" => ["m", "m", "missing dll", "version", ""], + "missingentrypoint" => ["e", "e", "missing entry point", "version", ""], + "missingordinal" => ["o", "o", "missing ordinal", "version", ""], + "missingsxs" => ["v", "v", "missing side-by-side dll version", "version", ""], + "stub" => ["u", "u", "stub Windows dll", "version", ""], + "native" => ["N", "N", "native Windows dll", "version", ""], + "loaderror258" => ["I", "I", "timed out while getting the test list", "version", ""], # Other status values - "skipped" => ["-", "s", "skipped by user request", ""], - "crash" => ["C", "C", "crash", "t"], - "258" => ["T", "T", "timeout", "t"], + "skipped" => ["-", "s", "skipped by user request", "", ""], + "crash" => ["C", "C", "crash", "t", ""], + "258" => ["T", "T", "timeout", "t", ""], );
-# Returns a tuple containing the symbol, CSS class, title and link type -# for the specified status. -sub get_status_html($) +# Returns a tuple containing the symbol, CSS class, title, link type and +# HTML attributes for the specified status. +sub get_status_html($$) { - my ($status) = @_; + my ($status, $failcolors) = @_;
return @{$status2html{$status}} if ($status2html{$status});
if ($status =~ /^[0-9]+$/) { - return ("F", "F", "$status failures", "t"); + return ("F", "F", "$status failures", "t", " style='background-color: $failcolors->{$status}'"); } if ($status =~ /^loaderror(.*)$/) { - return ("L", "L", "got error $1 while getting the test list", "version"); + return ("L", "L", "got error $1 while getting the test list", "version", ""); } - return ("?", "", "unknown status $status", "report"); + return ("?", "", "unknown status $status", "report", ""); }
sub write_patterns_list($$) @@ -481,6 +541,8 @@ sub write_patterns_list($$)
print $html "</div></div>\n";
+ compute_set_colors($test->{colors}); + print $html "<div class='test'>\n"; foreach my $reportdir (@sortedreports) { @@ -519,7 +581,7 @@ sub write_patterns_list($$) } else { - ($symbol, $class, $title, my $link) = get_status_html($status); + ($symbol, $class, $title, my $link, $attrs) = get_status_html($status, $test->{colors}); if ($link eq "t") { $tag = "a"; @@ -606,17 +668,23 @@ EOF # Build a list of test units that will appear on this page so we can # link them to each other. my $testnames = []; - unit: foreach my $testname (sort keys %tests) + foreach my $testname (sort keys %tests) { my $test = $tests{$testname}; + my $addtest; foreach my $testreport (values %{$test->{testreports}}) { - if ($testreport->{failed}) + next if (!$testreport->{failed}); + $addtest = 1; + + foreach my $status (values %{$testreport->{status}}) { - push @$testnames, $testname; - next unit; + next if ($status !~ /^[0-9]+$/); + next if ($status eq "258"); # timeouts have their own color + $test->{colors}->{$status} = undef; } } + push @$testnames, $testname if ($addtest); }
print $html "<h2>$title</h2>\n"; diff --git a/winetest/report.css b/winetest/report.css index cca1ba986..2cc466304 100644 --- a/winetest/report.css +++ b/winetest/report.css @@ -89,9 +89,7 @@ div.pattern :hover { color: black; text-decoration: underline; } .patC { /* crash */ background-color: #ff5555; } -.patF { /* failure(s) */ - background-color: #ff0000; -} +/* .patF failure(s) */ /* .patn not run for an unknown reason */ /* .patm missing dll */ .pate { /* missing entrypoint */
One page only shows the failure patterns of the Windows TestBot machines, and another just the Wine TestBot machines. This also cleanly separates the Windows and Wine failure modes.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- winetest/build-index | 4 ++-- winetest/build-patterns | 48 ++++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 12 deletions(-)
diff --git a/winetest/build-index b/winetest/build-index index 6a3356a25..e69ea1f28 100755 --- a/winetest/build-index +++ b/winetest/build-index @@ -220,7 +220,7 @@ foreach my $build (readdir(DIR)) { if ($build !~ /^[0-9a-f]{40}$/) { - if ($build !~ /^(?:..?|(?:errors|index|patterns).html|tests)$/) + if ($build !~ /^(?:..?|(?:errors|index|patterns.*).html|tests)$/) { error("'data/$build' is not a valid build directory\n"); } @@ -430,7 +430,7 @@ print OUT <<"EOF"; <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> </head> <body> -<div class="navbar"><a href="patterns.html">failure patterns</a></div> +<div class="navbar">Failure patterns: <a href="patterns.html">all</a> | <a href="patterns-tb-win.html">testbot windows</a> | <a href="patterns-tb-wine.html">testbot wine</a></div> <div class="main"> <h2>Wine test runs</h2> EOF diff --git a/winetest/build-patterns b/winetest/build-patterns index 9dc25816f..e87fabc2f 100755 --- a/winetest/build-patterns +++ b/winetest/build-patterns @@ -168,6 +168,8 @@ Actions:
Generated files: $workdir/data/patterns.html + $workdir/data/patterns-tb-win.html + $workdir/data/patterns-tb-wine.html
Exit: 0 - success @@ -206,7 +208,7 @@ foreach my $build (readdir(DIR)) { if ($build !~ /^[0-9a-f]{40}$/) { - if ($build !~ /^(?:..?|(?:errors|index|patterns).html|tests)$/) + if ($build !~ /^(?:..?|(?:errors|index|patterns.*).html|tests)$/) { error("'data/$build' is not a valid build directory\n"); } @@ -521,9 +523,9 @@ sub get_status_html($$) return ("?", "", "unknown status $status", "report", ""); }
-sub write_patterns_list($$) +sub write_patterns_list($$$$) { - my ($html, $testnames) = @_; + my ($html, $pagereports, $mainpage, $testnames) = @_;
for my $i (0..@$testnames-1) { @@ -532,6 +534,7 @@ sub write_patterns_list($$)
print $html "<div class='testfile' id='$testname'>\n"; print $html "<div class='updownbar'><a href='tests/$testname.html'>$testname</a>"; + print $html " <small>(<a href='$mainpage#$testname'>see all results</a></small>)" if ($mainpage); print $html "<div class='ralign'>";
my $href = $i ? $testnames->[$i-1] : ""; @@ -546,6 +549,7 @@ sub write_patterns_list($$) print $html "<div class='test'>\n"; foreach my $reportdir (@sortedreports) { + next if (!$pagereports->{$reportdir}); my $testreport = $test->{testreports}->{$reportdir}; next if (!$testreport->{failed}); print $html "<div class='pattern'>"; @@ -644,11 +648,11 @@ sub write_patterns_list($$) } }
-sub write_patterns_page($) +sub write_patterns_page($$$) { - my ($title) = (@_); + my ($pagereports, $subpage, $title) = (@_);
- my $filename = "data/patterns.html"; + my $filename = "data/patterns$subpage.html"; open(my $html, ">", "$filename.new") or die "could not open '$filename.new' for writing: $!";
print $html <<"EOF"; @@ -661,7 +665,7 @@ sub write_patterns_page($) <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> </head> <body> -<div class="navbar"><a href="..">index</a></div> +<div class="navbar"><a href="..">index</a> | Failure patterns: <a href="patterns.html">all</a> | <a href="patterns-tb-win.html">testbot windows</a> | <a href="patterns-tb-wine.html">testbot wine</a></div> <div class="main"> EOF
@@ -672,8 +676,9 @@ EOF { my $test = $tests{$testname}; my $addtest; - foreach my $testreport (values %{$test->{testreports}}) + foreach my $reportdir (keys %$pagereports) { + my $testreport = $test->{testreports}->{$reportdir}; next if (!$testreport->{failed}); $addtest = 1;
@@ -687,8 +692,9 @@ EOF push @$testnames, $testname if ($addtest); }
+ my $mainpage = $subpage ? "patterns.html" : ""; print $html "<h2>$title</h2>\n"; - write_patterns_list($html, $testnames); + write_patterns_list($html, $pagereports, $mainpage, $testnames); print $html "</div></body></html>\n"; close($html);
@@ -699,6 +705,28 @@ EOF } }
-write_patterns_page("Test failure patterns"); +write_patterns_page(%reports, "", "All test patterns"); + +my %tbreports; +foreach my $report (values %reports) +{ + next if ($report->{tag} !~ /^newtb-/); # ignore non-TestBot reports + if ($report->{is_rerun}) + { + # Skip reruns because they are infrequent and thus usually do not + # meaningfully contribute to the patterns. + # They can still be checked on the 'all reports' page. + next; + } + # Windows and Wine have different failure modes so put them on separate + # pages. + my $page = ($report->{platform} =~ /^(?:linux|mac|bsd|solaris|wine)$/) ? "wine" : "win"; + $tbreports{$page}->{$report->{dir}} = 1; +} + +write_patterns_page($tbreports{win}, "-tb-win", + "Windows test patterns (TestBot only)"); +write_patterns_page($tbreports{wine}, "-tb-wine", + "Wine test patterns (TestBot only)");
exit 0;
-
Francois Gouget