Highlight the new failure modes and put the test units that would otherwise go into the 'has always been failing' list in their own list. This should help catch new failures that pile on an already failing test unit.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- The goal here is to highlight test units that have had 2 failures forever (so not in the new list) and suddenly start having 4 failures, or that start crashing, timing out, etc.
Note that as implemented it will also highlight cases where some failures are fixed, for instance if the failure count goes from 7 to 4. But I think that's ok. --- winetest/build-patterns | 53 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns index a84e2a7ea..ea9cd4c1d 100755 --- a/winetest/build-patterns +++ b/winetest/build-patterns @@ -260,6 +260,10 @@ my %reports; # True if the test had enough successful results after the last failure to be # considered fixed. # +# - newmodes +# A hashtable of new failure modes, that its of status values present in the +# last $patternbuilds build results that are not present in older builds. +# # - testreports # A hashtable mapping report directory names to objects storing the results # for that test and report combination. Each testreport object has the @@ -695,6 +699,11 @@ sub get_status_html($$) return ("?", "", "unknown status $status", "report", ""); }
+sub cmpstatus +{ + return ($a =~ /^[0-9]+$/ and $b =~ /^[0-9]+$/) ? $a <=> $b : $a cmp $b; +} + sub write_patterns_list($$$$) { my ($html, $pagereports, $mainpage, $list) = @_; @@ -720,6 +729,17 @@ sub write_patterns_list($$$$) compute_set_colors($test->{colors});
print $html "<div class='test'>\n"; + if (%{$test->{newmodes}}) + { + print $html "New failure modes:"; + foreach my $status (sort cmpstatus keys %{$test->{newmodes}}) + { + my ($symbol, $class, $title, $_link, $attrs) = get_status_html($status, $test->{colors}); + print $html " <span class='pat$class'$attrs title='$title'><b><i>$symbol</i></b></span>"; + } + print $html "\n"; + } + foreach my $reportdir (@sortedreports) { next if (!$pagereports->{$reportdir}); @@ -727,6 +747,7 @@ sub write_patterns_list($$$$) next if (!$testreport->{failed}); print $html "<div class='pattern'>";
+ my $has_newmode; my ($range_symbol, $range_count) = ("", 0); my ($range_start, $range_end, $range_title); foreach my $build (@sortedbuilds) @@ -773,6 +794,11 @@ sub write_patterns_list($$$$) $attrs .= sprintf " href='%s/%s/%s.html#%s'", $build->{name}, $reportdir, $link, $dll; } + if ($test->{newmodes}->{$status}) + { + $symbol = "<b><i>$symbol</i></b>"; + $has_newmode = 1; + } }
if ($range_symbol eq $symbol) @@ -815,7 +841,9 @@ sub write_patterns_list($$$$) date_range($range_start, $range_end), $range_title, $range_symbol x $range_count; } - print $html "</div> $reportdir\n"; + my $label = $reportdir; + $label = "<b>$label</b>" if ($has_newmode); + print $html "</div> $label\n"; } print $html "</div></div>\n"; } @@ -848,6 +876,9 @@ EOF recent => {title => "Recent failures", desc => "have recent failures (that is failures that did not happen for the $patternbuilds oldest builds).", testnames => []}, + newmode => {title => "New failure modes", + desc => "failed in new ways in the $patternbuilds most recent builds.", + testnames => []}, regular => {title => "Regular failures", desc => "had failures.", testnames => []}, @@ -862,6 +893,7 @@ EOF my $first = @sortedbuilds; my $last = -1; my $fixed = 1; + my ($oldfailures, $newfailures) = ({}, {}); foreach my $reportdir (keys %$pagereports) { my $testreport = $test->{testreports}->{$reportdir}; @@ -876,8 +908,16 @@ EOF $testreport->{first}; $last = $replast if ($replast > $last);
- foreach my $status (values %{$testreport->{status}}) + for my $i (0..@sortedbuilds-1) { + my $build = $sortedbuilds[$i]; + my $status = $testreport->{status}->{$build->{name}}; + next if (!defined $status); + next if ($status eq "skipped"); + + my $failures = ($i < @sortedbuilds - $patternbuilds) ? $oldfailures : $newfailures; + $failures->{$status}++; + next if ($status !~ /^[0-9]+$/); next if ($status eq "258"); # timeouts have their own color $test->{colors}->{$status} = undef; @@ -885,9 +925,16 @@ EOF } next if ($last == -1); # no report had a pattern of interest
+ $test->{newmodes} = {}; + foreach my $status (keys %$newfailures) + { + next if ($oldfailures->{$status}); + $test->{newmodes}->{$status} = 1; + } $fixed = 1 if (($test->{fixed} || 0) >= $fixed_threshold); my $listid = ($fixed or $last < @sortedbuilds - $patternbuilds) ? "old" : ($first > $patternbuilds) ? "recent" : + %{$test->{newmodes}} ? "newmode" : "regular"; push @{$lists{$listid}->{testnames}}, $testname; } @@ -895,7 +942,7 @@ EOF # Generate the lists index (and up test unit links) print $html "<h1>$title</h1>\n"; print $html "<ul>\n"; - my @listids = ("recent", "regular", "old"); + my @listids = ("recent", "newmode", "regular", "old"); my $prevunit = ""; foreach my $listid (@listids) {