Module: tools
Branch: master
Commit: 9f40728cd785c04965642292963833d532070c6a
URL: https://source.winehq.org/git/tools.git/?a=commit;h=9f40728cd785c0496564229…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Mon May 3 14:59:00 2021 +0200
winetest/build-patterns: Highlight recent changes in the failure modes.
Highlight the new failure modes and put the test units that would
otherwise go into the 'has always been failing' list in their own list.
This should help catch new failures that pile on an already failing
test unit.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
winetest/build-patterns | 53 ++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns
index a84e2a7..ea9cd4c 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -260,6 +260,10 @@ my %reports;
# True if the test had enough successful results after the last failure to be
# considered fixed.
#
+# - newmodes
+# A hashtable of new failure modes, that its of status values present in the
+# last $patternbuilds build results that are not present in older builds.
+#
# - testreports
# A hashtable mapping report directory names to objects storing the results
# for that test and report combination. Each testreport object has the
@@ -695,6 +699,11 @@ sub get_status_html($$)
return ("?", "", "unknown status $status", "report", "");
}
+sub cmpstatus
+{
+ return ($a =~ /^[0-9]+$/ and $b =~ /^[0-9]+$/) ? $a <=> $b : $a cmp $b;
+}
+
sub write_patterns_list($$$$)
{
my ($html, $pagereports, $mainpage, $list) = @_;
@@ -720,6 +729,17 @@ sub write_patterns_list($$$$)
compute_set_colors($test->{colors});
print $html "<div class='test'>\n";
+ if (%{$test->{newmodes}})
+ {
+ print $html "New failure modes:";
+ foreach my $status (sort cmpstatus keys %{$test->{newmodes}})
+ {
+ my ($symbol, $class, $title, $_link, $attrs) = get_status_html($status, $test->{colors});
+ print $html " <span class='pat$class'$attrs title='$title'><b><i>$symbol</i></b></span>";
+ }
+ print $html "\n";
+ }
+
foreach my $reportdir (@sortedreports)
{
next if (!$pagereports->{$reportdir});
@@ -727,6 +747,7 @@ sub write_patterns_list($$$$)
next if (!$testreport->{failed});
print $html "<div class='pattern'>";
+ my $has_newmode;
my ($range_symbol, $range_count) = ("", 0);
my ($range_start, $range_end, $range_title);
foreach my $build (@sortedbuilds)
@@ -773,6 +794,11 @@ sub write_patterns_list($$$$)
$attrs .= sprintf " href='%s/%s/%s.html#%s'",
$build->{name}, $reportdir, $link, $dll;
}
+ if ($test->{newmodes}->{$status})
+ {
+ $symbol = "<b><i>$symbol</i></b>";
+ $has_newmode = 1;
+ }
}
if ($range_symbol eq $symbol)
@@ -815,7 +841,9 @@ sub write_patterns_list($$$$)
date_range($range_start, $range_end),
$range_title, $range_symbol x $range_count;
}
- print $html "</div> $reportdir\n";
+ my $label = $reportdir;
+ $label = "<b>$label</b>" if ($has_newmode);
+ print $html "</div> $label\n";
}
print $html "</div></div>\n";
}
@@ -848,6 +876,9 @@ EOF
recent => {title => "Recent failures",
desc => "have recent failures (that is failures that did not happen for the $patternbuilds oldest builds).",
testnames => []},
+ newmode => {title => "New failure modes",
+ desc => "failed in new ways in the $patternbuilds most recent builds.",
+ testnames => []},
regular => {title => "Regular failures",
desc => "had failures.",
testnames => []},
@@ -862,6 +893,7 @@ EOF
my $first = @sortedbuilds;
my $last = -1;
my $fixed = 1;
+ my ($oldfailures, $newfailures) = ({}, {});
foreach my $reportdir (keys %$pagereports)
{
my $testreport = $test->{testreports}->{$reportdir};
@@ -876,8 +908,16 @@ EOF
$testreport->{first};
$last = $replast if ($replast > $last);
- foreach my $status (values %{$testreport->{status}})
+ for my $i (0..@sortedbuilds-1)
{
+ my $build = $sortedbuilds[$i];
+ my $status = $testreport->{status}->{$build->{name}};
+ next if (!defined $status);
+ next if ($status eq "skipped");
+
+ my $failures = ($i < @sortedbuilds - $patternbuilds) ? $oldfailures : $newfailures;
+ $failures->{$status}++;
+
next if ($status !~ /^[0-9]+$/);
next if ($status eq "258"); # timeouts have their own color
$test->{colors}->{$status} = undef;
@@ -885,9 +925,16 @@ EOF
}
next if ($last == -1); # no report had a pattern of interest
+ $test->{newmodes} = {};
+ foreach my $status (keys %$newfailures)
+ {
+ next if ($oldfailures->{$status});
+ $test->{newmodes}->{$status} = 1;
+ }
$fixed = 1 if (($test->{fixed} || 0) >= $fixed_threshold);
my $listid = ($fixed or $last < @sortedbuilds - $patternbuilds) ? "old" :
($first > $patternbuilds) ? "recent" :
+ %{$test->{newmodes}} ? "newmode" :
"regular";
push @{$lists{$listid}->{testnames}}, $testname;
}
@@ -895,7 +942,7 @@ EOF
# Generate the lists index (and up test unit links)
print $html "<h1>$title</h1>\n";
print $html "<ul>\n";
- my @listids = ("recent", "regular", "old");
+ my @listids = ("recent", "newmode", "regular", "old");
my $prevunit = "";
foreach my $listid (@listids)
{
Module: tools
Branch: master
Commit: e5288e8eb055413071c707ed21768955b2de3ab7
URL: https://source.winehq.org/git/tools.git/?a=commit;h=e5288e8eb055413071c707e…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Mon May 3 14:45:11 2021 +0200
winetest/build-patterns: Fix detecting transitions in non-run-time failures.
Record the first failure type so changes can be detected later on.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
winetest/build-patterns | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns
index 460c17c..a84e2a7 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -480,12 +480,15 @@ foreach my $testname (keys %tests)
}
next if ($status eq "skipped"); # same as if WineTest was not run
- # It is not an error if the dll is missing (or other similar
- # error) _and_ it has always been so.
my $failtype = fail_type($status);
- next if ($testreport->{failtype} eq "" and $failtype ne "random");
-
- if ($testreport->{failtype} ne $failtype)
+ if ($testreport->{failtype} eq "" and $failtype ne "random")
+ {
+ # It is not an error if the dll is missing (or other similar
+ # error) _and_ it has always been so. So just record failtype
+ # for now.
+ $testreport->{failtype} = $failtype;
+ }
+ elsif ($testreport->{failtype} ne $failtype)
{
# Either there was no failure before; or the failure type
# changed in a way that cannot be explained by randomness; e.g.