[tools] winetest/build-patterns: Identify patterns with new and possibly fixed failures.

[Francois Gouget]

Show the patterns with new failures first, then those that always
failed, and finally those that do not have any recent failure and thus
may be fixed.
This should help catch new failures while they are fresh.

Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
---
 winetest/build-patterns | 113 ++++++++++++++++++++++++++++++++++------
 winetest/winetest.conf  |   3 ++
 2 files changed, 100 insertions(+), 16 deletions(-)

diff --git a/winetest/build-patterns b/winetest/build-patterns
index e87fabc2f..32bae6b31 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -31,7 +31,7 @@ sub BEGIN
     }
     unshift @INC, $1 if ($0 =~ m=^(/.*)/[^/]+$=);
 }
-use vars qw/$workdir $gitdir/;
+use vars qw/$workdir $gitdir $patternbuilds/;
 require "winetest.conf";
 
 my $name0=$0;
@@ -262,9 +262,19 @@ my %reports;
 #     should be included in that test's failure pattern.
 #
 #   - failtype
-#     The type of the most recent failure (see fail_type()).
+#     The type of the most recent failure (see fail_type(), last).
 #     In particular this distinguishes random failures from non-random ones.
 #
+#   - first
+#     The @sortedbuilds index of the oldest build with a failure matching
+#     failtype. So if failtype is 'random' this ignores any 'missing dll'
+#     failures that may have occurred before.
+#     scalar(@sortedbuilds) if there is no failure.
+#
+#   - last
+#     The @sortedbuilds index of the most recent build with a failure.
+#     -1 if there is none.
+#
 #   - status
 #     A hashtable of test results indexed by the build name.
 my %tests;
@@ -388,11 +398,16 @@ foreach my $testname (keys %tests)
         next if (!$testreport);
 
         # Record information about the failures for this report:
+        # - The index of the first (oldest) and last (most recent) failures.
+        #   Initialized for easy min / max calculations.
+        $testreport->{first} = @sortedbuilds;
+        $testreport->{last} = -1;
         # - Type of failure: random or not (missing dll, etc.)
         $testreport->{failtype} = "";
 
-        foreach my $build (@sortedbuilds)
+        for my $i (0..(a)sortedbuilds-1)
         {
+            my $build = $sortedbuilds[$i];
             my $status = $testreport->{status}->{$build->{name}};
             if (!defined $status)
             {
@@ -419,7 +434,13 @@ foreach my $testname (keys %tests)
                 # missing dll to the test running and crashing.
                 # Either way the relevant round of failures starts now.
                 $testreport->{failed} = 1;
-                last;
+                $testreport->{first} = $i;
+                $testreport->{last} = $i;
+                $testreport->{failtype} = $failtype;
+            }
+            else
+            {
+                $testreport->{last} = $i;
             }
         }
     }
@@ -525,8 +546,9 @@ sub get_status_html($$)
 
 sub write_patterns_list($$$$)
 {
-    my ($html, $pagereports, $mainpage, $testnames) = @_;
+    my ($html, $pagereports, $mainpage, $list) = @_;
 
+    my $testnames = $list->{testnames};
     for my $i (0..@$testnames-1)
     {
         my $testname = $testnames->[$i];
@@ -537,9 +559,9 @@ sub write_patterns_list($$$$)
         print $html "  <small>(<a href='$mainpage#$testname'>see all results</a></small>)" if ($mainpage);
         print $html "<div class='ralign'>";
 
-        my $href = $i ? $testnames->[$i-1] : "";
-        print $html "<a href='#$href'>↑</a>";
-        $href = $i+1 < @$testnames ? $testnames->[$i+1] : undef;
+        my $href = $i ? $testnames->[$i-1] : $list->{prevunit};
+        print $html "<a href='#$href'>↑</a>" if (defined $href);
+        $href = $i+1 < @$testnames ? $testnames->[$i+1] : $list->{nextunit};
         print $html "<a href='#$href'>↓</a>" if (defined $href);
 
         print $html "</div></div>\n";
@@ -669,18 +691,37 @@ sub write_patterns_page($$$)
 <div class="main">
 EOF
 
-    # Build a list of test units that will appear on this page so we can
-    # link them to each other.
-    my $testnames = [];
+    # Show the test units with recent failures first.
+    # Also use these lists to link the test units to each other.
+    my %lists = (
+        recent   => {title => "Recent failures",
+                     desc => "have recent failures (that is failures that did not happen for the $patternbuilds oldest builds).",
+                     testnames => []},
+        regular  => {title => "Regular failures",
+                     desc => "had failures.",
+                     testnames => []},
+        old      => {title => "Old failures",
+                     desc => "had no failure in the most recent $patternbuilds builds. Fixed or just rare failures?",
+                     testnames => []},
+    );
     foreach my $testname (sort keys %tests)
     {
         my $test = $tests{$testname};
-        my $addtest;
+
+        my $first = @sortedbuilds;
+        my $last = -1;
         foreach my $reportdir (keys %$pagereports)
         {
             my $testreport = $test->{testreports}->{$reportdir};
             next if (!$testreport->{failed});
-            $addtest = 1;
+
+            $first = $testreport->{first} if ($testreport->{first} < $first);
+            # For non-random failures we only care about the transition to
+            # the failure state, which is recorded in 'first'.
+            my $replast = $testreport->{failtype} eq "random" ? $testreport->{last} :
+                          $testreport->{first} == @sortedbuilds ? -1 :
+                          $testreport->{first};
+            $last = $replast if ($replast > $last);
 
             foreach my $status (values %{$testreport->{status}})
             {
@@ -689,12 +730,52 @@ EOF
                 $test->{colors}->{$status} = undef;
             }
         }
-        push @$testnames, $testname if ($addtest);
+        next if ($last == -1); # no report had a pattern of interest
+
+        my $listid = ($last < @sortedbuilds - $patternbuilds) ? "old" :
+                     ($first > $patternbuilds) ? "recent" :
+                     "regular";
+        push @{$lists{$listid}->{testnames}}, $testname;
+    }
+
+    # Generate the lists index (and up test unit links)
+    print $html "<h1>$title</h1>\n";
+    print $html "<ul>\n";
+    my @listids = ("recent", "regular", "old");
+    my $prevunit = "";
+    foreach my $listid (@listids)
+    {
+        my $list = $lists{$listid};
+        my $count = @{$list->{testnames}};
+        next if (!$count);
+        print $html "<li><a href='#$listid'>$count test units</a> $list->{desc}</li>\n";
+        $list->{prevunit} = $prevunit;
+        $prevunit = $list->{testnames}->[-1];
+    }
+    print $html "</ul>\n";
+
+    # Link the last test unit of each list to the first one of the next list
+    my $nextunit;
+    foreach my $listid (reverse @listids)
+    {
+        my $list = $lists{$listid};
+        my $count = @{$list->{testnames}};
+        next if (!$count);
+        $list->{nextunit} = $nextunit;
+        $nextunit = $list->{testnames}->[0];
     }
 
+    # Generate the test unit lists themselves
     my $mainpage = $subpage ? "patterns.html" : "";
-    print $html "<h2>$title</h2>\n";
-    write_patterns_list($html, $pagereports, $mainpage, $testnames);
+    foreach my $listid (@listids)
+    {
+        my $list = $lists{$listid};
+        my $count = @{$list->{testnames}};
+        next if (!$count);
+        print $html "<h2 id='$listid'>$list->{title}</h2>\n";
+        write_patterns_list($html, $pagereports, $mainpage, $list);
+    }
+
     print $html "</div></body></html>\n";
     close($html);
 
diff --git a/winetest/winetest.conf b/winetest/winetest.conf
index 84824a2b3..f193f8eed 100644
--- a/winetest/winetest.conf
+++ b/winetest/winetest.conf
@@ -24,4 +24,7 @@ $maxunitsize = 32 * 1024;
 # This should be in line with programs\winetest\send.c
 $maxfilesize = 1.5 * 1024 * 1024;
 
+# The number of builds after which a failure is considered old / new
+$patternbuilds = 10;
+
 1;                              # keep require happy
-- 
2.20.1