Francois Gouget : winetest/build-patterns: Detect fixed failures from the last failure day.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 1528f95b83cd13e118e2609cfc3752063ef22ae7
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=1528f95b83cd13e118e2609c...

Author: Francois Gouget 
Date:   Fri Apr 30 14:57:14 2021 +0200

winetest/build-patterns: Detect fixed failures from the last failure day.

Instead of analyzing the failures of a single report across time, this
analyzes the results of the last build with failures across reports.
This is then compared to the failure-free results of the next days to
determine the probability that the failure has been fixed.

This allows detecting when short-lived issues that impacted many test
configurations are fixed.

Signed-off-by: Francois Gouget 
Signed-off-by: Alexandre Julliard

---

winetest/build-patterns | 78 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 76 insertions(+), 2 deletions(-)

diff --git a/winetest/build-patterns b/winetest/build-patterns
index 6a14fc3..460c17c 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -252,6 +252,14 @@ my %reports;
 # - colors
 #   A hashtable of colors indexed by failure count.
 #
+# - last
+#   The @sortedbuilds index of the most recent build for which a report has a
+#   failure. -1 if there is none.
+#
+# - fixed
+#   True if the test had enough successful results after the last failure to be
+#   considered fixed.
+#
 # - testreports
 #   A hashtable mapping report directory names to objects storing the results
 #   for that test and report combination. Each testreport object has the
@@ -389,7 +397,7 @@ my @sortedreports = sort cmpreports keys %reports;

#
-# Analyze single-report patterns
+# Analyze single-report and single-build patterns
 #

sub fail_type($)
@@ -434,6 +442,9 @@ sub get_fix_probability($$$)
 foreach my $testname (keys %tests)
 {
     my $test = $tests{$testname};
+    $test->{last} = -1;
+    $test->{fixed} = 1;
+
     foreach my $reportdir (@sortedreports)
     {
         my $testreport = $test->{testreports}->{$reportdir};
@@ -498,9 +509,18 @@ foreach my $testname (keys %tests)
                 $testreport->{failruns} += $testreport->{fixedruns} + 1;
                 $testreport->{fixedruns} = 0;
             }
+            if ($test->{last} < $testreport->{last})
+            {
+                $test->{last} = $testreport->{last};
+            }
+
         }
         next if (!$testreport->{failed});
-        next if (!$testreport->{fixedruns});
+        if (!$testreport->{fixedruns})
+        {
+            $test->{fixed} = 0;
+            next;
+        }

if ($testreport->{failtype} eq "random")
         {
@@ -518,7 +538,60 @@ foreach my $testname (keys %tests)
             # it is fixed.
             $testreport->{fixed} = 1;
         }
+        $test->{fixed} = 0 if ($testreport->{fixed} < $fixed_threshold);
+    }
+    if ($test->{last} == @sortedbuilds-1)
+    {
+        $test->{fixed} = 0;
+        next;
+    }
+
+    # No need for further analysis if this test unit seems fixed already
+    next if ($test->{fixed});
+    next if ($test->{last} < @sortedbuilds - $patternbuilds);
+
+    # Analyze the pattern formed by the results from the build of the latest
+    # failure: sometimes a commit will cause failures in many reports and is
+    # fixed soon after (e.g. next day). In such a case analysing the reports
+    # individually does not provide enough evidence that the bug is fixed until
+    # many days later. But analysing the results on the last failure build can
+    # yield evidence of a high failure rate so that the fix is obvious in the
+    # next few days.
+    my $lastbuild = $sortedbuilds[$test->{last}];
+    my ($failures, $failruns, $fixedruns);
+    foreach my $reportdir (keys %{$test->{testreports}})
+    {
+        my $testreport = $test->{testreports}->{$reportdir};
+        next if (!$testreport->{failed});
+
+        my $status = $testreport->{status}->{$lastbuild->{name}};
+        if (!defined $status and
+            $lastbuild->{hasreport}->{$reportdir} and
+            $lastbuild->{hastest}->{$testname})
+        {
+            $failruns++;
+            next;
+        }
+
+        my $failtype = fail_type($status);
+        next if ($failtype ne "random");
+        $failures++;
+        $failruns++;
+
+        for my $i ($test->{last}+1..@sortedbuilds-1)
+        {
+            my $build = $sortedbuilds[$i];
+            $status = $testreport->{status}->{$build->{name}};
+            if (!defined $status and
+                $build->{hasreport}->{$reportdir} and
+                $build->{hastest}->{$testname})
+            {
+                $fixedruns++;
+            }
+        }
     }
+    next if (!$fixedruns); # no evidence of the issue being fixed
+    $test->{fixed} = get_fix_probability($failures, $failruns, $fixedruns);
 }

@@ -809,6 +882,7 @@ EOF
         }
         next if ($last == -1); # no report had a pattern of interest

+        $fixed = 1 if (($test->{fixed} || 0) >= $fixed_threshold);
         my $listid = ($fixed or $last < @sortedbuilds - $patternbuilds) ? "old" :
                      ($first > $patternbuilds) ? "recent" :
                      "regular";