Francois Gouget : testbot: Match the failures with the test logs.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 1b238d27711cf9b62fb5f304d19610c0472c7b05
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=1b238d27711cf9b62fb5f304...

Author: Francois Gouget 
Date:   Wed Jun 15 18:21:35 2022 +0200

testbot: Match the failures with the test logs.

This augments the .errors files with information mapping errors to known
failures.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=48912
Signed-off-by: Francois Gouget 
Signed-off-by: Alexandre Julliard

---

testbot/lib/WineTestBot/LogUtils.pm | 180 +++++++++++++++++++++++++++++++++++-
 1 file changed, 178 insertions(+), 2 deletions(-)

diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 2e3de83b..b8aaaa43 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -36,7 +36,10 @@ our @EXPORT = qw(GetLogFileNames GetLogLabel
 use Algorithm::Diff;
 use File::Basename;

+use ObjectModel::Collection; # For CombineKey()
 use WineTestBot::Config; # For $MaxUnitSize
+use WineTestBot::Failures;
+use WineTestBot::Tasks; # FIXME Hack for SaveLogFailures()
 use WineTestBot::Utils; # For LocaleName()

@@ -187,6 +190,12 @@ sub _AddLogError($$$;$$)
 Returns a hashtable containing a summary of the task log:
 =over

+=item LogName
+The log file basename.
+
+=item LogPath
+The full log file path.
+
 =item Type
 'tests' if the task ran Wine tests and 'build' otherwise.

@@ -913,6 +922,10 @@ copy of that line (with CR/LF converted to a simple LF).
 The format for new error lines is identical to that for old errors but with a
 different type.

+=item f <errindex> <failureid1>...
+Failure lines contain the index of the error in the error group and a list of
+the known failures they match (normally at most one).
+
 =back
 =back
 =cut
@@ -981,6 +994,10 @@ sub LoadLogErrorsFromFh($$)
     {
       _AddLogError($LogInfo, $LogInfo->{CurGroup}, $Value, $Property, "new");
     }
+    elsif ($Type eq "f")
+    {
+      $LogInfo->{CurGroup}->{Failures}->{$Property} = [ split / /, $Value ];
+    }
     else
     {
       $LogInfo->{BadLog} = "$LogInfo->{LineNo}: Found an unknown line type ($Type)";
@@ -1058,8 +1075,12 @@ sub _WriteLogErrorsToFh($$)
     print $Fh "g $Group->{LineNo} $GroupName\n";
     foreach my $Index (0..$#{$Group->{Errors}})
     {
+      my $LineNo = $Group->{LineNos}->[$Index];
       my $IsNew = $Group->{IsNew}->[$Index] ? "n" : "o";
-      print $Fh "$IsNew $Group->{LineNos}->[$Index] $Group->{Errors}->[$Index]\n";
+      print $Fh "$IsNew $LineNo $Group->{Errors}->[$Index]\n";
+
+      my $Failures = $Group->{Failures}->{$Index};
+      print $Fh "f $Index @$Failures\n" if ($Failures);
     }
   }
 }
@@ -1299,6 +1320,158 @@ sub TagNewErrors($$)
   }
 }

+=pod
+=over 12
+
+=item C<MatchLogFailures()>
+
+Checks the errors against known failures.
+
+The $LogInfo structure is augmented with the following fields:
+=over
+
+=item ErrGroups
+=over
+
+=item Failures
+A hashtable mapping error indices to the list of matching known
+failure ids.
+
+=back
+=back
+
+Returns a hashtable containing a summary of the log failures:
+=over
+
+=item LogName
+The log file basename.
+
+=item Collection
+A collection containing the relevant Failure objects.
+
+=item Failures
+A hashtable indexed by the failure ids. Each entry contains:
+
+=over
+
+=item Failure
+The failure object.
+
+=item NewCount
+A count errors matched by this known failure that were tagged as new.
+
+=item OldCount
+A count errors matched by this known failure that were tagged as old.
+
+=back
+
+=back
+=back
+=cut
+
+sub MatchLogFailures($$)
+{
+  my ($LogInfo, $Task) = @_;
+
+  my $LogFailures = {
+    Task => $Task,
+    LogName => $LogInfo->{LogName}
+  };
+  return $LogFailures if (!$LogInfo->{ErrCount});
+
+  my %FailureTree;
+  my $ConfigName = $Task->VMName .":$LogInfo->{LogName}";
+
+  $LogFailures->{Collection} = CreateFailures();
+  foreach my $Failure (@{$LogFailures->{Collection}->GetItems()})
+  {
+    # Ignore failures that don't apply to this configuration
+    my $ConfigRegExp = $Failure->ConfigRegExp;
+    my $Match = eval { $ConfigRegExp and $ConfigName =~ /$ConfigRegExp/ };
+    next if (!$Match);
+
+    my $UnitFailures = $FailureTree{$Failure->ErrorGroup}->{$Failure->TestUnit} ||= [];
+    push @$UnitFailures, $Failure;
+  }
+
+  foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+  {
+    next if (!$FailureTree{$GroupName});
+
+    my $Group = $LogInfo->{ErrGroups}->{$GroupName};
+    foreach my $ErrIndex (0..$#{$Group->{Errors}})
+    {
+      my $Line = $Group->{Errors}->[$ErrIndex];
+      my $TestUnit = $Line =~ /^([_a-z0-9]+).c:\d+:/ ? $1 : "";
+      my $UnitFailures = $FailureTree{$GroupName}->{$TestUnit};
+      next if (!$UnitFailures);
+
+      foreach my $UnitFailure (@$UnitFailures)
+      {
+        my $RegExp = $UnitFailure->FailureRegExp;
+        my $Match = eval { $RegExp and $Line =~ /$RegExp/ };
+        next if (!$Match);
+
+        my $LineFailures = $Group->{Failures}->{$ErrIndex} ||= [];
+        push @$LineFailures, $UnitFailure->Id;
+
+        my $LogFailure = $LogFailures->{Failures}->{$UnitFailure->Id};
+        if (!$LogFailure)
+        {
+          $LogFailure = $LogFailures->{Failures}->{$UnitFailure->Id} =
+                        { Failure => $UnitFailure };
+        }
+        my $Count = $Group->{IsNew}->[$ErrIndex] ? "NewCount" : "OldCount";
+        $LogFailure->{$Count}++;
+      }
+    }
+  }
+
+  return $LogFailures;
+}
+
+=pod
+=over 12
+
+=item C<SaveLogFailures()>
+
+Updates the TaskFailures objects for the task and log specified by the
+$LogFailures structure.
+
+Note that this implies deleting any preexisting TaskFailure to avoid leaving
+obsolete data.
+
+=back
+=cut
+
+sub SaveLogFailures($)
+{
+  my ($LogFailures) = @_;
+
+  # FIXME $Task->Failures->AddFilter() cannot be undone and impacts every
+  #       future use of $Task->Failures. So add the filter on a throw away
+  #       clone to not end up with a nonsensical filter.
+  my $TaskFailures = $LogFailures->{Task}->Failures->Clone();
+  $TaskFailures->AddFilter("TaskLog", [$LogFailures->{LogName}]);
+  my $ErrMessage = $TaskFailures->DeleteAll();
+  return $ErrMessage if (defined $ErrMessage);
+  return undef if (!$LogFailures->{Failures});
+
+  foreach my $LogFailure (values %{$LogFailures->{Failures}})
+  {
+    my $TaskFailure = $LogFailure->{Failure}->TaskFailures->Add();
+    my $OldKey = $TaskFailure->GetKey();
+    $TaskFailure->Task($LogFailures->{Task});
+    $TaskFailure->TaskLog($LogFailures->{LogName});
+    $TaskFailure->KeyChanged($OldKey, $TaskFailure->GetKey());
+    $TaskFailure->NewCount($LogFailure->{NewCount});
+    $TaskFailure->OldCount($LogFailure->{OldCount});
+  }
+
+  (my $_ErrKey, my $_ErrProperty, $ErrMessage) = $LogFailures->{Collection}->Save();
+  return $ErrMessage;
+}
+

#
 # Log errors caching [Part 2]
@@ -1322,8 +1495,11 @@ sub CreateLogErrorsCache($;$)
     # Don't mark the errors as new if there is no reference WineTest report
     # as this would cause false positives.
   }
+  my $LogFailures = MatchLogFailures($LogInfo, $Task) if ($Task);

-  return _SaveLogErrors($LogInfo);
+  my $ErrMessage = _SaveLogErrors($LogInfo);
+  $ErrMessage ||= SaveLogFailures($LogFailures) if ($Task);
+  return $ErrMessage;
 }