[25/25] testbot/LogUtils: Deduplicate the latest WineTest reports.

[Francois Gouget]

There is no need to keep old logs if they don't contain errors that are
already present in the latest one. It cuts down on the number of reports
that test results need to be compared to to detect new failures.
---
 testbot/bin/UpdateTaskLogs          |  6 ++
 testbot/lib/WineTestBot/LogUtils.pm | 92 ++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs
index e1746e09a..553bda989 100755
--- a/testbot/bin/UpdateTaskLogs
+++ b/testbot/bin/UpdateTaskLogs
@@ -197,6 +197,9 @@ sub BuildErrorsCache($$$$;$)
   return CreateLogErrorsCache($LogInfo, $Task);
 }

+# The list of files in latest/. This includes files deleted for redundancy.
+my %LatestFiles;
+
 sub DoUpdateLatestReport($$$)
 {
   my ($Task, $ReportName, $SrcReportPath) = @_;
@@ -205,6 +208,8 @@ sub DoUpdateLatestReport($$$)
   my $SrcIsRef = ($SrcReportPath =~ /-job\d+-/);
   my $RefReportName = $SrcIsRef ? basename($SrcReportPath) :
                                   $Task->GetRefReportName($ReportName);
+  return 0 if ($LatestFiles{$RefReportName});
+  $LatestFiles{$RefReportName} = 1;

my $Rc = 0;
   my $LatestReportPath = "$DataDir/latest/$RefReportName";
@@ -506,6 +511,7 @@ sub ProcessLatestReports()
     }
     else
     {
+      $LatestFiles{$RefReportName} = 1;
       if (!-f "$LatestReportPath.errors")
       {
         # Build the missing .errors file
diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 037387277..6177c666f 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -1013,9 +1013,9 @@ sub MarkAllErrorsAsNew($)

=item C<_GetLineKey()>

-This is a helper for TagNewErrors(). It reformats the log lines so they
-can meaningfully be compared to the reference log even if line numbers change,
-etc.
+This is a helper for _DeduplicateLatestReport() and TagNewErrors(). It
+reformats the log lines so they can meaningfully be compared to the reference
+log even if line numbers change, etc.

=back
 =cut
@@ -1235,6 +1235,89 @@ sub SnapshotLatestReport($$)
   return @ErrMessages;
 }

+sub _IsReportRedundant($$)
+{
+  my ($RefInfo, $LogInfo) = @_;
+
+  return undef if (($RefInfo->{ErrCount} || 0) < ($LogInfo->{ErrCount} || 0));
+  return 1 if (($LogInfo->{ErrCount} || 0) == 0);
+
+  foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+  {
+    my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+    my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+    return undef if (!$RefGroup);
+
+    my $Diff = Algorithm::Diff->new($RefGroup->{Errors}, $LogGroup->{Errors},
+                                    { keyGen => &_GetLineKey });
+    while ($Diff->Next())
+    {
+      # Same() > 0 => Items(2) > 0 despite there being no new lines
+      if (!$Diff->Same() and $Diff->Items(2) > 0)
+      {
+        # This old log has errors not present in the latest one so keep it
+        return undef;
+      }
+    }
+  }
+
+  return 1;
+}
+
+sub _DeduplicateLatestReport($$$)
+{
+  my ($RefReportName, $VMName, $ReportName) = @_;
+
+  my $LatestGlob = "$DataDir/latest/$VMName-job*-$ReportName";
+  my @LatestPaths = glob("$LatestGlob");
+  return undef if (@LatestPaths <= 1);
+
+  my $RefReportPath = "$DataDir/latest/$RefReportName";
+  my $RefInfo = LoadLogErrors($RefReportPath);
+  return $RefInfo->{BadLog} if (defined $RefInfo->{BadLog});
+
+  my %LatestAges;
+  foreach my $LogPath (@LatestPaths)
+  {
+    $LatestAges{$LogPath} = -M $LogPath || 0;
+  }
+  my $RefAge = $LatestAges{$RefReportPath};
+
+  my $ErrMessage;
+  foreach my $LogPath (sort { $LatestAges{$a} <=> $LatestAges{$b} } @LatestPaths)
+  {
+    my $LogName = basename($LogPath);
+    next if ($LogName eq $RefReportName);
+    next if ($LogName !~ /^(\Q$VMName\E-job[0-9]+-\Q$ReportName\E)$/);
+    $LogName = $1; # untaint
+    $LogPath = "$DataDir/latest/$LogName";
+
+    my $LogInfo = LoadLogErrors($LogPath);
+    if ($LogInfo->{BadLog})
+    {
+      # Take note of the error but continue to try deduplicating
+      $ErrMessage = $LogInfo->{BadLog};
+      next;
+    }
+    if ($RefAge < $LatestAges{$LogPath})
+    {
+      if (_IsReportRedundant($RefInfo, $LogInfo))
+      {
+        unlink $LogPath, "$LogPath.errors";
+      }
+    }
+    else
+    {
+      if (_IsReportRedundant($LogInfo, $RefInfo))
+      {
+        unlink $RefReportPath, "$RefReportPath.errors";
+        last;
+      }
+    }
+  }
+  return $ErrMessage;
+}
+
 sub UpdateLatestReport($$$)
 {
   my ($Task, $ReportName, $SrcReportPath) = @_;
@@ -1255,6 +1338,9 @@ sub UpdateLatestReport($$$)
     }
   }

+  my $ErrMessage = _DeduplicateLatestReport($RefReportName, $Task->VM->Name, $ReportName);
+  push @ErrMessages, $ErrMessage if (defined $ErrMessage);
+
   return @ErrMessages;
 }