[PATCH 1/2] testbot/JobDetails: Highlight new errors in the full log too.

[Francois Gouget]

Extend $LogInfo to store the 1-based source line number of each error.
For extra errors (i.e. errors not present in the source log), set the 
line number to 0.

Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
---
 testbot/lib/WineTestBot/LogUtils.pm | 20 +++++++++----
 testbot/web/JobDetails.pl           | 46 +++++++++++++++++++++++++----
 testbot/web/WineTestBot.css         |  1 +
 3 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 8187826e34..75d283381c 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -718,16 +718,17 @@ sub _AddLogGroup($$)
   if (!$LogInfo->{ErrGroups}->{$GroupName})
   {
     push @{$LogInfo->{ErrGroupNames}}, $GroupName;
-    $LogInfo->{ErrGroups}->{$GroupName} = { Errors => [] };
+    $LogInfo->{ErrGroups}->{$GroupName} = { LineNos => [], Errors => [] };
   }
   return $LogInfo->{ErrGroups}->{$GroupName};
 }
 
-sub _AddLogError($$$)
+sub _AddLogError($$$;$)
 {
-  my ($LogInfo, $ErrGroup, $Line) = @_;
+  my ($LogInfo, $ErrGroup, $Line, $LineNo) = @_;
 
   push @{$ErrGroup->{Errors}}, $Line;
+  push @{$ErrGroup->{LineNos}}, $LineNo || 0;
   $LogInfo->{ErrCount}++;
 }
 
@@ -756,8 +757,13 @@ An array containing the names of all the error groups.
 A hashtable indexed by the error group name. Each entry contains:
 
 =over
+
 =item Errors
 An array containing the error messages.
+
+=item LineNos
+An array containing the line number of the error in the log file.
+
 =back
 
 =back
@@ -793,8 +799,10 @@ sub GetLogErrors($)
     $LogInfo->{ErrCount} ||= 0;
     my $CurrentModule = "";
     my $CurrentGroup;
+    my $LineNo = 0;
     foreach my $Line (<$LogFile>)
     {
+      $LineNo++;
       $Line =~ s/\s*$//;
       if ($IsReport and $Line =~ /^([_.a-z0-9-]+):[_a-z0-9]* start /)
       {
@@ -814,7 +822,7 @@ sub GetLogErrors($)
       {
         $CurrentGroup = _AddLogGroup($LogInfo, $CurrentModule);
       }
-      _AddLogError($LogInfo, $CurrentGroup, $Line);
+      _AddLogError($LogInfo, $CurrentGroup, $Line, $LineNo);
     }
     close($LogFile);
   }
@@ -830,8 +838,10 @@ sub GetLogErrors($)
     $LogInfo->{ErrCount} ||= 0;
     # Add the related extra errors
     my $CurrentGroup;
+    my $LineNo = 0;
     foreach my $Line (<$LogFile>)
     {
+      $LineNo++;
       $Line =~ s/\s*$//;
       if (!$CurrentGroup)
       {
@@ -840,7 +850,7 @@ sub GetLogErrors($)
         my $GroupName = $IsReport ? "Report errors" : "Task errors";
         $CurrentGroup = _AddLogGroup($LogInfo, $GroupName);
       }
-      _AddLogError($LogInfo, $CurrentGroup, $Line);
+      _AddLogError($LogInfo, $CurrentGroup, $Line, $LineNo);
     }
     close($LogFile);
   }
diff --git a/testbot/web/JobDetails.pl b/testbot/web/JobDetails.pl
index 2909eb7dbc..f1ffd498ed 100644
--- a/testbot/web/JobDetails.pl
+++ b/testbot/web/JobDetails.pl
@@ -351,19 +351,42 @@ sub GetErrorCategory($)
   return "error";
 }
 
-sub GenerateFullLog($$$;$)
+sub GenerateFullLog($$$$;$)
 {
-  my ($self, $FileName, $HideLog, $Header) = @_;
+  my ($self, $FileName, $HideLog, $LogInfo, $Header) = @_;
+
+  my %NewLineNos;
+  if ($LogInfo and $LogInfo->{NewCount})
+  {
+    foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+    {
+      my $Group = $LogInfo->{ErrGroups}->{$GroupName};
+      next if (!$Group->{NewCount});
+      if (($FileName =~ /\.err$/) ==
+          ($GroupName =~ /^(?:Report errors|Task errors|TestBot errors)$/))
+      {
+        for my $ErrIndex (0..@{$Group->{Errors}} - 1)
+        {
+          if ($Group->{IsNew}->[$ErrIndex])
+          {
+            $NewLineNos{$Group->{LineNos}->[$ErrIndex]} = 1;
+          }
+        }
+      }
+    }
+  }
 
   my $GetCategory = $FileName =~ /\.err$/ ? \&GetErrorCategory :
                     $FileName =~ /\.report$/ ? \&GetReportLineCategory :
                     \&GetLogLineCategory;
 
+  my $LineNo = 0;
   my $IsEmpty = 1;
   if (open(my $LogFile, "<", $FileName))
   {
     foreach my $Line (<$LogFile>)
     {
+      $LineNo++;
       $Line =~ s/\s*$//;
       if ($IsEmpty)
       {
@@ -372,8 +395,8 @@ sub GenerateFullLog($$$;$)
         $IsEmpty = 0;
       }
 
-      my $Category = $GetCategory->($Line);
       my $Html = $self->escapeHTML($Line);
+      my $Category = $NewLineNos{$LineNo} ? "fullnew" : $GetCategory->($Line);
       if ($Category ne "none")
       {
         $Html =~ s~^(.*\S)\s*\r?$~<span class='log-$Category'>$1</span>~;
@@ -463,11 +486,24 @@ EOF
       # Show this log in full, highlighting the important lines
       #
 
+      my $LogInfo;
+      my $LogFileName = "$TaskDir/$MoreInfo->{Full}";
+      if ($MoreInfo->{Full} =~ /\.report$/)
+      {
+        $LogInfo = GetLogErrors($LogFileName);
+        if ($LogInfo->{ErrCount})
+        {
+          # Identify new errors in test reports
+          my $RefReportPath = $StepTask->GetFullFileName($StepTask->GetRefReportName($MoreInfo->{Full}));
+          TagNewErrors($RefReportPath, $LogInfo);
+        }
+      }
+
       my ($Action, $Url) = $self->GetMoreInfoLink($Key, GetLogLabel($MoreInfo->{Full}), "Full", $MoreInfo->{Full});
       $Url = $self->CGI->escapeHTML($Url);
       my $HideLog = $Action eq "Hide" ? " ondblclick='HideLog(event, \"$Url\")'" : "";
 
-      my $LogIsEmpty = $self->GenerateFullLog("$TaskDir/$MoreInfo->{Full}", $HideLog);
+      my $LogIsEmpty = $self->GenerateFullLog($LogFileName, $HideLog, $LogInfo);
       my $EmptyDiag;
       if ($LogIsEmpty)
       {
@@ -490,7 +526,7 @@ EOF
       my $ErrHeader = $MoreInfo->{Full} =~ /\.report/ ? "report" : "task";
       $ErrHeader = "old $ErrHeader" if ($MoreInfo->{Full} =~ /^old_/);
       $ErrHeader = "<div class='HrTitle'>". ucfirst($ErrHeader) ." errors<div class='HrLine'></div></div>";
-      my $ErrIsEmpty = $self->GenerateFullLog("$TaskDir/$MoreInfo->{Full}.err", $HideLog, $ErrHeader);
+      my $ErrIsEmpty = $self->GenerateFullLog("$LogFileName.err", $HideLog, $LogInfo, $ErrHeader);
       print $EmptyDiag if ($ErrIsEmpty and defined $EmptyDiag);
     }
     else
diff --git a/testbot/web/WineTestBot.css b/testbot/web/WineTestBot.css
index f588355833..9a23f5fa82 100644
--- a/testbot/web/WineTestBot.css
+++ b/testbot/web/WineTestBot.css
@@ -402,6 +402,7 @@ pre
 .log-boterror { color: #cc0052; }
 .log-diag { color: #e56300; }
 .log-new { color: #e56e00; font-weight: bold; }
+.log-fullnew { color: red; font-weight: bold; }
 
 a.title { color:inherit; text-decoration: none; }
 
-- 
2.20.1

[PATCH 2/2] testbot/LogUtils: Keep track of the first line of each error group.

[Francois Gouget]

This will make it possible to jump to the first line of a test unit that
contains failures.

Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
---

This also makes group names fit perfectly into the future log errors 
cache format ;-)

 testbot/lib/WineTestBot/LogUtils.pm | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 75d283381c..4d0d99bfab 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -709,16 +709,20 @@ sub _DumpErrors($$)
   }
 }
 
-sub _AddLogGroup($$)
+sub _AddLogGroup($$;$)
 {
-  my ($LogInfo, $GroupName) = @_;
+  my ($LogInfo, $GroupName, $LineNo) = @_;
 
   # In theory the error group names are all unique. But, just in case, make
   # sure we don't overwrite $LogInfo->{ErrGroups}->{$GroupName}.
   if (!$LogInfo->{ErrGroups}->{$GroupName})
   {
     push @{$LogInfo->{ErrGroupNames}}, $GroupName;
-    $LogInfo->{ErrGroups}->{$GroupName} = { LineNos => [], Errors => [] };
+    $LogInfo->{ErrGroups}->{$GroupName} = {
+      LineNo => $LineNo || 0,
+      LineNos => [],
+      Errors => []
+    };
   }
   return $LogInfo->{ErrGroups}->{$GroupName};
 }
@@ -758,6 +762,10 @@ A hashtable indexed by the error group name. Each entry contains:
 
 =over
 
+=item LineNo
+The line number of the start of this error group. Note that this is normally
+different from the line of the first error in that group.
+
 =item Errors
 An array containing the error messages.
 
@@ -798,7 +806,7 @@ sub GetLogErrors($)
   {
     $LogInfo->{ErrCount} ||= 0;
     my $CurrentModule = "";
-    my $CurrentGroup;
+    my ($CurrentGroup, $GroupLineNo);
     my $LineNo = 0;
     foreach my $Line (<$LogFile>)
     {
@@ -807,6 +815,7 @@ sub GetLogErrors($)
       if ($IsReport and $Line =~ /^([_.a-z0-9-]+):[_a-z0-9]* start /)
       {
         $CurrentModule = $1;
+        $GroupLineNo = $LineNo;
         $CurrentGroup = undef;
         next;
       }
@@ -820,7 +829,7 @@ sub GetLogErrors($)
       }
       if (!$CurrentGroup)
       {
-        $CurrentGroup = _AddLogGroup($LogInfo, $CurrentModule);
+        $CurrentGroup = _AddLogGroup($LogInfo, $CurrentModule, $GroupLineNo || $LineNo);
       }
       _AddLogError($LogInfo, $CurrentGroup, $Line, $LineNo);
     }
@@ -848,7 +857,7 @@ sub GetLogErrors($)
         # Note: $GroupName must not depend on the previous content as this
         #       would break diffs.
         my $GroupName = $IsReport ? "Report errors" : "Task errors";
-        $CurrentGroup = _AddLogGroup($LogInfo, $GroupName);
+        $CurrentGroup = _AddLogGroup($LogInfo, $GroupName, $LineNo);
       }
       _AddLogError($LogInfo, $CurrentGroup, $Line, $LineNo);
     }
-- 
2.20.1