Francois Gouget : testbot/TestWTBS: Allow checking the new task counters.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 41f41b29546d13b4c325d273338b880f3b4a141f
URL:    https://gitlab.winehq.org/winehq/tools/-/commit/41f41b29546d13b4c325d273338b...

Author: Francois Gouget 
Date:   Thu Nov  3 03:15:51 2022 +0100

testbot/TestWTBS: Allow checking the new task counters.

---

testbot/tests/TestWTBS | 103 ++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 77 insertions(+), 26 deletions(-)

diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 33074a82..59909287 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -368,7 +368,7 @@ sub LoadTestInfo($)
     {
       # Ignore these
     }
-    elsif ($Entry !~ /^(?:CurGroup|eCount|eNew|MsgGroups|MsgGroupNames|LineNo|LogName|LogPath)$/)
+    elsif ($Entry !~ /^(?:CurGroup|eCount|eNew|kCount|kNew|MsgGroups|MsgGroupNames|LineNo|LogName|LogPath)$/)
     {
       fail("$FileName: $Entry is not a valid property name");
     }
@@ -400,6 +400,22 @@ sub LoadTestInfo($)
     SetDefault($TestInfo, "tasks", "Status", "completed");
     SetDefault($TestInfo, "tasks", "HasTask", 1);
   }
+  foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64", "wine", "wine:build", "wine:test", "wine:module")
+  {
+    my $TaskInfo = $TestInfo->{$TaskType};
+    if (defined $TaskInfo->{TestFailures})
+    {
+      SetDefault($TestInfo, $TaskType, "NewTestFailures", $TaskInfo->{TestFailures});
+      if (CheckValue($TaskInfo->{TestFailures}))
+      {
+        SetDefault($TestInfo, $TaskType, "FailedTestUnits", $TaskInfo->{TestFailures} ? 1 : 0);
+      }
+    }
+    if (defined $TaskInfo->{Warnings})
+    {
+      SetDefault($TestInfo, $TaskType, "NewWarnings", $TaskInfo->{Warnings});
+    }
+  }
   foreach my $BuildType ("build", "wine:build")
   {
     next if (!defined $TestInfo->{$BuildType}->{TestUnits});
@@ -913,10 +929,22 @@ still queued or running or have been canceled are automatically skipped to
 avoid false positives.
 By default the Status field should be 'completed'.

+=item <tasks.NewWarnings>
+=item <tasks.Warnings>
+=item <tasks.NewTestFailures>
 =item <tasks.TestFailures>

-Checks the value of the task's TestFailures field.
-By default the TestFailures field is not checked.
+Checks the value of the corresponding task's field.
+By default none of these fields are checked.
+If Warnings is set and not NewWarnings, then it is assumed it should have the
+same value. The same applies to TestFailures and NewTestFailures.
+
+=item <tasks.FailedTestUnits>
+
+Checks the value of the corresponding task's field. More precisely:
+* FailedTestUnits == undef && TestFailures == undef -> not checked
+* FailedTestUnits == undef && TestFailures == 0     -> expect 0
+* FailedTestUnits == undef && TestFailures != 0     -> expect 1

=item <tasks.HasTimeout>

@@ -1000,7 +1028,7 @@ sub CheckTask($$$$)
   my $CheckTimeouts = ($Task->Status eq "completed" and
                        CheckValue($TaskInfo->{HasTimeout}));

-  my ($ReportFailures, $IgnoreFailures);
+  my (%Ignore, %Counts);
   my ($ReportCount, $TimeoutCount, $NewFailures) = (0, 0, 0);
   foreach my $LogName (@{GetLogFileNames($Task->GetDir())})
   {
@@ -1028,16 +1056,36 @@ sub CheckTask($$$$)
         map { $TestUnits->{$MissionType}->{$_} = 1 } (keys %$ReportTestUnits);
       }

-      if (CheckValue($MissionInfo->{TestFailures}))
+      foreach my $Counter ("NewWarnings", "Warnings", "NewTestFailures", "TestFailures")
+      {
+        if (CheckValue($MissionInfo->{$Counter}))
+        {
+          my $LogCount = $Counter eq "NewWarnings" ? ($LogInfo->{wNew} || 0) - ($LogInfo->{wKnownNew} || 0) :
+              $Counter eq "Warnings" ? $LogInfo->{wCount} :
+              $Counter eq "NewFailures" ? ($LogInfo->{eNew} || 0) - ($LogInfo->{eKnownNew} || 0) :
+              $Counter eq "Failures" ? $LogInfo->{eCount} :
+              -1;
+          ok($LogCount <= $MissionInfo->{$Counter},
+             "Check $Counter of $LogName in task ". TaskKeyStr($Task))
+              or diag("report $Counter count = $LogCount, expected at most $MissionInfo->{$Counter}");
+          $Counts{$Counter} += $MissionInfo->{$Counter};
+        }
+        else
+        {
+          $Ignore{$Counter} = 1;
+        }
+      }
+      if (CheckValue($MissionInfo->{FailedTestUnits}))
       {
-        ok(($LogInfo->{eCount} || 0) <= $MissionInfo->{TestFailures},
-           "Check Failures of $LogName in task ". TaskKeyStr($Task))
-            or diag("report error count = ", ($LogInfo->{eCount} || 0), ", expected at most $MissionInfo->{TestFailures}");
-        $ReportFailures += $MissionInfo->{TestFailures};
+        my $LogCount = @{$LogInfo->{FailedTestUnits} || []};
+        ok($LogCount == $MissionInfo->{FailedTestUnits},
+           "Check FailedTestUnits of $LogName in task ". TaskKeyStr($Task))
+            or diag("report FailedTestUnits count = $LogCount expected $MissionInfo->{FailedTestUnits}");
+        $Counts{FailedTestUnits} ||= $MissionInfo->{FailedTestUnits};
       }
       else
       {
-        $IgnoreFailures = 1;
+        $Ignore{FailedTestUnits} = 1;
       }
     }
     elsif ($LogName =~ /^testbot./)
@@ -1062,24 +1110,27 @@ sub CheckTask($$$$)
   }
   if ($Task->Status eq "completed")
   {
-    if ($IsTestTask and ($ReportCount or CheckValue($TaskInfo->{TestUnits})))
+    if ($ReportCount)
     {
-      if ($IgnoreFailures)
-      {
-        # For some mission types (typically test=module) the failure count
-        # is unpredictable and cannot be checked.
-        ; # Nothing to do
-      }
-      elsif ($ReportCount)
-      {
-        # The task's failure count must be compared to the sum of the failure
-        # counts of each test report.
-        is($Task->TestFailures, $ReportFailures, "Check Failures of task ". TaskKeyStr($Task));
-      }
-      elsif (CheckValue($TaskInfo->{TestFailures}))
+      foreach my $Counter ("NewWarnings", "Warnings", "NewTestFailures", "TestFailures", "FailedTestUnits")
       {
-        # There is no report if the VM crashed / rebooted
-        is($Task->TestFailures, $TaskInfo->{TestFailures}, "Check Failures of task ". TaskKeyStr($Task));
+        if ($Ignore{$Counter})
+        {
+          # For some mission types (typically test=module) the counter values
+          # are unpredictable and cannot be checked.
+          ; # Nothing to do
+        }
+        elsif ($ReportCount)
+        {
+          # The task's (warning / failure) count must be compared to the sum
+          # of the counts for each report.
+          is($Task->$Counter || 0, $Counts{$Counter}, "Check $Counter of task ". TaskKeyStr($Task));
+        }
+        elsif (CheckValue($TaskInfo->{$Counter}))
+        {
+          # There is no report if the VM crashed / rebooted
+          is($Task->$Counter, $TaskInfo->{$Counter}, "Check $Counter of task ". TaskKeyStr($Task));
+        }
       }
     }
     elsif (!$IsTestTask and $ReportCount)