[tools 1/3] testbot/TestWTBS: Check the result and error emails.
When a job completes the TestBot sends an email with the results and another with a summary of the errors in case of new failures. This patch allows checking that these emails did get sent if appropriate and seem to contain the expected data. For this the emails should be saved to a file in mbox format and that file passed to TestWTBS using the --mbox option. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/tests/TestWTBS | 187 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 175 insertions(+), 12 deletions(-) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 098008110..58cbf3041 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -107,6 +107,7 @@ sub check_opt_val($$) } my $OptJobs; +my $OptMbox; while (@ARGV) { my $Arg = shift @ARGV; @@ -114,6 +115,10 @@ while (@ARGV) { $OptJobs = check_opt_val($Arg, $OptJobs); } + elsif ($Arg eq "--mbox") + { + $OptMbox = check_opt_val($Arg, $OptMbox); + } elsif ($Arg eq "--help") { $Usage = 0; @@ -156,7 +161,7 @@ if (defined $Usage) error("try '$name0 --help' for more information\n"); exit $Usage; } - print "Usage: $name0 [--jobs RANGES] [--help]\n"; + print "Usage: $name0 [--jobs RANGES] [--mbox FILE] [--help]\n"; print "\n"; print "Tests the Patches subject parser.\n"; print "\n"; @@ -166,6 +171,7 @@ if (defined $Usage) print " these jobs to be ignored. Each range is either a single job id,\n"; print " or of the form 'FIRST..LAST' where FIRST and LAST are either the\n"; print " empty string or a job id.\n"; + print " --mbox FILE Check the TestBot's emails in this mbox-format file.\n"; print " --help Shows this usage message.\n"; exit 0; } @@ -417,6 +423,114 @@ sub GetReportTestUnits($) } +# +# Load the TestBot emails +# + +my %Emails; + +sub CloseEmail($) +{ + my ($State) = @_; + + # Avoid undefined values to simplify diagnostic messages + $State->{HasFailedEmail} ||= 0; + $State->{HasResEmail} ||= 0; + + if (!$State->{JobId}) + { + foreach my $Field ("HasFailedEmail", "HasResEmail") + { + next if (!$State->{$Field}); + fail("Unexpected $Field property found on line $State->{$Field}"); + } + foreach my $Hash ("FailedVMs", "ResVMs") + { + next if (!$State->{$Hash}); + my $Lines = join(" ", map { "$_:$State->{$Hash}->{$_}" } sort keys %{$State->{$Hash}}); + fail("Unexpected $Hash property found on lines: $Lines"); + } + } + else + { + my $Email = $Emails{$State->{JobId}}; + $Emails{$State->{JobId}} = $Email = {} if (!$Email); + + $Email->{HasFailedEmail} ||= $State->{HasFailedEmail}; + $Email->{FailedVMs} ||= $State->{FailedVMs}; + + $Email->{HasResEmail} ||= $State->{HasResEmail}; + $Email->{ResVMs} ||= $State->{ResVMs}; + } +} + +sub LoadMbox($) +{ + my ($MboxFilename) = @_; + + if (open(my $MboxFh, "<", $MboxFilename)) + { + my ($State, $LineNo, $Empty); + foreach my $Line (<$MboxFh>) + { + chomp $Line; + $LineNo++; + if ($Line eq "") + { + $Empty = 1; + next; + } + if ($Line =~ m~^From ~) + { + CloseEmail($State); + $State = {}; + } + elsif ($Line =~ /^Subject: TestBot job (\d+) results:/) + { + $State->{JobId} = $1; + $State->{JobIdLine} = $LineNo; + $State->{HasResEmail} = $LineNo; + } + elsif ($Line =~ /JobDetails.pl\?Key=(\d+)$/) + { + my $JobId = $1; + if (defined $State->{JobId}) + { + is($JobId, $State->{JobId}, "Check mbox job ids on lines $LineNo and $State->{JobIdLine}"); + } + else + { + $State->{JobId} = $JobId; + $State->{JobIdLine} = $LineNo; + } + } + elsif ($Line =~ /^Content-Disposition: attachment; filename=([^-]+)-/) + { + my $VM = $1; + # This could match attachments in other emails + $State->{ResVMs}->{$VM}++ if ($State->{HasResEmail}); + } + elsif ($Line =~ /I found new failures/) + { + $State->{HasFailedEmail} = $LineNo; + } + elsif ($Line =~ /^=== (\w+) \(.*\) ===$/) + { + my $VM = $1; + $State->{FailedVMs}->{$VM}++; + } + $Empty = undef; + } + CloseEmail($State); + close($MboxFh); + } + else + { + fail("Could not open $MboxFilename for reading: $!"); + } +} + + # # Verify the Jobs and Tasks # @@ -627,11 +741,12 @@ sub CheckTask($$$$) $TestUnits->{$TaskType}->{"*skipped*"} = 1; } - my $ReportCount = 0; + my ($ReportCount, $NewFailures) = (0, 0); foreach my $LogName (@{GetLogFileNames($Task->GetDir())}) { my $LogPath = $Task->GetDir() ."/$LogName"; my $LogInfo = LoadLogErrors($LogPath); + $NewFailures += $LogInfo->{NewCount} || 0; my $LogType = "log"; if ($LogName =~ /\.report$/) @@ -662,6 +777,7 @@ sub CheckTask($$$$) # was run, i.e. $ReportCount, or take it as is if no report is available. is($Task->TestFailures, $TaskInfo->{TestFailures} * ($ReportCount || 1), "Check Failures of task ". TaskKeyStr($Task)); } + return $NewFailures; } =pod @@ -687,13 +803,13 @@ sub CheckJob($$) $Remarks =~ s/^\[\Q$PatchesMailingList\E\] //; is($Remarks, $JobInfo->{Remarks}, "Check Remarks for job ". $Job->Id); } - if (CheckValue($JobInfo->{Status})) + if (CheckValue($TestInfo->{patch}->{Disposition})) { - is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id); + fail("The '$TestInfo->{patch}->{Disposition}' patch disposition is incompatible with job ". $Job->Id); } - elsif (CheckValue($TestInfo->{patch}->{Disposition})) + elsif (CheckValue($JobInfo->{Status})) { - fail("The '$TestInfo->{patch}->{Disposition}' patch disposition is incompatible with job ". $Job->Id); + is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id); } } @@ -741,12 +857,15 @@ sub CheckJobTree($;$) my ($Job, $TestInfo) = @_; return if (!IsJobInRange($Job)); - return if ($CheckedJobs{$Job->Id}); - $CheckedJobs{$Job->Id} = 1; + my $JobId = $Job->Id; # Simplify use in strings + return if ($CheckedJobs{$JobId}); + $CheckedJobs{$JobId} = 1; - my $HasTask; + my ($HasTask, $HasNewFailures); my $TestUnits = { wine => {} }; + my $Email = $Emails{$JobId}; + my %FailedVMs; my $Steps = $Job->Steps; foreach my $Step (sort { $a->No <=> $b->No } @{$Job->Steps->GetItems()}) { @@ -766,7 +885,22 @@ sub CheckJobTree($;$) foreach my $Task (sort { $a->No <=> $b->No } @{$Step->Tasks->GetItems()}) { $HasTask->{$TaskType} = 1; - CheckTask($Task, $TaskType, $TestInfo, $TestUnits); + my $NewFailures = CheckTask($Task, $TaskType, $TestInfo, $TestUnits); + if ($NewFailures) + { + $FailedVMs{$Task->VM->Name}++; + $HasNewFailures = 1; + } + + if ($Emails{$JobId}) + { + my $VMName = $Task->VM->Name; + if ($Task->Status !~ /^(?:canceled|skipped)$/ and $Email->{HasResEmail}) + { + ok($Email->{ResVMs}->{$VMName}, "Expecting $VMName logs / reports in the job $JobId results email"); + } + } + if ($TaskType =~ /^win(?:32|64)$/) { my $TestUnit = $Step->FileName; @@ -776,6 +910,22 @@ sub CheckJobTree($;$) } } } + if ($Email->{HasFailedEmail}) + { + foreach my $VMName (sort keys %FailedVMs) + { + # The emails count each VM once per task report containing a new error + # while CheckJobTree counts them only once per task. + ok(($Email->{FailedVMs}->{$VMName} || 0) >= $FailedVMs{$VMName}, + "Check for $VMName logs / reports in the job $JobId new failures email") + or diag("Expected at least $FailedVMs{$VMName} $VMName instance in the email, got $Email->{FailedVMs}->{$VMName}"); + delete $Email->{FailedVMs}->{$VMName}; + } + foreach my $VMName (sort keys %{$Email->{FailedVMs}}) + { + fail("New failure email for job $JobId reported an extra VM: $VMName"); + } + } CheckJob($Job, $TestInfo); # Ignore manually submitted jobs because they allow the developer to pick @@ -790,7 +940,7 @@ sub CheckJobTree($;$) if (CheckValue($TypeInfo->{HasTask})) { $HasTask->{$Type} ||= 0; - is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job ". $Job->Id); + is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job $JobId"); } next if ($TestUnits->{$Type}->{"*skipped*"}); @@ -798,11 +948,18 @@ sub CheckJobTree($;$) { foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits}) { - ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job ". $Job->Id) + ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job $JobId") or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}})); } } } + + if ($OptMbox) + { + my $Email = $Emails{$JobId}; + ok($Email->{HasResEmail}, "Looking for results email for job $JobId"); + is(!!$Email->{HasFailedEmail}, !!($Job->Status ne "completed" or $HasNewFailures), "Check presence of a new failures email for job $JobId"); + } } } @@ -986,6 +1143,12 @@ if (!$HasBaseVM->{build}) delete $HasBaseVM->{win64}; } +if (defined $OptMbox) +{ + print "***** Loading emails *****\n"; + LoadMbox($OptMbox); +} + print "***** Checking Patches *****\n"; CheckPatches(); -- 2.20.1
End-of-line spaces should be rare but more importantly they don't materially change the error line and can make it hard to match errors in TestWTBS. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/lib/WineTestBot/LogUtils.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index 79714358c..7786494ea 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -1115,8 +1115,11 @@ sub _GetLineKey($) my ($Line) = @_; return undef if (!defined $Line); + # Remove end-of-line spaces + $Line =~ s/ +$//; + # Remove the line number - $Line =~ s/^([_a-z0-9]+\.c:)\d+:( Test (?:failed|succeeded inside todo block): )/$1$2/ + $Line =~ s/^([_a-z0-9]+\.c:)\d+:( Test (?:failed|succeeded inside todo block):)/$1$2/ # Remove the crash code address: it changes whenever the test is recompiled or $Line =~ s/^(Unhandled exception: .* code) \(0x[0-9a-fA-F]{8,16}\)\.$/$1/ -- 2.20.1
This is useful for checking that the TestBot assembled all the expected parts of a patch series, particularly when multiple patch series are pending. For instance: ----- TestWTBS ----- a patch.Grep WTBS [12]/4 a patch.GrepV WTBS [34]/4 ---- TestWTBS ----- Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/tests/TestWTBS | 52 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 58cbf3041..d0e7cabca 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -384,6 +384,12 @@ sub LoadTestInfo($) $TestInfo->{$TaskType}->{"$LogType.$GrepType"} = [ $Array ]; } } + + my $Array = $TestInfo->{patch}->{$GrepType}; + next if (!defined $Array); + next if (ref($Array) eq "ARRAY"); + fail("patch.$GrepType should be an array (a lines)"); + $TestInfo->{patch}->{$GrepType} = [ $Array ]; } # Automatically check webpatch.Status in simple cases @@ -643,18 +649,18 @@ sub CheckLogErrors($$$) } } -sub GrepTaskLog($$$$) +sub GrepFile($$$$;$) { - my ($Task, $LogName, $TaskInfo, $LogType) = @_; + my ($Key, $Dir, $FileName, $GrepInfo, $GrepPrefix) = @_; - my @Grep = @{$TaskInfo->{"$LogType.Grep"} || []}; - my @GrepV = @{$TaskInfo->{"$LogType.GrepV"} || []}; + my @Grep = @{$GrepInfo->{"${GrepPrefix}Grep"} || []}; + my @GrepV = @{$GrepInfo->{"${GrepPrefix}GrepV"} || []}; return if (!@Grep and !@GrepV); - if (open(my $LogFile, "<", $Task->GetDir() ."/$LogName")) + if (open(my $Fh, "<", "$Dir/$FileName")) { my $LineNo; - foreach my $Line (<$LogFile>) + foreach my $Line (<$Fh>) { $LineNo++; my $i = 0; @@ -662,7 +668,7 @@ sub GrepTaskLog($$$$) { if ($Line =~ /$GrepV[$i]/) { - fail(TaskKeyStr($Task) ."/$LogName:$LineNo should not match $GrepV[$i]"); + fail("$Key/$FileName:$LineNo should not match $GrepV[$i]"); splice @GrepV, $i, 1; next; } @@ -680,14 +686,21 @@ sub GrepTaskLog($$$$) } last if (!@Grep and !@GrepV); } - close($LogFile); + close($Fh); } foreach my $RegExp (@Grep) { - fail(TaskKeyStr($Task) ."/$LogName should match $RegExp"); + fail("$Key/$FileName should match $RegExp"); } } +sub GrepTaskLog($$$$) +{ + my ($Task, $LogName, $GrepInfo, $GrepPrefix) = @_; + + return GrepFile(TaskKeyStr($Task), $Task->GetDir(), $LogName, $GrepInfo, $GrepPrefix); +} + =pod =item <job.Status> @@ -769,7 +782,7 @@ sub CheckTask($$$$) CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"}, TaskKeyStr($Task) ."/$LogName"); } - GrepTaskLog($Task, $LogName, $TaskInfo, $LogType); + GrepTaskLog($Task, $LogName, $TaskInfo, "$LogType."); } if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures})) { @@ -850,6 +863,21 @@ space-separated list on the VMs matching the specified category. For instance: p win.TestUnits cabinet:extract cabinet:fdi +=item <patch.Grep> +=item <patch.GrepV> + +Verifies the presence or absence of specific regular expressions in the +job's patch file. Multiple regular expressions can be specified so this uses +the array type of line ('a'). + +For instance: +a patch.Grep WTBS [12]/4 +a patch.GrepV WTBS [34]/4 + +Assuming that the WTBS patch series contains specific strings that allow +identifying each part, this checks that the job's patch contains parts 1 and 2 +but not parts 3 and 4. + =cut sub CheckJobTree($;$) @@ -876,6 +904,10 @@ sub CheckJobTree($;$) $TestInfo = LoadTestInfo($Step->GetFullFileName()); return if (!$TestInfo); + if ($TestInfo->{patch}) + { + GrepFile($Job->Id, $Job->GetDir(), "patch.diff", $TestInfo->{patch}); + } } my $TaskType = $Step->Type eq "build" ? "build" : -- 2.20.1
participants (1)
-
Francois Gouget