When a job completes the TestBot sends an email with the results and another with a summary of the errors in case of new failures. This patch allows checking that these emails did get sent if appropriate and seem to contain the expected data. For this the emails should be saved to a file in mbox format and that file passed to TestWTBS using the --mbox option.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- testbot/tests/TestWTBS | 187 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 175 insertions(+), 12 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 098008110..58cbf3041 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -107,6 +107,7 @@ sub check_opt_val($$) }
my $OptJobs; +my $OptMbox; while (@ARGV) { my $Arg = shift @ARGV; @@ -114,6 +115,10 @@ while (@ARGV) { $OptJobs = check_opt_val($Arg, $OptJobs); } + elsif ($Arg eq "--mbox") + { + $OptMbox = check_opt_val($Arg, $OptMbox); + } elsif ($Arg eq "--help") { $Usage = 0; @@ -156,7 +161,7 @@ if (defined $Usage) error("try '$name0 --help' for more information\n"); exit $Usage; } - print "Usage: $name0 [--jobs RANGES] [--help]\n"; + print "Usage: $name0 [--jobs RANGES] [--mbox FILE] [--help]\n"; print "\n"; print "Tests the Patches subject parser.\n"; print "\n"; @@ -166,6 +171,7 @@ if (defined $Usage) print " these jobs to be ignored. Each range is either a single job id,\n"; print " or of the form 'FIRST..LAST' where FIRST and LAST are either the\n"; print " empty string or a job id.\n"; + print " --mbox FILE Check the TestBot's emails in this mbox-format file.\n"; print " --help Shows this usage message.\n"; exit 0; } @@ -417,6 +423,114 @@ sub GetReportTestUnits($) }
+# +# Load the TestBot emails +# + +my %Emails; + +sub CloseEmail($) +{ + my ($State) = @_; + + # Avoid undefined values to simplify diagnostic messages + $State->{HasFailedEmail} ||= 0; + $State->{HasResEmail} ||= 0; + + if (!$State->{JobId}) + { + foreach my $Field ("HasFailedEmail", "HasResEmail") + { + next if (!$State->{$Field}); + fail("Unexpected $Field property found on line $State->{$Field}"); + } + foreach my $Hash ("FailedVMs", "ResVMs") + { + next if (!$State->{$Hash}); + my $Lines = join(" ", map { "$_:$State->{$Hash}->{$_}" } sort keys %{$State->{$Hash}}); + fail("Unexpected $Hash property found on lines: $Lines"); + } + } + else + { + my $Email = $Emails{$State->{JobId}}; + $Emails{$State->{JobId}} = $Email = {} if (!$Email); + + $Email->{HasFailedEmail} ||= $State->{HasFailedEmail}; + $Email->{FailedVMs} ||= $State->{FailedVMs}; + + $Email->{HasResEmail} ||= $State->{HasResEmail}; + $Email->{ResVMs} ||= $State->{ResVMs}; + } +} + +sub LoadMbox($) +{ + my ($MboxFilename) = @_; + + if (open(my $MboxFh, "<", $MboxFilename)) + { + my ($State, $LineNo, $Empty); + foreach my $Line (<$MboxFh>) + { + chomp $Line; + $LineNo++; + if ($Line eq "") + { + $Empty = 1; + next; + } + if ($Line =~ m~^From ~) + { + CloseEmail($State); + $State = {}; + } + elsif ($Line =~ /^Subject: TestBot job (\d+) results:/) + { + $State->{JobId} = $1; + $State->{JobIdLine} = $LineNo; + $State->{HasResEmail} = $LineNo; + } + elsif ($Line =~ /JobDetails.pl?Key=(\d+)$/) + { + my $JobId = $1; + if (defined $State->{JobId}) + { + is($JobId, $State->{JobId}, "Check mbox job ids on lines $LineNo and $State->{JobIdLine}"); + } + else + { + $State->{JobId} = $JobId; + $State->{JobIdLine} = $LineNo; + } + } + elsif ($Line =~ /^Content-Disposition: attachment; filename=([^-]+)-/) + { + my $VM = $1; + # This could match attachments in other emails + $State->{ResVMs}->{$VM}++ if ($State->{HasResEmail}); + } + elsif ($Line =~ /I found new failures/) + { + $State->{HasFailedEmail} = $LineNo; + } + elsif ($Line =~ /^=== (\w+) (.*) ===$/) + { + my $VM = $1; + $State->{FailedVMs}->{$VM}++; + } + $Empty = undef; + } + CloseEmail($State); + close($MboxFh); + } + else + { + fail("Could not open $MboxFilename for reading: $!"); + } +} + + # # Verify the Jobs and Tasks # @@ -627,11 +741,12 @@ sub CheckTask($$$$) $TestUnits->{$TaskType}->{"*skipped*"} = 1; }
- my $ReportCount = 0; + my ($ReportCount, $NewFailures) = (0, 0); foreach my $LogName (@{GetLogFileNames($Task->GetDir())}) { my $LogPath = $Task->GetDir() ."/$LogName"; my $LogInfo = LoadLogErrors($LogPath); + $NewFailures += $LogInfo->{NewCount} || 0;
my $LogType = "log"; if ($LogName =~ /.report$/) @@ -662,6 +777,7 @@ sub CheckTask($$$$) # was run, i.e. $ReportCount, or take it as is if no report is available. is($Task->TestFailures, $TaskInfo->{TestFailures} * ($ReportCount || 1), "Check Failures of task ". TaskKeyStr($Task)); } + return $NewFailures; }
=pod @@ -687,13 +803,13 @@ sub CheckJob($$) $Remarks =~ s/^[\Q$PatchesMailingList\E] //; is($Remarks, $JobInfo->{Remarks}, "Check Remarks for job ". $Job->Id); } - if (CheckValue($JobInfo->{Status})) + if (CheckValue($TestInfo->{patch}->{Disposition})) { - is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id); + fail("The '$TestInfo->{patch}->{Disposition}' patch disposition is incompatible with job ". $Job->Id); } - elsif (CheckValue($TestInfo->{patch}->{Disposition})) + elsif (CheckValue($JobInfo->{Status})) { - fail("The '$TestInfo->{patch}->{Disposition}' patch disposition is incompatible with job ". $Job->Id); + is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id); } }
@@ -741,12 +857,15 @@ sub CheckJobTree($;$) my ($Job, $TestInfo) = @_; return if (!IsJobInRange($Job));
- return if ($CheckedJobs{$Job->Id}); - $CheckedJobs{$Job->Id} = 1; + my $JobId = $Job->Id; # Simplify use in strings + return if ($CheckedJobs{$JobId}); + $CheckedJobs{$JobId} = 1;
- my $HasTask; + my ($HasTask, $HasNewFailures); my $TestUnits = { wine => {} }; + my $Email = $Emails{$JobId};
+ my %FailedVMs; my $Steps = $Job->Steps; foreach my $Step (sort { $a->No <=> $b->No } @{$Job->Steps->GetItems()}) { @@ -766,7 +885,22 @@ sub CheckJobTree($;$) foreach my $Task (sort { $a->No <=> $b->No } @{$Step->Tasks->GetItems()}) { $HasTask->{$TaskType} = 1; - CheckTask($Task, $TaskType, $TestInfo, $TestUnits); + my $NewFailures = CheckTask($Task, $TaskType, $TestInfo, $TestUnits); + if ($NewFailures) + { + $FailedVMs{$Task->VM->Name}++; + $HasNewFailures = 1; + } + + if ($Emails{$JobId}) + { + my $VMName = $Task->VM->Name; + if ($Task->Status !~ /^(?:canceled|skipped)$/ and $Email->{HasResEmail}) + { + ok($Email->{ResVMs}->{$VMName}, "Expecting $VMName logs / reports in the job $JobId results email"); + } + } + if ($TaskType =~ /^win(?:32|64)$/) { my $TestUnit = $Step->FileName; @@ -776,6 +910,22 @@ sub CheckJobTree($;$) } } } + if ($Email->{HasFailedEmail}) + { + foreach my $VMName (sort keys %FailedVMs) + { + # The emails count each VM once per task report containing a new error + # while CheckJobTree counts them only once per task. + ok(($Email->{FailedVMs}->{$VMName} || 0) >= $FailedVMs{$VMName}, + "Check for $VMName logs / reports in the job $JobId new failures email") + or diag("Expected at least $FailedVMs{$VMName} $VMName instance in the email, got $Email->{FailedVMs}->{$VMName}"); + delete $Email->{FailedVMs}->{$VMName}; + } + foreach my $VMName (sort keys %{$Email->{FailedVMs}}) + { + fail("New failure email for job $JobId reported an extra VM: $VMName"); + } + } CheckJob($Job, $TestInfo);
# Ignore manually submitted jobs because they allow the developer to pick @@ -790,7 +940,7 @@ sub CheckJobTree($;$) if (CheckValue($TypeInfo->{HasTask})) { $HasTask->{$Type} ||= 0; - is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job ". $Job->Id); + is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job $JobId"); }
next if ($TestUnits->{$Type}->{"*skipped*"}); @@ -798,11 +948,18 @@ sub CheckJobTree($;$) { foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits}) { - ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job ". $Job->Id) + ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job $JobId") or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}})); } } } + + if ($OptMbox) + { + my $Email = $Emails{$JobId}; + ok($Email->{HasResEmail}, "Looking for results email for job $JobId"); + is(!!$Email->{HasFailedEmail}, !!($Job->Status ne "completed" or $HasNewFailures), "Check presence of a new failures email for job $JobId"); + } } }
@@ -986,6 +1143,12 @@ if (!$HasBaseVM->{build}) delete $HasBaseVM->{win64}; }
+if (defined $OptMbox) +{ + print "***** Loading emails *****\n"; + LoadMbox($OptMbox); +} + print "***** Checking Patches *****\n"; CheckPatches();