Don't assume that jobs coming from the patches website necessarily have
a sender email address. Update the patches website in all cases.
Also mark a patch as failed in the patches website when the job has
skipped tasks or TestBot errors.
Clearly identify the new failures in the report sent to the developer.
Provide more details about the job result, particularly TestBot errors,
the presence of skipped tasks, and consequences.
Call for help fixing preexisting failures.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
---
testbot/bin/WineSendLog.pl | 340 ++++++++++++++++++++++++-------------
testbot/tests/TestWTBS | 7 +-
2 files changed, 226 insertions(+), 121 deletions(-)
diff --git a/testbot/bin/WineSendLog.pl b/testbot/bin/WineSendLog.pl
index 9351d5d6f..3897ac034 100755
--- a/testbot/bin/WineSendLog.pl
+++ b/testbot/bin/WineSendLog.pl
@@ -173,146 +173,220 @@ sub SendLog($)
{
my ($Job) = @_;
- my $To = $WinePatchToOverride || $Job->GetEMailRecipient();
- if (! defined($To))
- {
- return;
- }
-
- my $StepsTasks = CreateStepsTasks(undef, $Job);
- my $SortedStepsTasks = $StepsTasks->GetSortedItems();
-
- my $JobURL = MakeOfficialURL("/JobDetails.pl?Key=". $Job->GetKey());
-
-
#
- # Send a job summary and all the logs as attachments to the developer
+ # Collect and count the new and old failures, etc.
#
- Debug("-------------------- Developer email --------------------\n");
- my $Sendmail;
- if ($Debug)
- {
- open($Sendmail, ">>&=", 1);
- }
- else
- {
- open($Sendmail, "|-", "/usr/sbin/sendmail -oi -t -odq");
- }
- print $Sendmail "From: $RobotEMail\n";
- print $Sendmail "To: $To\n";
- my $Subject = "TestBot job " . $Job->Id . " results";
- my $Description = $Job->GetDescription();
- if ($Description)
- {
- $Subject .= ": " . $Description;
- }
- print $Sendmail "Subject: $Subject\n";
- if ($Job->Patch and $Job->Patch->MessageId)
- {
- print $Sendmail "In-Reply-To: ", $Job->Patch->MessageId, "\n";
- print $Sendmail "References: ", $Job->Patch->MessageId, "\n";
- }
+ my (@New, $JobInfo);
+ my ($NewCount, $OldCount, $BotCount, $NotRun) = (0, 0, 0, 0);
+ my $FileType = "test executable";
- print $Sendmail "\n";
- print $Sendmail "VM Status Failures Command\n";
+ my $StepsTasks = CreateStepsTasks(undef, $Job);
+ my $SortedStepsTasks = $StepsTasks->GetSortedItems();
foreach my $StepTask (@$SortedStepsTasks)
{
- my $TestFailures = $StepTask->TestFailures;
- $TestFailures = "" if (!defined $TestFailures);
- my $Status = $StepTask->Status;
- $Status = $TestFailures ? "failed" : "success" if ($Status eq "completed");
- my $Cmd = "";
- $Cmd = $StepTask->FileName ." " if ($StepTask->FileType =~ /^exe/);
- $Cmd .= $StepTask->CmdLineArg if (defined $StepTask->CmdLineArg);
-
- printf $Sendmail "%-20s %-8s %-8s %s\n", $StepTask->VM->Name, $Status,
- $TestFailures, $Cmd;
- }
-
- print $Sendmail "\nThe full results can be found at:\n$JobURL\n\n";
+ if ($StepTask->Status =~ /^(?:canceled|skipped)$/)
+ {
+ $NotRun++;
+ next;
+ }
- # Print the job summary
- my $JobErrors;
- foreach my $StepTask (@$SortedStepsTasks)
- {
- my $Id = $StepTask->Id;
my $TaskDir = $StepTask->GetTaskDir();
my $LogNames = GetLogFileNames($TaskDir);
- $JobErrors->{$Id}->{LogNames} = $LogNames;
+ $JobInfo->{$StepTask->Id}->{LogNames} = $LogNames;
+ $JobInfo->{$StepTask->Id}->{NewCount} = 0;
+ $JobInfo->{$StepTask->Id}->{OldCount} = 0;
+ if ($StepTask->Status eq "boterror")
+ {
+ $JobInfo->{$StepTask->Id}->{BotCount} = 1;
+ $BotCount++;
+ }
+ $FileType = "patch" if ($StepTask->FileType eq "patch");
+
foreach my $LogName (@$LogNames)
{
my $LogInfo = LoadLogErrors("$TaskDir/$LogName");
next if (!defined $LogInfo->{BadLog} and !$LogInfo->{ErrCount});
- $JobErrors->{$Id}->{HasErrors} = 1;
- $JobErrors->{$Id}->{$LogName} = $LogInfo;
-
- print $Sendmail "\n=== ", GetTitle($StepTask, $LogName), " ===\n";
- print $Sendmail "$LogInfo->{BadLog}\n" if (defined $LogInfo->{BadLog});
+ $JobInfo->{$StepTask->Id}->{$LogName} = $LogInfo;
+ my $HasLogHeader;
foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
{
- print $Sendmail ($GroupName ? "\n$GroupName:\n" : "\n");
+ my $HasGroupHeader;
my $Group = $LogInfo->{ErrGroups}->{$GroupName};
- print $Sendmail "$_\n" for (@{$Group->{Errors}});
+ foreach my $ErrIndex (0..$#{$Group->{Errors}})
+ {
+ if ($Group->{IsNew}->[$ErrIndex])
+ {
+ if (!$HasLogHeader)
+ {
+ push @New, "\n=== ". GetTitle($StepTask, $LogName) ." ===\n";
+ $HasLogHeader = 1;
+ }
+ if (!$HasGroupHeader)
+ {
+ push @New, ($GroupName ? "\n$GroupName:\n" : "\n");
+ $HasGroupHeader = 1;
+ }
+ push @New, "$Group->{Errors}->[$ErrIndex]\n";
+ $JobInfo->{$StepTask->Id}->{NewCount}++;
+ }
+ else
+ {
+ $JobInfo->{$StepTask->Id}->{OldCount}++;
+ }
+ }
}
+ $NewCount += $JobInfo->{$StepTask->Id}->{NewCount};
+ $OldCount += $JobInfo->{$StepTask->Id}->{OldCount};
}
}
- close($Sendmail);
-
- # This is all for jobs submitted from the website
- if (!defined $Job->Patch)
- {
- Debug("Not a mailing list patch -> all done.\n");
- return;
- }
-
#
- # Build a job summary with only the new errors
+ # Send a job summary to the developer
#
- # Note that this may be a bit inaccurate right after a Wine commit if this
- # job's patch got compiled on top of the new Wine before all the reference
- # WineTest results were updated. This is made more likely by the job
- # priorities: high for Wine updates, and low for WineTest runs.
- # However in practice this would only be an issue if the patch reintroduced
- # an error that just disappeared in the latest Wine which is highly unlikely.
- my @Messages;
- foreach my $StepTask (@$SortedStepsTasks)
+ my $JobURL = MakeOfficialURL("/JobDetails.pl?Key=". $Job->GetKey());
+ my $To = $WinePatchToOverride || $Job->GetEMailRecipient();
+ if (defined $To)
{
- my $Id = $StepTask->Id;
- next if (!$JobErrors->{$Id}->{HasErrors});
+ Debug("-------------------- Developer email --------------------\n");
+ my $Sendmail;
+ if ($Debug)
+ {
+ open($Sendmail, ">>&=", 1);
+ }
+ else
+ {
+ open($Sendmail, "|-", "/usr/sbin/sendmail -oi -t -odq");
+ }
+ print $Sendmail "From: $RobotEMail\n";
+ print $Sendmail "To: $To\n";
+ my $Subject = "TestBot job " . $Job->Id . " results";
+ my $Description = $Job->GetDescription();
+ if ($Description)
+ {
+ $Subject .= ": " . $Description;
+ }
+ print $Sendmail "Subject: $Subject\n";
+ if ($Job->Patch and $Job->Patch->MessageId)
+ {
+ print $Sendmail "In-Reply-To: ", $Job->Patch->MessageId, "\n";
+ print $Sendmail "References: ", $Job->Patch->MessageId, "\n";
+ }
+ print $Sendmail "\nHi,\n\n";
- # Note: We could check $StepTask->Status for TestBot errors. However,
- # whether they are caused by the patch or not, they prevent the TestBot
- # from checking for new errors which justifies sending an email to the
- # mailing list so that the patch receives greater scrutiny.
+ if ($NewCount)
+ {
+ print $Sendmail <<"EOF";
+It looks like your $FileType introduces some new failures. Please
+investigate and fix them if they are indeed new. Note that rare
+failures and failures with always changing text (e.g. because of memory
+addresses) can cause false positives. If this is what happened, then
+fixing those would really help.
- foreach my $LogName (@{$JobErrors->{$Id}->{LogNames}})
+EOF
+ }
+ if ($BotCount)
{
- my $LogInfo = $JobErrors->{$Id}->{$LogName};
- # Skip if there are no errors
- next if (!$LogInfo->{NewCount});
+ print $Sendmail <<"EOF";
+$BotCount TestBot errors prevented a full analysis of your $FileType.
+If the test caused the operating system (e.g. Windows) to crash or
+reboot you will probably have to modify it to avoid that.
+Other issues should be reported to the TestBot administrators.
- push @Messages, "\n=== ". GetTitle($StepTask, $LogName) ." ===\n";
+EOF
+ }
+ elsif (!$NewCount and !$NotRun)
+ {
+ print $Sendmail <<"EOF";
+Congratulations!
+It looks like your $FileType passed the tests with flying colors.
- foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+EOF
+ }
+ if ($OldCount)
+ {
+ print $Sendmail <<"EOF";
+Some preexisting failures (not caused by your $FileType) happened.
+If you know how to fix them that would be helpful.
+
+EOF
+ }
+
+ print $Sendmail " Failures\n";
+ print $Sendmail "VM Status New / Old Command\n";
+ foreach my $StepTask (@$SortedStepsTasks)
+ {
+ next if ($StepTask->Status =~ /^(?:canceled|skipped)$/);
+
+ my $TestFailures = $StepTask->TestFailures;
+ $TestFailures = "" if (!defined $TestFailures);
+ my $Status = $StepTask->Status ne "completed" ? $StepTask->Status :
+ $TestFailures ? "failed" :
+ "success";
+ my $Cmd = "";
+ $Cmd = $StepTask->FileName ." " if ($StepTask->FileType =~ /^exe/);
+ $Cmd .= $StepTask->CmdLineArg if (defined $StepTask->CmdLineArg);
+
+ printf $Sendmail "%-20s %-8s %4s / %4s %s\n", $StepTask->VMName,
+ $Status, $JobInfo->{$StepTask->Id}->{NewCount},
+ $JobInfo->{$StepTask->Id}->{OldCount}, $Cmd;
+ }
+ if ($NotRun)
+ {
+ print $Sendmail <<"EOF";
+
+Note that $NotRun additional tasks could not be run due to previous
+errors. A full run may uncover more issues.
+
+EOF
+ }
+
+ print $Sendmail <<"EOF";
+
+The full results can be found at:
+$JobURL
+
+Your paranoid android.
+
+EOF
+
+ # Print the failures
+ foreach my $StepTask (@$SortedStepsTasks)
+ {
+ my $TaskInfo = $JobInfo->{$StepTask->Id};
+ my $TaskDir = $StepTask->GetTaskDir();
+ foreach my $LogName (@{$TaskInfo->{LogNames}})
{
- my $Group = $LogInfo->{ErrGroups}->{$GroupName};
- next if (!$Group->{NewCount});
+ my $LogInfo = LoadLogErrors("$TaskDir/$LogName");
+ next if (!defined $LogInfo->{BadLog} and !$LogInfo->{ErrCount});
- push @Messages, ($GroupName ? "\n$GroupName:\n" : "\n");
- foreach my $ErrIndex (0..$#{$Group->{Errors}})
+ print $Sendmail "\n=== ", GetTitle($StepTask, $LogName), " ===\n";
+ print $Sendmail "$LogInfo->{BadLog}\n" if (defined $LogInfo->{BadLog});
+
+ foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
{
- if ($Group->{IsNew}->[$ErrIndex])
+ print $Sendmail ($GroupName ? "\n$GroupName:\n" : "\n");
+ my $Group = $LogInfo->{ErrGroups}->{$GroupName};
+ foreach my $ErrIndex (0..$#{$Group->{Errors}})
{
- push @Messages, "$Group->{Errors}->[$ErrIndex]\n";
+ my $Prefix = $Group->{IsNew}->[$ErrIndex] ? "new" : "old";
+ print $Sendmail "[$Prefix] $Group->{Errors}->[$ErrIndex]\n";
}
}
}
}
+
+ close($Sendmail);
+ }
+
+ # This is all for jobs submitted from the website
+ if (!defined $Job->Patch)
+ {
+ Debug("Not a mailing list patch -> all done.\n");
+ return;
}
@@ -322,8 +396,13 @@ sub SendLog($)
Debug("\n-------------------- Mailing list email --------------------\n");
- if (@Messages)
+ if (!$NewCount)
{
+ Debug("Found no error to report to the mailing list\n");
+ }
+ else
+ {
+ my $Sendmail;
if ($Debug)
{
open($Sendmail, ">>&=", 1);
@@ -333,36 +412,63 @@ sub SendLog($)
open($Sendmail, "|-", "/usr/sbin/sendmail -oi -t -odq");
}
print $Sendmail "From: $RobotEMail\n";
- print $Sendmail "To: $To\n";
- print $Sendmail "Cc: $WinePatchCc\n";
+ if (defined $To)
+ {
+ print $Sendmail "To: $To\n";
+ print $Sendmail "Cc: $WinePatchCc\n";
+ }
+ else
+ {
+ print $Sendmail "To: $WinePatchCc\n";
+ }
print $Sendmail "Subject: Re: ", $Job->Patch->Subject, "\n";
if ($Job->Patch->MessageId)
{
print $Sendmail "In-Reply-To: ", $Job->Patch->MessageId, "\n";
print $Sendmail "References: ", $Job->Patch->MessageId, "\n";
}
- print $Sendmail <<"EOF";
+ print $Sendmail "\nHi,\n\n";
-Hi,
+ if ($NewCount)
+ {
+ print $Sendmail <<"EOF";
+It looks like your $FileType introduced the new failures shown below.
+Please investigate and fix them if they are indeed new. Note that rare
+failures and failures with always changing text (e.g. because of memory
+addresses) can cause false positives. If this is what happened, then
+fixing those would really help.
-While running your changed tests, I think I found new failures.
-Being a bot and all I'm not very good at pattern recognition, so I might be
-wrong, but could you please double-check?
+EOF
+ }
+ if ($BotCount)
+ {
+ print $Sendmail <<"EOF";
+$BotCount TestBot errors prevented a full analysis of your $FileType.
+If the test caused the operating system (e.g. Windows) to crash or
+reboot you will probably have to modify it to avoid that.
+Other issues should be reported to the TestBot administrators.
-Full results can be found at:
+EOF
+ }
+ if ($OldCount)
+ {
+ print $Sendmail <<"EOF";
+The tests also ran into some preexisting test failures. If you know how
+to fix them that would be helpful. See the TestBot job for the details:
+
+EOF
+ }
+ print $Sendmail <<"EOF";
+The full results can be found at:
$JobURL
Your paranoid android.
EOF
- print $Sendmail $_ for (@Messages);
+ print $Sendmail $_ for (@New);
close($Sendmail);
}
- else
- {
- Debug("Found no error to report to the mailing list\n");
- }
#
@@ -379,14 +485,14 @@ EOF
{
# Only take into account new errors to decide whether the job was
# successful or not.
- DebugTee($Result, "Status: ". (@Messages ? "Failed" : "OK") ."\n");
+ DebugTee($Result, "Status: ". ($NewCount + $BotCount + $NotRun ? "Failed" : "OK") ."\n");
DebugTee($Result, "Job-ID: ". $Job->Id ."\n");
DebugTee($Result, "URL: $JobURL\n");
foreach my $StepTask (@$SortedStepsTasks)
{
my $TaskDir = $StepTask->GetTaskDir();
- foreach my $LogName (@{$JobErrors->{$StepTask->Id}->{LogNames}})
+ foreach my $LogName (@{$JobInfo->{$StepTask->Id}->{LogNames}})
{
print $Result "=== ", GetTitle($StepTask, $LogName), " ===\n";
DumpLogAndErr($Result, "$TaskDir/$LogName");
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 139f011ef..5663f956c 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -611,13 +611,12 @@ sub LoadMbox($)
$State->{JobIdLine} = $LineNo;
}
}
- elsif ($Line =~ /^Content-Disposition: attachment; filename=([^-]+)-/)
+ elsif ($Line =~ /^([a-zA-Z0-9_]+)\s+(?:success|failed)\s+\d+/)
{
my $VM = $1;
- # This could match attachments in other emails
$State->{ResVMs}->{$VM}++ if ($State->{HasResEmail});
}
- elsif ($Line =~ /I found new failures/)
+ elsif ($Line =~ /\bintroduces .* new failures\b/)
{
$State->{HasFailedEmail} = $LineNo;
}
@@ -1190,7 +1189,7 @@ sub CheckJobTree($;$)
my $VMName = $Task->VM->Name;
if ($Task->Status !~ /^(?:canceled|skipped)$/ and $Email->{HasResEmail})
{
- ok($Email->{ResVMs}->{$VMName}, "Expecting $VMName logs / reports in the job $JobId results email");
+ ok($Email->{ResVMs}->{$VMName}, "Expecting a $VMName error summary in the job $JobId results email");
}
}
--
2.30.2
