Module: tools
Branch: master
Commit: 691925f164e837b968cd4426c5c92e8bd7805ac8
URL: https://source.winehq.org/git/tools.git/?a=commit;h=691925f164e837b968cd442…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Tue Dec 26 01:48:26 2017 +0100
testbot/web: Show failed Tasks on the Activity page.
This helps identify transient errors, TestBot errors, or why a task is
being re-run.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre …
[View More]Julliard <julliard(a)winehq.org>
---
testbot/lib/WineTestBot/Activity.pm | 57 ++++++++++++++++++++++++++++++++++++-
testbot/web/Activity.pl | 41 ++++++++++++++++++++++----
testbot/web/WineTestBot.css | 4 +++
3 files changed, 95 insertions(+), 7 deletions(-)
diff --git a/testbot/lib/WineTestBot/Activity.pm b/testbot/lib/WineTestBot/Activity.pm
index 31930ed..d66109c 100644
--- a/testbot/lib/WineTestBot/Activity.pm
+++ b/testbot/lib/WineTestBot/Activity.pm
@@ -66,6 +66,18 @@ describing the TestBot activity. The structure is as follows:
...
},
},
+ resultvms => {
+ <VMName1> => {
+ vm => <VMObject>,
+ result => <VMResult>,
+ tries => <Tries>,
+ maxtries => <MaxTries>,
+ details => <ResultDetails>,
+ },
+ <VMName2> => {
+ ...
+ },
+ },
},
<GroupNo2> => {
...
@@ -134,11 +146,24 @@ sub GetActivity($)
$VMStatus->{status} = $Status;
$VMStatus->{rows} = 1;
+ $VMStatus->{result} = "";
if ($Status eq "running")
{
$VMStatus->{job} = $Jobs->GetItem($Extra[0]);
$VMStatus->{step} = $VMStatus->{job}->Steps->GetItem($Extra[1]) if ($VMStatus->{job});
$VMStatus->{task} = $VMStatus->{step}->Tasks->GetItem($Extra[2]) if ($VMStatus->{step});
+ if ($VMStatus->{task})
+ {
+ if ($VMStatus->{task}->Status =~ /^(?:badpatch|badbuild|boterror)$/)
+ {
+ $VMStatus->{result} = $VMStatus->{task}->Status;
+ }
+ elsif ($VMStatus->{task}->Status eq "completed" and
+ $VMStatus->{task}->TestFailures)
+ {
+ $VMStatus->{result} = "failed";
+ }
+ }
}
elsif (@Extra)
{
@@ -148,6 +173,24 @@ sub GetActivity($)
$VMStatus->{details} = join(" ", @Extra);
}
}
+ elsif ($Record->Type eq "vmresult")
+ {
+ my ($RecordName, $RecordHost) = split / /, $Record->Name;
+ next if (!$VMs->ItemExists($RecordName));
+
+ my $ResultVMs = ( $Group->{resultvms} ||= {} );
+ my $VMResult = ( $ResultVMs->{$RecordName} ||= {} );
+
+ $VMResult->{host} = $RecordHost;
+ my ($Result, @Extras) = split / /, $Record->Value;
+ $VMResult->{result} = $Result;
+ if (@Extras >= 2 and $Extras[0] =~ /^\d+$/ and $Extras[1] =~ /^\d+$/)
+ {
+ $VMResult->{tries} = shift @Extras;
+ $VMResult->{maxtries} = shift @Extras;
+ }
+ $VMResult->{details} = join(" ", @Extras);
+ }
}
### Fill the holes in the table, compute end times, etc.
@@ -157,7 +200,8 @@ sub GetActivity($)
{
my $Group = $Activity->{$RecordGroup->Id};
my $StatusVMs = $Group->{statusvms};
- next if (!$StatusVMs);
+ my $ResultVMs = $Group->{resultvms};
+ next if (!$StatusVMs and !$ResultVMs);
if ($LastGroup)
{
$LastGroup->{end} = $Group->{start};
@@ -175,6 +219,17 @@ sub GetActivity($)
{
my $LastVMStatus = $LastStatusVMs{$VM->Name} ? $LastStatusVMs{$VM->Name}->{$VM->Name} : undef;
+ my $VMResult = $ResultVMs->{$VM->Name};
+ if ($VMResult and $LastVMStatus and $LastVMStatus->{status} ne "engine")
+ {
+ # Transfer the result to the relevant status object
+ $LastVMStatus->{result} = $VMResult->{result};
+ $LastVMStatus->{details} = $VMResult->{details};
+ $LastVMStatus->{tries} = $VMResult->{tries};
+ $LastVMStatus->{maxtries} = $VMResult->{maxtries};
+ }
+ next if (!$StatusVMs);
+
my $VMStatus = $StatusVMs->{$VM->Name};
if ($VMStatus)
{
diff --git a/testbot/web/Activity.pl b/testbot/web/Activity.pl
index 64a752b..5a730df 100644
--- a/testbot/web/Activity.pl
+++ b/testbot/web/Activity.pl
@@ -160,11 +160,26 @@ EOF
my $VMStatus = $Group->{statusvms}->{$VM->Name};
next if ($VMStatus->{merged});
- # Add borders to separate VM hosts and indicate anomalies.
+ # Add borders to separate VM hosts and indicate various anomalies.
print "<td class='Record Record-$VMStatus->{status}";
- my $Host = $VM->GetHost();
- print " Record-left" if ($Col > 0 and $SortedVMs[$Col-1]->GetHost() ne $Host);
- print " Record-right" if ($Col+1 < @SortedVMs and $SortedVMs[$Col+1]->GetHost() ne $Host);
+ if ($VMStatus->{result} eq "timeout")
+ {
+ print " Record-timeout";
+ }
+ elsif ($VMStatus->{result} eq "boterror")
+ {
+ print " Record-boterror";
+ }
+ elsif ($VMStatus->{result} eq "error")
+ {
+ print " Record-error";
+ }
+ else
+ {
+ my $Host = $VM->GetHost();
+ print " Record-left" if ($Col > 0 and $SortedVMs[$Col-1]->GetHost() ne $Host);
+ print " Record-right" if ($Col+1 < @SortedVMs and $SortedVMs[$Col+1]->GetHost() ne $Host);
+ }
print " Record-miss" if ($VMStatus->{mispredict});
print "'";
print " rowspan='$VMStatus->{rows}'" if ($VMStatus->{rows} > 1);
@@ -215,6 +230,17 @@ EOF
$Label = "<span class='RecordHost'>(on $Host)</span><br>$Label";
}
print "$Label ", _GetHtmlDuration($VMStatus->{end} - $VMStatus->{start});
+
+ my $Result = "";
+ if ($VMStatus->{status} ne "dirty")
+ {
+ $Result = $VMStatus->{result} if ($VMStatus->{result});
+ $Result .= " $VMStatus->{tries}/$VMStatus->{maxtries}" if ($VMStatus->{tries});
+ $Result .= ": $VMStatus->{details}" if ($VMStatus->{details});
+ $Result =~ s/^: //;
+ }
+ print "<br><span class='RecordResult'>$Result</span>" if ($Result);
+
print "</td>\n";
}
print "</tr>\n";
@@ -238,8 +264,11 @@ sub GenerateFooter($)
print "<span class='Record-running'>running</span> a task (in which case it links to it),<br>\n";
print "<span class='Record-dirty'>dirty</span> while the server is powering off the VM after a task or while it assesses its state on startup.</p>\n";
- print "<p>If no time is indicated then the VM remained in that state for less than 2 seconds. The tasks column indicates the number of runnable / queued tasks before that scheduling round. A long horizontal bar indicates the TestBot server was restarted.</p>\n";
- print "<p>This <span class='Record Record-idle Record-miss'>border</span> indicates that the server threw away the VM's current state without using it.</p>\n";
+ print "<p>If no time is indicated then the VM remained in that state for less than 2 seconds. The tasks column indicates the number of runnable / queued tasks before that scheduling round. A long horizontal bar indicates the TestBot server was restarted. </p>\n";
+ print "<p>This <span class='Record Record-running Record-timeout'>border</span> indicates that the task timed out,<br>\n";
+ print "this <span class='Record Record-running Record-error'>border</span> denotes a transient (network?) error so the task will be re-run,<br>\n";
+ print "and this <span class='Record Record-running Record-boterror'>border</span> indicates a TestBot error.<br>\n";
+ print "Finally this <span class='Record Record-idle Record-miss'>border</span> indicates that the server threw away the VM's current state without using it.</p>\n";
print "<p>The VM could also be <span class='Record-offline'>offline</span> due to a temporary issue,<br>\n";
print "or until the administrator can look at it for <span class='Record-maintenance'>maintenance</span>,<br>\n";
diff --git a/testbot/web/WineTestBot.css b/testbot/web/WineTestBot.css
index 55de9a3..4afbe31 100644
--- a/testbot/web/WineTestBot.css
+++ b/testbot/web/WineTestBot.css
@@ -348,6 +348,7 @@ td.Record { text-align: center; }
.RecordHost { font-size: smaller; }
.RecordJob { font-size: smaller; }
.RecordDuration { }
+.RecordResult { font-size: smaller; font-style: italic; }
.Record-start { }
.Record-off { color: #c0c0c0; }
@@ -371,4 +372,7 @@ td.Record { text-align: center; }
/* Special borders */
.Record.Record-left { border-left: thin solid #601919; }
.Record.Record-right { border-right: thin solid #601919; }
+.Record.Record-boterror { border-left: thick solid red; border-right: thick solid red; }
+.Record.Record-error { border-left: thick solid #990000; border-right: thick solid #990000; }
+.Record.Record-timeout { border-left: thick solid blue; border-right: thick solid blue; }
.Record.Record-miss { border-top: thick dashed #ff6600; }
[View Less]
Module: tools
Branch: master
Commit: 7bde1f1f231e5e7d6ada0d38ea7344a54a5428cb
URL: https://source.winehq.org/git/tools.git/?a=commit;h=7bde1f1f231e5e7d6ada0d3…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Tue Dec 26 01:48:06 2017 +0100
testbot: Add history records for mutable task results.
When a task fails due to a transient error it is re-run and the fact
that a transient error happened cannot be recovered from the final Task
object.
Similarly the final Task object …
[View More]does not clearly identify timeouts.
Furthermore all one can recover from the VM status history is that a
Task was run more than once but not the reason why.
So save abnormal task termination information in the history record.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/bin/WineRunBuild.pl | 20 ++++++++++++++++----
testbot/bin/WineRunReconfig.pl | 20 ++++++++++++++++----
testbot/bin/WineRunTask.pl | 19 ++++++++++++++++---
testbot/lib/WineTestBot/VMs.pm | 29 +++++++++++++++++++++++++++++
4 files changed, 77 insertions(+), 11 deletions(-)
diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl
index b91d594..e4295c8 100755
--- a/testbot/bin/WineRunBuild.pl
+++ b/testbot/bin/WineRunBuild.pl
@@ -45,6 +45,7 @@ use WineTestBot::Config;
use WineTestBot::Jobs;
use WineTestBot::VMs;
use WineTestBot::Log;
+use WineTestBot::RecordGroups;
use WineTestBot::Engine::Notify;
@@ -192,10 +193,13 @@ sub LogTaskError($)
umask($OldUMask);
}
-sub WrapUpAndExit($;$)
+sub WrapUpAndExit($;$$)
{
- my ($Status, $Retry) = @_;
+ my ($Status, $Retry, $Timeout) = @_;
my $NewVMStatus = $Status eq 'queued' ? 'offline' : 'dirty';
+ my $VMResult = $Status eq "boterror" ? "boterror" :
+ $Status eq "queued" ? "error" :
+ $Timeout ? "timeout" : "";
my $TestFailures;
my $Tries = $Task->TestFailures || 0;
@@ -219,6 +223,13 @@ sub WrapUpAndExit($;$)
LogTaskError("The previous $Tries run(s) terminated abnormally\n");
}
+ # Record result details that may be lost or overwritten by a later run
+ if ($VMResult)
+ {
+ $VMResult .= " $Tries $MaxTaskTries" if ($Retry);
+ $VM->RecordResult(undef, $VMResult);
+ }
+
# Update the Task and Job
$Task->Status($Status);
$Task->TestFailures($TestFailures);
@@ -373,7 +384,7 @@ if (!$Pid)
# log before giving up
#
-my ($NewStatus, $ErrMessage, $TAError);
+my ($NewStatus, $ErrMessage, $TAError, $TaskTimedOut);
Debug(Elapsed($Start), " Waiting for the script (", $Task->Timeout, "s timeout)\n");
if (!defined $TA->Wait($Pid, $Task->Timeout, 60))
{
@@ -382,6 +393,7 @@ if (!defined $TA->Wait($Pid, $Task->Timeout, 60))
{
$ErrMessage = "The build timed out\n";
$NewStatus = "badbuild";
+ $TaskTimedOut = 1;
}
else
{
@@ -480,4 +492,4 @@ $TA->Disconnect();
# Wrap up
#
-WrapUpAndExit($NewStatus);
+WrapUpAndExit($NewStatus, undef, $TaskTimedOut);
diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl
index ff99fb7..4eefa17 100755
--- a/testbot/bin/WineRunReconfig.pl
+++ b/testbot/bin/WineRunReconfig.pl
@@ -45,6 +45,7 @@ use WineTestBot::Config;
use WineTestBot::Jobs;
use WineTestBot::VMs;
use WineTestBot::Log;
+use WineTestBot::RecordGroups;
use WineTestBot::Engine::Notify;
@@ -192,11 +193,14 @@ sub LogTaskError($)
umask($OldUMask);
}
-sub WrapUpAndExit($;$)
+sub WrapUpAndExit($;$$)
{
- my ($Status, $Retry) = @_;
+ my ($Status, $Retry, $Timeout) = @_;
my $NewVMStatus = $Status eq 'queued' ? 'offline' :
$Status eq 'completed' ? 'idle' : 'dirty';
+ my $VMResult = $Status eq "boterror" ? "boterror" :
+ $Status eq "queued" ? "error" :
+ $Timeout ? "timeout" : "";
my $TestFailures;
my $Tries = $Task->TestFailures || 0;
@@ -220,6 +224,13 @@ sub WrapUpAndExit($;$)
LogTaskError("The previous $Tries run(s) terminated abnormally\n");
}
+ # Record result details that may be lost or overwritten by a later run
+ if ($VMResult)
+ {
+ $VMResult .= " $Tries $MaxTaskTries" if ($Retry);
+ $VM->RecordResult(undef, $VMResult);
+ }
+
# Update the Task and Job
$Task->Status($Status);
$Task->TestFailures($TestFailures);
@@ -338,7 +349,7 @@ if (!$Pid)
# log before giving up
#
-my ($NewStatus, $ErrMessage, $TAError);
+my ($NewStatus, $ErrMessage, $TAError, $TaskTimedOut);
Debug(Elapsed($Start), " Waiting for the script (", $Task->Timeout, "s timeout)\n");
if (!defined $TA->Wait($Pid, $Task->Timeout, 60))
{
@@ -347,6 +358,7 @@ if (!defined $TA->Wait($Pid, $Task->Timeout, 60))
{
$ErrMessage = "The build timed out\n";
$NewStatus = "badbuild";
+ $TaskTimedOut = 1;
}
else
{
@@ -470,4 +482,4 @@ if ($NewStatus eq 'completed')
# Wrap up
#
-WrapUpAndExit($NewStatus);
+WrapUpAndExit($NewStatus, undef, $TaskTimedOut);
diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl
index 3807a18..65faf72 100755
--- a/testbot/bin/WineRunTask.pl
+++ b/testbot/bin/WineRunTask.pl
@@ -44,6 +44,7 @@ use WineTestBot::Config;
use WineTestBot::Jobs;
use WineTestBot::VMs;
use WineTestBot::Log;
+use WineTestBot::RecordGroups;
use WineTestBot::Engine::Notify;
@@ -219,10 +220,13 @@ sub LogTaskError($)
umask($OldUMask);
}
-sub WrapUpAndExit($;$$)
+sub WrapUpAndExit($;$$$)
{
- my ($Status, $TestFailures, $Retry) = @_;
+ my ($Status, $TestFailures, $Retry, $Timeout) = @_;
my $NewVMStatus = $Status eq 'queued' ? 'offline' : 'dirty';
+ my $VMResult = $Status eq "boterror" ? "boterror" :
+ $Status eq "queued" ? "error" :
+ $Timeout ? "timeout" : "";
Debug(Elapsed($Start), " Taking a screenshot\n");
TakeScreenshot($VM, $FullScreenshotFileName);
@@ -248,6 +252,13 @@ sub WrapUpAndExit($;$$)
LogTaskError("The previous $Tries run(s) terminated abnormally\n");
}
+ # Record result details that may be lost or overwritten by a later run
+ if ($VMResult)
+ {
+ $VMResult .= " $Tries $MaxTaskTries" if ($Retry);
+ $VM->RecordResult(undef, $VMResult);
+ }
+
# Update the Task and Job
$Task->Status($Status);
$Task->TestFailures($TestFailures);
@@ -489,6 +500,7 @@ if (!defined $TA->Wait($Pid, $Timeout, $Keepalive))
}
}
+my $TimedOut;
Debug(Elapsed($Start), " Retrieving the report file to '$FullLogFileName'\n");
if ($TA->GetFile($RptFileName, $FullLogFileName))
{
@@ -712,6 +724,7 @@ if ($TA->GetFile($RptFileName, $FullLogFileName))
# so record the failure but don't add an error message.
$LogFailures++;
$CurrentIsBroken = 1;
+ $TimedOut = ($Step->Type ne "suite");
}
elsif ((!$Pid and !%CurrentPids) or
($Pid and !$CurrentPids{$Pid} and !$CurrentPids{0}))
@@ -767,4 +780,4 @@ FatalTAError(undef, $TAError, $PossibleCrash) if (defined $TAError);
# Wrap up
#
-WrapUpAndExit($NewStatus, $TaskFailures);
+WrapUpAndExit($NewStatus, $TaskFailures, undef, $TaskTimedOut || $TimedOut);
diff --git a/testbot/lib/WineTestBot/VMs.pm b/testbot/lib/WineTestBot/VMs.pm
index 961be52..2ba6b4b 100644
--- a/testbot/lib/WineTestBot/VMs.pm
+++ b/testbot/lib/WineTestBot/VMs.pm
@@ -566,6 +566,35 @@ sub RecordStatus($$;$)
}
}
+=pod
+=over 12
+
+=item C<RecordStatus()>
+
+Adds a Record of the specified VM process result.
+
+Also resets the last known VM status so a new one will be recorded even if
+it matches the current one, for instance if a new revert is started after the
+first one failed.
+
+=back
+=cut
+
+sub RecordResult($$$)
+{
+ my ($self, $Records, $Result) = @_;
+
+ if ($Records)
+ {
+ $Records->AddRecord("vmresult", $self->GetRecordName(), $Result);
+ }
+ else
+ {
+ SaveRecord("vmresult", $self->GetRecordName(), $Result);
+ }
+ delete $_VMStatuses{$self->Name};
+}
+
package WineTestBot::VMs;
[View Less]