Francois Gouget : testbot: Add a message type for flaky failures.
Module: tools Branch: master Commit: 617f337d8616a94db3a7b37b829a538bab87a564 URL: https://gitlab.winehq.org/winehq/tools/-/commit/617f337d8616a94db3a7b37b829a... Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Wed Sep 7 16:19:19 2022 +0200 testbot: Add a message type for flaky failures. In the .errors files flaky failures are identified as 'k' messages (instead of 'e' for regular failures). Flaky failures are also identified as old or new based on their presence in the reference reports and shown accordingly on the JobDetails page (the corresponding CSS class names now systematically contain the message type and prefix it with "new" for new messages). Before this patch new flaky failures were flagged as regular new failures which caused some false positives. Now WineSendLog can identify flaky failures and not count them as new failures, thus fixing the false positives. --- testbot/bin/WineRunTask.pl | 4 ++-- testbot/bin/WineRunWineTest.pl | 4 ++-- testbot/bin/WineSendLog.pl | 7 +++++-- testbot/lib/WineTestBot/LogUtils.pm | 24 ++++++++++++------------ testbot/web/JobDetails.pl | 16 +++++++++------- testbot/web/WineTestBot.css | 4 +++- 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 85ba5df5..2eec9c73 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -562,8 +562,8 @@ if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); LogTaskError("$_\n") for (@$ErrMessages); - # $LogInfo->{eCount} can legitimately be undefined in case of a timeout - $TaskFailures += $LogInfo->{eCount} || 0; + # The message counters can legitimately be undefined in case of a timeout + $TaskFailures += ($LogInfo->{eCount} || 0) + ($LogInfo->{kCount} || 0); my $LogErrMsg = CreateLogErrorsCache($LogInfo, $Task); LogTaskError("$LogErrMsg\n") if (defined $LogErrMsg); } diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index 6b49677c..985fdc2a 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -578,8 +578,8 @@ foreach my $RptFileName (@$ReportNames) my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); LogTaskError("$_\n") for (@$ErrMessages); - # $LogInfo->{eCount} can legitimately be undefined in case of a timeout - $TaskFailures += $LogInfo->{eCount} || 0; + # The message counters can legitimately be undefined in case of a timeout + $TaskFailures += ($LogInfo->{eCount} || 0) + ($LogInfo->{kCount} || 0); my $LogErrMsg = CreateLogErrorsCache($LogInfo, $Task); LogTaskError("$LogErrMsg\n") if (defined $LogErrMsg); } diff --git a/testbot/bin/WineSendLog.pl b/testbot/bin/WineSendLog.pl index 0246f89e..ab7b90c0 100755 --- a/testbot/bin/WineSendLog.pl +++ b/testbot/bin/WineSendLog.pl @@ -233,7 +233,8 @@ sub SendLog($) } $TaskInfo->{OldCount}++; } - elsif ($Group->{IsNew}->[$MsgIndex]) + elsif ($Group->{Types}->[$MsgIndex] eq "e" and + $Group->{IsNew}->[$MsgIndex]) { if (!$HasLogHeader) { @@ -248,7 +249,8 @@ sub SendLog($) push @New, "$Group->{Messages}->[$MsgIndex]\n"; $TaskInfo->{NewCount}++; } - else + elsif ($Group->{Types}->[$MsgIndex] eq "e" or + $Group->{Types}->[$MsgIndex] eq "k") # flaky => old failure { $TaskInfo->{OldCount}++; } @@ -403,6 +405,7 @@ EOF my $LineNo = $Group->{LineNos}->[$MsgIndex]; my $Bugs = $LogInfo->{Bugs}->{$LineNo}; my $Prefix = $Bugs ? join(" ", sort { $a <=> $b } @$Bugs) : + $Group->{Types}->[$MsgIndex] eq "k" ? "flk" : $Group->{IsNew}->[$MsgIndex] ? "new" : "old"; print $Sendmail "[$Prefix] $Group->{Messages}->[$MsgIndex]\n"; diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index e7f147be..06751e78 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -497,9 +497,9 @@ sub _CloseTestUnit($$$) } } -sub _AddReportMsg($$$$) +sub _AddReportMsg($$$$$) { - my ($LogInfo, $Cur, $LineNo, $Line) = @_; + my ($LogInfo, $Cur, $LineNo, $Line, $Type) = @_; # Make the timeout messages more user-friendly my $ErrLine = $Line; @@ -513,7 +513,7 @@ sub _AddReportMsg($$$$) { $Cur->{Group} = _AddLogGroup($LogInfo, $Cur->{Dll}, $Cur->{GroupLineNo} || $LineNo); } - _AddLogMsg($LogInfo, $Cur->{Group}, $ErrLine, $LineNo, "e"); + _AddLogMsg($LogInfo, $Cur->{Group}, $ErrLine, $LineNo, $Type); } =pod @@ -589,7 +589,7 @@ sub ParseWineTestReport($$$) elsif ($InDlls == 1 and $Line =~ /^\s+\S+=(?:dll is native|load error \d+)$/) { - _AddReportMsg($LogInfo, $Cur, $LineNo, $Line); + _AddReportMsg($LogInfo, $Cur, $LineNo, $Line, "e"); $Cur->{LineFailures}++; } elsif ($Line =~ /^Test output:$/) @@ -626,7 +626,7 @@ sub ParseWineTestReport($$$) $Line =~ /^([_a-z0-9]+)\.c:\d+:[0-9.]* Test (?:failed|succeeded inside todo block): /) { _CheckUnit($LogInfo, $Cur, $1, "failure"); - _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'"); + _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'", "e"); $Cur->{LineFailures}++; } elsif (($Cur->{Unit} ne "" and @@ -634,7 +634,7 @@ sub ParseWineTestReport($$$) $Line =~ /^([_a-z0-9]+)\.c:\d+:[0-9.]* Test (?:marked flaky|succeeded inside flaky todo block): /) { _CheckUnit($LogInfo, $Cur, $1, "flaky"); - _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'"); + _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'", "k"); $Cur->{LineFlaky}++; } elsif (($Cur->{Unit} ne "" and @@ -663,7 +663,7 @@ sub ParseWineTestReport($$$) $Cur->{SummaryFailures}++; $LogInfo->{IsWineTest} = 1; - _AddReportMsg($LogInfo, $Cur, $LineNo, $Line); + _AddReportMsg($LogInfo, $Cur, $LineNo, $Line, "e"); $Cur->{LineFailures}++; } elsif (($Cur->{Unit} ne "" and @@ -686,7 +686,7 @@ sub ParseWineTestReport($$$) $Cur->{Pids}->{$Pid || 0} = 1; $Cur->{SummaryFailures}++; } - _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'"); + _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'", "e"); $Cur->{LineFailures}++; } } @@ -697,7 +697,7 @@ sub ParseWineTestReport($$$) # This also replaces a test summary line. The pid is unknown so use 0. $Cur->{Pids}->{0} = 1; $Cur->{SummaryFailures}++; - _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'"); + _AddReportMsg($LogInfo, $Cur, $LineNo, "$&$'", "e"); $Cur->{LineFailures}++; } } @@ -715,7 +715,7 @@ sub ParseWineTestReport($$$) $Cur->{Pids}->{$Pid || 0} = 1; $Cur->{SummaryFailures}++; } - _AddReportMsg($LogInfo, $Cur, $LineNo, $&); + _AddReportMsg($LogInfo, $Cur, $LineNo, $&, "e"); $Cur->{LineFailures}++; } } @@ -769,7 +769,7 @@ sub ParseWineTestReport($$$) # so record the failure but don't add an error message. $Cur->{IsBroken} = 1; $LogInfo->{TimeoutCount}++; - _AddReportMsg($LogInfo, $Cur, $LineNo, $&); + _AddReportMsg($LogInfo, $Cur, $LineNo, $&, "e"); } elsif ((!$Pid and !%{$Cur->{Pids}}) or ($Pid and !$Cur->{Pids}->{$Pid} and !$Cur->{Pids}->{0})) @@ -1014,7 +1014,7 @@ sub LoadLogErrorsFromFh($$) $LogInfo->{BadLog} = "$LogInfo->{LineNo}: Got a $Type line with no group"; return $Line; } - elsif ($Type eq "e" or $Type eq "E") + elsif ($Type eq "e" or $Type eq "E" or $Type eq "k" or $Type eq "K") { _AddLogMsg($LogInfo, $LogInfo->{CurGroup}, $Value, $Property, $Type); } diff --git a/testbot/web/JobDetails.pl b/testbot/web/JobDetails.pl index ce51ca74..eed9eb30 100644 --- a/testbot/web/JobDetails.pl +++ b/testbot/web/JobDetails.pl @@ -425,7 +425,9 @@ sub GenerateFullLog($$$$$) my $LineNo = $Group->{LineNos}->[$MsgIndex]; if ($LineNo) { - $ErrCategory{$LineNo} = $Group->{IsNew}->[$MsgIndex] ? "new" : "error"; + $ErrCategory{$LineNo} = ($Group->{IsNew}->[$MsgIndex] ? "new" : ""). + ($Group->{Types}->[$MsgIndex] eq "k" ? "flaky" : + "error"); $ErrFailures{$LineNo} = $Group->{Failures}->{$MsgIndex}; if ($ErrFailures{$LineNo}) @@ -438,7 +440,7 @@ sub GenerateFullLog($$$$$) } } $LogBugs{-2} ||= [$LineNo, "First failure"]; - if ($ErrCategory{$LineNo} eq "new" and !$ErrFailures{$LineNo}) + if ($ErrCategory{$LineNo} eq "newerror" and !$ErrFailures{$LineNo}) { $LogBugs{-1} ||= [$LineNo, "First untracked new failure"]; $LogBugs{10000000} = [$LineNo, "Last untracked new failure"]; @@ -522,7 +524,7 @@ sub GenerateFullLog($$$$$) } my $Line = $Group->{Messages}->[$MsgIndex]; - my $Category = $Group->{IsNew}->[$MsgIndex] ? "new" : "error"; + my $Category = $Group->{IsNew}->[$MsgIndex] ? "newerror" : "error"; my $Html = $self->escapeHTML($Line); if ($Group->{Failures}->{$MsgIndex}) { @@ -695,10 +697,10 @@ EOF foreach my $Line (@{$Group->{Messages}}) { my $Html = $self->escapeHTML($Line); - if ($Group->{IsNew}->[$MsgIndex]) - { - $Html = "<span class='log-new'>$Html</span>"; - } + my $Category = ($Group->{IsNew}->[$MsgIndex] ? "new" : ""). + ($Group->{Types}->[$MsgIndex] eq "k" ? "flaky" : + "error"); + $Html = "<span class='log-$Category'>$Html</span>"; my $FailureIds = $Group->{Failures}->{$MsgIndex}; if ($FailureIds) { diff --git a/testbot/web/WineTestBot.css b/testbot/web/WineTestBot.css index c40b8546..e686e6f5 100644 --- a/testbot/web/WineTestBot.css +++ b/testbot/web/WineTestBot.css @@ -402,10 +402,12 @@ pre .log-info { background-color: #d9ffcc; } .log-skip { color: blue; } .log-todo { color: #d08000; } +.log-flaky { color: magenta; } +.log-newflaky { color: magenta; font-weight: bold; } .log-error { color: red; } +.log-newerror { color: red; font-weight: bold; } .log-boterror { color: #cc0052; } .log-diag { color: #e56300; } -.log-new { color: red; font-weight: bold; } .bug-missing { color: red; font-weight: bold; } .bug-closed { color: red; }
participants (1)
-
Alexandre Julliard