From: Francisco Casas fcasas@codeweavers.com
After running 'make check', this script can be used to see which tests are not passing, showing the test line and a [XP], [XF], or [F] tag, for each backend.
It can be used like this:
python3 lightboard.py [-b] <vkd3d_build_path>/tests/hlsl --- tools/lightboard.py | 141 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tools/lightboard.py
diff --git a/tools/lightboard.py b/tools/lightboard.py new file mode 100644 index 00000000..a1bba7b3 --- /dev/null +++ b/tools/lightboard.py @@ -0,0 +1,141 @@ +# +# Utility script to present HLSL individual shader_runner test results from +# log files. +# +# Copyright 2022 Francisco Casas for CodeWeavers +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA +# + +import os +from sys import argv + +import re + +generic_re = re.compile(r"^\w+:\d+: Section [.*?], line (\d+): \w+.*$") +xfail_re = re.compile(r"^\w+:\d+: Section [.*?], line \d+: Todo:.*$") +xpass_re = re.compile(r"^\w+:\d+: Section [.*?], line \d+: Todo succeeded:.*$") +fail_re = re.compile(r"^\w+:\d+: Section [.*?], line \d+: Test failed:.*$") +general_fail_re = re.compile(r"^FAIL .* (exit status: (\d+))$") +compiling_re = re.compile(r"^\w+:.*: Compiling shaders with (\S+) and executing with (\S+)") +assert_re = re.compile(r"^.*?Assertion `.*?' failed.$"); + +RED = "\033[31;1m" +FAINT = "\033[0;2m" +GREEN = "\033[32m" +YELLOW = "\033[33m" +RESET = "\033[0m" + +print_backends = False + +for arg in argv[:]: + if arg == "-b": + print_backends = True + argv.remove(arg) + +if len(argv) != 2: + print("usage: python3 %s [-b] <vkd3d_build_path>/tests/hlsl "%argv[0]) + exit(1) + + +directory = argv[1] + +files = sorted(list(os.listdir(directory))) + +for fname in files: + fname = os.path.join(directory, fname) + + # Check that it is indeed a file, and ends with .log + if not os.path.isfile(fname): + continue + if fname.split('.')[-1] != "log": + continue + + str = "" + show_str = False + + backend = "" + + # Line number of the last match + last_line_n = -1 + + with open(fname, "r", encoding="latin-1") as f: + for line in f: + line = line.strip() + + # Empty lines reset last_line_n + if line == "": + last_line_n = -1 + continue + + match = compiling_re.match(line) + if match: + backend = match.groups()[0] + " & " + match.groups()[1] + " " + if print_backends: + if len(str) > 0: + str += "\n" + str += " " + str += "%s%-22s%s"%(FAINT, backend, RESET) + else: + if len(str) == 0: + str += " " + else: + str += " |" + last_line_n = -1 + continue + + match = generic_re.match(line) + if match: + line_n = int(match.groups()[0]) + show_str = True + + if line_n != last_line_n: + str += " %s%d%s"%(FAINT, line_n, RESET) + last_line_n = line_n + any_xf = False + + # We expect one of these to match + if xfail_re.match(line): + str += "%s[XF]%s"%(GREEN, RESET) + any_xf = True + elif xpass_re.match(line): + if any_xf: + # XP after XF is just a coincidence + str += "%s[XP]%s"%(YELLOW, RESET) + else: + str += "%s[XP]%s"%(RED, RESET) + elif fail_re.match(line): + str += "%s[F]%s"%(RED, RESET) + else: + str += "%s[?]%s"%(RED, RESET) + + match = assert_re.match(line) + if match: + str += " %s[AF]%s%s"%(RED, RESET, ':'.join(line.split(":")[1:])) + show_str = True + last_line_n = -1 + + match = general_fail_re.match(line) + if match: + exists = int(match.groups()[0]) + if exists > 1: + str += " %s[EXIT %d]%s"%(RED, exists, RESET) + show_str = True + last_line_n = -1 + + if show_str: + print(fname + " :") + print(str) +