This depends on !412, and might need some finishing touches before being ready.
-- v6: ci: Run shader tests on Windows. ci: Run cross tests on Windows.
From: Giovanni Mascellani gmascellani@codeweavers.com
A driver program is introduced to coordinate test running on Windows, similarly to what "make test" does on Linux and macOS. --- .gitignore | 1 + .gitlab-ci.yml | 2 + Makefile.am | 27 ++++++- gitlab/README | 10 ++- gitlab/build-crosstest | 5 +- gitlab/test.yml | 35 +++++++++ tests/driver.c | 173 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 gitlab/test.yml create mode 100644 tests/driver.c
diff --git a/.gitignore b/.gitignore index 841220ee6..5c35199a5 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ vkd3d-*.tar.xz *.tab.c *.tab.h *.trs +*.txt *.yy.c *~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cc59d764..276c654ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,9 @@ stages: - image - build + - test
include: - local: "/gitlab/image.yml" - local: "/gitlab/build.yml" + - local: "/gitlab/test.yml" diff --git a/Makefile.am b/Makefile.am index 2821ddc6f..4e3a43bb8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -480,6 +480,9 @@ shader_runner_cross_sources = \ $(srcdir)/tests/shader_runner_d3d11.c \ $(srcdir)/tests/shader_runner_d3d12.c
+driver_cross_sources = \ + $(srcdir)/tests/driver.c + if HAVE_CROSSTARGET32 CROSS32_CC = @CROSSCC32@ CROSS32_DLLTOOL = @CROSSTARGET32@-dlltool @@ -495,6 +498,7 @@ endif CROSS32_FILES = $(CROSS32_EXEFILES) if BUILD_TESTS CROSS32_FILES += tests/shader_runner.cross32.exe +CROSS32_FILES += tests/driver.cross32.exe endif
CLEANFILES += $(CROSS32_IMPLIBS) $(CROSS32_FILES) @@ -517,6 +521,11 @@ tests/shader_runner.cross32.exe: $(shader_runner_cross_sources) $(CROSS32_IMPLIB $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS32_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po
+tests/driver.cross32.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!.exe$$!!'`; \ + $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest32: endif @@ -536,6 +545,7 @@ endif CROSS64_FILES = $(CROSS64_EXEFILES) if BUILD_TESTS CROSS64_FILES += tests/shader_runner.cross64.exe +CROSS64_FILES += tests/driver.cross64.exe endif
CLEANFILES += $(CROSS64_IMPLIBS) $(CROSS64_FILES) @@ -558,12 +568,25 @@ tests/shader_runner.cross64.exe: $(shader_runner_cross_sources) $(CROSS64_IMPLIB $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS64_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po
+tests/driver.cross64.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!.exe$$!!'`; \ + $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest64: endif
-.PHONY: crosstest crosstest32 crosstest64 -crosstest: crosstest32 crosstest64 +tests/crosstests.txt: FORCE + $(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@ + +crosstest-lists: tests/crosstests.txt +CLEANFILES += tests/crosstests.txt + +FORCE: + +.PHONY: crosstest crosstest32 crosstest64 crosstest-lists FORCE +crosstest: crosstest32 crosstest64 crosstest-lists
if BUILD_DOC @DX_RULES@ diff --git a/gitlab/README b/gitlab/README index 04fd61119..310912be2 100644 --- a/gitlab/README +++ b/gitlab/README @@ -19,9 +19,10 @@ MoltenVK as the Vulkan driver. The llvmpipe and macOS jobs are currently allowed to fail.
Additionally, MinGW is used to build PE binaries for both vkd3d and -its crosstests, for both 32 and 64 bit. These builds are not currently -tested (but the pipeline still fails if the compilation is not -successful). +its crosstests, for both 32 and 64 bit. The PE crosstests are executed +on Windows 10 to check that behavior imposed by the tests corresponds +to Microsoft's D3D12 implementation. The rendering backend is +currently Window's WARP software implementation.
The testing logs are available as CI artifacts, as well as the PE modules built by the crosstest and MinGW jobs. @@ -58,3 +59,6 @@ environment for running the tests. All the software required to compile and run the tests will therefore have to be installed directly on the host system. Complete instructions to setup the macOS are currently not available. + +Finally, a runner tagged with `win10-21h2' must be available and +submit jobs to a Windows 10 virtual machine. diff --git a/gitlab/build-crosstest b/gitlab/build-crosstest index 4a1341a09..da59bc8da 100755 --- a/gitlab/build-crosstest +++ b/gitlab/build-crosstest @@ -19,6 +19,9 @@ cd build touch ../pipeline_failed
mkdir -p ../artifacts/$COMMIT -rsync -Rr config.log tests/*.exe ../artifacts/$COMMIT +rsync -Rr config.log tests/*.txt tests/*.exe ../artifacts/$COMMIT + +# Make the driver easily available to the Windows CI job +cp tests/driver.cross64.exe ../artifacts
git reset --hard diff --git a/gitlab/test.yml b/gitlab/test.yml new file mode 100644 index 000000000..97bebba53 --- /dev/null +++ b/gitlab/test.yml @@ -0,0 +1,35 @@ +test-win-64: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "64" + artifacts: + when: always + paths: + - artifacts + +test-win-32: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "32" + artifacts: + when: always + paths: + - artifacts diff --git a/tests/driver.c b/tests/driver.c new file mode 100644 index 000000000..dd8092723 --- /dev/null +++ b/tests/driver.c @@ -0,0 +1,173 @@ +/* + * Copyright 2023 Giovanni Mascellani for CodeWeavers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include <stdio.h> +#include <stdbool.h> +#include <windows.h> + +static bool run_program(const char *cmdline, const char *log_filename) +{ + HANDLE log = INVALID_HANDLE_VALUE; + SECURITY_ATTRIBUTES attrs = {0}; + PROCESS_INFORMATION info = {0}; + STARTUPINFOA startup = {0}; + char cmdline2[1024]; + DWORD exit_code; + bool ret = true; + + strcpy(cmdline2, cmdline); + + attrs.nLength = sizeof(attrs); + attrs.bInheritHandle = TRUE; + + log = CreateFileA(log_filename, GENERIC_WRITE, 0, &attrs, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (log == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot create log file %s, last error %ld.\n", log_filename, GetLastError()); + ret = false; + goto out; + } + + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdInput = INVALID_HANDLE_VALUE; + startup.hStdOutput = log; + startup.hStdError = log; + + if (!CreateProcessA(NULL, cmdline2, NULL, NULL, TRUE, 0, NULL, NULL, &startup, &info)) + { + fprintf(stderr, "Cannot create process %s, last error %ld.\n", cmdline2, GetLastError()); + ret = false; + goto out; + } + + if (WaitForSingleObject(info.hProcess, INFINITE) != WAIT_OBJECT_0) + { + fprintf(stderr, "Cannot wait for process termination, last error %ld.\n", GetLastError()); + ret = false; + goto out; + } + + if (!GetExitCodeProcess(info.hProcess, &exit_code)) + { + fprintf(stderr, "Cannot retrive the process exit code, last error %ld.\n", GetLastError()); + ret = false; + goto out; + } + + ret = exit_code == 0; + + printf("%s: %s\n", ret ? "PASS" : "FAIL", cmdline); + +out: + if (info.hProcess && !CloseHandle(info.hProcess)) + fprintf(stderr, "Cannot close process, last error %ld.\n", GetLastError()); + if (info.hThread && !CloseHandle(info.hThread)) + fprintf(stderr, "Cannot close thread, last error %ld.\n", GetLastError()); + if (log != INVALID_HANDLE_VALUE && !CloseHandle(log)) + fprintf(stderr, "Cannot close log file, last error %ld.\n", GetLastError()); + + return ret; +} + +static bool run_tests_for_directory(const char *commit_dir) +{ + char cmdline[1024], log_filename[1024], list_filename[1024], line[1024]; + unsigned int success_count = 0, test_count = 0; + const char *test_arch = getenv("TEST_ARCH"); + FILE *list_file; + bool ret = true; + + if (!test_arch) + test_arch = "64"; + + printf("Building %s\n", commit_dir); + printf("---\n"); + + sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir); + list_file = fopen(list_filename, "r"); + + if (!list_file) + { + fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno); + ret = false; + } + else + { + while (fgets(line, sizeof(line), list_file)) + { + size_t len = strlen(line); + + if (line[len - 1] == '\n') + line[len - 1] = '\0'; + + sprintf(cmdline, "artifacts/%s/%s.cross%s.exe", commit_dir, line, test_arch); + sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line); + ++test_count; + success_count += !!run_program(cmdline, log_filename); + } + + fclose(list_file); + } + + printf("=======\n"); + printf("Summary\n"); + printf("=======\n"); + + printf("# TOTAL: %u\n", test_count); + printf("# PASS: %u\n", success_count); + printf("# FAIL: %u\n", test_count - success_count); + + if (test_count != success_count) + ret = false; + + return ret; +} + +int wmain(void) +{ + WIN32_FIND_DATAA find_data; + HANDLE find_handle; + bool ret = true; + + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); + + find_handle = FindFirstFileA("artifacts/*-*", &find_data); + if (find_handle == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot list commits, last error %ld.\n", GetLastError()); + ret = false; + } + else + { + do + { + ret &= run_tests_for_directory(find_data.cFileName); + } while (FindNextFileA(find_handle, &find_data)); + + if (GetLastError() != ERROR_NO_MORE_FILES) + { + fprintf(stderr, "Cannot list tests, last error %ld.\n", GetLastError()); + ret = false; + } + + FindClose(find_handle); + } + + return !ret; +}
From: Giovanni Mascellani gmascellani@codeweavers.com
--- Makefile.am | 7 +++++-- tests/driver.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-)
diff --git a/Makefile.am b/Makefile.am index 4e3a43bb8..def1db6f5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -580,8 +580,11 @@ endif tests/crosstests.txt: FORCE $(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@
-crosstest-lists: tests/crosstests.txt -CLEANFILES += tests/crosstests.txt +tests/shader_tests.txt: FORCE + $(AM_V_GEN) for i in $(vkd3d_shader_tests) ; do echo $$i ; done > $@ + +crosstest-lists: tests/crosstests.txt tests/shader_tests.txt +CLEANFILES += tests/crosstests.txt tests/shader_tests.txt
FORCE:
diff --git a/tests/driver.c b/tests/driver.c index dd8092723..70e455016 100644 --- a/tests/driver.c +++ b/tests/driver.c @@ -19,19 +19,37 @@ #include <stdio.h> #include <stdbool.h> #include <windows.h> +#include <shlobj.h>
static bool run_program(const char *cmdline, const char *log_filename) { + char cmdline2[1024], log_dirname[1024], *file_part; HANDLE log = INVALID_HANDLE_VALUE; SECURITY_ATTRIBUTES attrs = {0}; PROCESS_INFORMATION info = {0}; STARTUPINFOA startup = {0}; - char cmdline2[1024]; DWORD exit_code; bool ret = true; + int res;
strcpy(cmdline2, cmdline);
+ if (GetFullPathNameA(log_filename, sizeof(log_dirname), log_dirname, &file_part) == 0) + { + fprintf(stderr, "Cannot extract the directory name for path %s, last error %ld.\n", log_filename, GetLastError()); + ret = false; + goto out; + } + *file_part = '\0'; + + res = SHCreateDirectoryExA(NULL, log_dirname, NULL); + if (res != ERROR_SUCCESS && res != ERROR_ALREADY_EXISTS) + { + fprintf(stderr, "Cannot create log directory %s, error %d.\n", log_dirname, res); + ret = false; + goto out; + } + attrs.nLength = sizeof(attrs); attrs.bInheritHandle = TRUE;
@@ -99,6 +117,36 @@ static bool run_tests_for_directory(const char *commit_dir) printf("Building %s\n", commit_dir); printf("---\n");
+ sprintf(list_filename, "artifacts/%s/tests/shader_tests.txt", commit_dir); + list_file = fopen(list_filename, "r"); + + if (!list_file) + { + fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno); + ret = false; + } + else + { + while (fgets(line, sizeof(line), list_file)) + { + size_t len = strlen(line); + + if (line[len - 1] == '\n') + line[--len] = '\0'; + + sprintf(cmdline, "artifacts/%s/tests/shader_runner.cross%s.exe %s", commit_dir, test_arch, line); + + /* Remove the .shader_test suffix. */ + line[len - 12] = '\0'; + sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line); + + ++test_count; + success_count += !!run_program(cmdline, log_filename); + } + + fclose(list_file); + } + sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir); list_file = fopen(list_filename, "r");
This merge request was approved by Henri Verbeet.
Alexandre Julliard (@julliard) commented about tests/driver.c:
goto out;- }
- if (WaitForSingleObject(info.hProcess, INFINITE) != WAIT_OBJECT_0)
- {
fprintf(stderr, "Cannot wait for process termination, last error %ld.\n", GetLastError());ret = false;goto out;- }
- if (!GetExitCodeProcess(info.hProcess, &exit_code))
- {
fprintf(stderr, "Cannot retrive the process exit code, last error %ld.\n", GetLastError());ret = false;goto out;- }
This should have a (short) timeout and then kill the process, like winetest.exe does. We can't have a runaway test block the runner for an hour.
On Fri Oct 20 20:14:30 2023 +0000, Alexandre Julliard wrote:
This should have a (short) timeout and then kill the process, like winetest.exe does. We can't have a runaway test block the runner for an hour.
Yeah, it makes sense. Actually, it already happens every now and then with the macOS tests, so it wouldn't be bad to have something similar there too.
-
Alexandre Julliard (＠julliard) -
Giovanni Mascellani
-
Giovanni Mascellani (＠giomasce)
-
Henri Verbeet (＠hverbeet)