This depends on !412, and might need some finishing touches before being ready.
-- v8: ci: Run shader tests on Windows. ci: Run cross tests on Windows.
From: Giovanni Mascellani gmascellani@codeweavers.com
A driver program is introduced to coordinate test running on Windows, similarly to what "make test" does on Linux and macOS. --- .gitignore | 1 + .gitlab-ci.yml | 2 + Makefile.am | 27 +++++- gitlab/README | 10 +- gitlab/build-crosstest | 5 +- gitlab/test.yml | 35 +++++++ tests/driver.c | 204 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 gitlab/test.yml create mode 100644 tests/driver.c
diff --git a/.gitignore b/.gitignore index 841220ee6..5c35199a5 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ vkd3d-*.tar.xz *.tab.c *.tab.h *.trs +*.txt *.yy.c *~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cc59d764..276c654ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,9 @@ stages: - image - build + - test
include: - local: "/gitlab/image.yml" - local: "/gitlab/build.yml" + - local: "/gitlab/test.yml" diff --git a/Makefile.am b/Makefile.am index 2821ddc6f..4e3a43bb8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -480,6 +480,9 @@ shader_runner_cross_sources = \ $(srcdir)/tests/shader_runner_d3d11.c \ $(srcdir)/tests/shader_runner_d3d12.c
+driver_cross_sources = \ + $(srcdir)/tests/driver.c + if HAVE_CROSSTARGET32 CROSS32_CC = @CROSSCC32@ CROSS32_DLLTOOL = @CROSSTARGET32@-dlltool @@ -495,6 +498,7 @@ endif CROSS32_FILES = $(CROSS32_EXEFILES) if BUILD_TESTS CROSS32_FILES += tests/shader_runner.cross32.exe +CROSS32_FILES += tests/driver.cross32.exe endif
CLEANFILES += $(CROSS32_IMPLIBS) $(CROSS32_FILES) @@ -517,6 +521,11 @@ tests/shader_runner.cross32.exe: $(shader_runner_cross_sources) $(CROSS32_IMPLIB $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS32_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po
+tests/driver.cross32.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!.exe$$!!'`; \ + $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest32: endif @@ -536,6 +545,7 @@ endif CROSS64_FILES = $(CROSS64_EXEFILES) if BUILD_TESTS CROSS64_FILES += tests/shader_runner.cross64.exe +CROSS64_FILES += tests/driver.cross64.exe endif
CLEANFILES += $(CROSS64_IMPLIBS) $(CROSS64_FILES) @@ -558,12 +568,25 @@ tests/shader_runner.cross64.exe: $(shader_runner_cross_sources) $(CROSS64_IMPLIB $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS64_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po
+tests/driver.cross64.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!.exe$$!!'`; \ + $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest64: endif
-.PHONY: crosstest crosstest32 crosstest64 -crosstest: crosstest32 crosstest64 +tests/crosstests.txt: FORCE + $(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@ + +crosstest-lists: tests/crosstests.txt +CLEANFILES += tests/crosstests.txt + +FORCE: + +.PHONY: crosstest crosstest32 crosstest64 crosstest-lists FORCE +crosstest: crosstest32 crosstest64 crosstest-lists
if BUILD_DOC @DX_RULES@ diff --git a/gitlab/README b/gitlab/README index 04fd61119..310912be2 100644 --- a/gitlab/README +++ b/gitlab/README @@ -19,9 +19,10 @@ MoltenVK as the Vulkan driver. The llvmpipe and macOS jobs are currently allowed to fail.
Additionally, MinGW is used to build PE binaries for both vkd3d and -its crosstests, for both 32 and 64 bit. These builds are not currently -tested (but the pipeline still fails if the compilation is not -successful). +its crosstests, for both 32 and 64 bit. The PE crosstests are executed +on Windows 10 to check that behavior imposed by the tests corresponds +to Microsoft's D3D12 implementation. The rendering backend is +currently Window's WARP software implementation.
The testing logs are available as CI artifacts, as well as the PE modules built by the crosstest and MinGW jobs. @@ -58,3 +59,6 @@ environment for running the tests. All the software required to compile and run the tests will therefore have to be installed directly on the host system. Complete instructions to setup the macOS are currently not available. + +Finally, a runner tagged with `win10-21h2' must be available and +submit jobs to a Windows 10 virtual machine. diff --git a/gitlab/build-crosstest b/gitlab/build-crosstest index 4a1341a09..da59bc8da 100755 --- a/gitlab/build-crosstest +++ b/gitlab/build-crosstest @@ -19,6 +19,9 @@ cd build touch ../pipeline_failed
mkdir -p ../artifacts/$COMMIT -rsync -Rr config.log tests/*.exe ../artifacts/$COMMIT +rsync -Rr config.log tests/*.txt tests/*.exe ../artifacts/$COMMIT + +# Make the driver easily available to the Windows CI job +cp tests/driver.cross64.exe ../artifacts
git reset --hard diff --git a/gitlab/test.yml b/gitlab/test.yml new file mode 100644 index 000000000..97bebba53 --- /dev/null +++ b/gitlab/test.yml @@ -0,0 +1,35 @@ +test-win-64: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "64" + artifacts: + when: always + paths: + - artifacts + +test-win-32: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "32" + artifacts: + when: always + paths: + - artifacts diff --git a/tests/driver.c b/tests/driver.c new file mode 100644 index 000000000..3ecdae2d8 --- /dev/null +++ b/tests/driver.c @@ -0,0 +1,204 @@ +/* + * Copyright 2023 Giovanni Mascellani for CodeWeavers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include <stdio.h> +#include <stdbool.h> +#include <windows.h> + +#define TIMEOUT_MS (10 * 1000) +#define MAX_TIMEOUT_COUNT 3 + +enum program_result +{ + PROGRAM_RESULT_SUCCESS, + PROGRAM_RESULT_TIMEOUT, + PROGRAM_RESULT_FAILURE, +}; + +static enum program_result run_program(const char *cmdline, const char *log_filename) +{ + enum program_result ret = PROGRAM_RESULT_SUCCESS; + HANDLE log = INVALID_HANDLE_VALUE; + SECURITY_ATTRIBUTES attrs = {0}; + PROCESS_INFORMATION info = {0}; + DWORD exit_code, wait_result; + STARTUPINFOA startup = {0}; + char cmdline2[1024]; + + strcpy(cmdline2, cmdline); + + attrs.nLength = sizeof(attrs); + attrs.bInheritHandle = TRUE; + + log = CreateFileA(log_filename, GENERIC_WRITE, 0, &attrs, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (log == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot create log file %s, last error %ld.\n", log_filename, GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdInput = INVALID_HANDLE_VALUE; + startup.hStdOutput = log; + startup.hStdError = log; + + if (!CreateProcessA(NULL, cmdline2, NULL, NULL, TRUE, 0, NULL, NULL, &startup, &info)) + { + fprintf(stderr, "Cannot create process %s, last error %ld.\n", cmdline2, GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + wait_result = WaitForSingleObject(info.hProcess, TIMEOUT_MS); + if (wait_result == WAIT_TIMEOUT) + { + fprintf(stderr, "Process timed out, terminating it.\n"); + ret = PROGRAM_RESULT_TIMEOUT; + + if (!TerminateProcess(info.hProcess, 1)) + { + fprintf(stderr, "Cannot terminate process, last error %ld.\n", GetLastError()); + goto out; + } + + wait_result = WaitForSingleObject(info.hProcess, INFINITE); + } + + if (wait_result != WAIT_OBJECT_0) + { + fprintf(stderr, "Cannot wait for process termination, last error %ld.\n", GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + if (!GetExitCodeProcess(info.hProcess, &exit_code)) + { + fprintf(stderr, "Cannot retrive the process exit code, last error %ld.\n", GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + ret = exit_code == 0 ? PROGRAM_RESULT_SUCCESS : PROGRAM_RESULT_FAILURE; + + printf("%s: %s\n", ret ? "PASS" : "FAIL", cmdline); + +out: + if (info.hProcess && !CloseHandle(info.hProcess)) + fprintf(stderr, "Cannot close process, last error %ld.\n", GetLastError()); + if (info.hThread && !CloseHandle(info.hThread)) + fprintf(stderr, "Cannot close thread, last error %ld.\n", GetLastError()); + if (log != INVALID_HANDLE_VALUE && !CloseHandle(log)) + fprintf(stderr, "Cannot close log file, last error %ld.\n", GetLastError()); + + return ret; +} + +static bool run_tests_for_directory(const char *commit_dir) +{ + char cmdline[1024], log_filename[1024], list_filename[1024], line[1024]; + unsigned int success_count = 0, test_count = 0, timeout_count = 0; + const char *test_arch = getenv("TEST_ARCH"); + enum program_result result; + FILE *list_file; + bool ret = true; + + if (!test_arch) + test_arch = "64"; + + printf("Building %s\n", commit_dir); + printf("---\n"); + + sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir); + list_file = fopen(list_filename, "r"); + + if (!list_file) + { + fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno); + ret = false; + } + else + { + while (fgets(line, sizeof(line), list_file) && timeout_count < MAX_TIMEOUT_COUNT) + { + size_t len = strlen(line); + + if (line[len - 1] == '\n') + line[len - 1] = '\0'; + + sprintf(cmdline, "artifacts/%s/%s.cross%s.exe", commit_dir, line, test_arch); + sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line); + ++test_count; + result = run_program(cmdline, log_filename); + success_count += result == PROGRAM_RESULT_SUCCESS; + timeout_count += result == PROGRAM_RESULT_TIMEOUT; + } + + fclose(list_file); + } + + if (timeout_count >= MAX_TIMEOUT_COUNT) + fprintf(stderr, "Too many timeouts, aborting tests.\n"); + + printf("=======\n"); + printf("Summary\n"); + printf("=======\n"); + + printf("# TOTAL: %u\n", test_count); + printf("# PASS: %u\n", success_count); + printf("# FAIL: %u\n", test_count - success_count); + + if (test_count != success_count) + ret = false; + + return ret; +} + +int wmain(void) +{ + WIN32_FIND_DATAA find_data; + HANDLE find_handle; + bool ret = true; + + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); + + find_handle = FindFirstFileA("artifacts/*-*", &find_data); + if (find_handle == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot list commits, last error %ld.\n", GetLastError()); + ret = false; + } + else + { + do + { + ret &= run_tests_for_directory(find_data.cFileName); + } while (FindNextFileA(find_handle, &find_data)); + + if (GetLastError() != ERROR_NO_MORE_FILES) + { + fprintf(stderr, "Cannot list tests, last error %ld.\n", GetLastError()); + ret = false; + } + + FindClose(find_handle); + } + + return !ret; +}
From: Giovanni Mascellani gmascellani@codeweavers.com
--- Makefile.am | 7 +++++-- tests/driver.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-)
diff --git a/Makefile.am b/Makefile.am index 4e3a43bb8..def1db6f5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -580,8 +580,11 @@ endif tests/crosstests.txt: FORCE $(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@
-crosstest-lists: tests/crosstests.txt -CLEANFILES += tests/crosstests.txt +tests/shader_tests.txt: FORCE + $(AM_V_GEN) for i in $(vkd3d_shader_tests) ; do echo $$i ; done > $@ + +crosstest-lists: tests/crosstests.txt tests/shader_tests.txt +CLEANFILES += tests/crosstests.txt tests/shader_tests.txt
FORCE:
diff --git a/tests/driver.c b/tests/driver.c index 3ecdae2d8..002447892 100644 --- a/tests/driver.c +++ b/tests/driver.c @@ -19,6 +19,7 @@ #include <stdio.h> #include <stdbool.h> #include <windows.h> +#include <shlobj.h>
#define TIMEOUT_MS (10 * 1000) #define MAX_TIMEOUT_COUNT 3 @@ -32,16 +33,33 @@ enum program_result
static enum program_result run_program(const char *cmdline, const char *log_filename) { + char cmdline2[1024], log_dirname[1024], *file_part; enum program_result ret = PROGRAM_RESULT_SUCCESS; HANDLE log = INVALID_HANDLE_VALUE; SECURITY_ATTRIBUTES attrs = {0}; PROCESS_INFORMATION info = {0}; DWORD exit_code, wait_result; STARTUPINFOA startup = {0}; - char cmdline2[1024]; + int res;
strcpy(cmdline2, cmdline);
+ if (GetFullPathNameA(log_filename, sizeof(log_dirname), log_dirname, &file_part) == 0) + { + fprintf(stderr, "Cannot extract the directory name for path %s, last error %ld.\n", log_filename, GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + *file_part = '\0'; + + res = SHCreateDirectoryExA(NULL, log_dirname, NULL); + if (res != ERROR_SUCCESS && res != ERROR_ALREADY_EXISTS) + { + fprintf(stderr, "Cannot create log directory %s, error %d.\n", log_dirname, res); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + attrs.nLength = sizeof(attrs); attrs.bInheritHandle = TRUE;
@@ -125,6 +143,38 @@ static bool run_tests_for_directory(const char *commit_dir) printf("Building %s\n", commit_dir); printf("---\n");
+ sprintf(list_filename, "artifacts/%s/tests/shader_tests.txt", commit_dir); + list_file = fopen(list_filename, "r"); + + if (!list_file) + { + fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno); + ret = false; + } + else + { + while (fgets(line, sizeof(line), list_file) && timeout_count < MAX_TIMEOUT_COUNT) + { + size_t len = strlen(line); + + if (line[len - 1] == '\n') + line[--len] = '\0'; + + sprintf(cmdline, "artifacts/%s/tests/shader_runner.cross%s.exe %s", commit_dir, test_arch, line); + + /* Remove the .shader_test suffix. */ + line[len - 12] = '\0'; + sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line); + + ++test_count; + result = run_program(cmdline, log_filename); + success_count += result == PROGRAM_RESULT_SUCCESS; + timeout_count += result == PROGRAM_RESULT_TIMEOUT; + } + + fclose(list_file); + } + sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir); list_file = fopen(list_filename, "r");
-
Giovanni Mascellani -
Giovanni Mascellani (＠giomasce)