From: Józef Kucia jkucia@codeweavers.com
In preparation for the following commmit.
Sequence numbers allows us to more easily track progress on queues in comparison to tracking each VkFence individually.
Signed-off-by: Józef Kucia jkucia@codeweavers.com ---
Even, if the queue submit count is on the order of hundreds per frame, it shouldn't overrun 64-bit sequence number.
--- libs/vkd3d/command.c | 92 ++++++++++++++++++++++++++++++++++---- libs/vkd3d/device.c | 6 +-- libs/vkd3d/vkd3d_private.h | 19 +++++++- 3 files changed, 103 insertions(+), 14 deletions(-)
diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c index ddd36d407c8b..2febca67e1ce 100644 --- a/libs/vkd3d/command.c +++ b/libs/vkd3d/command.c @@ -38,10 +38,17 @@ HRESULT vkd3d_queue_create(struct d3d12_device *device, return hresult_from_errno(rc); }
+ object->completed_sequence_number = 0; + object->submitted_sequence_number = 0; + object->vk_family_index = family_index; object->vk_queue_flags = properties->queueFlags; object->timestamp_bits = properties->timestampValidBits;
+ object->semaphores = NULL; + object->semaphores_size = 0; + object->semaphore_count = 0; + VK_CALL(vkGetDeviceQueue(device->vk_device, family_index, 0, &object->vk_queue));
TRACE("Created queue %p for queue family index %u.\n", object, family_index); @@ -51,8 +58,23 @@ HRESULT vkd3d_queue_create(struct d3d12_device *device, return S_OK; }
-void vkd3d_queue_destroy(struct vkd3d_queue *queue) +void vkd3d_queue_destroy(struct vkd3d_queue *queue, struct d3d12_device *device) { + const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs; + unsigned int i; + int rc; + + if ((rc = pthread_mutex_lock(&queue->mutex))) + ERR("Failed to lock mutex, error %d.\n", rc); + + for (i = 0; i < queue->semaphore_count; ++i) + VK_CALL(vkDestroySemaphore(device->vk_device, queue->semaphores[i].vk_semaphore, NULL)); + + vkd3d_free(queue->semaphores); + + if (!rc) + pthread_mutex_unlock(&queue->mutex); + pthread_mutex_destroy(&queue->mutex); vkd3d_free(queue); } @@ -103,9 +125,46 @@ static VkResult vkd3d_queue_wait_idle(struct vkd3d_queue *queue, return vr; }
+static void vkd3d_queue_update_sequence_number(struct vkd3d_queue *queue, + uint64_t sequence_number, struct d3d12_device *device) +{ + const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs; + uint64_t completed_sequence_number; + unsigned int i; + int rc; + + if ((rc = pthread_mutex_lock(&queue->mutex))) + { + ERR("Failed to lock mutex, error %d.\n", rc); + return; + } + + completed_sequence_number = queue->completed_sequence_number; + queue->completed_sequence_number = max(sequence_number, queue->completed_sequence_number); + + TRACE("Sequence number %"PRIu64" -> %"PRIu64".\n", completed_sequence_number, queue->completed_sequence_number); + + for (i = 0; i < queue->semaphore_count; ++i) + { + if (queue->semaphores[i].sequence_number > queue->completed_sequence_number) + break; + + VK_CALL(vkDestroySemaphore(device->vk_device, queue->semaphores[i].vk_semaphore, NULL)); + } + if (i > 0) + { + TRACE("Destroyed %u Vulkan semaphores.\n", i); + queue->semaphore_count -= i; + memmove(queue->semaphores, &queue->semaphores[i], queue->semaphore_count * sizeof(*queue->semaphores)); + } + + pthread_mutex_unlock(&queue->mutex); +} + /* Fence worker thread */ static HRESULT vkd3d_enqueue_gpu_fence(struct vkd3d_fence_worker *worker, - VkFence vk_fence, struct d3d12_fence *fence, uint64_t value) + VkFence vk_fence, struct d3d12_fence *fence, uint64_t value, + struct vkd3d_queue *queue, uint64_t queue_sequence_number) { int rc;
@@ -135,6 +194,8 @@ static HRESULT vkd3d_enqueue_gpu_fence(struct vkd3d_fence_worker *worker, worker->vk_fences[worker->fence_count] = vk_fence; worker->fences[worker->fence_count].fence = fence; worker->fences[worker->fence_count].value = value; + worker->fences[worker->fence_count].queue = queue; + worker->fences[worker->fence_count].queue_sequence_number = queue_sequence_number; ++worker->fence_count;
pthread_cond_signal(&worker->cond); @@ -204,9 +265,12 @@ static void vkd3d_wait_for_gpu_fences(struct vkd3d_fence_worker *worker) if (!(vr = VK_CALL(vkGetFenceStatus(device->vk_device, vk_fence)))) { struct vkd3d_waiting_fence *current = &worker->fences[i]; + TRACE("Signaling fence %p value %#"PRIx64".\n", current->fence, current->value); if (FAILED(hr = d3d12_fence_signal(current->fence, current->value, vk_fence))) ERR("Failed to signal D3D12 fence, hr %#x.\n", hr); + + vkd3d_queue_update_sequence_number(current->queue, current->queue_sequence_number, device); continue; }
@@ -5103,9 +5167,11 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue * VkSemaphore vk_semaphore = VK_NULL_HANDLE; VkSemaphoreCreateInfo semaphore_info; VkFence vk_fence = VK_NULL_HANDLE; + struct vkd3d_queue *vkd3d_queue; struct d3d12_device *device; struct d3d12_fence *fence; VkSubmitInfo submit_info; + uint64_t sequence_number; VkQueue vk_queue; VkResult vr; HRESULT hr; @@ -5114,6 +5180,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
device = command_queue->device; vk_procs = &device->vk_procs; + vkd3d_queue = command_queue->vkd3d_queue;
fence = unsafe_impl_from_ID3D12Fence(fence_iface);
@@ -5144,17 +5211,23 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue * submit_info.signalSemaphoreCount = 1; submit_info.pSignalSemaphores = &vk_semaphore;
- if (!(vk_queue = vkd3d_queue_acquire(command_queue->vkd3d_queue))) + if (!(vk_queue = vkd3d_queue_acquire(vkd3d_queue))) { - ERR("Failed to acquire queue %p.\n", command_queue->vkd3d_queue); + ERR("Failed to acquire queue %p.\n", vkd3d_queue); hr = E_FAIL; goto fail; } - vr = VK_CALL(vkQueueSubmit(vk_queue, 1, &submit_info, vk_fence)); - vkd3d_queue_release(command_queue->vkd3d_queue); + + if ((vr = VK_CALL(vkQueueSubmit(vk_queue, 1, &submit_info, vk_fence))) >= 0) + sequence_number = ++vkd3d_queue->submitted_sequence_number; + + vkd3d_queue_release(vkd3d_queue); + + assert(sequence_number != ~(uint64_t)0); + if (vr < 0) { - WARN("Failed to submit, vr %d.\n", vr); + WARN("Failed to submit signal operation, vr %d.\n", vr); goto fail_vkresult; }
@@ -5164,7 +5237,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue * vr = VK_CALL(vkGetFenceStatus(device->vk_device, vk_fence)); if (vr == VK_NOT_READY) { - if (SUCCEEDED(hr = vkd3d_enqueue_gpu_fence(&device->fence_worker, vk_fence, fence, value))) + if (SUCCEEDED(hr = vkd3d_enqueue_gpu_fence(&device->fence_worker, vk_fence, fence, value, vkd3d_queue, sequence_number))) vk_fence = VK_NULL_HANDLE; } else if (vr == VK_SUCCESS) @@ -5172,6 +5245,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue * TRACE("Already signaled %p, value %#"PRIx64".\n", fence, value); hr = d3d12_fence_signal(fence, value, vk_fence); vk_fence = VK_NULL_HANDLE; + vkd3d_queue_update_sequence_number(vkd3d_queue, sequence_number, device); } else { @@ -5182,7 +5256,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue * if (vk_fence || vk_semaphore) { /* In case of an unexpected failure, try to safely destroy Vulkan objects. */ - vkd3d_queue_wait_idle(command_queue->vkd3d_queue, vk_procs); + vkd3d_queue_wait_idle(vkd3d_queue, vk_procs); goto fail; }
diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c index 41f7ea18d688..cb2934a23afb 100644 --- a/libs/vkd3d/device.c +++ b/libs/vkd3d/device.c @@ -1425,12 +1425,12 @@ struct vkd3d_device_queue_info static void d3d12_device_destroy_vkd3d_queues(struct d3d12_device *device) { if (device->direct_queue) - vkd3d_queue_destroy(device->direct_queue); + vkd3d_queue_destroy(device->direct_queue, device); if (device->compute_queue && device->compute_queue != device->direct_queue) - vkd3d_queue_destroy(device->compute_queue); + vkd3d_queue_destroy(device->compute_queue, device); if (device->copy_queue && device->copy_queue != device->direct_queue && device->copy_queue != device->compute_queue) - vkd3d_queue_destroy(device->copy_queue); + vkd3d_queue_destroy(device->copy_queue, device);
device->direct_queue = NULL; device->compute_queue = NULL; diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h index 09ac4e01b7c7..f934d0ba9154 100644 --- a/libs/vkd3d/vkd3d_private.h +++ b/libs/vkd3d/vkd3d_private.h @@ -157,7 +157,9 @@ struct vkd3d_fence_worker struct vkd3d_waiting_fence { struct d3d12_fence *fence; - UINT64 value; + uint64_t value; + struct vkd3d_queue *queue; + uint64_t queue_sequence_number; } *fences; size_t fences_size;
@@ -882,17 +884,30 @@ struct vkd3d_queue { /* Access to VkQueue must be externally synchronized. */ pthread_mutex_t mutex; + VkQueue vk_queue; + + uint64_t completed_sequence_number; + uint64_t submitted_sequence_number; + uint32_t vk_family_index; VkQueueFlags vk_queue_flags; uint32_t timestamp_bits; + + struct + { + VkSemaphore vk_semaphore; + uint64_t sequence_number; + } *semaphores; + size_t semaphores_size; + size_t semaphore_count; };
VkQueue vkd3d_queue_acquire(struct vkd3d_queue *queue) DECLSPEC_HIDDEN; HRESULT vkd3d_queue_create(struct d3d12_device *device, uint32_t family_index, const VkQueueFamilyProperties *properties, struct vkd3d_queue **queue) DECLSPEC_HIDDEN; -void vkd3d_queue_destroy(struct vkd3d_queue *queue) DECLSPEC_HIDDEN; +void vkd3d_queue_destroy(struct vkd3d_queue *queue, struct d3d12_device *device) DECLSPEC_HIDDEN; void vkd3d_queue_release(struct vkd3d_queue *queue) DECLSPEC_HIDDEN;
/* ID3D12CommandQueue */