[PATCH vkd3d 6/8] vkd3d: Introduce queue sequence numbers to track unused Vulkan semaphores.

[Józef Kucia]

From: Józef Kucia

In preparation for the following commmit.

Sequence numbers allows us to more easily track progress on queues in
comparison to tracking each VkFence individually.

Signed-off-by: Józef Kucia 
---

Even, if the queue submit count is on the order of hundreds per frame,
it shouldn't overrun 64-bit sequence number.

---
 libs/vkd3d/command.c       | 92 ++++++++++++++++++++++++++++++++++----
 libs/vkd3d/device.c        |  6 +--
 libs/vkd3d/vkd3d_private.h | 19 +++++++-
 3 files changed, 103 insertions(+), 14 deletions(-)

diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
index ddd36d407c8b..2febca67e1ce 100644
--- a/libs/vkd3d/command.c
+++ b/libs/vkd3d/command.c
@@ -38,10 +38,17 @@ HRESULT vkd3d_queue_create(struct d3d12_device *device,
         return hresult_from_errno(rc);
     }

+    object->completed_sequence_number = 0;
+    object->submitted_sequence_number = 0;
+
     object->vk_family_index = family_index;
     object->vk_queue_flags = properties->queueFlags;
     object->timestamp_bits = properties->timestampValidBits;

+    object->semaphores = NULL;
+    object->semaphores_size = 0;
+    object->semaphore_count = 0;
+
     VK_CALL(vkGetDeviceQueue(device->vk_device, family_index, 0, &object->vk_queue));

TRACE("Created queue %p for queue family index %u.\n", object, family_index);
@@ -51,8 +58,23 @@ HRESULT vkd3d_queue_create(struct d3d12_device *device,
     return S_OK;
 }

-void vkd3d_queue_destroy(struct vkd3d_queue *queue)
+void vkd3d_queue_destroy(struct vkd3d_queue *queue, struct d3d12_device *device)
 {
+    const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
+    unsigned int i;
+    int rc;
+
+    if ((rc = pthread_mutex_lock(&queue->mutex)))
+        ERR("Failed to lock mutex, error %d.\n", rc);
+
+    for (i = 0; i < queue->semaphore_count; ++i)
+        VK_CALL(vkDestroySemaphore(device->vk_device, queue->semaphores[i].vk_semaphore, NULL));
+
+    vkd3d_free(queue->semaphores);
+
+    if (!rc)
+        pthread_mutex_unlock(&queue->mutex);
+
     pthread_mutex_destroy(&queue->mutex);
     vkd3d_free(queue);
 }
@@ -103,9 +125,46 @@ static VkResult vkd3d_queue_wait_idle(struct vkd3d_queue *queue,
     return vr;
 }

+static void vkd3d_queue_update_sequence_number(struct vkd3d_queue *queue,
+        uint64_t sequence_number, struct d3d12_device *device)
+{
+    const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
+    uint64_t completed_sequence_number;
+    unsigned int i;
+    int rc;
+
+    if ((rc = pthread_mutex_lock(&queue->mutex)))
+    {
+        ERR("Failed to lock mutex, error %d.\n", rc);
+        return;
+    }
+
+    completed_sequence_number = queue->completed_sequence_number;
+    queue->completed_sequence_number = max(sequence_number, queue->completed_sequence_number);
+
+    TRACE("Sequence number %"PRIu64" -> %"PRIu64".\n", completed_sequence_number, queue->completed_sequence_number);
+
+    for (i = 0; i < queue->semaphore_count; ++i)
+    {
+        if (queue->semaphores[i].sequence_number > queue->completed_sequence_number)
+            break;
+
+        VK_CALL(vkDestroySemaphore(device->vk_device, queue->semaphores[i].vk_semaphore, NULL));
+    }
+    if (i > 0)
+    {
+        TRACE("Destroyed %u Vulkan semaphores.\n", i);
+        queue->semaphore_count -= i;
+        memmove(queue->semaphores, &queue->semaphores[i], queue->semaphore_count * sizeof(*queue->semaphores));
+    }
+
+    pthread_mutex_unlock(&queue->mutex);
+}
+
 /* Fence worker thread */
 static HRESULT vkd3d_enqueue_gpu_fence(struct vkd3d_fence_worker *worker,
-        VkFence vk_fence, struct d3d12_fence *fence, uint64_t value)
+        VkFence vk_fence, struct d3d12_fence *fence, uint64_t value,
+        struct vkd3d_queue *queue, uint64_t queue_sequence_number)
 {
     int rc;

@@ -135,6 +194,8 @@ static HRESULT vkd3d_enqueue_gpu_fence(struct vkd3d_fence_worker *worker,
     worker->vk_fences[worker->fence_count] = vk_fence;
     worker->fences[worker->fence_count].fence = fence;
     worker->fences[worker->fence_count].value = value;
+    worker->fences[worker->fence_count].queue = queue;
+    worker->fences[worker->fence_count].queue_sequence_number = queue_sequence_number;
     ++worker->fence_count;

pthread_cond_signal(&worker->cond);
@@ -204,9 +265,12 @@ static void vkd3d_wait_for_gpu_fences(struct vkd3d_fence_worker *worker)
         if (!(vr = VK_CALL(vkGetFenceStatus(device->vk_device, vk_fence))))
         {
             struct vkd3d_waiting_fence *current = &worker->fences[i];
+
             TRACE("Signaling fence %p value %#"PRIx64".\n", current->fence, current->value);
             if (FAILED(hr = d3d12_fence_signal(current->fence, current->value, vk_fence)))
                 ERR("Failed to signal D3D12 fence, hr %#x.\n", hr);
+
+            vkd3d_queue_update_sequence_number(current->queue, current->queue_sequence_number, device);
             continue;
         }

@@ -5103,9 +5167,11 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
     VkSemaphore vk_semaphore = VK_NULL_HANDLE;
     VkSemaphoreCreateInfo semaphore_info;
     VkFence vk_fence = VK_NULL_HANDLE;
+    struct vkd3d_queue *vkd3d_queue;
     struct d3d12_device *device;
     struct d3d12_fence *fence;
     VkSubmitInfo submit_info;
+    uint64_t sequence_number;
     VkQueue vk_queue;
     VkResult vr;
     HRESULT hr;
@@ -5114,6 +5180,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *

device = command_queue->device;
     vk_procs = &device->vk_procs;
+    vkd3d_queue = command_queue->vkd3d_queue;

fence = unsafe_impl_from_ID3D12Fence(fence_iface);

@@ -5144,17 +5211,23 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
     submit_info.signalSemaphoreCount = 1;
     submit_info.pSignalSemaphores = &vk_semaphore;

-    if (!(vk_queue = vkd3d_queue_acquire(command_queue->vkd3d_queue)))
+    if (!(vk_queue = vkd3d_queue_acquire(vkd3d_queue)))
     {
-        ERR("Failed to acquire queue %p.\n", command_queue->vkd3d_queue);
+        ERR("Failed to acquire queue %p.\n", vkd3d_queue);
         hr = E_FAIL;
         goto fail;
     }
-    vr = VK_CALL(vkQueueSubmit(vk_queue, 1, &submit_info, vk_fence));
-    vkd3d_queue_release(command_queue->vkd3d_queue);
+
+    if ((vr = VK_CALL(vkQueueSubmit(vk_queue, 1, &submit_info, vk_fence))) >= 0)
+        sequence_number = ++vkd3d_queue->submitted_sequence_number;
+
+    vkd3d_queue_release(vkd3d_queue);
+
+    assert(sequence_number != ~(uint64_t)0);
+
     if (vr < 0)
     {
-        WARN("Failed to submit, vr %d.\n", vr);
+        WARN("Failed to submit signal operation, vr %d.\n", vr);
         goto fail_vkresult;
     }

@@ -5164,7 +5237,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
     vr = VK_CALL(vkGetFenceStatus(device->vk_device, vk_fence));
     if (vr == VK_NOT_READY)
     {
-        if (SUCCEEDED(hr = vkd3d_enqueue_gpu_fence(&device->fence_worker, vk_fence, fence, value)))
+        if (SUCCEEDED(hr = vkd3d_enqueue_gpu_fence(&device->fence_worker, vk_fence, fence, value, vkd3d_queue, sequence_number)))
             vk_fence = VK_NULL_HANDLE;
     }
     else if (vr == VK_SUCCESS)
@@ -5172,6 +5245,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
         TRACE("Already signaled %p, value %#"PRIx64".\n", fence, value);
         hr = d3d12_fence_signal(fence, value, vk_fence);
         vk_fence = VK_NULL_HANDLE;
+        vkd3d_queue_update_sequence_number(vkd3d_queue, sequence_number, device);
     }
     else
     {
@@ -5182,7 +5256,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
     if (vk_fence || vk_semaphore)
     {
         /* In case of an unexpected failure, try to safely destroy Vulkan objects. */
-        vkd3d_queue_wait_idle(command_queue->vkd3d_queue, vk_procs);
+        vkd3d_queue_wait_idle(vkd3d_queue, vk_procs);
         goto fail;
     }

diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c
index 41f7ea18d688..cb2934a23afb 100644
--- a/libs/vkd3d/device.c
+++ b/libs/vkd3d/device.c
@@ -1425,12 +1425,12 @@ struct vkd3d_device_queue_info
 static void d3d12_device_destroy_vkd3d_queues(struct d3d12_device *device)
 {
     if (device->direct_queue)
-        vkd3d_queue_destroy(device->direct_queue);
+        vkd3d_queue_destroy(device->direct_queue, device);
     if (device->compute_queue && device->compute_queue != device->direct_queue)
-        vkd3d_queue_destroy(device->compute_queue);
+        vkd3d_queue_destroy(device->compute_queue, device);
     if (device->copy_queue && device->copy_queue != device->direct_queue
             && device->copy_queue != device->compute_queue)
-        vkd3d_queue_destroy(device->copy_queue);
+        vkd3d_queue_destroy(device->copy_queue, device);

device->direct_queue = NULL;
     device->compute_queue = NULL;
diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
index 09ac4e01b7c7..f934d0ba9154 100644
--- a/libs/vkd3d/vkd3d_private.h
+++ b/libs/vkd3d/vkd3d_private.h
@@ -157,7 +157,9 @@ struct vkd3d_fence_worker
     struct vkd3d_waiting_fence
     {
         struct d3d12_fence *fence;
-        UINT64 value;
+        uint64_t value;
+        struct vkd3d_queue *queue;
+        uint64_t queue_sequence_number;
     } *fences;
     size_t fences_size;

@@ -882,17 +884,30 @@ struct vkd3d_queue
 {
     /* Access to VkQueue must be externally synchronized. */
     pthread_mutex_t mutex;
+
     VkQueue vk_queue;
+
+    uint64_t completed_sequence_number;
+    uint64_t submitted_sequence_number;
+
     uint32_t vk_family_index;
     VkQueueFlags vk_queue_flags;
     uint32_t timestamp_bits;
+
+    struct
+    {
+        VkSemaphore vk_semaphore;
+        uint64_t sequence_number;
+    } *semaphores;
+    size_t semaphores_size;
+    size_t semaphore_count;
 };

VkQueue vkd3d_queue_acquire(struct vkd3d_queue *queue) DECLSPEC_HIDDEN;
 HRESULT vkd3d_queue_create(struct d3d12_device *device,
         uint32_t family_index, const VkQueueFamilyProperties *properties,
         struct vkd3d_queue **queue) DECLSPEC_HIDDEN;
-void vkd3d_queue_destroy(struct vkd3d_queue *queue) DECLSPEC_HIDDEN;
+void vkd3d_queue_destroy(struct vkd3d_queue *queue, struct d3d12_device *device) DECLSPEC_HIDDEN;
 void vkd3d_queue_release(struct vkd3d_queue *queue) DECLSPEC_HIDDEN;

/* ID3D12CommandQueue */

[Henri Verbeet]

Signed-off-by: Henri Verbeet