[PATCH 5/5] wined3d: Use command buffer IDs for synchronisation in wined3d_buffer_gl_sync_apple().

[Henri Verbeet]

Analogous to the Vulkan adapter. This is slightly awkward in OpenGL
because it doesn't have explicit command buffers like Vulkan, but
calling wined3d_context_gl_submit_command_fence() on swapchain present
works well enough in practice. The main advantage of this approach is that it
avoids using a separate fence for each usage of each bo.

Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>
---
 dlls/wined3d/adapter_gl.c      |   2 +
 dlls/wined3d/buffer.c          |  61 ++--------------
 dlls/wined3d/context.c         |   4 -
 dlls/wined3d/context_gl.c      | 130 +++++++++++++++++++++++++++------
 dlls/wined3d/query.c           |   2 +-
 dlls/wined3d/swapchain.c       |   1 +
 dlls/wined3d/wined3d_private.h |  34 +++++++--
 7 files changed, 146 insertions(+), 88 deletions(-)

diff --git a/dlls/wined3d/adapter_gl.c b/dlls/wined3d/adapter_gl.c
index cb382cd6734..4781622d530 100644
--- a/dlls/wined3d/adapter_gl.c
+++ b/dlls/wined3d/adapter_gl.c
@@ -4256,6 +4256,8 @@ static HRESULT adapter_gl_create_device(struct wined3d *wined3d, const struct wi
     if (!(device_gl = heap_alloc_zero(sizeof(*device_gl))))
         return E_OUTOFMEMORY;
 
+    device_gl->current_fence_id = 1;
+
     if (FAILED(hr = wined3d_device_init(&device_gl->d, wined3d, adapter->ordinal, device_type, focus_window,
             flags, surface_alignment, levels, level_count, adapter->gl_info.supported, device_parent)))
     {
diff --git a/dlls/wined3d/buffer.c b/dlls/wined3d/buffer.c
index 61e35dc1e45..82051453ef1 100644
--- a/dlls/wined3d/buffer.c
+++ b/dlls/wined3d/buffer.c
@@ -187,11 +187,6 @@ static void wined3d_buffer_gl_destroy_buffer_object(struct wined3d_buffer_gl *bu
     wined3d_context_gl_destroy_bo(context_gl, &buffer_gl->bo);
     buffer_gl->b.buffer_object = 0;
 
-    if (buffer_gl->b.fence)
-    {
-        wined3d_fence_destroy(buffer_gl->b.fence);
-        buffer_gl->b.fence = NULL;
-    }
     buffer_gl->b.flags &= ~WINED3D_BUFFER_APPLESYNC;
 }
 
@@ -746,9 +741,9 @@ void * CDECL wined3d_buffer_get_parent(const struct wined3d_buffer *buffer)
 static void wined3d_buffer_gl_sync_apple(struct wined3d_buffer_gl *buffer_gl,
         uint32_t flags, struct wined3d_context_gl *context_gl)
 {
+    struct wined3d_device_gl *device_gl = wined3d_device_gl(buffer_gl->b.resource.device);
     const struct wined3d_gl_info *gl_info = context_gl->gl_info;
-    enum wined3d_fence_result ret;
-    HRESULT hr;
+    struct wined3d_bo_gl *bo = &buffer_gl->bo;
 
     /* No fencing needs to be done if the app promises not to overwrite
      * existing data. */
@@ -759,59 +754,17 @@ static void wined3d_buffer_gl_sync_apple(struct wined3d_buffer_gl *buffer_gl,
     {
         wined3d_buffer_gl_bind(buffer_gl, context_gl);
 
-        GL_EXTCALL(glBufferData(buffer_gl->bo.binding, buffer_gl->b.resource.size, NULL, buffer_gl->bo.usage));
+        GL_EXTCALL(glBufferData(bo->binding, buffer_gl->b.resource.size, NULL, bo->usage));
         checkGLcall("glBufferData");
-        return;
-    }
-
-    if (!buffer_gl->b.fence)
-    {
-        TRACE("Creating fence for buffer %p.\n", buffer_gl);
-
-        if (FAILED(hr = wined3d_fence_create(buffer_gl->b.resource.device, &buffer_gl->b.fence)))
-        {
-            if (hr == WINED3DERR_NOTAVAILABLE)
-                FIXME("Fences not supported, dropping async buffer locks.\n");
-            else
-                ERR("Failed to create fence, hr %#x.\n", hr);
-            goto drop_fence;
-        }
-
-        /* Since we don't know about old draws a glFinish is needed once */
-        gl_info->gl_ops.gl.p_glFinish();
+        bo->command_fence_id = 0;
         return;
     }
 
     TRACE("Synchronizing buffer %p.\n", buffer_gl);
-    ret = wined3d_fence_wait(buffer_gl->b.fence, buffer_gl->b.resource.device);
-    switch (ret)
-    {
-        case WINED3D_FENCE_NOT_STARTED:
-        case WINED3D_FENCE_OK:
-            /* All done */
-            return;
-
-        case WINED3D_FENCE_WRONG_THREAD:
-            WARN("Cannot synchronize buffer lock due to a thread conflict.\n");
-            goto drop_fence;
-
-        default:
-            ERR("wined3d_fence_wait() returned %u, dropping async buffer locks.\n", ret);
-            goto drop_fence;
-    }
 
-drop_fence:
-    if (buffer_gl->b.fence)
-    {
-        wined3d_fence_destroy(buffer_gl->b.fence);
-        buffer_gl->b.fence = NULL;
-    }
-
-    gl_info->gl_ops.gl.p_glFinish();
-    wined3d_buffer_gl_bind(buffer_gl, context_gl);
-    GL_EXTCALL(glBufferParameteriAPPLE(buffer_gl->bo.binding, GL_BUFFER_SERIALIZED_MODIFY_APPLE, GL_TRUE));
-    checkGLcall("glBufferParameteriAPPLE(buffer_gl->buffer_type_hint, GL_BUFFER_SERIALIZED_MODIFY_APPLE, GL_TRUE)");
-    buffer_gl->b.flags &= ~WINED3D_BUFFER_APPLESYNC;
+    if (bo->command_fence_id == device_gl->current_fence_id)
+        wined3d_context_gl_submit_command_fence(context_gl);
+    wined3d_context_gl_wait_command_fence(context_gl, bo->command_fence_id);
 }
 
 static void buffer_mark_used(struct wined3d_buffer *buffer)
diff --git a/dlls/wined3d/context.c b/dlls/wined3d/context.c
index ee2659ce23a..b52df0bd11e 100644
--- a/dlls/wined3d/context.c
+++ b/dlls/wined3d/context.c
@@ -271,7 +271,6 @@ void context_update_stream_info(struct wined3d_context *context, const struct wi
     wined3d_stream_info_from_declaration(stream_info, state, d3d_info);
 
     stream_info->all_vbo = 1;
-    context->buffer_fence_count = 0;
     for (i = 0, map = stream_info->use_map; map; map >>= 1, ++i)
     {
         struct wined3d_stream_info_element *element;
@@ -312,9 +311,6 @@ void context_update_stream_info(struct wined3d_context *context, const struct wi
         if (!element->data.buffer_object)
             stream_info->all_vbo = 0;
 
-        if (buffer->fence)
-            context->buffer_fences[context->buffer_fence_count++] = buffer->fence;
-
         TRACE("Load array %u %s.\n", i, debug_bo_address(&element->data));
     }
 
diff --git a/dlls/wined3d/context_gl.c b/dlls/wined3d/context_gl.c
index 12e5e088713..ee1adfcf559 100644
--- a/dlls/wined3d/context_gl.c
+++ b/dlls/wined3d/context_gl.c
@@ -1374,6 +1374,10 @@ static void wined3d_context_gl_cleanup(struct wined3d_context_gl *context_gl)
 
     if (context_gl->valid)
     {
+        wined3d_context_gl_submit_command_fence(context_gl);
+        wined3d_context_gl_wait_command_fence(context_gl,
+                wined3d_device_gl(context_gl->c.device)->current_fence_id - 1);
+
         if (context_gl->dummy_arbfp_prog)
             GL_EXTCALL(glDeleteProgramsARB(1, &context_gl->dummy_arbfp_prog));
 
@@ -1422,6 +1426,7 @@ static void wined3d_context_gl_cleanup(struct wined3d_context_gl *context_gl)
 
         checkGLcall("context cleanup");
     }
+    heap_free(context_gl->submitted.fences);
     heap_free(context_gl->free_pipeline_statistics_queries);
     heap_free(context_gl->free_so_statistics_queries);
     heap_free(context_gl->free_timestamp_queries);
@@ -2571,6 +2576,80 @@ void wined3d_context_gl_bind_texture(struct wined3d_context_gl *context_gl, GLen
     checkGLcall("bind texture");
 }
 
+static void wined3d_context_gl_poll_fences(struct wined3d_context_gl *context_gl)
+{
+    struct wined3d_device_gl *device_gl = wined3d_device_gl(context_gl->c.device);
+    struct wined3d_command_fence_gl *f;
+    SIZE_T i;
+
+    for (i = 0; i < context_gl->submitted.fence_count; ++i)
+    {
+        f = &context_gl->submitted.fences[i];
+
+        if (f->id > device_gl->completed_fence_id)
+        {
+            if (wined3d_fence_test(f->fence, &device_gl->d, 0) != WINED3D_FENCE_OK)
+                continue;
+            device_gl->completed_fence_id = f->id;
+        }
+
+        wined3d_fence_destroy(f->fence);
+        if (i != context_gl->submitted.fence_count - 1)
+            *f = context_gl->submitted.fences[context_gl->submitted.fence_count - 1];
+        --context_gl->submitted.fence_count;
+    }
+}
+
+void wined3d_context_gl_wait_command_fence(struct wined3d_context_gl *context_gl, uint64_t id)
+{
+    struct wined3d_device_gl *device_gl = wined3d_device_gl(context_gl->c.device);
+    enum wined3d_fence_result ret;
+    SIZE_T i;
+
+    if (id <= device_gl->completed_fence_id
+            || id > device_gl->current_fence_id) /* In case the fence ID wrapped. */
+        return;
+
+    for (i = 0; i < context_gl->submitted.fence_count; ++i)
+    {
+        if (context_gl->submitted.fences[i].id != id)
+            continue;
+
+        if ((ret = wined3d_fence_wait(context_gl->submitted.fences[i].fence, &device_gl->d)) != WINED3D_FENCE_OK)
+            ERR("Failed to wait for command fence with id 0x%s, ret %#x.\n", wine_dbgstr_longlong(id), ret);
+        wined3d_context_gl_poll_fences(context_gl);
+        return;
+    }
+
+    ERR("Failed to find fence for command fence with id 0x%s.\n", wine_dbgstr_longlong(id));
+}
+
+void wined3d_context_gl_submit_command_fence(struct wined3d_context_gl *context_gl)
+{
+    struct wined3d_device_gl *device_gl = wined3d_device_gl(context_gl->c.device);
+    struct wined3d_command_fence_gl *f;
+    HRESULT hr;
+
+    if (!wined3d_array_reserve((void **)&context_gl->submitted.fences, &context_gl->submitted.fences_size,
+            context_gl->submitted.fence_count + 1, sizeof(*context_gl->submitted.fences)))
+        ERR("Failed to grow submitted command buffer array.\n");
+
+    f = &context_gl->submitted.fences[context_gl->submitted.fence_count++];
+    f->id = device_gl->current_fence_id;
+    if (FAILED(hr = wined3d_fence_create(&device_gl->d, &f->fence)))
+        ERR("Failed to create fence, hr %#x.\n", hr);
+    wined3d_fence_issue(f->fence, &device_gl->d);
+
+    /* We don't expect this to ever happen, but handle it anyway. */
+    if (!++device_gl->current_fence_id)
+    {
+        wined3d_context_gl_wait_command_fence(context_gl, device_gl->current_fence_id - 1);
+        device_gl->completed_fence_id = 0;
+        device_gl->current_fence_id = 1;
+    }
+    wined3d_context_gl_poll_fences(context_gl);
+}
+
 void *wined3d_context_gl_map_bo_address(struct wined3d_context_gl *context_gl,
         const struct wined3d_bo_address *data, size_t size, uint32_t flags)
 {
@@ -2728,6 +2807,8 @@ bool wined3d_context_gl_create_bo(struct wined3d_context_gl *context_gl, GLsizei
     bo->id = id;
     bo->binding = binding;
     bo->usage = usage;
+    bo->command_fence_id = 0;
+
     return true;
 }
 
@@ -3605,7 +3686,7 @@ static BOOL context_apply_draw_state(struct wined3d_context *context,
     const struct wined3d_gl_info *gl_info = context_gl->gl_info;
     const struct wined3d_fb_state *fb = &state->fb;
     unsigned int i, base;
-    WORD map;
+    uint32_t map;
 
     context->uses_fbo_attached_resources = 0;
 
@@ -3638,24 +3719,36 @@ static BOOL context_apply_draw_state(struct wined3d_context *context,
     {
         context_update_stream_info(context, state);
     }
-    else
+
+    map = context->stream_info.use_map;
+    while (map)
     {
-        for (i = 0, map = context->stream_info.use_map; map; map >>= 1, ++i)
-        {
-            if (map & 1)
-                wined3d_buffer_load(state->streams[context->stream_info.elements[i].stream_idx].buffer,
-                        context, state);
-        }
-        /* Loading the buffers above may have invalidated the stream info. */
-        if (isStateDirty(context, STATE_STREAMSRC))
-            context_update_stream_info(context, state);
+        const struct wined3d_stream_info_element *e;
+        struct wined3d_buffer_gl *buffer_gl;
+
+        e = &context->stream_info.elements[wined3d_bit_scan(&map)];
+        buffer_gl = wined3d_buffer_gl(state->streams[e->stream_idx].buffer);
+
+        wined3d_buffer_load(&buffer_gl->b, context, state);
+        wined3d_context_gl_reference_bo(context_gl, &buffer_gl->bo);
     }
+    /* Loading the buffers above may have invalidated the stream info. */
+    if (wined3d_context_is_graphics_state_dirty(context, STATE_STREAMSRC))
+        context_update_stream_info(context, state);
+
     if (indexed && state->index_buffer)
     {
+        struct wined3d_buffer_gl *buffer_gl = wined3d_buffer_gl(state->index_buffer);
+
         if (context->stream_info.all_vbo)
-            wined3d_buffer_load(state->index_buffer, context, state);
+        {
+            wined3d_buffer_load(&buffer_gl->b, context, state);
+            wined3d_context_gl_reference_bo(context_gl, &buffer_gl->bo);
+        }
         else
-            wined3d_buffer_load_sysmem(state->index_buffer, context);
+        {
+            wined3d_buffer_load_sysmem(&buffer_gl->b, context);
+        }
     }
 
     for (i = 0, base = 0; i < ARRAY_SIZE(context->dirty_graphics_states); ++i)
@@ -4475,7 +4568,6 @@ void draw_primitive(struct wined3d_device *device, const struct wined3d_state *s
     const struct wined3d_stream_info *stream_info;
     struct wined3d_rendertarget_view *dsv, *rtv;
     struct wined3d_stream_info si_emulated;
-    struct wined3d_fence *ib_fence = NULL;
     const struct wined3d_gl_info *gl_info;
     struct wined3d_context_gl *context_gl;
     struct wined3d_context *context;
@@ -4573,14 +4665,9 @@ void draw_primitive(struct wined3d_device *device, const struct wined3d_state *s
     {
         struct wined3d_buffer *index_buffer = state->index_buffer;
         if (!index_buffer->buffer_object || !stream_info->all_vbo)
-        {
             idx_data = index_buffer->resource.heap_memory;
-        }
         else
-        {
-            ib_fence = index_buffer->fence;
             idx_data = NULL;
-        }
         idx_data = (const BYTE *)idx_data + state->index_offset;
 
         if (state->index_format == WINED3DFMT_R16_UINT)
@@ -4719,11 +4806,6 @@ void draw_primitive(struct wined3d_device *device, const struct wined3d_state *s
         checkGLcall("disable rasterizer discard");
     }
 
-    if (ib_fence)
-        wined3d_fence_issue(ib_fence, device);
-    for (i = 0; i < context->buffer_fence_count; ++i)
-        wined3d_fence_issue(context->buffer_fences[i], device);
-
     context_release(context);
 
     TRACE("Draw completed.\n");
diff --git a/dlls/wined3d/query.c b/dlls/wined3d/query.c
index 5aca4a2c5a4..9e0e22b32e5 100644
--- a/dlls/wined3d/query.c
+++ b/dlls/wined3d/query.c
@@ -180,7 +180,7 @@ static BOOL wined3d_fence_supported(const struct wined3d_gl_info *gl_info)
     return gl_info->supported[ARB_SYNC] || gl_info->supported[NV_FENCE] || gl_info->supported[APPLE_FENCE];
 }
 
-static enum wined3d_fence_result wined3d_fence_test(const struct wined3d_fence *fence,
+enum wined3d_fence_result wined3d_fence_test(const struct wined3d_fence *fence,
         struct wined3d_device *device, DWORD flags)
 {
     const struct wined3d_gl_info *gl_info;
diff --git a/dlls/wined3d/swapchain.c b/dlls/wined3d/swapchain.c
index 7e46e5c6150..ffffc18aa84 100644
--- a/dlls/wined3d/swapchain.c
+++ b/dlls/wined3d/swapchain.c
@@ -559,6 +559,7 @@ static void swapchain_gl_present(struct wined3d_swapchain *swapchain,
 
     /* call wglSwapBuffers through the gl table to avoid confusing the Steam overlay */
     gl_info->gl_ops.wgl.p_wglSwapBuffers(context_gl->dc);
+    wined3d_context_gl_submit_command_fence(context_gl);
 
     wined3d_swapchain_gl_rotate(swapchain, context);
 
diff --git a/dlls/wined3d/wined3d_private.h b/dlls/wined3d/wined3d_private.h
index 3dcd3e26846..a7baff857d0 100644
--- a/dlls/wined3d/wined3d_private.h
+++ b/dlls/wined3d/wined3d_private.h
@@ -1540,6 +1540,8 @@ struct wined3d_bo_gl
     GLuint id;
     GLenum binding;
     GLenum usage;
+
+    uint64_t command_fence_id;
 };
 
 static inline GLuint wined3d_bo_gl_id(uintptr_t bo)
@@ -1842,6 +1844,8 @@ void wined3d_fence_destroy(struct wined3d_fence *fence) DECLSPEC_HIDDEN;
 void wined3d_fence_issue(struct wined3d_fence *fence, struct wined3d_device *device) DECLSPEC_HIDDEN;
 enum wined3d_fence_result wined3d_fence_wait(const struct wined3d_fence *fence,
         struct wined3d_device *device) DECLSPEC_HIDDEN;
+enum wined3d_fence_result wined3d_fence_test(const struct wined3d_fence *fence,
+        struct wined3d_device *device, DWORD flags) DECLSPEC_HIDDEN;
 
 /* Direct3D terminology with little modifications. We do not have an issued
  * state because only the driver knows about it, but we have a created state
@@ -2105,10 +2109,6 @@ struct wined3d_context
 
     struct wined3d_stream_info stream_info;
 
-    /* Fences for GL_APPLE_flush_buffer_range */
-    struct wined3d_fence *buffer_fences[MAX_ATTRIBS];
-    unsigned int buffer_fence_count;
-
     unsigned int viewport_count;
     unsigned int scissor_rect_count;
 };
@@ -2127,6 +2127,12 @@ void context_update_stream_info(struct wined3d_context *context, const struct wi
 HRESULT wined3d_context_no3d_init(struct wined3d_context *context_no3d,
         struct wined3d_swapchain *swapchain) DECLSPEC_HIDDEN;
 
+struct wined3d_command_fence_gl
+{
+    uint64_t id;
+    struct wined3d_fence *fence;
+};
+
 struct wined3d_context_gl
 {
     struct wined3d_context c;
@@ -2214,6 +2220,13 @@ struct wined3d_context_gl
     GLfloat colour[4], fog_start, fog_end, fog_colour[4];
 
     GLuint dummy_arbfp_prog;
+
+    struct
+    {
+        struct wined3d_command_fence_gl *fences;
+        SIZE_T fences_size;
+        SIZE_T fence_count;
+    } submitted;
 };
 
 static inline struct wined3d_context_gl *wined3d_context_gl(struct wined3d_context *context)
@@ -2287,6 +2300,7 @@ struct wined3d_context_gl *wined3d_context_gl_reacquire(struct wined3d_context_g
 void wined3d_context_gl_release(struct wined3d_context_gl *context_gl) DECLSPEC_HIDDEN;
 BOOL wined3d_context_gl_set_current(struct wined3d_context_gl *context_gl) DECLSPEC_HIDDEN;
 void wined3d_context_gl_set_draw_buffer(struct wined3d_context_gl *context_gl, GLenum buffer) DECLSPEC_HIDDEN;
+void wined3d_context_gl_submit_command_fence(struct wined3d_context_gl *context_gl) DECLSPEC_HIDDEN;
 void wined3d_context_gl_texture_update(struct wined3d_context_gl *context_gl,
         const struct wined3d_texture_gl *texture_gl) DECLSPEC_HIDDEN;
 void wined3d_context_gl_unload_tex_coords(const struct wined3d_context_gl *context_gl) DECLSPEC_HIDDEN;
@@ -2294,6 +2308,7 @@ void wined3d_context_gl_unmap_bo_address(struct wined3d_context_gl *context_gl,
         unsigned int range_count, const struct wined3d_range *ranges) DECLSPEC_HIDDEN;
 void wined3d_context_gl_update_stream_sources(struct wined3d_context_gl *context_gl,
         const struct wined3d_state *state) DECLSPEC_HIDDEN;
+void wined3d_context_gl_wait_command_fence(struct wined3d_context_gl *context_gl, uint64_t id) DECLSPEC_HIDDEN;
 
 struct wined3d_command_buffer_vk
 {
@@ -3803,6 +3818,9 @@ struct wined3d_device_gl
 
     /* Textures for when no other textures are bound. */
     struct wined3d_dummy_textures dummy_textures;
+
+    uint64_t completed_fence_id;
+    uint64_t current_fence_id;
 };
 
 static inline struct wined3d_device_gl *wined3d_device_gl(struct wined3d_device *device)
@@ -4789,7 +4807,6 @@ struct wined3d_buffer
 
     struct wined3d_range *maps;
     SIZE_T maps_size, modified_areas;
-    struct wined3d_fence *fence;
 
     /* conversion stuff */
     UINT decl_change_count, full_conversion_count;
@@ -6298,6 +6315,13 @@ static inline bool wined3d_primitive_type_is_list(enum wined3d_primitive_type t)
             || t == WINED3D_PT_PATCH;
 }
 
+static inline void wined3d_context_gl_reference_bo(struct wined3d_context_gl *context_gl, struct wined3d_bo_gl *bo_gl)
+{
+    struct wined3d_device_gl *device_gl = wined3d_device_gl(context_gl->c.device);
+
+    bo_gl->command_fence_id = device_gl->current_fence_id;
+}
+
 /* The WNDCLASS-Name for the fake window which we use to retrieve the GL capabilities */
 #define WINED3D_OPENGL_WINDOW_CLASS_NAME "WineD3D_OpenGL"
 
-- 
2.20.1

[Matteo Bruni]

Signed-off-by: Matteo Bruni <mbruni(a)codeweavers.com>
---
With the caveat that I haven't actually tested it, I trust you did :)

I REALLY like the per-frame fencing. I wish I had thought of it
myself...