[PATCH v5 4/4] winegstreamer: Call wg_format_get_max_size() to get the video decoder output sample size.

[Conor McCarthy]

From: Conor McCarthy

Windows uses a smaller alignment than GStreamer for some formats, for
example NV12. This means we cannot use MFCalculateImageSize() to get the
output sample size. Commit 7b79e3a87b1e switched to calling it instead of
GetOutputStreamInfo() to fix some game bugs.

Fixes video decode failure in a Resident Evil Village cutscene.
---
 dlls/winegstreamer/video_decoder.c | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/dlls/winegstreamer/video_decoder.c b/dlls/winegstreamer/video_decoder.c
index b588e080004..598d44cea9d 100644
--- a/dlls/winegstreamer/video_decoder.c
+++ b/dlls/winegstreamer/video_decoder.c
@@ -102,6 +102,7 @@ struct video_decoder
     wg_transform_t wg_transform;
     struct wg_transform_attrs wg_transform_attrs;
     struct wg_sample_queue *wg_sample_queue;
+    struct wg_format wg_output_format; /* for calculating sample size */

IMFVideoSampleAllocatorEx *allocator;
     BOOL allocator_initialized;
@@ -754,6 +755,13 @@ static HRESULT WINAPI transform_SetOutputType(IMFTransform *iface, DWORD id, IMF
     else
         hr = try_create_wg_transform(decoder, output_type);

+    mf_media_type_to_wg_format(output_type, &decoder->wg_output_format);
+    if (!decoder->wg_output_format.major_type)
+    {
+        FIXME("Failed to get output wg format.\n");
+        hr = MF_E_INVALIDMEDIATYPE;
+    }
+
     IMFMediaType_Release(output_type);

if (FAILED(hr))
@@ -943,9 +951,8 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     UINT32 sample_size;
     LONGLONG duration;
     IMFSample *sample;
-    UINT64 frame_size, frame_rate;
+    UINT64 frame_rate;
     bool preserve_timestamps;
-    GUID subtype;
     DWORD size;
     HRESULT hr;

@@ -961,15 +968,13 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     if (!(sample = samples->pSample) && !(decoder->output_info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES))
         return E_INVALIDARG;

-    if (FAILED(hr = IMFMediaType_GetGUID(decoder->output_type, &MF_MT_SUBTYPE, &subtype)))
-        return hr;
-    if (FAILED(hr = IMFMediaType_GetUINT64(decoder->output_type, &MF_MT_FRAME_SIZE, &frame_size)))
-        return hr;
-    if (FAILED(hr = MFCalculateImageSize(&subtype, frame_size >> 32, (UINT32)frame_size, &sample_size)))
-        return hr;
-
     if (decoder->output_info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)
     {
+        /* GetOutputStreamInfo() is bugged for some games, and MFCalculateImageSize()
+        * for some formats uses a smaller stride alignment than GStreamer does */
+        if (!(sample_size = wg_format_get_max_size(&decoder->wg_output_format)))
+            return E_INVALIDARG;
+
         if (decoder->temp_buffer)
         {
             if (FAILED(IMFMediaBuffer_GetMaxLength(decoder->temp_buffer, &size)) || size < sample_size)
@@ -989,6 +994,9 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
         }
     }

+    /* In cases where the caller supplies the sample, and GStreamer requires a
+     * larger stride alignment than Windows, e.g. NV12, this call may fail, and
+     * the output buffer would need to be converted to the smaller alignment. */
     if (SUCCEEDED(hr = wg_transform_read_mf(decoder->wg_transform, sample, &samples->dwStatus, &preserve_timestamps)))
     {
         wg_sample_queue_flush(decoder->wg_sample_queue, false);