[PATCH v2 5/6] winegstreamer: Check H264 ProcessOutput sample against actual image size.

[Rémi Bernon]

From: Rémi Bernon

Instead of maximum output sample size returned from GetOutputStreamInfo.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=45988
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=47084
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=49715
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=52183
Signed-off-by: Rémi Bernon 
---
 dlls/mf/tests/mf.c                |  2 +-
 dlls/winegstreamer/h264_decoder.c | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/dlls/mf/tests/mf.c b/dlls/mf/tests/mf.c
index 7b598ba7dac..77cf40e971b 100644
--- a/dlls/mf/tests/mf.c
+++ b/dlls/mf/tests/mf.c
@@ -7254,7 +7254,7 @@ static void test_h264_decoder(void)

status = 0;
     memset(&output, 0, sizeof(output));
-    output.pSample = create_sample(NULL, actual_width * actual_height * 2);
+    output.pSample = create_sample(NULL, actual_width * actual_height * 3 / 2);
     hr = IMFTransform_ProcessOutput(transform, 0, 1, &output, &status);
     ok(hr == S_OK, "ProcessOutput returned %#lx\n", hr);
     ok(output.dwStreamID == 0, "got dwStreamID %lu\n", output.dwStreamID);
diff --git a/dlls/winegstreamer/h264_decoder.c b/dlls/winegstreamer/h264_decoder.c
index 2ece7dee5b1..561a3c1578c 100644
--- a/dlls/winegstreamer/h264_decoder.c
+++ b/dlls/winegstreamer/h264_decoder.c
@@ -581,10 +581,11 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
         MFT_OUTPUT_DATA_BUFFER *samples, DWORD *status)
 {
     struct h264_decoder *decoder = impl_from_IMFTransform(iface);
-    MFT_OUTPUT_STREAM_INFO info;
     struct wg_sample *wg_sample;
     struct wg_format wg_format;
+    UINT32 sample_size;
     UINT64 frame_rate;
+    GUID subtype;
     HRESULT hr;

TRACE("iface %p, flags %#lx, count %lu, samples %p, status %p.\n", iface, flags, count, samples, status);
@@ -592,9 +593,6 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     if (count != 1)
         return E_INVALIDARG;

-    if (FAILED(hr = IMFTransform_GetOutputStreamInfo(iface, 0, &info)))
-        return hr;
-
     if (!decoder->wg_transform)
         return MF_E_TRANSFORM_TYPE_NOT_SET;

@@ -602,10 +600,16 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     samples[0].dwStatus = 0;
     if (!samples[0].pSample) return E_INVALIDARG;

+    if (FAILED(hr = IMFMediaType_GetGUID(decoder->output_type, &MF_MT_SUBTYPE, &subtype)))
+        return hr;
+    if (FAILED(hr = MFCalculateImageSize(&subtype, decoder->wg_format.u.video.width,
+            decoder->wg_format.u.video.height, &sample_size)))
+        return hr;
+
     if (FAILED(hr = wg_sample_create_mf(samples[0].pSample, &wg_sample)))
         return hr;

-    if (wg_sample->max_size < info.cbSize)
+    if (wg_sample->max_size < sample_size)
     {
         wg_sample_release(wg_sample);
         return MF_E_BUFFERTOOSMALL;