Rémi Bernon : winegstreamer: Check H264 ProcessOutput sample against actual image size.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: 7b79e3a87b1e1a9478e92d6ea3b5973da4388a82
URL:    https://source.winehq.org/git/wine.git/?a=commit;h=7b79e3a87b1e1a9478e92d6ea...

Author: Rémi Bernon 
Date:   Thu Jun 23 13:33:33 2022 +0200

winegstreamer: Check H264 ProcessOutput sample against actual image size.

Instead of maximum output sample size returned from GetOutputStreamInfo.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=45988
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=47084
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=49715
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=52183
Signed-off-by: Rémi Bernon

---

dlls/mf/tests/mf.c                |  4 ++--
 dlls/winegstreamer/h264_decoder.c | 14 +++++++++-----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/dlls/mf/tests/mf.c b/dlls/mf/tests/mf.c
index 1d4f29e9361..9c87460b115 100644
--- a/dlls/mf/tests/mf.c
+++ b/dlls/mf/tests/mf.c
@@ -7671,7 +7671,7 @@ static void test_h264_decoder(void)

status = 0;
     memset(&output, 0, sizeof(output));
-    output.pSample = create_sample(NULL, actual_width * actual_height * 2);
+    output.pSample = create_sample(NULL, nv12_frame_len);
     hr = IMFTransform_ProcessOutput(transform, 0, 1, &output, &status);
     ok(hr == S_OK, "ProcessOutput returned %#lx\n", hr);
     ok(output.dwStreamID == 0, "got dwStreamID %lu\n", output.dwStreamID);
@@ -7715,7 +7715,7 @@ static void test_h264_decoder(void)
     for (i = 0; i < actual_aperture.Area.cy; ++i)
     {
         memset(data + actual_width * i + actual_aperture.Area.cx, 0xcd, actual_width - actual_aperture.Area.cx);
-        memset(data + actual_width * (actual_height + i) + actual_aperture.Area.cx, 0xcd, actual_width - actual_aperture.Area.cx);
+        memset(data + actual_width * (actual_height + i / 2) + actual_aperture.Area.cx, 0xcd, actual_width - actual_aperture.Area.cx);
     }
     memset(data + actual_width * actual_aperture.Area.cy, 0xcd, (actual_height - actual_aperture.Area.cy) * actual_width);
     memset(data + actual_width * (actual_height + actual_aperture.Area.cy / 2), 0xcd, (actual_height - actual_aperture.Area.cy) / 2 * actual_width);
diff --git a/dlls/winegstreamer/h264_decoder.c b/dlls/winegstreamer/h264_decoder.c
index 471ad49d8e6..43b8f83a20c 100644
--- a/dlls/winegstreamer/h264_decoder.c
+++ b/dlls/winegstreamer/h264_decoder.c
@@ -581,10 +581,11 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
         MFT_OUTPUT_DATA_BUFFER *samples, DWORD *status)
 {
     struct h264_decoder *decoder = impl_from_IMFTransform(iface);
-    MFT_OUTPUT_STREAM_INFO info;
     struct wg_sample *wg_sample;
     struct wg_format wg_format;
+    UINT32 sample_size;
     UINT64 frame_rate;
+    GUID subtype;
     HRESULT hr;

TRACE("iface %p, flags %#lx, count %lu, samples %p, status %p.\n", iface, flags, count, samples, status);
@@ -592,9 +593,6 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     if (count != 1)
         return E_INVALIDARG;

-    if (FAILED(hr = IMFTransform_GetOutputStreamInfo(iface, 0, &info)))
-        return hr;
-
     if (!decoder->wg_transform)
         return MF_E_TRANSFORM_TYPE_NOT_SET;

@@ -602,10 +600,16 @@ static HRESULT WINAPI transform_ProcessOutput(IMFTransform *iface, DWORD flags,
     samples[0].dwStatus = 0;
     if (!samples[0].pSample) return E_INVALIDARG;

+    if (FAILED(hr = IMFMediaType_GetGUID(decoder->output_type, &MF_MT_SUBTYPE, &subtype)))
+        return hr;
+    if (FAILED(hr = MFCalculateImageSize(&subtype, decoder->wg_format.u.video.width,
+            decoder->wg_format.u.video.height, &sample_size)))
+        return hr;
+
     if (FAILED(hr = wg_sample_create_mf(samples[0].pSample, &wg_sample)))
         return hr;

-    if (wg_sample->max_size < info.cbSize)
+    if (wg_sample->max_size < sample_size)
     {
         wg_sample_release(wg_sample);
         return MF_E_BUFFERTOOSMALL;