Zebediah Figura : quartz/vmr9: Return E_INVALIDARG if both texture and offscreen flags are passed to AllocateSurfaceHelper().

[Alexandre Julliard]

Module: wine
Branch: master
Commit: 7f0347bb04a2be2fcd5ebab826439c6f0d6a5703
URL:    https://source.winehq.org/git/wine.git/?a=commit;h=7f0347bb04a2be2fcd5ebab82...

Author: Zebediah Figura <z.figura12(a)gmail.com>
Date:   Tue Jul 21 17:49:57 2020 -0500

quartz/vmr9: Return E_INVALIDARG if both texture and offscreen flags are passed to AllocateSurfaceHelper().

Signed-off-by: Zebediah Figura <z.figura12(a)gmail.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>

---

 dlls/quartz/tests/vmr9.c | 7 +++++++
 dlls/quartz/vmr9.c       | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/dlls/quartz/tests/vmr9.c b/dlls/quartz/tests/vmr9.c
index a2fd5b7686..092ee87132 100644
--- a/dlls/quartz/tests/vmr9.c
+++ b/dlls/quartz/tests/vmr9.c
@@ -2760,6 +2760,13 @@ static void test_allocate_surface_helper(void)
 
     IDirect3DSurface9_Release(surfaces[0]);
 
+    info.Format = D3DFMT_A8R8G8B8;
+    info.dwFlags = VMR9AllocFlag_OffscreenSurface | VMR9AllocFlag_TextureSurface;
+    count = 1;
+    hr = IVMRSurfaceAllocatorNotify9_AllocateSurfaceHelper(notify, &info, &count, surfaces);
+    ok(hr == E_INVALIDARG, "Got hr %#x.\n", hr);
+    ok(count == 1, "Got count %u.\n", count);
+
 out:
     IVMRSurfaceAllocatorNotify9_Release(notify);
     ref = IBaseFilter_Release(filter);
diff --git a/dlls/quartz/vmr9.c b/dlls/quartz/vmr9.c
index 00cd6bbce8..f66500086d 100644
--- a/dlls/quartz/vmr9.c
+++ b/dlls/quartz/vmr9.c
@@ -2042,6 +2042,13 @@ static HRESULT WINAPI VMR9SurfaceAllocatorNotify_AllocateSurfaceHelper(IVMRSurfa
             allocinfo->dwFlags, allocinfo->dwWidth, allocinfo->dwHeight,
             allocinfo->Format, allocinfo->Format, allocinfo->Pool, allocinfo->MinBuffers);
 
+    if ((allocinfo->dwFlags & VMR9AllocFlag_TextureSurface)
+            && (allocinfo->dwFlags & VMR9AllocFlag_OffscreenSurface))
+    {
+        WARN("Invalid flags specified; returning E_INVALIDARG.\n");
+        return E_INVALIDARG;
+    }
+
     if (!allocinfo->Format)
     {
         IDirect3DSurface9 *backbuffer;