[PATCH 0/2] MR5935: winemac.drv: Use ScreenCaptureKit for grabbing window snapshots
The old method will trigger a permission prompt in Sequoia. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935
From: Tim Clem <tclem(a)codeweavers.com> In preparation for using a different method on macOS 14.4+. --- dlls/winemac.drv/cocoa_window.m | 92 +++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/dlls/winemac.drv/cocoa_window.m b/dlls/winemac.drv/cocoa_window.m index caf06ff338f..89844dac173 100644 --- a/dlls/winemac.drv/cocoa_window.m +++ b/dlls/winemac.drv/cocoa_window.m @@ -2323,48 +2323,19 @@ - (BOOL) canProvideSnapshot return (self.windowNumber > 0 && ![self isEmptyShaped]); } - - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force + /* Create an image of the given window using the CGWindowList API. The + returned image must be released by the caller. */ + - (CGImageRef) windowListSnapshotForWindow:(WineWindow *)window { - if (![self isEmptyShaped]) - return; - - NSTimeInterval now = [[NSProcessInfo processInfo] systemUptime]; - if (!force && now < lastDockIconSnapshot + 1) - return; - - if (window) - { - if (![window canProvideSnapshot]) - return; - } - else - { - CGFloat bestArea; - for (WineWindow* childWindow in self.childWindows) - { - if (![childWindow isKindOfClass:[WineWindow class]] || ![childWindow canProvideSnapshot]) - continue; - - NSSize size = childWindow.frame.size; - CGFloat area = size.width * size.height; - if (!window || area > bestArea) - { - window = childWindow; - bestArea = area; - } - } - - if (!window) - return; - } - const void* windowID = (const void*)(uintptr_t)(CGWindowID)window.windowNumber; CFArrayRef windowIDs = CFArrayCreate(NULL, &windowID, 1, NULL); CGImageRef windowImage = CGWindowListCreateImageFromArray(CGRectNull, windowIDs, kCGWindowImageBoundsIgnoreFraming); CFRelease(windowIDs); - if (!windowImage) - return; + return windowImage; + } + - (void) drawDockTileWithImage:(CGImageRef)windowImage + { NSImage* appImage = [NSApp applicationIconImage]; if (!appImage) appImage = [NSImage imageNamed:NSImageNameApplicationIcon]; @@ -2398,8 +2369,6 @@ - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force [dockIcon unlockFocus]; - CGImageRelease(windowImage); - NSImageView* imageView = (NSImageView*)self.dockTile.contentView; if (![imageView isKindOfClass:[NSImageView class]]) { @@ -2409,7 +2378,52 @@ - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force } imageView.image = dockIcon; [self.dockTile display]; - lastDockIconSnapshot = now; + } + + - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force + { + CGImageRef windowImage; + + if (![self isEmptyShaped]) + return; + + NSTimeInterval now = [[NSProcessInfo processInfo] systemUptime]; + if (!force && now < lastDockIconSnapshot + 1) + return; + + if (window) + { + if (![window canProvideSnapshot]) + return; + } + else + { + CGFloat bestArea; + for (WineWindow* childWindow in self.childWindows) + { + if (![childWindow isKindOfClass:[WineWindow class]] || ![childWindow canProvideSnapshot]) + continue; + + NSSize size = childWindow.frame.size; + CGFloat area = size.width * size.height; + if (!window || area > bestArea) + { + window = childWindow; + bestArea = area; + } + } + + if (!window) + return; + } + + windowImage = [self windowListSnapshotForWindow:window]; + if (windowImage) + { + [self drawDockTileWithImage:windowImage]; + CGImageRelease(windowImage); + lastDockIconSnapshot = now; + } } - (void) checkEmptyShaped -- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/5935
From: Tim Clem <tclem(a)codeweavers.com> The CG methods for capturing screen contents are deprecated and trigger a permissions prompt in Sequoia. ScreenCaptureKit's shareable content APIs are the replacement, and particularly -getCurrentProcessShareableContentWithCompletionHandler: does not require any permissions for capturing images within the same app. See 496b001ae0b for why we have to do this manually in the first place. --- configure.ac | 4 ++ dlls/winemac.drv/Makefile.in | 3 +- dlls/winemac.drv/cocoa_window.m | 94 ++++++++++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 7fe5477daf7..75d5f937acf 100644 --- a/configure.ac +++ b/configure.ac @@ -633,6 +633,10 @@ case $host_os in AC_SUBST(APPKIT_LIBS,"-framework AppKit") AC_SUBST(SECURITY_LIBS,"-framework Security -framework CoreFoundation") AC_SUBST(SYSTEMCONFIGURATION_LIBS,"-framework SystemConfiguration") + AC_LANG_PUSH([Objective C]) + AC_CHECK_HEADER([ScreenCaptureKit/ScreenCaptureKit.h], + [AC_SUBST(SCREENCAPTUREKIT_LIBS,"-framework ScreenCaptureKit")]) + AC_LANG_POP([Objective C]) WINELOADER_LDFLAGS="-Wl,-segalign,0x1000,-pagezero_size,0x1000,-sectcreate,__TEXT,__info_plist,loader/wine_info.plist" diff --git a/dlls/winemac.drv/Makefile.in b/dlls/winemac.drv/Makefile.in index e3e71c5f4cc..5680dc7c50e 100644 --- a/dlls/winemac.drv/Makefile.in +++ b/dlls/winemac.drv/Makefile.in @@ -12,7 +12,8 @@ UNIX_LIBS = \ -framework OpenGL \ -framework QuartzCore \ -framework Security \ - $(METAL_LIBS) + $(METAL_LIBS) \ + $(SCREENCAPTUREKIT_LIBS) SOURCES = \ clipboard.c \ diff --git a/dlls/winemac.drv/cocoa_window.m b/dlls/winemac.drv/cocoa_window.m index 89844dac173..9ab3b317530 100644 --- a/dlls/winemac.drv/cocoa_window.m +++ b/dlls/winemac.drv/cocoa_window.m @@ -25,6 +25,9 @@ #import <CoreVideo/CoreVideo.h> #import <Metal/Metal.h> #import <QuartzCore/QuartzCore.h> +#ifdef MAC_OS_VERSION_14_4 +#import <ScreenCaptureKit/ScreenCaptureKit.h> +#endif #import "cocoa_window.h" @@ -2334,6 +2337,69 @@ - (CGImageRef) windowListSnapshotForWindow:(WineWindow *)window return windowImage; } +#ifdef MAC_OS_VERSION_14_4 + /* Create an image of the given window using the ScreenCaptureKit shareable + content APIs. The completion handler block is called on the main thread. + The image passed to the block must be released by the caller. In the case + of an error, the block is called with NULL. */ + - (void) shareableContentSnapshotForWindow:(WineWindow *)window + withCompletionHandler:(void (^)(CGImageRef image))completion + { + [SCShareableContent getCurrentProcessShareableContentWithCompletionHandler:^(SCShareableContent *shareableContent, NSError *error) { + dispatch_async(dispatch_get_main_queue(), ^{ + SCWindow *scWindow; + SCContentFilter *filter; + SCStreamConfiguration *streamConfig; + + if (!shareableContent) + { + completion(NULL); + return; + } + + for (SCWindow *scw in shareableContent.windows) + { + if (scw.windowID == window.windowNumber) + { + scWindow = scw; + break; + } + } + + if (!scWindow) + { + completion(NULL); + return; + } + + filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:scWindow]; + + streamConfig = [[SCStreamConfiguration alloc] init]; + streamConfig.width = NSWidth(window.frame); + streamConfig.height = NSHeight(window.frame); + streamConfig.scalesToFit = YES; + streamConfig.showsCursor = NO; + streamConfig.ignoreShadowsSingleWindow = YES; + streamConfig.ignoreGlobalClipSingleWindow = YES; + streamConfig.includeChildWindows = NO; + + [SCScreenshotManager captureImageWithFilter:filter configuration:streamConfig completionHandler:^(CGImageRef sampleBuffer, NSError *error) { + [filter release]; + [streamConfig release]; + + /* We do *not* own the returned image. */ + if (sampleBuffer) + CGImageRetain(sampleBuffer); + + dispatch_async(dispatch_get_main_queue(), ^{ + completion(sampleBuffer); + }); + }]; + }); + }]; + } +#endif + - (void) drawDockTileWithImage:(CGImageRef)windowImage { NSImage* appImage = [NSApp applicationIconImage]; @@ -2382,8 +2448,6 @@ - (void) drawDockTileWithImage:(CGImageRef)windowImage - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force { - CGImageRef windowImage; - if (![self isEmptyShaped]) return; @@ -2417,12 +2481,28 @@ - (void) grabDockIconSnapshotFromWindow:(WineWindow*)window force:(BOOL)force return; } - windowImage = [self windowListSnapshotForWindow:window]; - if (windowImage) +#ifdef MAC_OS_VERSION_14_4 + if ([SCShareableContent respondsToSelector:@selector(getCurrentProcessShareableContentWithCompletionHandler:)]) { - [self drawDockTileWithImage:windowImage]; - CGImageRelease(windowImage); - lastDockIconSnapshot = now; + [self shareableContentSnapshotForWindow:window withCompletionHandler:^(CGImageRef windowImage) { + if (windowImage) + { + [self drawDockTileWithImage:windowImage]; + CGImageRelease(windowImage); + lastDockIconSnapshot = now; + } + }]; + } + else +#endif + { + CGImageRef windowImage = [self windowListSnapshotForWindow:window]; + if (windowImage) + { + [self drawDockTileWithImage:windowImage]; + CGImageRelease(windowImage); + lastDockIconSnapshot = now; + } } } -- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/5935
Hi, It looks like your patch introduced the new failures shown below. Please investigate and fix them before resubmitting your patch. If they are not new, fixing them anyway would help a lot. Otherwise please ask for the known failures list to be updated. The full results can be found at: https://testbot.winehq.org/JobDetails.pl?Key=146579 Your paranoid android. === build (build log) === error: patch failed: dlls/winemac.drv/cocoa_window.m:2334 Task: Patch failed to apply === debian11 (build log) === error: patch failed: dlls/winemac.drv/cocoa_window.m:2334 Task: Patch failed to apply === debian11b (build log) === error: patch failed: dlls/winemac.drv/cocoa_window.m:2334 Task: Patch failed to apply
I tested this out by commenting out most of `grabDockIconSnapshotFromWindow:force:` and just setting `window = self`, so minimizing always results in a screenshot of the window being used for the dock tile. With the ScreenCaptureKit code path (on 14.5), when minimizing I only saw the `getCurrentProcessShareableContentWithCompletionHandler:` completion handler run (and its dispatch to the main thread). The `captureImageWithFilter:` completion handler then doesn't run until I un-minimize the window, which is very weird. This means that the first minimize of a window doesn't set the dock tile, every subsequent minimize then shows the dock tile image that should have been used for the previous minimize. I'm not sure why this is, maybe the async-ness means that the window is minimized by the time `captureImageWithFilter:` is called, and it just won't capture an image until the window is restored? Maybe the `getCurrentProcessShareableContentWithCompletionHandler:` completion handler could do everything there instead of dispatching back to the main thread (which seems to cause a 0.5-1 sec delay on my heavily loaded machine)? Also, it seems like !5798 is causing issues with minimize thumbnails, I had to revert that to get correct minimize thumbnails (with screenshot or non-screenshot methods). -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74634
On Fri Jun 28 16:59:24 2024 +0000, Brendan Shanks wrote:
I tested this out by commenting out most of `grabDockIconSnapshotFromWindow:force:` and just setting `window = self`, so minimizing always results in a screenshot of the window being used for the dock tile. With the ScreenCaptureKit code path (on 14.5), when minimizing I only saw the `getCurrentProcessShareableContentWithCompletionHandler:` completion handler run (and its dispatch to the main thread). The `captureImageWithFilter:` completion handler then doesn't run until I un-minimize the window, which is very weird. This means that the first minimize of a window doesn't set the dock tile, every subsequent minimize then shows the dock tile image that should have been used for the previous minimize. I'm not sure why this is, maybe the async-ness means that the window is minimized by the time `captureImageWithFilter:` is called, and it just won't capture an image until the window is restored? Maybe the `getCurrentProcessShareableContentWithCompletionHandler:` completion handler could do everything there instead of dispatching back to the main thread (which seems to cause a 0.5-1 sec delay on my heavily loaded machine)? Also, it seems like !5798 is causing issues with minimize thumbnails, I had to revert that to get correct minimize thumbnails (with screenshot or non-screenshot methods). Yikes! Thanks for reviewing. I wasn't seeing any of those issues on Sonoma or Sequoia, with the same sort of testing procedure. I was hoping the async-ness wouldn't be an problem, but I guess it is.
I originally had the `getCurrentProcessShareableContentWithCompletionHandler:` do everything without the dispatch, but since it wasn't making a difference for me I just did everything on the main thread. I'll try to reproduce locally and see what helps. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74672
In MacOSX15.sdk (Xcode16) it’s not just depreciated it’s marked as removed so results in a compilation error. We’d also need a version of this for wine-9.0.x or wine won’t be buildable using Xcode16. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74749
On Sat Jun 29 21:52:08 2024 +0000, Dean M Greer wrote:
In MacOSX15.sdk (Xcode16) it’s not just depreciated it’s marked as removed so results in a compilation error. We’d also need a version of this for wine-9.0.x or wine won’t be buildable using Xcode16. In my testing, if you build with Xcode 16 but target an older macOS version (set `MACOSX_DEPLOYMENT_TARGET=14.0` or older) those errors will just be warnings instead.
-- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74758
On Sun Jun 30 01:37:36 2024 +0000, Brendan Shanks wrote:
In my testing, if you build with Xcode 16 but target an older macOS version (set `MACOSX_DEPLOYMENT_TARGET=14.0` or older) those errors will just be warnings instead. The Winehq packages already target 10.15 so those won’t be an issue, I’ll alter the macports packages to account for this then as usually they don’t like forcing an older deployment target without a valid reason.
-- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74759
On Fri Jun 28 16:59:24 2024 +0000, Tim Clem wrote:
Yikes! Thanks for reviewing. I wasn't seeing any of those issues on Sonoma or Sequoia, with the same sort of testing procedure. I was hoping the async-ness wouldn't be an problem, but I guess it is. I originally had the `getCurrentProcessShareableContentWithCompletionHandler:` do everything without the dispatch, but since it wasn't making a difference for me I just did everything on the main thread. I'll try to reproduce locally and see what helps. I'm seeing these issues now (though I didn't have to revert !5798). Not sure how I missed them earlier.
I tried doing everything in the background thread that `getCurrentProcessShareableContent` is called back on, but it didn't help - `captureImageWithFilter:` still doesn't call its block until the window is restored. Doing something heinous like blocking the main thread until the SCK calls complete actually does work and grab a snapshot in time, but I'd really rather not do that. Not sure what our best option is. Maybe Apple can make SCK able to capture miniaturized windows, or perhaps they can make `CGWindowListCreateImageFromArray` not prompt for permissions if you're asking for images of windows from your own app. Any thoughts? -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74815
On Mon Jul 1 16:42:43 2024 +0000, Tim Clem wrote:
I'm seeing these issues now (though I didn't have to revert !5798). Not sure how I missed them earlier. I tried doing everything in the background thread that `getCurrentProcessShareableContent` is called back on, but it didn't help - `captureImageWithFilter:` still doesn't call its block until the window is restored. Doing something heinous like blocking the main thread until the SCK calls complete actually does work and grab a snapshot in time, but I'd really rather not do that. Not sure what our best option is. Maybe Apple can make SCK able to capture miniaturized windows, or perhaps they can make `CGWindowListCreateImageFromArray` not prompt for permissions if you're asking for images of windows from your own app. Any thoughts? Actually, I just had a realization - we're not going to be capturing minimized windows, we're going to be capturing *hidden* ones. In the actual use case for this code, we're trying to grab an image of a child window that's about to be hidden because its zero-sized parent is miniaturizing; we're not trying to snapshot the parent itself. The hacky way we're testing isn't representative of what would really happen.
The good news is that snapshotting a hidden window does seem to work without a delay. I'll try to find a real-world application that relies on this behavior, or make one myself. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_74817
On Mon Jul 1 16:53:05 2024 +0000, Tim Clem wrote:
Actually, I just had a realization - we're not going to be capturing minimized windows, we're going to be capturing *hidden* ones. In the actual use case for this code, we're trying to grab an image of a child window that's about to be hidden because its zero-sized parent is miniaturizing; we're not trying to snapshot the parent itself. The hacky way we're testing isn't representative of what would really happen. The good news is that snapshotting a hidden window does seem to work without a delay. I'll try to find a real-world application that relies on this behavior, or make one myself. Ok, here's an application that exercises this functionality: https://www.hydrocad.net/flex/download.htm. Note that I had issues running it on Apple Silicon for some reason I did not debug; it works fine on Intel.
And it's bad news. Capturing a hidden/hiding child window does not do what we want. It *does* call back the `captureImageWithFilter:` block immediately, but the image is not of the window. Barring some intervention from Apple, I don't think this API is going to work for us. I'll keep investigating, but I think we're either going to have to deal with the permissions prompt or disable the functionality for Sequoia. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_75592
This merge request was closed by Tim Clem. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935
For what it's worth, I haven't seen any permission prompts for CGWindowListCreateImageFromArray yet. Perhaps Apple added special handling for the case of you asking for screenshots of your own apps' windows. Regardless, ScreenCaptureKit definitely has issues with minimized/minimizing windows, so it probably isn't appropriate for us, or at least we can't just slot it in to the current flow. I'm going to close this and we can revisit if permission prompts start happening. -- https://gitlab.winehq.org/wine/wine/-/merge_requests/5935#note_117707
participants (5)
-
Brendan Shanks (＠bshanks) -
Dean M Greer (＠Gcenx)
-
Marvin -
Tim Clem
-
Tim Clem (＠tclem)