From ef52113073fe4eed10d408efeeb86dd1c4acd22e Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 20 Oct 2024 14:39:20 -0400 Subject: [PATCH] Resolve WGC technical limitation (#303) --- docs/tutorial.md | 1 - ruff.toml | 4 + scripts/requirements.txt | 18 +- .../WindowsGraphicsCaptureMethod.py | 29 +-- src/capture_method/__init__.py | 16 +- src/d3d11.py | 226 ++++++++++++++++++ src/region_selection.py | 61 ++--- src/utils.py | 41 ---- 8 files changed, 290 insertions(+), 106 deletions(-) create mode 100644 src/d3d11.py diff --git a/docs/tutorial.md b/docs/tutorial.md index 5c40d99f..b9040ddf 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -43,7 +43,6 @@ - **Windows Graphics Capture** (fast, most compatible, capped at 60fps) Only available in Windows 10.0.17134 and up. - Due to current technical limitations, Windows versions below 10.0.0.17763 require having at least one audio or video Capture Device connected and enabled. Allows recording UWP apps, Hardware Accelerated and Exclusive Fullscreen windows. Adds a yellow border on Windows 10 (not on Windows 11). Caps at around 60 FPS. diff --git a/ruff.toml b/ruff.toml index d6536ecd..bbefddaf 100644 --- a/ruff.toml +++ b/ruff.toml @@ -146,6 +146,10 @@ max-branches = 15 # Issues with using a star-imported name will be caught by type-checkers. "F405", # may be undefined, or defined from star imports ] +"src/d3d11.py" = [ + # Following windows API/ctypes like naming conventions + "N801", # invalid-class-name +] [lint.flake8-tidy-imports.banned-api] "cv2.imread".msg = """\ diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1897d5a6..69d458a5 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -17,9 +17,11 @@ PySide6-Essentials>=6.6.0 # Python 3.12 support scipy>=1.11.2 # Python 3.12 support tomli-w>=1.1.0 # Typing fixes typing-extensions>=4.4.0 # @override decorator support + # # Build and compile resources pyinstaller>=5.13 # Python 3.12 support + # # https://peps.python.org/pep-0508/#environment-markers # @@ -27,16 +29,16 @@ pyinstaller>=5.13 # Python 3.12 support comtypes<1.4.5 ; sys_platform == 'win32' # https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/807 pygrabber>=0.2 ; sys_platform == 'win32' # Completed types pywin32>=301 ; sys_platform == 'win32' -winrt-Windows.AI.MachineLearning>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support +typed-D3DShot[numpy]>=1.0.1 ; sys_platform == 'win32' winrt-Windows.Foundation>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Graphics.Capture>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Graphics.Capture.Interop>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Graphics.DirectX>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Graphics.DirectX.Direct3D11>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Graphics.Imaging>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support +winrt-Windows.Graphics.Capture>=2.3.0 ; sys_platform == 'win32' # Python 3.13 support +winrt-Windows.Graphics.Capture.Interop>=2.3.0 ; sys_platform == 'win32' # Python 3.13 support +winrt-Windows.Graphics.DirectX>=2.3.0 ; sys_platform == 'win32' # Python 3.13 support +winrt-Windows.Graphics.DirectX.Direct3D11>=2.3.0 ; sys_platform == 'win32' # Python 3.13 support +winrt-Windows.Graphics.DirectX.Direct3D11.Interop>=2.3.0 ; sys_platform == 'win32' winrt-Windows.Graphics>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -winrt-Windows.Media.Capture>=2.2.0 ; sys_platform == 'win32' # Python 3.13 support -typed-D3DShot[numpy]>=1.0.1 ; sys_platform == 'win32' +winrt-Windows.Graphics.Imaging>=2.3.0 ; sys_platform == 'win32' # Python 3.13 support + # # Linux-only dependencies PyScreeze ; sys_platform == 'linux' diff --git a/src/capture_method/WindowsGraphicsCaptureMethod.py b/src/capture_method/WindowsGraphicsCaptureMethod.py index d599c2e6..aa905a32 100644 --- a/src/capture_method/WindowsGraphicsCaptureMethod.py +++ b/src/capture_method/WindowsGraphicsCaptureMethod.py @@ -14,23 +14,19 @@ from winrt.windows.graphics.capture.interop import create_for_window from winrt.windows.graphics.directx import DirectXPixelFormat from winrt.windows.graphics.directx.direct3d11 import IDirect3DSurface +from winrt.windows.graphics.directx.direct3d11.interop import ( + create_direct3d11_device_from_dxgi_device, +) from winrt.windows.graphics.imaging import BitmapBufferAccessMode, SoftwareBitmap from capture_method.CaptureMethodBase import CaptureMethodBase -from utils import ( - BGRA_CHANNEL_COUNT, - WGC_MIN_BUILD, - WINDOWS_BUILD_NUMBER, - get_direct3d_device, - is_valid_hwnd, -) +from d3d11 import D3D11_CREATE_DEVICE_FLAG, D3D_DRIVER_TYPE, D3D11CreateDevice +from utils import BGRA_CHANNEL_COUNT, WGC_MIN_BUILD, WINDOWS_BUILD_NUMBER, is_valid_hwnd if TYPE_CHECKING: from AutoSplit import AutoSplit WGC_NO_BORDER_MIN_BUILD = 20348 -LEARNING_MODE_DEVICE_BUILD = 17763 -"""https://learn.microsoft.com/en-us/uwp/api/windows.ai.machinelearning.learningmodeldevice""" async def convert_d3d_surface_to_software_bitmap(surface: IDirect3DSurface | None): @@ -42,13 +38,11 @@ class WindowsGraphicsCaptureMethod(CaptureMethodBase): short_description = "fast, most compatible, capped at 60fps" description = f""" Only available in Windows 10.0.{WGC_MIN_BUILD} and up. -Due to current technical limitations, Windows versions below 10.0.0.{LEARNING_MODE_DEVICE_BUILD} -require having at least one audio or video Capture Device connected and enabled. Allows recording UWP apps, Hardware Accelerated and Exclusive Fullscreen windows. Adds a yellow border on Windows 10 (not on Windows 11). Caps at around 60 FPS.""" - size: SizeInt32 + size: "SizeInt32" frame_pool: Direct3D11CaptureFramePool | None = None session: GraphicsCaptureSession | None = None """This is stored to prevent session from being garbage collected""" @@ -59,11 +53,16 @@ def __init__(self, autosplit: "AutoSplit"): if not is_valid_hwnd(autosplit.hwnd): return + dxgi, *_ = D3D11CreateDevice( + DriverType=D3D_DRIVER_TYPE.HARDWARE, + Flags=D3D11_CREATE_DEVICE_FLAG.BGRA_SUPPORT, + ) + direct3d_device = create_direct3d11_device_from_dxgi_device(dxgi.value) item = create_for_window(autosplit.hwnd) frame_pool = Direct3D11CaptureFramePool.create_free_threaded( - get_direct3d_device(), + direct3d_device, DirectXPixelFormat.B8_G8_R8_A8_UINT_NORMALIZED, - 1, + 1, # number_of_buffers item.size, ) if not frame_pool: @@ -114,6 +113,8 @@ def get_frame(self) -> MatLike | None: return None # We were too fast and the next frame wasn't ready yet + # TODO: Consider "add_frame_arrive" instead ! + # https://github.com/pywinrt/pywinrt/blob/5bf1ac5ff4a77cf343e11d7c841c368fa9235d81/samples/screen_capture/__main__.py#L67-L78 if not frame: return self.last_converted_frame diff --git a/src/capture_method/__init__.py b/src/capture_method/__init__.py index 6d25cda1..a494e0f5 100644 --- a/src/capture_method/__init__.py +++ b/src/capture_method/__init__.py @@ -10,13 +10,7 @@ from capture_method.CaptureMethodBase import CaptureMethodBase from capture_method.VideoCaptureDeviceCaptureMethod import VideoCaptureDeviceCaptureMethod -from utils import ( - WGC_MIN_BUILD, - WINDOWS_BUILD_NUMBER, - first, - get_input_device_resolution, - try_get_direct3d_device, -) +from utils import WGC_MIN_BUILD, WINDOWS_BUILD_NUMBER, first, get_input_device_resolution if sys.platform == "win32": from _ctypes import COMError # noqa: PLC2701 # comtypes is untyped @@ -125,12 +119,8 @@ def get(self, key: CaptureMethodEnum, default: object = None, /): CAPTURE_METHODS = CaptureMethodDict() if sys.platform == "win32": - if ( # Windows Graphics Capture requires a minimum Windows Build - WINDOWS_BUILD_NUMBER >= WGC_MIN_BUILD - # Our current implementation of Windows Graphics Capture - # does not ensure we can get an ID3DDevice - and try_get_direct3d_device() - ): + # Windows Graphics Capture requires a minimum Windows Build + if WINDOWS_BUILD_NUMBER >= WGC_MIN_BUILD: CAPTURE_METHODS[CaptureMethodEnum.WINDOWS_GRAPHICS_CAPTURE] = WindowsGraphicsCaptureMethod CAPTURE_METHODS[CaptureMethodEnum.BITBLT] = BitBltCaptureMethod try: # Test for laptop cross-GPU Desktop Duplication issue diff --git a/src/d3d11.py b/src/d3d11.py new file mode 100644 index 00000000..e9e4c5bb --- /dev/null +++ b/src/d3d11.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2024 David Lechner +import sys + +if sys.platform != "win32": + raise OSError + +import ctypes +import enum +import uuid +from ctypes import wintypes +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ctypes import _FuncPointer # pyright: ignore[reportPrivateUsage] + + +### +# https://github.com/pywinrt/pywinrt/blob/main/samples/screen_capture/iunknown.py +### + + +class GUID(ctypes.Structure): + _fields_ = ( + ("Data1", ctypes.c_ulong), + ("Data2", ctypes.c_ushort), + ("Data3", ctypes.c_ushort), + ("Data4", ctypes.c_ubyte * 8), + ) + + +class IUnknown(ctypes.c_void_p): + QueryInterface = ctypes.WINFUNCTYPE( + # _CData is incompatible with int + int, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ctypes.POINTER(GUID), + ctypes.POINTER(wintypes.LPVOID), + )(0, "QueryInterface") + AddRef = ctypes.WINFUNCTYPE(wintypes.ULONG)(1, "AddRef") + Release = ctypes.WINFUNCTYPE(wintypes.ULONG)(2, "Release") + + def query_interface(self, iid: uuid.UUID | str) -> "IUnknown": + if isinstance(iid, str): + iid = uuid.UUID(iid) + + ppv = wintypes.LPVOID() + _iid = GUID.from_buffer_copy(iid.bytes_le) + ret = self.QueryInterface(self, ctypes.byref(_iid), ctypes.byref(ppv)) + + if ret: + raise ctypes.WinError(ret) + + return IUnknown(ppv.value) + + def __del__(self): + IUnknown.Release(self) + + +### +# https://github.com/pywinrt/pywinrt/blob/main/samples/screen_capture/d3d11.py +### + + +__all__ = [ + "D3D11_CREATE_DEVICE_FLAG", + "D3D_DRIVER_TYPE", + "D3D_FEATURE_LEVEL", + "D3D11CreateDevice", +] + +IN = 1 +OUT = 2 + +# https://learn.microsoft.com/en-us/windows/win32/api/d3dcommon/ne-d3dcommon-d3d_driver_type +# +# typedef enum D3D_DRIVER_TYPE { +# D3D_DRIVER_TYPE_UNKNOWN = 0, +# D3D_DRIVER_TYPE_HARDWARE, +# D3D_DRIVER_TYPE_REFERENCE, +# D3D_DRIVER_TYPE_NULL, +# D3D_DRIVER_TYPE_SOFTWARE, +# D3D_DRIVER_TYPE_WARP +# } ; + + +class D3D_DRIVER_TYPE(enum.IntEnum): + UNKNOWN = 0 + HARDWARE = 1 + REFERENCE = 2 + NULL = 3 + SOFTWARE = 4 + WARP = 5 + + +# https://learn.microsoft.com/en-us/windows/win32/api/d3d11/ne-d3d11-d3d11_create_device_flag +# +# typedef enum D3D11_CREATE_DEVICE_FLAG { +# D3D11_CREATE_DEVICE_SINGLETHREADED = 0x1, +# D3D11_CREATE_DEVICE_DEBUG = 0x2, +# D3D11_CREATE_DEVICE_SWITCH_TO_REF = 0x4, +# D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS = 0x8, +# D3D11_CREATE_DEVICE_BGRA_SUPPORT = 0x20, +# D3D11_CREATE_DEVICE_DEBUGGABLE = 0x40, +# D3D11_CREATE_DEVICE_PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY = 0x80, +# D3D11_CREATE_DEVICE_DISABLE_GPU_TIMEOUT = 0x100, +# D3D11_CREATE_DEVICE_VIDEO_SUPPORT = 0x800 +# } ; + + +class D3D11_CREATE_DEVICE_FLAG(enum.IntFlag): + SINGLETHREADED = 0x1 + DEBUG = 0x2 + SWITCH_TO_REF = 0x4 + PREVENT_INTERNAL_THREADING_OPTIMIZATIONS = 0x8 + BGRA_SUPPORT = 0x20 + DEBUGGABLE = 0x40 + PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY = 0x80 + DISABLE_GPU_TIMEOUT = 0x100 + VIDEO_SUPPORT = 0x800 + + +# https://learn.microsoft.com/en-us/windows/win32/api/d3dcommon/ne-d3dcommon-d3d_feature_level +# +# typedef enum D3D_FEATURE_LEVEL { +# D3D_FEATURE_LEVEL_1_0_GENERIC, +# D3D_FEATURE_LEVEL_1_0_CORE, +# D3D_FEATURE_LEVEL_9_1, +# D3D_FEATURE_LEVEL_9_2, +# D3D_FEATURE_LEVEL_9_3, +# D3D_FEATURE_LEVEL_10_0, +# D3D_FEATURE_LEVEL_10_1, +# D3D_FEATURE_LEVEL_11_0, +# D3D_FEATURE_LEVEL_11_1, +# D3D_FEATURE_LEVEL_12_0, +# D3D_FEATURE_LEVEL_12_1, +# D3D_FEATURE_LEVEL_12_2 +# } ; + + +class D3D_FEATURE_LEVEL(enum.IntEnum): + LEVEL_1_0_GENERIC = 0x1000 + LEVEL_1_0_CORE = 0x1001 + LEVEL_9_1 = 0x9100 + LEVEL_9_2 = 0x9200 + LEVEL_9_3 = 0x9300 + LEVEL_10_0 = 0xA000 + LEVEL_10_1 = 0xA100 + LEVEL_11_0 = 0xB000 + LEVEL_11_1 = 0xB100 + LEVEL_12_0 = 0xC000 + LEVEL_12_1 = 0xC100 + LEVEL_12_2 = 0xC200 + + +# not sure where this is officially defined or if the value would ever change + +D3D11_SDK_VERSION = 7 + +# https://learn.microsoft.com/en-us/windows/win32/api/d3d11/nf-d3d11-d3d11createdevice +# +# HRESULT D3D11CreateDevice( +# [in, optional] IDXGIAdapter *pAdapter, +# D3D_DRIVER_TYPE DriverType, +# HMODULE Software, +# UINT Flags, +# [in, optional] const D3D_FEATURE_LEVEL *pFeatureLevels, +# UINT FeatureLevels, +# UINT SDKVersion, +# [out, optional] ID3D11Device **ppDevice, +# [out, optional] D3D_FEATURE_LEVEL *pFeatureLevel, +# [out, optional] ID3D11DeviceContext **ppImmediateContext +# ); + + +def errcheck( + result: int, + _func: "_FuncPointer", # Actually WinFunctionType but that's an internal class + args: tuple[ + IUnknown | None, # IDXGIAdapter + D3D_DRIVER_TYPE, + wintypes.HMODULE | None, + D3D11_CREATE_DEVICE_FLAG, + D3D_FEATURE_LEVEL | None, + int, + int, + IUnknown, # ID3D11Device + wintypes.UINT, + IUnknown, # ID3D11DeviceContext + ], +): + if result: + raise ctypes.WinError(result) + + return (args[7], D3D_FEATURE_LEVEL(args[8].value), args[9]) + + +D3D11CreateDevice = ctypes.WINFUNCTYPE( + # _CData is incompatible with int + int, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + wintypes.LPVOID, + wintypes.UINT, + wintypes.LPVOID, + wintypes.UINT, + ctypes.POINTER(wintypes.UINT), + wintypes.UINT, + wintypes.UINT, + ctypes.POINTER(IUnknown), + ctypes.POINTER(wintypes.UINT), + ctypes.POINTER(IUnknown), +)( + ("D3D11CreateDevice", ctypes.windll.d3d11), + ( + (IN, "pAdapter", None), + (IN, "DriverType", D3D_DRIVER_TYPE.UNKNOWN), + (IN, "Software", None), + (IN, "Flags", 0), + (IN, "pFeatureLevels", None), + (IN, "FeatureLevels", 0), + (IN, "SDKVersion", D3D11_SDK_VERSION), + (OUT, "ppDevice"), + (OUT, "pFeatureLevel"), + (OUT, "ppImmediateContext"), + ), +) +# _CData is incompatible with int +D3D11CreateDevice.errcheck = errcheck # type: ignore[assignment] # pyright: ignore[reportAttributeAccessIssue] diff --git a/src/region_selection.py b/src/region_selection.py index c8f92c8e..b21e3fc5 100644 --- a/src/region_selection.py +++ b/src/region_selection.py @@ -32,9 +32,6 @@ SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, ) - from winrt._winrt import initialize_with_window # noqa: PLC2701 - from winrt.windows.foundation import AsyncStatus, IAsyncOperation - from winrt.windows.graphics.capture import GraphicsCaptureItem, GraphicsCapturePicker if sys.platform == "linux": from Xlib.display import Display @@ -79,32 +76,38 @@ def get_top_window_at(x: int, y: int): # TODO: For later as a different picker option -def __select_graphics_item(autosplit: "AutoSplit"): # pyright: ignore [reportUnusedFunction] - """Uses the built-in GraphicsCapturePicker to select the Window.""" - if sys.platform != "win32": - raise OSError - - def callback(async_operation: IAsyncOperation[GraphicsCaptureItem], async_status: AsyncStatus): - try: - if async_status != AsyncStatus.COMPLETED: - return - except SystemError as exception: - # HACK: can happen when closing the GraphicsCapturePicker - if str(exception).endswith("returned a result with an error set"): - return - raise - item = async_operation.get_results() - if not item: - return - autosplit.settings_dict["captured_window_title"] = item.display_name - autosplit.capture_method.reinitialize() - - picker = GraphicsCapturePicker() - initialize_with_window(picker, autosplit.effectiveWinId()) - async_operation = picker.pick_single_item_async() - # None if the selection is canceled - if async_operation: - async_operation.completed = callback +# def __select_graphics_item(autosplit: "AutoSplit"): +# """Uses the built-in GraphicsCapturePicker to select the Window.""" +# if sys.platform != "win32": +# raise OSError +# from winrt._winrt import initialize_with_window +# from winrt.windows.foundation import AsyncStatus, IAsyncOperation +# from winrt.windows.graphics.capture import GraphicsCaptureItem, GraphicsCapturePicker +# +# def callback( +# async_operation: IAsyncOperation[GraphicsCaptureItem], +# async_status: AsyncStatus, +# ): +# try: +# if async_status != AsyncStatus.COMPLETED: +# return +# except SystemError as exception: +# # HACK: can happen when closing the GraphicsCapturePicker +# if str(exception).endswith("returned a result with an error set"): +# return +# raise +# item = async_operation.get_results() +# if not item: +# return +# autosplit.settings_dict["captured_window_title"] = item.display_name +# autosplit.capture_method.reinitialize() +# +# picker = GraphicsCapturePicker() +# initialize_with_window(picker, autosplit.effectiveWinId()) +# async_operation = picker.pick_single_item_async() +# # None if the selection is canceled +# if async_operation: +# async_operation.completed = callback def select_region(autosplit: "AutoSplit"): diff --git a/src/utils.py b/src/utils.py index f82018a3..38a66921 100644 --- a/src/utils.py +++ b/src/utils.py @@ -25,8 +25,6 @@ import win32gui import win32ui from pygrabber.dshow_graph import FilterGraph - from winrt.windows.ai.machinelearning import LearningModelDevice, LearningModelDeviceKind - from winrt.windows.media.capture import MediaCapture STARTUPINFO: TypeAlias = subprocess.STARTUPINFO else: @@ -198,45 +196,6 @@ def get_or_create_eventloop(): return asyncio.get_event_loop() -def get_direct3d_device(): - if sys.platform != "win32": - raise OSError("Direct3D Device is only available on Windows") - - # Note: Must create in the same thread (can't use a global) - # otherwise when ran from LiveSplit it will raise: - # OSError: The application called an interface that was marshalled for a different thread - media_capture = MediaCapture() - - async def init_mediacapture(): - await media_capture.initialize_async() - - asyncio.run(init_mediacapture()) - direct_3d_device = ( - media_capture.media_capture_settings - and media_capture.media_capture_settings.direct3_d11_device - ) - if not direct_3d_device: - try: - # May be problematic? - # https://github.com/pywinrt/python-winsdk/issues/11#issuecomment-1315345318 - direct_3d_device = LearningModelDevice( - LearningModelDeviceKind.DIRECT_X_HIGH_PERFORMANCE - ).direct3_d11_device - # TODO: Unknown potential error, I don't have an older Win10 machine to test. - except BaseException: # noqa: S110,BLE001 - pass - if not direct_3d_device: - raise OSError("Unable to initialize a Direct3D Device.") - return direct_3d_device - - -def try_get_direct3d_device(): - try: - return get_direct3d_device() - except OSError: - return None - - def try_input_device_access(): """Same as `make_uinput` in `keyboard/_nixcommon.py`.""" if sys.platform != "linux":