Skip to content

Commit

Permalink
backend, rendervulkan: Use DRM format for output format on backend
Browse files Browse the repository at this point in the history 
Fixes running on my Thinkpad X13s
  • Loading branch information
@misyltoad
misyltoad committed Sep 7, 2024
1 parent 7bbc2cd commit 6eaee77
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 47 deletions.
6 changes: 3 additions & 3 deletions src/Backends/DRMBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3174,10 +3174,10 @@ namespace gamescope
// thus: newLayout is ignored.
return VK_IMAGE_LAYOUT_GENERAL;
}
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override
{
*pPrimaryPlaneFormat = DRMFormatToVulkan( g_nDRMFormat, false );
*pOverlayPlaneFormat = DRMFormatToVulkan( g_nDRMFormatOverlay, false );
*pPrimaryPlaneFormat = g_nDRMFormat;
*pOverlayPlaneFormat = g_nDRMFormatOverlay;
}
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override
{
Expand Down
6 changes: 3 additions & 3 deletions src/Backends/HeadlessBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,10 @@ namespace gamescope
{
return VK_IMAGE_LAYOUT_GENERAL;
}
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override
{
*pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
*pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM;
*pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 );
*pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM );
}
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override
{
Expand Down
6 changes: 3 additions & 3 deletions src/Backends/OpenVRBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,10 +532,10 @@ namespace gamescope
{
return VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
}
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override
{
*pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
*pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM;
*pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 );
*pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM );
}
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override
{
Expand Down
8 changes: 4 additions & 4 deletions src/Backends/SDLBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ namespace gamescope
virtual std::span<const char *const> GetInstanceExtensions() const override;
virtual std::span<const char *const> GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const override;
virtual VkImageLayout GetPresentLayout() const override;
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override;
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override;
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override;

virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) override;
Expand Down Expand Up @@ -351,10 +351,10 @@ namespace gamescope
return VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
}

void CSDLBackend::GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const
void CSDLBackend::GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const
{
*pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
*pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM;
*pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 );
*pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM );
}

bool CSDLBackend::ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const
Expand Down
36 changes: 23 additions & 13 deletions src/Backends/WaylandBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ namespace gamescope
virtual std::span<const char *const> GetInstanceExtensions() const override;
virtual std::span<const char *const> GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const override;
virtual VkImageLayout GetPresentLayout() const override;
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override;
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override;
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override;

virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) override;
Expand Down Expand Up @@ -1542,23 +1542,33 @@ namespace gamescope
return VK_IMAGE_LAYOUT_GENERAL;
}

void CWaylandBackend::GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const
void CWaylandBackend::GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const
{
VkFormat u8BitFormat = VK_FORMAT_UNDEFINED;
if ( SupportsFormat( DRM_FORMAT_ARGB8888 ) )
u8BitFormat = VK_FORMAT_B8G8R8A8_UNORM;
else if ( SupportsFormat( DRM_FORMAT_ABGR8888 ) )
u8BitFormat = VK_FORMAT_R8G8B8A8_UNORM;
// Prefer opaque for composition on the Wayland backend.

VkFormat u10BitFormat = VK_FORMAT_UNDEFINED;
if ( SupportsFormat( DRM_FORMAT_ABGR2101010 ) )
u10BitFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
uint32_t u8BitFormat = DRM_FORMAT_INVALID;
if ( SupportsFormat( DRM_FORMAT_XRGB8888 ) )
u8BitFormat = DRM_FORMAT_XRGB8888;
else if ( SupportsFormat( DRM_FORMAT_XBGR8888 ) )
u8BitFormat = DRM_FORMAT_XBGR8888;
else if ( SupportsFormat( DRM_FORMAT_ARGB8888 ) )
u8BitFormat = DRM_FORMAT_ARGB8888;
else if ( SupportsFormat( DRM_FORMAT_ABGR8888 ) )
u8BitFormat = DRM_FORMAT_ABGR8888;

uint32_t u10BitFormat = DRM_FORMAT_INVALID;
if ( SupportsFormat( DRM_FORMAT_XBGR2101010 ) )
u10BitFormat = DRM_FORMAT_XBGR2101010;
else if ( SupportsFormat( DRM_FORMAT_XRGB2101010 ) )
u10BitFormat = DRM_FORMAT_XRGB2101010;
else if ( SupportsFormat( DRM_FORMAT_ABGR2101010 ) )
u10BitFormat = DRM_FORMAT_ABGR2101010;
else if ( SupportsFormat( DRM_FORMAT_ARGB2101010 ) )
u10BitFormat = VK_FORMAT_A2R10G10B10_UNORM_PACK32;
u10BitFormat = DRM_FORMAT_ARGB2101010;

assert( u8BitFormat != VK_FORMAT_UNDEFINED );
assert( u8BitFormat != DRM_FORMAT_INVALID );

*pPrimaryPlaneFormat = u10BitFormat != VK_FORMAT_UNDEFINED ? u10BitFormat : u8BitFormat;
*pPrimaryPlaneFormat = u10BitFormat != DRM_FORMAT_INVALID ? u10BitFormat : u8BitFormat;
*pOverlayPlaneFormat = u8BitFormat;
}

Expand Down
2 changes: 1 addition & 1 deletion src/backend.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ namespace gamescope
virtual std::span<const char *const> GetInstanceExtensions() const = 0;
virtual std::span<const char *const> GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const = 0;
virtual VkImageLayout GetPresentLayout() const = 0;
virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const = 0;
virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const = 0;
virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const = 0;

virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) = 0;
Expand Down
37 changes: 19 additions & 18 deletions src/rendervulkan.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3112,12 +3112,13 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput )
if ( surfaceFormat == formatCount )
return false;

pOutput->outputFormat = pOutput->surfaceFormats[ surfaceFormat ].format;
VkFormat eVkFormat = pOutput->surfaceFormats[ surfaceFormat ].format;
pOutput->uOutputFormat = VulkanFormatToDRM( pOutput->surfaceFormats[ surfaceFormat ].format );

VkFormat formats[2] =
{
ToSrgbVulkanFormat( pOutput->outputFormat ),
ToLinearVulkanFormat( pOutput->outputFormat ),
ToSrgbVulkanFormat( eVkFormat ),
ToLinearVulkanFormat( eVkFormat ),
};

VkImageFormatListCreateInfo usageListInfo = {
Expand All @@ -3126,15 +3127,15 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput )
.pViewFormats = formats,
};

vk_log.infof("Creating Gamescope nested swapchain with format %u and colorspace %u", pOutput->outputFormat, pOutput->surfaceFormats[surfaceFormat].colorSpace);
vk_log.infof("Creating Gamescope nested swapchain with format %u and colorspace %u", eVkFormat, pOutput->surfaceFormats[surfaceFormat].colorSpace);

VkSwapchainCreateInfoKHR createInfo = {
.sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
.pNext = formats[0] != formats[1] ? &usageListInfo : nullptr,
.flags = formats[0] != formats[1] ? VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR : (VkSwapchainCreateFlagBitsKHR )0,
.surface = pOutput->surface,
.minImageCount = imageCount,
.imageFormat = pOutput->outputFormat,
.imageFormat = eVkFormat,
.imageColorSpace = pOutput->surfaceFormats[surfaceFormat].colorSpace,
.imageExtent = {
.width = g_nOutputWidth,
Expand Down Expand Up @@ -3163,7 +3164,7 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput )
{
pOutput->outputImages[i] = new CVulkanTexture();

if ( !pOutput->outputImages[i]->BInitFromSwapchain(swapchainImages[i], g_nOutputWidth, g_nOutputHeight, pOutput->outputFormat))
if ( !pOutput->outputImages[i]->BInitFromSwapchain(swapchainImages[i], g_nOutputWidth, g_nOutputHeight, eVkFormat))
return false;
}

Expand Down Expand Up @@ -3220,26 +3221,26 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput )
pOutput->outputImagesPartialOverlay[1] = nullptr;
pOutput->outputImagesPartialOverlay[2] = nullptr;

VkFormat format = pOutput->outputFormat;
uint32_t uDRMFormat = pOutput->uOutputFormat;

pOutput->outputImages[0] = new CVulkanTexture();
bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags );
bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImages[1] = new CVulkanTexture();
bSuccess = pOutput->outputImages[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags );
bSuccess = pOutput->outputImages[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImages[2] = new CVulkanTexture();
bSuccess = pOutput->outputImages[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags );
bSuccess = pOutput->outputImages[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
Expand All @@ -3249,28 +3250,28 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput )
// Oh no.
pOutput->temporaryHackyBlankImage = vulkan_create_debug_blank_texture();

if ( pOutput->outputFormatOverlay != VK_FORMAT_UNDEFINED && !kDisablePartialComposition )
if ( pOutput->uOutputFormatOverlay != VK_FORMAT_UNDEFINED && !kDisablePartialComposition )
{
VkFormat partialFormat = pOutput->outputFormatOverlay;
uint32_t uPartialDRMFormat = pOutput->uOutputFormatOverlay;

pOutput->outputImagesPartialOverlay[0] = new CVulkanTexture();
bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() );
bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImagesPartialOverlay[1] = new CVulkanTexture();
bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() );
bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImagesPartialOverlay[2] = new CVulkanTexture();
bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() );
bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
Expand Down Expand Up @@ -3356,15 +3357,15 @@ bool vulkan_make_output()
}
else
{
GetBackend()->GetPreferredOutputFormat( &pOutput->outputFormat, &pOutput->outputFormatOverlay );
GetBackend()->GetPreferredOutputFormat( &pOutput->uOutputFormat, &pOutput->uOutputFormatOverlay );

if ( pOutput->outputFormat == VK_FORMAT_UNDEFINED )
if ( pOutput->uOutputFormat == DRM_FORMAT_INVALID )
{
vk_log.errorf( "failed to find Vulkan format suitable for KMS" );
return false;
}

if ( pOutput->outputFormatOverlay == VK_FORMAT_UNDEFINED )
if ( pOutput->uOutputFormatOverlay == DRM_FORMAT_INVALID )
{
vk_log.errorf( "failed to find Vulkan format suitable for KMS partial overlays" );
return false;
Expand Down
4 changes: 2 additions & 2 deletions src/rendervulkan.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,8 +508,8 @@ struct VulkanOutput_t
std::vector<gamescope::OwningRc<CVulkanTexture>> outputImagesPartialOverlay;
gamescope::OwningRc<CVulkanTexture> temporaryHackyBlankImage;

VkFormat outputFormat = VK_FORMAT_UNDEFINED;
VkFormat outputFormatOverlay = VK_FORMAT_UNDEFINED;
uint32_t uOutputFormat = DRM_FORMAT_INVALID;
uint32_t uOutputFormatOverlay = DRM_FORMAT_INVALID;

std::array<gamescope::OwningRc<CVulkanTexture>, 2> pScreenshotImages;

Expand Down

0 comments on commit 6eaee77

Please sign in to comment.