GBM-allocated scan-out buffers for gamescope's DRM backend on NVIDIA (opt-in: gamescope_drm_gbm_scanout=1). Nine commits from https://github.com/farnoy/gamescope (branch nvidia-gbm-wayland: 5cf847274, ffb525cf7, 211f4ca26, 117b593ea, 8da9459ff, cb0378edc, 4e879fbf7, 0c7fddbf3, ae7b99df3), as one diff against gamescope 3.16.31. Upstream issue: https://github.com/ValveSoftware/gamescope/issues/2309. Tested on an RTX 5080 (driver 615.78.08): Steam's gaming UI is clean with it and shows artifacts without it. Applies with `git apply` (it ignores these lines); newer gamescope versions need a rebase. diff --git a/README.md b/README.md index 0f53c60..7de4241 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If running RadeonSI clients with older cards (GFX8 and below), currently have to Dependent first (**Debian/Debian-based**): ``` -apt install meson ninja-build pkg-config cmake libpipewire-0.3-dev hwdata libx11-dev libwayland-dev libvulkan-dev wayland-protocols libx11-xcb-dev libxdamage-dev libxcomposite-dev libxcursor-dev libxxf86vm-dev libxtst-dev libxres-dev libxmu-dev libxkbcommon-dev libcap-dev libsdl2-dev libavif-dev libpixman-1-dev liblcms2-dev libseat-dev libinput-dev xwayland libxcb-composite0-dev libxcb-ewmh-dev libxcb-icccm4-dev libxcb-res0-dev glslang-tools libluajit-5.1-dev libcatch2-dev +apt install meson ninja-build pkg-config cmake libpipewire-0.3-dev hwdata libx11-dev libwayland-dev libvulkan-dev libgbm-dev wayland-protocols libx11-xcb-dev libxdamage-dev libxcomposite-dev libxcursor-dev libxxf86vm-dev libxtst-dev libxres-dev libxmu-dev libxkbcommon-dev libcap-dev libsdl2-dev libavif-dev libpixman-1-dev liblcms2-dev libseat-dev libinput-dev xwayland libxcb-composite0-dev libxcb-ewmh-dev libxcb-icccm4-dev libxcb-res0-dev glslang-tools libluajit-5.1-dev libcatch2-dev ``` Build with: diff --git a/src/Backends/DRMBackend.cpp b/src/Backends/DRMBackend.cpp index 9aaab7b..9b40f02 100644 --- a/src/Backends/DRMBackend.cpp +++ b/src/Backends/DRMBackend.cpp @@ -56,6 +56,10 @@ #include "libdisplay-info/cta.h" #include "wlr_end.hpp" +#if HAVE_GBM +#include "GbmScanoutBuffer.h" +#endif + #include "gamescope-control-protocol.h" extern int g_nPreferredOutputWidth; @@ -75,6 +79,19 @@ gamescope::ConVar cv_drm_debug_disable_color_range( "drm_debug_disable_col gamescope::ConVar cv_drm_debug_disable_explicit_sync( "drm_debug_disable_explicit_sync", false, "Force disable explicit sync on the DRM backend." ); gamescope::ConVar cv_drm_debug_disable_in_fence_fd( "drm_debug_disable_in_fence_fd", false, "Force disable IN_FENCE_FD being set to avoid over-synchronization on the DRM backend." ); +gamescope::ConVar cv_drm_gbm_scanout( "drm_gbm_scanout", false, "Allocate scanout buffers with GBM and import them into Vulkan. Fixes corrupted scanout on the NVIDIA proprietary driver.", + []( gamescope::ConVar &cvar ) + { + // The callback also runs for writes of the unchanged value; only an + // actual toggle warrants rebuilding the output images. + static bool s_bLastValue = false; + if ( cvar == s_bLastValue ) + return; + s_bLastValue = cvar; + + g_bForceOutputImageRemake = true; + }); + gamescope::ConVar cv_drm_allow_dynamic_modes_for_external_display( "drm_allow_dynamic_modes_for_external_display", false, "Allow dynamic mode/refresh rate switching for external displays." ); int HackyDRMPresent( const FrameInfo_t *pFrameInfo, bool bAsync ); @@ -113,6 +130,13 @@ struct drm_t { bool allow_modifiers; struct wlr_drm_format_set formats; + // GBM device on a dup of the KMS fd, for backend-allocated scanout + // buffers. Buffers keep the device alive via this shared_ptr; the + // deleter destroys the device and closes its dup'd fd, so buffer + // teardown stays valid even after drm->fd itself is closed. + std::shared_ptr gbm; + bool bIsNvidia = false; + std::vector< std::unique_ptr< gamescope::CDRMPlane > > planes; std::vector< std::unique_ptr< gamescope::CDRMCRTC > > crtcs; std::unordered_map< uint32_t, gamescope::CDRMConnector > connectors; @@ -547,10 +571,11 @@ namespace gamescope ~CDRMFb(); uint32_t GetFbId() const { return m_uFbId; } - + private: uint32_t m_uFbId = 0; }; + } uint32_t g_nDRMFormat = DRM_FORMAT_INVALID; @@ -1546,6 +1571,28 @@ bool init_drm(struct drm_t *drm, int width, int height, int refresh) drm->allow_modifiers = true; } + if ( drmVersion *pVersion = drmGetVersion( drm->fd ) ) + { + drm->bIsNvidia = pVersion->name && !strcmp( pVersion->name, "nvidia-drm" ); + drm_log.infof( "KMS driver: %s", pVersion->name ? pVersion->name : "unknown" ); + drmFreeVersion( pVersion ); + } + +#if HAVE_GBM + // Create the GBM device on a dup of the KMS fd so buffer teardown + // (gbm_bo_destroy) stays valid even after wlsession_close_kms() has + // closed drm->fd; textures holding GBM buffers can outlive the backend + // on shutdown. The deleter closes the dup'd fd after the device. + drm->gbm = gamescope::CreateGbmDevice( drm->fd ); + if ( !drm->gbm ) + drm_log.errorf( "GBM scanout buffers will be unavailable." ); + + if ( drm->bIsNvidia && !cv_drm_gbm_scanout ) + { + drm_log.infof( "nvidia-drm detected: scanout of Vulkan-allocated buffers is known to corrupt on this driver. Consider enabling GBM scanout buffers with gamescope_drm_gbm_scanout=1." ); + } +#endif + g_bSupportsAsyncFlips = drmGetCap(drm->fd, DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP, &cap) == 0 && cap != 0; if (!g_bSupportsAsyncFlips) drm_log.errorf("Immediate flips are not supported by the KMS driver"); @@ -1842,6 +1889,12 @@ void finish_drm(struct drm_t *drm) drm->crtcs.clear(); drm->connectors.clear(); + // Drop our reference to the GBM device. Outstanding scanout buffers + // keep it alive via their own references; the device owns a dup of the + // KMS fd, so late gbm_bo_destroy calls stay valid after + // wlsession_close_kms() below. + drm->gbm = nullptr; + // Signal the page-flip handler thread to exit and join it so it won't be // using the DRM fd while we clean it up. Closing the pipe write end @@ -4227,6 +4280,27 @@ namespace gamescope return drm_fbid_from_dmabuf( &g_DRM, pDmaBuf ); } + virtual bool SupportsExternalScanoutBuffers() const override + { +#if HAVE_GBM + return g_DRM.gbm != nullptr && g_DRM.allow_modifiers && cv_drm_gbm_scanout; +#else + return false; +#endif + } + + virtual std::shared_ptr CreateScanoutBuffer( uint32_t uWidth, uint32_t uHeight, uint32_t uDrmFormat, std::span ulModifiers, bool bLinear ) override + { +#if HAVE_GBM + if ( !g_DRM.gbm ) + return nullptr; + + return CreateGbmScanoutBuffer( g_DRM.gbm, uWidth, uHeight, uDrmFormat, ulModifiers, bLinear ); +#else + return nullptr; +#endif + } + virtual bool UsesModifiers() const override { return g_DRM.allow_modifiers; diff --git a/src/Backends/DeferredBackend.h b/src/Backends/DeferredBackend.h index 7f76122..a7385b3 100644 --- a/src/Backends/DeferredBackend.h +++ b/src/Backends/DeferredBackend.h @@ -163,6 +163,25 @@ namespace gamescope return m_pChild->ImportDmabufToBackend( pDmaBuf ); } + virtual bool SupportsExternalScanoutBuffers() const override + { + { + std::shared_lock lock{ m_mutInit }; + if ( m_bInittedChild ) + return m_pChild->SupportsExternalScanoutBuffers(); + } + return false; + } + virtual std::shared_ptr CreateScanoutBuffer( uint32_t uWidth, uint32_t uHeight, uint32_t uDrmFormat, std::span ulModifiers, bool bLinear ) override + { + { + std::shared_lock lock{ m_mutInit }; + if ( m_bInittedChild ) + return m_pChild->CreateScanoutBuffer( uWidth, uHeight, uDrmFormat, ulModifiers, bLinear ); + } + return nullptr; + } + virtual bool UsesModifiers() const override { return true; diff --git a/src/Backends/GbmScanoutBuffer.cpp b/src/Backends/GbmScanoutBuffer.cpp new file mode 100644 index 0000000..401f679 --- /dev/null +++ b/src/Backends/GbmScanoutBuffer.cpp @@ -0,0 +1,158 @@ +#include "GbmScanoutBuffer.h" + +#include "drm_include.h" +#include "log.hpp" + +#include +#include +#include + +#include + +namespace gamescope +{ + static LogScope s_GbmLog( "gbm" ); + + class CGbmScanoutBuffer final : public IBackendScanoutBuffer + { + public: + CGbmScanoutBuffer( std::shared_ptr pDevice, gbm_bo *pBo, const wlr_dmabuf_attributes &attrs ) + : m_pDevice{ std::move( pDevice ) } + , m_pBo{ pBo } + , m_Attrs{ attrs } + { + } + + ~CGbmScanoutBuffer() + { + wlr_dmabuf_attributes_finish( &m_Attrs ); + gbm_bo_destroy( m_pBo ); + } + + const wlr_dmabuf_attributes *GetDmabufAttributes() const override { return &m_Attrs; } + + private: + // Declared before m_pBo so the device outlives the bo. + std::shared_ptr m_pDevice; + gbm_bo *m_pBo = nullptr; + wlr_dmabuf_attributes m_Attrs = {}; + }; + + std::shared_ptr CreateGbmDevice( int nFd ) + { + int nGbmFd = dup( nFd ); + if ( nGbmFd < 0 ) + { + s_GbmLog.errorf_errno( "Failed to dup fd for GBM" ); + return nullptr; + } + + gbm_device *pDevice = gbm_create_device( nGbmFd ); + if ( !pDevice ) + { + close( nGbmFd ); + s_GbmLog.errorf( "Failed to create GBM device" ); + return nullptr; + } + + s_GbmLog.infof( "Created GBM device (backend: %s)", gbm_device_get_backend_name( pDevice ) ); + return std::shared_ptr( pDevice, [ nGbmFd ]( gbm_device *pDevice ){ gbm_device_destroy( pDevice ); close( nGbmFd ); } ); + } + + std::shared_ptr CreateGbmScanoutBuffer( const std::shared_ptr &pDevice, uint32_t uWidth, uint32_t uHeight, uint32_t uDrmFormat, std::span ulModifiers, bool bLinear ) + { + if ( !pDevice ) + return nullptr; + + uint32_t uFlags = GBM_BO_USE_RENDERING | GBM_BO_USE_SCANOUT; + + bool bForceLinearModifier = false; + struct gbm_bo *pBo = nullptr; + if ( bLinear ) + { + // GBM rejects GBM_BO_USE_LINEAR combined with an explicit + // modifier list (EINVAL in Mesa), so allocate directly. + pBo = gbm_bo_create( pDevice.get(), uWidth, uHeight, uDrmFormat, uFlags | GBM_BO_USE_LINEAR ); + bForceLinearModifier = true; + } + else if ( !ulModifiers.empty() ) + { + pBo = gbm_bo_create_with_modifiers2( pDevice.get(), uWidth, uHeight, uDrmFormat, ulModifiers.data(), ulModifiers.size(), uFlags ); + + // Some GBM implementations cannot allocate with an explicit + // modifier list at all. If only LINEAR was requested, a plain + // allocation with GBM_BO_USE_LINEAR is equivalent. + if ( !pBo && ulModifiers.size() == 1 && ulModifiers[0] == DRM_FORMAT_MOD_LINEAR ) + { + pBo = gbm_bo_create( pDevice.get(), uWidth, uHeight, uDrmFormat, uFlags | GBM_BO_USE_LINEAR ); + bForceLinearModifier = true; + } + } + + if ( !pBo ) + { + s_GbmLog.errorf( "GBM allocation failed for %ux%u format 0x%x", uWidth, uHeight, uDrmFormat ); + return nullptr; + } + + wlr_dmabuf_attributes attrs = + { + .width = int32_t( uWidth ), + .height = int32_t( uHeight ), + .format = uDrmFormat, + .modifier = bForceLinearModifier ? DRM_FORMAT_MOD_LINEAR : gbm_bo_get_modifier( pBo ), + }; + + const int nPlaneCount = gbm_bo_get_plane_count( pBo ); + bool bSuccess = nPlaneCount >= 1 && nPlaneCount <= WLR_DMABUF_MAX_PLANES; + + if ( bSuccess && !bForceLinearModifier && !Algorithm::Contains( ulModifiers, attrs.modifier ) ) + { + s_GbmLog.errorf( "GBM returned modifier 0x%" PRIx64 " which was not in the allowed list", attrs.modifier ); + bSuccess = false; + } + + // Only count a plane in n_planes once we own its fd, so that + // wlr_dmabuf_attributes_finish never closes an fd we didn't acquire. + for ( int i = 0; bSuccess && i < nPlaneCount; i++ ) + { + int nFd = gbm_bo_get_fd_for_plane( pBo, i ); + if ( nFd < 0 ) + { + s_GbmLog.errorf_errno( "gbm_bo_get_fd_for_plane failed" ); + bSuccess = false; + break; + } + attrs.fd[i] = nFd; + attrs.offset[i] = gbm_bo_get_offset( pBo, i ); + attrs.stride[i] = gbm_bo_get_stride_for_plane( pBo, i ); + attrs.n_planes = i + 1; + } + bSuccess = bSuccess && attrs.n_planes == nPlaneCount; + + // Vulkan's dmabuf memory import is single-fd: all planes must + // reference the same underlying buffer. + for ( int i = 1; bSuccess && i < attrs.n_planes; i++ ) + { + struct stat first, other; + if ( fstat( attrs.fd[0], &first ) != 0 || fstat( attrs.fd[i], &other ) != 0 || + first.st_dev != other.st_dev || first.st_ino != other.st_ino ) + { + s_GbmLog.errorf( "GBM returned a disjoint multi-plane bo, cannot import" ); + bSuccess = false; + } + } + + if ( !bSuccess ) + { + wlr_dmabuf_attributes_finish( &attrs ); + gbm_bo_destroy( pBo ); + return nullptr; + } + + s_GbmLog.debugf( "allocated %ux%u format 0x%x modifier 0x%" PRIx64 " (%d planes)", + uWidth, uHeight, uDrmFormat, attrs.modifier, attrs.n_planes ); + + return std::make_shared( pDevice, pBo, attrs ); + } +} diff --git a/src/Backends/GbmScanoutBuffer.h b/src/Backends/GbmScanoutBuffer.h new file mode 100644 index 0000000..9dbf3b2 --- /dev/null +++ b/src/Backends/GbmScanoutBuffer.h @@ -0,0 +1,20 @@ +#pragma once + +#include "backend.h" + +#include +#include +#include + +struct gbm_device; + +namespace gamescope +{ + // Creates a GBM device on a dup of nFd. The device owns the dup, so buffers + // that hold the device keep working after the caller closes nFd. + std::shared_ptr CreateGbmDevice( int nFd ); + + // Allocates a scanout-capable buffer with GBM_BO_USE_SCANOUT. ulModifiers + // must not contain DRM_FORMAT_MOD_INVALID; returns nullptr on failure. + std::shared_ptr CreateGbmScanoutBuffer( const std::shared_ptr &pDevice, uint32_t uWidth, uint32_t uHeight, uint32_t uDrmFormat, std::span ulModifiers, bool bLinear ); +} diff --git a/src/backend.h b/src/backend.h index bd57c42..12e4361 100644 --- a/src/backend.h +++ b/src/backend.h @@ -290,6 +290,17 @@ namespace gamescope virtual ~IBackendPlane() = default; }; + // A scanout-capable buffer allocated by the backend rather than by Vulkan + // (GBM on the DRM backend). Owns the underlying allocation and the fds + // inside its wlr_dmabuf_attributes; callers must dup() any fds they keep. + class IBackendScanoutBuffer + { + public: + virtual ~IBackendScanoutBuffer() {} + + virtual const wlr_dmabuf_attributes *GetDmabufAttributes() const = 0; + }; + class CBaseBackendFb : public IBackendFb { public: @@ -341,6 +352,19 @@ namespace gamescope // Rc manages acquire/release of buffer to/from client while imported. virtual OwningRc ImportDmabufToBackend( wlr_dmabuf_attributes *pDmaBuf ) = 0; + // Whether the backend can allocate scanout-capable buffers itself + // (GBM on the DRM backend) for the renderer to import, instead of the + // renderer allocating through Vulkan and exporting. + virtual bool SupportsExternalScanoutBuffers() const { return false; } + // ulModifiers: final allowed modifier list -- already intersected by the + // caller across KMS support ( GetSupportedModifiers ) and Vulkan + // importability for every format that will alias this buffer. + // Never contains DRM_FORMAT_MOD_INVALID. + virtual std::shared_ptr CreateScanoutBuffer( uint32_t uWidth, uint32_t uHeight, uint32_t uDrmFormat, std::span ulModifiers, bool bLinear ) + { + return nullptr; + } + virtual bool UsesModifiers() const = 0; virtual std::span GetSupportedModifiers( uint32_t uDrmFormat ) const = 0; inline bool SupportsFormat( uint32_t uDrmFormat ) const diff --git a/src/meson.build b/src/meson.build index ae3ac16..12fe4ae 100644 --- a/src/meson.build +++ b/src/meson.build @@ -11,6 +11,8 @@ dep_xmu = dependency('xmu') dep_xi = dependency('xi') drm_dep = dependency('libdrm', version: '>= 2.4.113', required: get_option('drm_backend')) +# gbm_bo_create_with_modifiers2 needs Mesa >= 21.3 +gbm_dep = dependency('gbm', version: '>= 21.3', required: false) eis_dep = dependency('libeis-1.0', required : get_option('input_emulation')) wayland_server = dependency('wayland-server', version: '>=1.21') @@ -149,6 +151,11 @@ if sdl2_dep.found() endif gamescope_cpp_args += '-DHAVE_DRM=@0@'.format(drm_dep.found().to_int()) +have_gbm = drm_dep.found() and gbm_dep.found() +if have_gbm + src += 'Backends/GbmScanoutBuffer.cpp' +endif +gamescope_cpp_args += '-DHAVE_GBM=@0@'.format(have_gbm.to_int()) gamescope_cpp_args += '-DHAVE_SDL2=@0@'.format(sdl2_dep.found().to_int()) gamescope_cpp_args += '-DHAVE_AVIF=@0@'.format(avif_dep.found().to_int()) gamescope_cpp_args += '-DHAVE_LIBCAP=@0@'.format(cap_dep.found().to_int()) @@ -196,7 +203,7 @@ gamescope_version = configure_file( include_directories : [reshade_include, sol2_include], dependencies: [ dep_wayland, dep_x11, dep_xdamage, dep_xcomposite, dep_xrender, dep_xext, dep_xfixes, - dep_xxf86vm, dep_xres, glm_dep, drm_dep, wayland_server, + dep_xxf86vm, dep_xres, glm_dep, drm_dep, gbm_dep, wayland_server, xkbcommon, thread_dep, sdl2_dep, wlroots_dep, vulkan_dep, liftoff_dep, dep_xtst, dep_xmu, cap_dep, epoll_dep, pipewire_dep, librt_dep, stb_dep, displayinfo_dep, openvr_dep, dep_xcursor, avif_dep, dep_xi, diff --git a/src/rendervulkan.cpp b/src/rendervulkan.cpp index 54e4efc..cd5769c 100644 --- a/src/rendervulkan.cpp +++ b/src/rendervulkan.cpp @@ -2080,44 +2080,109 @@ static VkImageViewType VulkanImageTypeToViewType(VkImageType type) } } -bool CVulkanTexture::BInit( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA /* = nullptr */, uint32_t contentWidth /* = 0 */, uint32_t contentHeight /* = 0 */, CVulkanTexture *pExistingImageToReuseMemory, gamescope::OwningRc pBackendFb ) +static VkImageUsageFlags VulkanUsageFromCreateFlags( const CVulkanTexture::createFlags &flags ) { - m_pBackendFb = std::move( pBackendFb ); - m_drmFormat = drmFormat; - VkResult res = VK_ERROR_INITIALIZATION_FAILED; - - VkImageTiling tiling = (flags.bMappable || flags.bLinear) ? VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; VkImageUsageFlags usage = 0; - VkMemoryPropertyFlags properties; if ( flags.bSampled == true ) - { usage |= VK_IMAGE_USAGE_SAMPLED_BIT; - } if ( flags.bStorage == true ) - { usage |= VK_IMAGE_USAGE_STORAGE_BIT; - } if ( flags.bColorAttachment == true ) - { usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - } - if ( flags.bFlippable == true ) + if ( flags.bTransferSrc == true ) + usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + + if ( flags.bTransferDst == true ) + usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; + + return usage; +} + +static bool BSupportsExternalScanoutModifier( uint32_t drmFormat, VkImageType imageType, VkImageUsageFlags usage, uint64_t ulModifier ); +static std::vector GetExternalScanoutModifiers( std::span drmFormats, const CVulkanTexture::createFlags &flags ); + +bool CVulkanTexture::BInit( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA /* = nullptr */, uint32_t contentWidth /* = 0 */, uint32_t contentHeight /* = 0 */, CVulkanTexture *pExistingImageToReuseMemory, gamescope::OwningRc pBackendFb, std::shared_ptr pScanoutBuffer ) +{ + // A retry only makes sense when the external scanout buffer was chosen + // internally, not handed in by the caller (who then owns the fallback). + const bool bCanRetryWithoutExternal = pDMA == nullptr && pScanoutBuffer == nullptr; + + if ( BInitInternal( width, height, depth, drmFormat, flags, pDMA, contentWidth, contentHeight, pExistingImageToReuseMemory, std::move( pBackendFb ), std::move( pScanoutBuffer ), true ) ) + return true; + + if ( bCanRetryWithoutExternal && m_pScanoutBuffer != nullptr ) { - flags.bExportable = true; + vk_log.errorf( "Failed to init texture from external scanout buffer, retrying with Vulkan allocation" ); + ReleaseResources(); + return BInitInternal( width, height, depth, drmFormat, flags, nullptr, contentWidth, contentHeight, pExistingImageToReuseMemory, nullptr, nullptr, false ); } - if ( flags.bTransferSrc == true ) - { - usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + return false; +} + +bool CVulkanTexture::BInitInternal( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA, uint32_t contentWidth, uint32_t contentHeight, CVulkanTexture *pExistingImageToReuseMemory, gamescope::OwningRc pBackendFb, std::shared_ptr pScanoutBuffer, bool bAllowExternalScanout ) +{ + m_pBackendFb = std::move( pBackendFb ); + m_pScanoutBuffer = std::move( pScanoutBuffer ); + m_drmFormat = drmFormat; + VkResult res = VK_ERROR_INITIALIZATION_FAILED; + + VkImageTiling tiling = (flags.bMappable || flags.bLinear) ? VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; + VkImageUsageFlags usage = VulkanUsageFromCreateFlags( flags ); + VkMemoryPropertyFlags properties; + + // Flippable images are allocated by the backend where supported (GBM on + // the DRM backend): scanout memory needs placement only the backend's + // allocator can guarantee (physically contiguous vidmem on NVIDIA). + // Output images alias two formats in one buffer and are handled by + // vulkan_make_output_images_external instead. + wlr_dmabuf_attributes externalScanoutAttrs = {}; + if ( bAllowExternalScanout && + flags.bFlippable && pDMA == nullptr && m_pScanoutBuffer == nullptr && + m_pBackendFb == nullptr && + !flags.bOutputImage && !flags.bMappable && pExistingImageToReuseMemory == nullptr && + flags.imageType == VK_IMAGE_TYPE_2D && depth == 1 && + GetBackend()->SupportsExternalScanoutBuffers() && g_device.supportsModifiers() ) + { + std::vector modifiers; + if ( flags.bLinear ) + { + // Only LINEAR is acceptable here, so probe just that instead of + // enumerating every KMS modifier. + if ( gamescope::Algorithm::Contains( GetBackend()->GetSupportedModifiers( drmFormat ), uint64_t( DRM_FORMAT_MOD_LINEAR ) ) && + BSupportsExternalScanoutModifier( drmFormat, flags.imageType, usage, DRM_FORMAT_MOD_LINEAR ) ) + modifiers = { DRM_FORMAT_MOD_LINEAR }; + } + else + { + const uint32_t uFormats[1] = { drmFormat }; + modifiers = GetExternalScanoutModifiers( uFormats, flags ); + } + + if ( !modifiers.empty() ) + m_pScanoutBuffer = GetBackend()->CreateScanoutBuffer( width, height, drmFormat, modifiers, flags.bLinear ); + + if ( m_pScanoutBuffer ) + { + externalScanoutAttrs = *m_pScanoutBuffer->GetDmabufAttributes(); + pDMA = &externalScanoutAttrs; + } + else + { + vk_log.debugf( "External scanout buffer unavailable for format 0x%x, using Vulkan allocation", drmFormat ); + } } - if ( flags.bTransferDst == true ) + if ( flags.bFlippable == true && pDMA == nullptr ) { - usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; + // Flippable images we allocate ourselves get exported to the backend. + // A flippable image created from an existing dmabuf (eg. a + // backend-allocated scanout buffer) is imported instead. + flags.bExportable = true; } if ( flags.bMappable == true ) @@ -2320,7 +2385,8 @@ bool CVulkanTexture::BInit( uint32_t width, uint32_t height, uint32_t depth, uin vk_errorf( res, "vkCreateImage failed" ); return false; } - + m_bOwnsImage = true; + VkMemoryRequirements memRequirements; g_device.vk.GetImageMemoryRequirements(g_device.device(), m_vkImage, &memRequirements); @@ -2383,6 +2449,37 @@ bool CVulkanTexture::BInit( uint32_t width, uint32_t height, uint32_t depth, uin return false; } + // A dmabuf import must pick a memory type from the mask + // vkGetMemoryFdPropertiesKHR reports for the FD, intersected + // with the image's memory requirements - not just any type + // satisfying the requirements. + VkMemoryFdPropertiesKHR fdProps = { + .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR, + }; + res = g_device.vk.GetMemoryFdPropertiesKHR( g_device.device(), VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fdProps ); + if ( res != VK_SUCCESS ) + { + vk_errorf( res, "vkGetMemoryFdPropertiesKHR failed" ); + close( fd ); + return false; + } + + const uint32_t uImportTypeBits = memRequirements.memoryTypeBits & fdProps.memoryTypeBits; + int32_t nImportMemoryTypeIndex = g_device.findMemoryType( properties, uImportTypeBits ); + if ( nImportMemoryTypeIndex < 0 ) + { + // Fall back to any permitted type; a valid import beats the + // preferred property flags. + nImportMemoryTypeIndex = g_device.findMemoryType( 0, uImportTypeBits ); + } + if ( nImportMemoryTypeIndex < 0 ) + { + vk_log.errorf( "No compatible memory type for dmabuf import" ); + close( fd ); + return false; + } + allocInfo.memoryTypeIndex = uint32_t( nImportMemoryTypeIndex ); + // Memory already provided by pDMA importMemoryInfo = { .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR, @@ -2547,13 +2644,43 @@ bool CVulkanTexture::BInit( uint32_t width, uint32_t height, uint32_t depth, uin m_dmabuf = dmabuf; } + else if ( pDMA != nullptr && flags.bFlippable == true ) + { + // Flippable image backed by an external (backend-allocated) dmabuf: + // keep our own copy of the attributes for pipewire/screenshots and + // for fbid creation. We own these fds; they are closed by + // wlr_dmabuf_attributes_finish in ~CVulkanTexture. + m_dmabuf = *pDMA; + m_dmabuf.n_planes = 0; + for ( int i = 0; i < pDMA->n_planes; i++ ) + { + m_dmabuf.fd[i] = dup( pDMA->fd[i] ); + if ( m_dmabuf.fd[i] < 0 ) + { + vk_log.errorf_errno( "dup failed" ); + return false; + } + m_dmabuf.n_planes = i + 1; + } + } - if ( flags.bFlippable == true ) + if ( flags.bFlippable == true && m_pBackendFb == nullptr ) { m_pBackendFb = GetBackend()->ImportDmabufToBackend( &m_dmabuf ); + if ( pDMA != nullptr && m_pBackendFb == nullptr ) + { + // For external scanout buffers the whole point is scanning out; + // fail so the caller can fall back to Vulkan allocation. + vk_log.errorf( "Failed to import external scanout buffer to backend" ); + return false; + } } - bool bHasAlpha = pDMA ? DRMFormatHasAlpha( pDMA->format ) : true; + // Self-allocated scanout buffers (m_pScanoutBuffer) must behave exactly like + // the export path above: X-channel formats map to alpha-bearing VkFormats + // with identity swizzle, and storage usage stays allowed. Only genuinely + // client-imported buffers take the SWIZZLE_ONE/no-storage handling. + bool bHasAlpha = ( pDMA && !m_pScanoutBuffer ) ? DRMFormatHasAlpha( pDMA->format ) : true; if (!bHasAlpha ) { @@ -2742,9 +2869,10 @@ CVulkanTexture::CVulkanTexture( void ) { } -CVulkanTexture::~CVulkanTexture( void ) +void CVulkanTexture::ReleaseResources() { wlr_dmabuf_attributes_finish( &m_dmabuf ); + m_dmabuf = {}; if ( m_pMappedData != nullptr && m_vkImageMemory ) { @@ -2764,24 +2892,47 @@ CVulkanTexture::~CVulkanTexture( void ) m_linearView = VK_NULL_HANDLE; } + if ( m_lumaView != VK_NULL_HANDLE ) + { + g_device.vk.DestroyImageView( g_device.device(), m_lumaView, nullptr ); + m_lumaView = VK_NULL_HANDLE; + } + + if ( m_chromaView != VK_NULL_HANDLE ) + { + g_device.vk.DestroyImageView( g_device.device(), m_chromaView, nullptr ); + m_chromaView = VK_NULL_HANDLE; + } + if ( m_pBackendFb != nullptr ) m_pBackendFb = nullptr; - if ( m_vkImageMemory != VK_NULL_HANDLE ) + // Images we created are destroyed even when the memory isn't ours + // (aliased textures, or an init that failed before allocating memory). + // Swapchain-owned images are never destroyed here. + if ( m_vkImage != VK_NULL_HANDLE && m_bOwnsImage ) { - if ( m_vkImage != VK_NULL_HANDLE ) - { - g_device.vk.DestroyImage( g_device.device(), m_vkImage, nullptr ); - m_vkImage = VK_NULL_HANDLE; - } + g_device.vk.DestroyImage( g_device.device(), m_vkImage, nullptr ); + } + m_vkImage = VK_NULL_HANDLE; + m_bOwnsImage = false; + if ( m_vkImageMemory != VK_NULL_HANDLE ) + { g_device.vk.FreeMemory( g_device.device(), m_vkImageMemory, nullptr ); m_vkImageMemory = VK_NULL_HANDLE; } + m_pScanoutBuffer = nullptr; + m_bInitialized = false; } +CVulkanTexture::~CVulkanTexture( void ) +{ + ReleaseResources(); +} + int CVulkanTexture::memoryFence() { const VkMemoryGetFdInfoKHR memory_get_fd_info = { @@ -3341,6 +3492,139 @@ bool vulkan_remake_swapchain( void ) return bRet; } +// Whether Vulkan can create + import a dmabuf image with this +// format/modifier and usage. Uses the same srgb/linear mutable-format view +// pair the real image in BInitInternal is created with. +static bool BSupportsExternalScanoutModifier( uint32_t drmFormat, VkImageType imageType, VkImageUsageFlags usage, uint64_t ulModifier ) +{ + VkImageCreateInfo imageInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .imageType = imageType, + .format = DRMFormatToVulkan( drmFormat, false ), + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, + .usage = usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + }; + + std::array formats = { + DRMFormatToVulkan( drmFormat, false ), + DRMFormatToVulkan( drmFormat, true ), + }; + + VkImageFormatListCreateInfo formatList = { + .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO, + .viewFormatCount = (uint32_t)formats.size(), + .pViewFormats = formats.data(), + }; + + if ( formats[0] != formats[1] ) + { + formatList.pNext = std::exchange( imageInfo.pNext, &formatList ); + imageInfo.flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; + } + + VkExternalImageFormatProperties externalImageProperties = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES, + }; + + VkResult res = getModifierProps( &imageInfo, ulModifier, &externalImageProperties ); + return res == VK_SUCCESS && + !!( externalImageProperties.externalMemoryProperties.externalMemoryFeatures & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT ); +} + +// Modifiers that (a) the backend can scan out for every format in drmFormats, +// and (b) Vulkan can create + import a dmabuf image with, using the usage +// implied by the given createFlags, for every format in drmFormats. +// This is the "plane ∩ renderer" intersection other compositors (eg. KWin) +// use when allocating scanout buffers through GBM, extended across all +// formats that will alias the same buffer. +static std::vector GetExternalScanoutModifiers( std::span drmFormats, const CVulkanTexture::createFlags &flags ) +{ + std::vector modifiers; + if ( drmFormats.empty() ) + return modifiers; + + const VkImageUsageFlags usage = VulkanUsageFromCreateFlags( flags ); + + for ( uint64_t modifier : GetBackend()->GetSupportedModifiers( drmFormats[0] ) ) + { + if ( modifier == DRM_FORMAT_MOD_INVALID ) + continue; + + bool bCompatible = true; + for ( uint32_t drmFormat : drmFormats ) + { + if ( !gamescope::Algorithm::Contains( GetBackend()->GetSupportedModifiers( drmFormat ), modifier ) || + !BSupportsExternalScanoutModifier( drmFormat, flags.imageType, usage, modifier ) ) + { + bCompatible = false; + break; + } + } + + if ( bCompatible ) + modifiers.push_back( modifier ); + } + + return modifiers; +} + +// Allocate the output images through the backend (GBM on DRM) and import +// them into Vulkan, instead of allocating with Vulkan and exporting. +// Fixes scanout on drivers that need special placement for scanout memory +// (NVIDIA needs physical contiguity, which only its GBM path guarantees). +static bool vulkan_make_output_images_external( VulkanOutput_t *pOutput, const CVulkanTexture::createFlags &outputImageflags, uint32_t uOutputWidth, uint32_t uOutputHeight ) +{ + uint32_t uFormats[2] = { pOutput->uOutputFormat, DRM_FORMAT_INVALID }; + size_t nFormats = 1; + + const bool bOverlay = pOutput->uOutputFormatOverlay != VK_FORMAT_UNDEFINED && !kDisablePartialComposition; + if ( bOverlay ) + uFormats[nFormats++] = pOutput->uOutputFormatOverlay; + + std::vector modifiers = GetExternalScanoutModifiers( std::span( uFormats, nFormats ), outputImageflags ); + if ( modifiers.empty() ) + { + vk_log.errorf( "No modifier is scanout-capable and Vulkan-importable for all output formats" ); + return false; + } + + for ( size_t i = 0; i < 3; i++ ) + { + std::shared_ptr pScanoutBuffer = + GetBackend()->CreateScanoutBuffer( uOutputWidth, uOutputHeight, pOutput->uOutputFormat, modifiers, false ); + if ( !pScanoutBuffer ) + return false; + + // BInit never takes ownership of these attrs' fds; it dups what it keeps. + wlr_dmabuf_attributes attrs = *pScanoutBuffer->GetDmabufAttributes(); + + pOutput->outputImages[i] = new CVulkanTexture(); + if ( !pOutput->outputImages[i]->BInit( uOutputWidth, uOutputHeight, 1u, pOutput->uOutputFormat, outputImageflags, &attrs, 0, 0, nullptr, nullptr, pScanoutBuffer ) ) + return false; + + if ( bOverlay ) + { + // The overlay image is a second import of the same buffer with + // the overlay format: identical modifier and plane layout + // (both formats are 32bpp, so strides match), its own fbid, and + // its own kernel reference on the underlying memory. Both + // textures share the IBackendScanoutBuffer. + attrs = *pScanoutBuffer->GetDmabufAttributes(); + attrs.format = pOutput->uOutputFormatOverlay; + + pOutput->outputImagesPartialOverlay[i] = new CVulkanTexture(); + if ( !pOutput->outputImagesPartialOverlay[i]->BInit( uOutputWidth, uOutputHeight, 1u, pOutput->uOutputFormatOverlay, outputImageflags, &attrs, 0, 0, nullptr, nullptr, pScanoutBuffer ) ) + return false; + } + } + + return true; +} + static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) { CVulkanTexture::createFlags outputImageflags; @@ -3360,14 +3644,30 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) pOutput->outputImagesPartialOverlay[1] = nullptr; pOutput->outputImagesPartialOverlay[2] = nullptr; - uint32_t uDRMFormat = pOutput->uOutputFormat; - // Output images are physical-oriented; the panel scans out unrotated. uint32_t uOutputWidth = g_nOutputWidth; uint32_t uOutputHeight = g_nOutputHeight; if ( g_uOutputRotation & 1u ) std::swap( uOutputWidth, uOutputHeight ); + if ( GetBackend()->SupportsExternalScanoutBuffers() && g_device.supportsModifiers() ) + { + if ( vulkan_make_output_images_external( pOutput, outputImageflags, uOutputWidth, uOutputHeight ) ) + { + // Oh no. + pOutput->temporaryHackyBlankImage = vulkan_create_debug_blank_texture(); + return true; + } + + vk_log.errorf( "Failed to create backend (GBM) scanout buffers, falling back to Vulkan allocation" ); + for ( auto &pImage : pOutput->outputImages ) + pImage = nullptr; + for ( auto &pImage : pOutput->outputImagesPartialOverlay ) + pImage = nullptr; + } + + uint32_t uDRMFormat = pOutput->uOutputFormat; + pOutput->outputImages[0] = new CVulkanTexture(); bool bSuccess = pOutput->outputImages[0]->BInit( uOutputWidth, uOutputHeight, 1u, uDRMFormat, outputImageflags ); if ( bSuccess != true ) diff --git a/src/rendervulkan.hpp b/src/rendervulkan.hpp index fa519db..4f0bff0 100644 --- a/src/rendervulkan.hpp +++ b/src/rendervulkan.hpp @@ -161,7 +161,7 @@ public: VkImageType imageType; }; - bool BInit( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA = nullptr, uint32_t contentWidth = 0, uint32_t contentHeight = 0, CVulkanTexture *pExistingImageToReuseMemory = nullptr, gamescope::OwningRc pBackendFb = nullptr ); + bool BInit( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA = nullptr, uint32_t contentWidth = 0, uint32_t contentHeight = 0, CVulkanTexture *pExistingImageToReuseMemory = nullptr, gamescope::OwningRc pBackendFb = nullptr, std::shared_ptr pScanoutBuffer = nullptr ); bool BInitFromSwapchain( VkImage image, uint32_t width, uint32_t height, VkFormat format ); uint32_t IncRef(); @@ -211,9 +211,17 @@ public: uint32_t queueFamily = VK_QUEUE_FAMILY_IGNORED; private: + bool BInitInternal( uint32_t width, uint32_t height, uint32_t depth, uint32_t drmFormat, createFlags flags, wlr_dmabuf_attributes *pDMA, uint32_t contentWidth, uint32_t contentHeight, CVulkanTexture *pExistingImageToReuseMemory, gamescope::OwningRc pBackendFb, std::shared_ptr pScanoutBuffer, bool bAllowExternalScanout ); + // Frees everything BInitInternal may have created, so a failed init can + // be retried. Also used by the destructor. + void ReleaseResources(); + bool m_bInitialized = false; bool m_bExternal = false; bool m_bOutputImage = false; + // False for swapchain images: those are owned by the swapchain and must + // never be passed to vkDestroyImage. + bool m_bOwnsImage = false; uint32_t m_drmFormat = DRM_FORMAT_INVALID; @@ -251,6 +259,12 @@ private: EStreamColorspace m_streamColorspace = k_EStreamColorspace_Unknown; struct wlr_dmabuf_attributes m_dmabuf = {}; + + // Set when the underlying memory was allocated by the backend (GBM on + // the DRM backend) and imported into Vulkan, rather than allocated by + // Vulkan. Keeps the backend allocation alive for the texture's lifetime; + // aliasing textures (eg. the partial-overlay output images) share it. + std::shared_ptr m_pScanoutBuffer; }; struct vec2_t @@ -788,6 +802,7 @@ static inline uint32_t div_roundup(uint32_t x, uint32_t y) VK_FUNC(GetImageMemoryRequirements) \ VK_FUNC(GetImageSubresourceLayout) \ VK_FUNC(GetMemoryFdKHR) \ + VK_FUNC(GetMemoryFdPropertiesKHR) \ VK_FUNC(GetQueryPoolResults) \ VK_FUNC(GetSemaphoreCounterValue) \ VK_FUNC(GetSwapchainImagesKHR) \ diff --git a/src/steamcompmgr.cpp b/src/steamcompmgr.cpp index 2ff296b..2cb5eb5 100644 --- a/src/steamcompmgr.cpp +++ b/src/steamcompmgr.cpp @@ -161,6 +161,7 @@ static lut3d_t g_tmpLut3d; extern int g_nDynamicRefreshHz; bool g_bForceHDRSupportDebug = false; +std::atomic g_bForceOutputImageRemake = { false }; extern float g_flInternalDisplayBrightnessNits; extern float g_flHDRItmSdrNits; extern float g_flHDRItmTargetNits; @@ -10255,13 +10256,21 @@ steamcompmgr_main(int argc, char **argv) // Pick our width/height for this potential frame, regardless of how it might change later // At some point we might even add proper locking so we get real updates atomically instead // of whatever jumble of races the below might cause over a couple of frames + // Consume the remake request exactly once, before evaluating the other + // conditions: a concurrent request (eg. the drm_gbm_scanout convar + // callback on the Wayland thread) must never be cleared unseen, only + // exchanged out - a write racing past this point stays set for the + // next iteration. + const bool bForceOutputImageRemake = g_bForceOutputImageRemake.exchange( false ); + if ( currentOutputWidth != g_nOutputWidth || currentOutputHeight != g_nOutputHeight || currentOutputRefresh != g_nOutputRefresh || currentOutputRotation != g_uOutputRotation || currentHDROutput != g_bOutputHDREnabled || currentHDRCapable != bOutputHDRCapable || - currentHDRForce != g_bForceHDRSupportDebug ) + currentHDRForce != g_bForceHDRSupportDebug || + bForceOutputImageRemake ) { if ( g_nXWaylandCount > 1 ) { diff --git a/src/steamcompmgr.hpp b/src/steamcompmgr.hpp index c1d1ab0..fc7a521 100644 --- a/src/steamcompmgr.hpp +++ b/src/steamcompmgr.hpp @@ -22,6 +22,7 @@ void steamcompmgr_main(int argc, char **argv); #include "vblankmanager.hpp" #include "mangoapp_control.h" +#include #include #include #include @@ -45,6 +46,10 @@ static const uint32_t g_zposMuraCorrection = 5; extern bool g_bHDRItmEnable; extern bool g_bForceHDRSupportDebug; +// Request a rebuild of the output images on the next steamcompmgr iteration, +// eg. after toggling scanout buffer allocation strategy at runtime. +extern std::atomic g_bForceOutputImageRemake; + extern EStreamColorspace g_ForcedNV12ColorSpace; struct CursorBarrierInfo