drm: conform to both renderable and scanoutable formats in scanout bu…

…ffers fixes #28
hyprwm · Jul 24, 2024 · 353dc1b · 353dc1b
1 parent 744a383
commit 353dc1b
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 4 deletions.
diff --git a/src/allocator/GBM.cpp b/src/allocator/GBM.cpp
@@ -68,9 +68,8 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
                                   std::format("GBM: Allocating a buffer: size {}, format {}, cursor: {}, multigpu: {}, scanout: {}", attrs.size, fourccToName(attrs.format), CURSOR,
                                               MULTIGPU, params.scanout)));
 
-    const auto            FORMATS = CURSOR ?
-                   swapchain->backendImpl->getCursorFormats() :
-                   (swapchain->backendImpl->getRenderableFormats().size() == 0 ? swapchain->backendImpl->getRenderFormats() : swapchain->backendImpl->getRenderableFormats());
+    const auto            FORMATS    = CURSOR ? swapchain->backendImpl->getCursorFormats() : swapchain->backendImpl->getRenderFormats();
+    const auto            RENDERABLE = swapchain->backendImpl->getRenderableFormats();
 
     std::vector<uint64_t> explicitModifiers;
 
@@ -95,6 +94,21 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
             if (m == DRM_FORMAT_MOD_INVALID)
                 continue;
 
+            if (!RENDERABLE.empty() && params.scanout && !CURSOR && !MULTIGPU) {
+                // regular scanout plane, check if the format is renderable
+                auto rformat = std::find_if(RENDERABLE.begin(), RENDERABLE.end(), [f](const auto& e) { return e.drmFormat == f.drmFormat; });
+
+                if (rformat == RENDERABLE.end()) {
+                    TRACE(allocator->backend->log(AQ_LOG_TRACE, std::format("GBM: Dropping format {} as it's not renderable", fourccToName(f.drmFormat))));
+                    break;
+                }
+
+                if (std::find(rformat->modifiers.begin(), rformat->modifiers.end(), m) == rformat->modifiers.end()) {
+                    TRACE(allocator->backend->log(AQ_LOG_TRACE, std::format("GBM: Dropping modifier 0x{:x} as it's not renderable", m)));
+                    continue;
+                }
+            }
+
             explicitModifiers.push_back(m);
         }
     }

diff --git a/src/backend/drm/DRM.cpp b/src/backend/drm/DRM.cpp
@@ -507,7 +507,7 @@ void Aquamarine::CDRMBackend::buildGlFormats(const std::vector<SGLFormat>& fmts)
         if (fmt.external)
             continue;
 
-        if (auto it = std::find_if(result.begin(), result.end(), [fmt] (const auto& e) { return fmt.drmFormat == e.drmFormat; }); it != result.end()) {
+        if (auto it = std::find_if(result.begin(), result.end(), [fmt](const auto& e) { return fmt.drmFormat == e.drmFormat; }); it != result.end()) {
             it->modifiers.emplace_back(fmt.modifier);
             continue;
         }