fix(linux/wlr): resilient GBM allocation for headless NVIDIA

GBM buffer allocation with GBM_BO_USE_RENDERING|GBM_BO_USE_LINEAR
fails on headless NVIDIA render nodes. Try progressively relaxed
flag combinations before falling back to SHM.

Also prefer DRM render nodes over primary nodes in CUDA device
lookup — primary nodes require DRM master which is unavailable
on headless setups.

Tested on RTX 4060 Ti headless (TrueNAS, no monitor):
- GBM succeeds with GBM_BO_USE_RENDERING flag alone
- VRAM capture path works (zero-copy DMA-BUF → EGL → CUDA → NVENC)
- SHM fallback still catches cases where all GBM combos fail

Author: atassis

src/platform/linux/cuda.cpp

@@ -248,19 +248,36 @@ namespace cuda {
       return std::tolower(c);
     });
 
-    // Look for the name of the primary node in sysfs
+    // Look for DRM nodes in sysfs — prefer render node (doesn't require DRM master)
     try {
       char sysfs_path[PATH_MAX];
       std::snprintf(sysfs_path, sizeof(sysfs_path), "/sys/bus/pci/devices/%s/drm", pci_bus_id.data());
       fs::path sysfs_dir {sysfs_path};
+
+      // First pass: look for render node (renderD*)
+      for (auto &entry : fs::directory_iterator {sysfs_dir}) {
+        auto file = entry.path().filename();
+        auto filestring = file.generic_string();
+        if (std::string_view {filestring}.substr(0, 7) != "renderD"sv) {
+          continue;
+        }
+
+        BOOST_LOG(debug) << "Found DRM render node: "sv << filestring;
+
+        fs::path dri_path {"/dev/dri"sv};
+        auto device_path = dri_path / file;
+        return open(device_path.c_str(), O_RDWR);
+      }
+
+      // Second pass: fallback to primary node (card*)
       for (auto &entry : fs::directory_iterator {sysfs_dir}) {
         auto file = entry.path().filename();
         auto filestring = file.generic_string();
         if (std::string_view {filestring}.substr(0, 4) != "card"sv) {
           continue;
         }
 
-        BOOST_LOG(debug) << "Found DRM primary node: "sv << filestring;
+        BOOST_LOG(debug) << "Found DRM primary node (fallback): "sv << filestring;
 
         fs::path dri_path {"/dev/dri"sv};
         auto device_path = dri_path / file;

src/platform/linux/wayland.cpp

@@ -485,9 +485,36 @@ namespace wl {
       return false;
     }
 
-    current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR);
+    // Try progressively relaxed GBM usage flags.
+    // NVIDIA GBM on headless render nodes may not support all combinations.
+    static const struct {
+      uint32_t flags;
+      const char *desc;
+    } flag_combos[] = {
+      { GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR, "RENDERING|LINEAR" },
+      { GBM_BO_USE_RENDERING, "RENDERING" },
+      { GBM_BO_USE_LINEAR, "LINEAR" },
+      { 0, "none" },
+    };
+
+    for (const auto &combo : flag_combos) {
+      current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, combo.flags);
+      if (current_bo) {
+        static bool flags_logged = false;
+        if (!flags_logged) {
+          BOOST_LOG(info) << "[wayland] GBM buffer created with flags: "sv << combo.desc
+                          << " format=0x"sv << std::hex << dmabuf_info.format << std::dec
+                          << " "sv << dmabuf_info.width << "x"sv << dmabuf_info.height;
+          flags_logged = true;
+        }
+        break;
+      }
+    }
+
     if (!current_bo) {
-      BOOST_LOG(error) << "Failed to create GBM buffer"sv;
+      BOOST_LOG(error) << "Failed to create GBM buffer with any flag combination"sv
+                       << " format=0x"sv << std::hex << dmabuf_info.format << std::dec
+                       << " "sv << dmabuf_info.width << "x"sv << dmabuf_info.height;
       return false;
     }