xwayland: fix crash when trying to initialize without Xwayland installed (#9077)

src/xwayland/XWayland.cpp

@@ -1,21 +1,39 @@
 #include "XWayland.hpp"
+#include "../Compositor.hpp"
 #include "../debug/Log.hpp"
+#include "../helpers/fs/FsUtils.hpp"
 
-CXWayland::CXWayland(const bool enabled) {
+CXWayland::CXWayland(const bool wantsEnabled) {
 #ifndef NO_XWAYLAND
+    // Disable Xwayland and clean up if the user disabled it.
+    if (!wantsEnabled) {
+        Debug::log(LOG, "XWayland has been disabled, cleaning up...");
+        for (auto& w : g_pCompositor->m_vWindows) {
+            if (!w->m_bIsX11)
+                continue;
+            g_pCompositor->closeWindow(w);
+        }
+        unsetenv("DISPLAY");
+        m_enabled = false;
+        return;
+    }
+
+    if (!NFsUtils::executableExistsInPath("Xwayland")) {
+        // If Xwayland doesn't exist, don't try to start it.
+        Debug::log(LOG, "Unable to find XWayland; not starting it.");
+        return;
+    }
+
     Debug::log(LOG, "Starting up the XWayland server");
 
     pServer = std::make_unique<CXWaylandServer>();
 
-    if (!enabled) {
-        unsetenv("DISPLAY");
-        return;
-    }
-
     if (!pServer->create()) {
         Debug::log(ERR, "XWayland failed to start: it will not work.");
         return;
     }
+
+    m_enabled = true;
 #else
     Debug::log(LOG, "Not starting XWayland: disabled at compile time");
 #endif
@@ -31,3 +49,7 @@ void CXWayland::setCursor(unsigned char* pixData, uint32_t stride, const Vector2
     pWM->setCursor(pixData, stride, size, hotspot);
 #endif
 }
+
+bool CXWayland::enabled() {
+    return m_enabled;
+}

src/xwayland/XWayland.hpp

@@ -17,18 +17,22 @@ class CXWM;
 
 class CXWayland {
   public:
-    CXWayland(const bool enabled);
+    CXWayland(const bool wantsEnabled);
 
 #ifndef NO_XWAYLAND
     std::unique_ptr<CXWaylandServer> pServer;
     std::unique_ptr<CXWM>            pWM;
 #endif
+    bool enabled();
 
     void setCursor(unsigned char* pixData, uint32_t stride, const Vector2D& size, const Vector2D& hotspot);
 
     struct {
         CSignal newSurface;
     } events;
+
+  private:
+    bool m_enabled = false;
 };
 
 inline std::unique_ptr<CXWayland>                g_pXWayland;