desktop/window: fix floating windows being auto-grouped (#13475)

---------

Co-authored-by: Aqa-Ib <16420574+Aqa-Ib@users.noreply.github.com>

hyprtester/src/tests/clients/child-window.cpp

@@ -118,6 +118,33 @@ static bool test() {
     Tests::killAllWindows();
     EXPECT(Tests::windowCount(), 0);
 
+    // test that child windows (shouldBeFloated) are not auto-grouped
+    NLog::log("{}Test child windows are not auto-grouped", Colors::GREEN);
+    auto kitty = Tests::spawnKitty();
+    if (!kitty) {
+        NLog::log("{}Error: kitty did not spawn", Colors::RED);
+        return false;
+    }
+
+    // create group and enable auto-grouping
+    OK(getFromSocket("/dispatch togglegroup"));
+    OK(getFromSocket("/keyword group:auto_group true"));
+
+    SClient client2;
+    if (!startClient(client2))
+        return false;
+
+    EXPECT(Tests::windowCount(), 2);
+    createChild(client2);
+    EXPECT(Tests::windowCount(), 3);
+
+    // child has set_parent so shouldBeFloated returns true, it should not be auto-grouped
+    EXPECT_COUNT_STRING(getFromSocket("/clients"), "grouped: 0", 1);
+
+    stopClient(client2);
+    Tests::killAllWindows();
+    EXPECT(Tests::windowCount(), 0);
+
     return !ret;
 }
 

hyprtester/src/tests/main/groups.cpp

@@ -254,6 +254,46 @@ static bool test() {
         EXPECT(Tests::windowCount(), 0);
     }
 
+    // test that we deny a floated window getting auto-grouped into a tiled group.
+    NLog::log("{}Test that we deny a floated window getting auto-grouped into a tiled group.", Colors::GREEN);
+
+    OK(getFromSocket("/keyword windowrule[kitty-tiled]:match:class kitty_tiled"));
+    OK(getFromSocket("/keyword windowrule[kitty-tiled]:tile yes"));
+    auto kittyProcE = Tests::spawnKitty("kitty_tiled");
+    if (!kittyProcE) {
+        NLog::log("{}Error: kitty did not spawn", Colors::RED);
+        return false;
+    }
+    OK(getFromSocket("/dispatch togglegroup"));
+
+    OK(getFromSocket("/keyword windowrule[kitty-floated]:match:class kitty_floated"));
+    OK(getFromSocket("/keyword windowrule[kitty-floated]:float yes"));
+    auto kittyProcF = Tests::spawnKitty("kitty_floated");
+    if (!kittyProcF) {
+        NLog::log("{}Error: kitty did not spawn", Colors::RED);
+        return false;
+    }
+
+    EXPECT(Tests::windowCount(), 2);
+
+    {
+        auto clients  = getFromSocket("/clients");
+        auto classPos = clients.find("class: kitty_floated");
+        if (classPos == std::string::npos) {
+            NLog::log("{}Could not find kitty_floated in clients output", Colors::RED);
+            ret = 1;
+        } else {
+            auto entryStart  = clients.rfind("Window ", classPos);
+            auto entryEnd    = clients.find("\n\n", classPos);
+            auto windowEntry = clients.substr(entryStart, entryEnd - entryStart);
+            EXPECT_CONTAINS(windowEntry, "floating: 1");
+            EXPECT_CONTAINS(windowEntry, "grouped: 0");
+        }
+    }
+
+    Tests::killAllWindows();
+    EXPECT(Tests::windowCount(), 0);
+
     return !ret;
 }
 

src/desktop/view/Window.cpp

@@ -1951,7 +1951,8 @@ void CWindow::mapWindow() {
         && Desktop::focusState()->window()                                                 // focused window exists
         && canBeGroupedInto(Desktop::focusState()->window()->m_group)                      // we can group
         && Desktop::focusState()->window()->m_workspace == m_workspace                     // workspaces match, we're not opening on another ws
-        && !isModal() && !(parent() && m_isFloating) && !isX11OverrideRedirect() // not a modal, floating child or X11 OR
+        && !g_pXWaylandManager->shouldBeFloated(m_self.lock()) && !isX11OverrideRedirect() // not a window that should float or X11
+        && !(m_isFloating && !Desktop::focusState()->window()->m_isFloating)               // do not auto-group a floated window into a tiled group
     ) {
         // add to group if we are focused on one
         Desktop::focusState()->window()->m_group->add(m_self.lock());