From 610c59dc34225bad1c610388e43e8937276e43ff Mon Sep 17 00:00:00 2001
From: Vaxry <vaxry@vaxry.net>
Date: Sat, 27 Dec 2025 20:18:50 +0100
Subject: [PATCH] opengl: properly combine transforms in renderTexture

ref #12666
---
 src/helpers/math/Math.cpp | 82 ++++++++++++++++++++++++++++++++++-----
 src/helpers/math/Math.hpp |  3 +-
 src/render/OpenGL.cpp     |  9 +++--
 3 files changed, 79 insertions(+), 15 deletions(-)

diff --git a/src/helpers/math/Math.cpp b/src/helpers/math/Math.cpp
index 0f7a5d14..d10997b5 100644
--- a/src/helpers/math/Math.cpp
+++ b/src/helpers/math/Math.cpp
@@ -1,19 +1,37 @@
 #include "Math.hpp"
 #include "../memory/Memory.hpp"
+#include "../../macros.hpp"
 
-Hyprutils::Math::eTransform Math::wlTransformToHyprutils(wl_output_transform t) {
+#include <unordered_map>
+#include <array>
+
+using namespace Math;
+
+// FIXME: expose in hu
+static std::unordered_map<eTransform, Mat3x3> transforms = {
+    {HYPRUTILS_TRANSFORM_NORMAL, std::array<float, 9>{1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_90, std::array<float, 9>{0.0f, 1.0f, 0.0f, -1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_180, std::array<float, 9>{-1.0f, 0.0f, 0.0f, 0.0f, -1.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_270, std::array<float, 9>{0.0f, -1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_FLIPPED, std::array<float, 9>{-1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_FLIPPED_90, std::array<float, 9>{0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_FLIPPED_180, std::array<float, 9>{1.0f, 0.0f, 0.0f, 0.0f, -1.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+    {HYPRUTILS_TRANSFORM_FLIPPED_270, std::array<float, 9>{0.0f, -1.0f, 0.0f, -1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}},
+};
+
+eTransform Math::wlTransformToHyprutils(wl_output_transform t) {
     switch (t) {
-        case WL_OUTPUT_TRANSFORM_NORMAL: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_NORMAL;
-        case WL_OUTPUT_TRANSFORM_180: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_180;
-        case WL_OUTPUT_TRANSFORM_90: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_90;
-        case WL_OUTPUT_TRANSFORM_270: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_270;
-        case WL_OUTPUT_TRANSFORM_FLIPPED: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_FLIPPED;
-        case WL_OUTPUT_TRANSFORM_FLIPPED_180: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_FLIPPED_180;
-        case WL_OUTPUT_TRANSFORM_FLIPPED_270: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_FLIPPED_270;
-        case WL_OUTPUT_TRANSFORM_FLIPPED_90: return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_FLIPPED_90;
+        case WL_OUTPUT_TRANSFORM_NORMAL: return eTransform::HYPRUTILS_TRANSFORM_NORMAL;
+        case WL_OUTPUT_TRANSFORM_180: return eTransform::HYPRUTILS_TRANSFORM_180;
+        case WL_OUTPUT_TRANSFORM_90: return eTransform::HYPRUTILS_TRANSFORM_90;
+        case WL_OUTPUT_TRANSFORM_270: return eTransform::HYPRUTILS_TRANSFORM_270;
+        case WL_OUTPUT_TRANSFORM_FLIPPED: return eTransform::HYPRUTILS_TRANSFORM_FLIPPED;
+        case WL_OUTPUT_TRANSFORM_FLIPPED_180: return eTransform::HYPRUTILS_TRANSFORM_FLIPPED_180;
+        case WL_OUTPUT_TRANSFORM_FLIPPED_270: return eTransform::HYPRUTILS_TRANSFORM_FLIPPED_270;
+        case WL_OUTPUT_TRANSFORM_FLIPPED_90: return eTransform::HYPRUTILS_TRANSFORM_FLIPPED_90;
         default: break;
     }
-    return Hyprutils::Math::eTransform::HYPRUTILS_TRANSFORM_NORMAL;
+    return eTransform::HYPRUTILS_TRANSFORM_NORMAL;
 }
 
 wl_output_transform Math::invertTransform(wl_output_transform tr) {
@@ -22,3 +40,47 @@ wl_output_transform Math::invertTransform(wl_output_transform tr) {
 
     return tr;
 }
+
+static bool matEq(const Mat3x3& a, const Mat3x3& b) {
+    for (size_t i = 0; i < 9; ++i) {
+        const float Δ = std::fabs(a.getMatrix()[i] - b.getMatrix()[i]);
+        if (Δ > 1e-6) // eps
+            return false;
+    }
+    return true;
+}
+
+static eTransform composeInternal(eTransform a, eTransform b) {
+    const auto& A      = transforms.at(a);
+    const auto& B      = transforms.at(b);
+    const auto  RESULT = Mat3x3{A}.multiply(B);
+
+    for (const auto& [t, M] : transforms) {
+        if (matEq(M, RESULT))
+            return t;
+    }
+
+    return eTransform::HYPRUTILS_TRANSFORM_NORMAL;
+}
+
+eTransform Math::composeTransform(eTransform a, eTransform b) {
+    static std::array<std::array<eTransform, 8>, 8> lookup;
+    static bool                                     once = true;
+
+    if (once) {
+        once = false;
+
+        // bake the composition table
+        static_assert(HYPRUTILS_TRANSFORM_FLIPPED_270 == 7);
+        for (size_t i = 0; i <= HYPRUTILS_TRANSFORM_FLIPPED_270 /* 7 */; ++i) {
+            for (size_t j = 0; j <= HYPRUTILS_TRANSFORM_FLIPPED_270 /* 7 */; ++j) {
+                lookup[i][j] = composeInternal(sc<eTransform>(i), sc<eTransform>(j));
+            }
+        }
+    }
+
+    RASSERT(a >= HYPRUTILS_TRANSFORM_NORMAL && a <= HYPRUTILS_TRANSFORM_FLIPPED_270, "Invalid transform a in composeTransform");
+    RASSERT(b >= HYPRUTILS_TRANSFORM_NORMAL && b <= HYPRUTILS_TRANSFORM_FLIPPED_270, "Invalid transform b in composeTransform");
+
+    return lookup[a][b];
+}
diff --git a/src/helpers/math/Math.hpp b/src/helpers/math/Math.hpp
index c4baba3c..cc181434 100644
--- a/src/helpers/math/Math.hpp
+++ b/src/helpers/math/Math.hpp
@@ -14,4 +14,5 @@ namespace Math {
 
     eTransform               wlTransformToHyprutils(wl_output_transform t);
     wl_output_transform      invertTransform(wl_output_transform tr);
-}
+    eTransform               composeTransform(eTransform a, eTransform b);
+}
\ No newline at end of file
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 9a5d6ad0..33e380e1 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -1640,10 +1640,11 @@ void CHyprOpenGLImpl::renderTextureInternal(SP<CTexture> tex, const CBox& box, c
     static const auto PCURSORTIMEOUT = CConfigValue<Hyprlang::FLOAT>("cursor:inactive_timeout");
 
     // get the needed transform for this texture
-    const bool TRANSFORMS_MATCH = Math::wlTransformToHyprutils(m_renderData.pMonitor->m_transform) == tex->m_transform; // FIXME: combine them properly!!!
-    eTransform TRANSFORM        = HYPRUTILS_TRANSFORM_NORMAL;
-    if (m_monitorTransformEnabled || TRANSFORMS_MATCH)
-        TRANSFORM = Math::wlTransformToHyprutils(Math::invertTransform(m_renderData.pMonitor->m_transform));
+    const auto                  MONITOR_INVERTED = Math::wlTransformToHyprutils(Math::invertTransform(m_renderData.pMonitor->m_transform));
+    Hyprutils::Math::eTransform TRANSFORM        = tex->m_transform;
+
+    if (m_monitorTransformEnabled)
+        TRANSFORM = Math::composeTransform(MONITOR_INVERTED, TRANSFORM);
 
     Mat3x3     matrix   = m_renderData.monitorProjection.projectBox(newBox, TRANSFORM, newBox.rot);
     Mat3x3     glMatrix = m_renderData.projection.copy().multiply(matrix);