internal: introduce new types to avoid unsigned int rollover and signed int overflow (#7216)

* framebuffer: avoid gluint overflow

GLuint was being initialized to -1 and rolling over to unsigned int max,
its defined behaviour but very unnecessery. add a bool and use it for
checking if allocated or not.

* opengl: avoid gluint rollover

-1 rolls over to unsigned int max, use 0xFF instead.

* core: big uint64_t to int type conversion

there were a few uint64_t to int implicit conversions overflowing int
and causing UB, make all monitor/workspaces/windows use the new
typedefs. also fix the various related 64 to 32 implicit conversions
going around found with -Wshorten-64-to-32

src/render/Framebuffer.cpp

@@ -12,9 +12,10 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     uint32_t glFormat = FormatUtils::drmFormatToGL(drmFormat);
     uint32_t glType   = FormatUtils::glFormatToType(glFormat);
 
-    if (m_iFb == (uint32_t)-1) {
+    if (!m_iFbAllocated) {
         firstAlloc = true;
         glGenFramebuffers(1, &m_iFb);
+        m_iFbAllocated = true;
     }
 
     if (m_cTex->m_iTexID == 0) {
@@ -88,12 +89,12 @@ void CFramebuffer::bind() {
 }
 
 void CFramebuffer::release() {
-    if (m_iFb != (uint32_t)-1 && m_iFb)
+    if (m_iFbAllocated)
         glDeleteFramebuffers(1, &m_iFb);
 
     m_cTex->destroyTexture();
-    m_iFb   = -1;
-    m_vSize = Vector2D();
+    m_iFbAllocated = false;
+    m_vSize        = Vector2D();
 }
 
 CFramebuffer::~CFramebuffer() {
@@ -101,5 +102,5 @@ CFramebuffer::~CFramebuffer() {
 }
 
 bool CFramebuffer::isAllocated() {
-    return m_iFb != (GLuint)-1;
+    return m_iFbAllocated;
 }

src/render/Framebuffer.hpp

@@ -18,7 +18,8 @@ class CFramebuffer {
     Vector2D     m_vSize;
 
     SP<CTexture> m_cTex;
-    GLuint       m_iFb = -1;
+    GLuint       m_iFb;
+    bool         m_iFbAllocated{false};
 
     SP<CTexture> m_pStencilTex;
 };

src/render/OpenGL.cpp

@@ -1247,14 +1247,14 @@ void CHyprOpenGLImpl::renderRectWithBlur(CBox* box, const CColor& col, int round
 
     glEnable(GL_STENCIL_TEST);
 
-    glStencilFunc(GL_ALWAYS, 1, -1);
+    glStencilFunc(GL_ALWAYS, 1, 0xFF);
     glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
 
     glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
     renderRect(box, CColor(0, 0, 0, 0), round);
     glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
 
-    glStencilFunc(GL_EQUAL, 1, -1);
+    glStencilFunc(GL_EQUAL, 1, 0xFF);
     glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
 
     scissor(box);
@@ -1269,7 +1269,7 @@ void CHyprOpenGLImpl::renderRectWithBlur(CBox* box, const CColor& col, int round
     glClearStencil(0);
     glClear(GL_STENCIL_BUFFER_BIT);
     glDisable(GL_STENCIL_TEST);
-    glStencilMask(-1);
+    glStencilMask(0xFF);
     glStencilFunc(GL_ALWAYS, 1, 0xFF);
     scissor((CBox*)nullptr);
 
@@ -1802,12 +1802,12 @@ CFramebuffer* CHyprOpenGLImpl::blurMainFramebufferWithDamage(float a, CRegion* o
     CRegion tempDamage{damage};
 
     // and draw
-    for (int i = 1; i <= *PBLURPASSES; ++i) {
+    for (auto i = 1; i <= *PBLURPASSES; ++i) {
         tempDamage = damage.copy().scale(1.f / (1 << i));
         drawPass(&m_RenderData.pCurrentMonData->m_shBLUR1, &tempDamage); // down
     }
 
-    for (int i = *PBLURPASSES - 1; i >= 0; --i) {
+    for (auto i = *PBLURPASSES - 1; i >= 0; --i) {
         tempDamage = damage.copy().scale(1.f / (1 << i));                // when upsampling we make the region twice as big
         drawPass(&m_RenderData.pCurrentMonData->m_shBLUR2, &tempDamage); // up
     }
@@ -2091,7 +2091,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(SP<CTexture> tex, CBox* pBox, float
 
     glEnable(GL_STENCIL_TEST);
 
-    glStencilFunc(GL_ALWAYS, 1, -1);
+    glStencilFunc(GL_ALWAYS, 1, 0xFF);
     glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
 
     glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
@@ -2101,7 +2101,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(SP<CTexture> tex, CBox* pBox, float
         renderTexture(tex, pBox, a, round, true, true); // discard opaque
     glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
 
-    glStencilFunc(GL_EQUAL, 1, -1);
+    glStencilFunc(GL_EQUAL, 1, 0xFF);
     glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
 
     // stencil done. Render everything.
@@ -2124,7 +2124,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(SP<CTexture> tex, CBox* pBox, float
     glDisable(GL_STENCIL_TEST);
     renderTextureInternalWithDamage(tex, pBox, a, &texDamage, round, false, false, true, true);
 
-    glStencilMask(-1);
+    glStencilMask(0xFF);
     glStencilFunc(GL_ALWAYS, 1, 0xFF);
     scissor((CBox*)nullptr);
 }

src/render/Renderer.cpp

@@ -1658,7 +1658,7 @@ void CHyprRenderer::arrangeLayerArray(CMonitor* pMonitor, const std::vector<PHLL
     }
 }
 
-void CHyprRenderer::arrangeLayersForMonitor(const int& monitor) {
+void CHyprRenderer::arrangeLayersForMonitor(const MONITORID& monitor) {
     const auto PMONITOR = g_pCompositor->getMonitorFromID(monitor);
 
     if (!PMONITOR)

src/render/Renderer.hpp

@@ -49,7 +49,7 @@ class CHyprRenderer {
     ~CHyprRenderer();
 
     void                            renderMonitor(CMonitor* pMonitor);
-    void                            arrangeLayersForMonitor(const int&);
+    void                            arrangeLayersForMonitor(const MONITORID&);
     void                            damageSurface(SP<CWLSurfaceResource>, double, double, double scale = 1.0);
     void                            damageWindow(PHLWINDOW, bool forceFull = false);
     void                            damageBox(CBox*, bool skipFrameSchedule = false);