From 65d4a16afd7d84742751d3f9b3b0976738cac136 Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Fri, 27 Oct 2023 23:48:55 -0400
Subject: [PATCH] renderer_vulkan: fix cropping for presentation

---
 .../nvnflinger/buffer_transform_flags.h       |  2 +
 .../renderer_vulkan/vk_blit_screen.cpp        | 97 ++++++++++---------
 2 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/src/core/hle/service/nvnflinger/buffer_transform_flags.h b/src/core/hle/service/nvnflinger/buffer_transform_flags.h
index 67aa5dad63..ffe579718f 100644
--- a/src/core/hle/service/nvnflinger/buffer_transform_flags.h
+++ b/src/core/hle/service/nvnflinger/buffer_transform_flags.h
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include "common/common_funcs.h"
 #include "common/common_types.h"
 
 namespace Service::android {
@@ -21,5 +22,6 @@ enum class BufferTransformFlags : u32 {
     /// Rotate source image 270 degrees clockwise
     Rotate270 = 0x07,
 };
+DECLARE_ENUM_FLAG_OPERATORS(BufferTransformFlags);
 
 } // namespace Service::android
diff --git a/src/video_core/renderer_vulkan/vk_blit_screen.cpp b/src/video_core/renderer_vulkan/vk_blit_screen.cpp
index 52fc142d19..459ab32c24 100644
--- a/src/video_core/renderer_vulkan/vk_blit_screen.cpp
+++ b/src/video_core/renderer_vulkan/vk_blit_screen.cpp
@@ -1395,63 +1395,72 @@ void BlitScreen::SetUniformData(BufferData& data, const Layout::FramebufferLayou
         MakeOrthographicMatrix(static_cast<f32>(layout.width), static_cast<f32>(layout.height));
 }
 
-void BlitScreen::SetVertexData(BufferData& data, const Tegra::FramebufferConfig& framebuffer,
-                               const Layout::FramebufferLayout layout) const {
-    const auto& framebuffer_transform_flags = framebuffer.transform_flags;
-    const auto& framebuffer_crop_rect = framebuffer.crop_rect;
+static Common::Rectangle<f32> NormalizeCrop(Common::Rectangle<int> crop,
+                                            const Tegra::FramebufferConfig& framebuffer) {
+    f32 left, top, right, bottom;
 
-    static constexpr Common::Rectangle<f32> texcoords{0.f, 0.f, 1.f, 1.f};
-    auto left = texcoords.left;
-    auto right = texcoords.right;
+    if (!crop.IsEmpty()) {
+        // If crop rectangle is not empty, apply properties from rectangle.
+        left = static_cast<f32>(crop.left);
+        top = static_cast<f32>(crop.top);
+        right = static_cast<f32>(crop.right);
+        bottom = static_cast<f32>(crop.bottom);
+    } else {
+        // Otherwise, fall back to framebuffer dimensions.
+        left = 0;
+        top = 0;
+        right = static_cast<f32>(framebuffer.width);
+        bottom = static_cast<f32>(framebuffer.height);
+    }
 
-    switch (framebuffer_transform_flags) {
-    case Service::android::BufferTransformFlags::Unset:
-        break;
-    case Service::android::BufferTransformFlags::FlipV:
-        // Flip the framebuffer vertically
-        left = texcoords.right;
-        right = texcoords.left;
-        break;
-    default:
+    // Apply transformation flags.
+    auto framebuffer_transform_flags = framebuffer.transform_flags;
+
+    if (True(framebuffer_transform_flags & Service::android::BufferTransformFlags::FlipH)) {
+        // Switch left and right.
+        std::swap(left, right);
+    }
+    if (True(framebuffer_transform_flags & Service::android::BufferTransformFlags::FlipV)) {
+        // Switch top and bottom.
+        std::swap(top, bottom);
+    }
+
+    framebuffer_transform_flags &= ~Service::android::BufferTransformFlags::FlipH;
+    framebuffer_transform_flags &= ~Service::android::BufferTransformFlags::FlipV;
+    if (True(framebuffer_transform_flags)) {
         UNIMPLEMENTED_MSG("Unsupported framebuffer_transform_flags={}",
                           static_cast<u32>(framebuffer_transform_flags));
-        break;
     }
 
-    UNIMPLEMENTED_IF(framebuffer_crop_rect.left != 0);
+    return Common::Rectangle<f32>(left, top, right, bottom);
+}
 
-    f32 left_start{};
-    if (framebuffer_crop_rect.Top() > 0) {
-        left_start = static_cast<f32>(framebuffer_crop_rect.Top()) /
-                     static_cast<f32>(framebuffer_crop_rect.Bottom());
-    }
-    f32 scale_u = static_cast<f32>(framebuffer.width) / static_cast<f32>(screen_info.width);
-    f32 scale_v = static_cast<f32>(framebuffer.height) / static_cast<f32>(screen_info.height);
-    // Scale the output by the crop width/height. This is commonly used with 1280x720 rendering
-    // (e.g. handheld mode) on a 1920x1080 framebuffer.
-    if (!fsr) {
-        if (framebuffer_crop_rect.GetWidth() > 0) {
-            scale_u = static_cast<f32>(framebuffer_crop_rect.GetWidth()) /
-                      static_cast<f32>(screen_info.width);
-        }
-        if (framebuffer_crop_rect.GetHeight() > 0) {
-            scale_v = static_cast<f32>(framebuffer_crop_rect.GetHeight()) /
-                      static_cast<f32>(screen_info.height);
-        }
-    }
+void BlitScreen::SetVertexData(BufferData& data, const Tegra::FramebufferConfig& framebuffer,
+                               const Layout::FramebufferLayout layout) const {
+    // Get the normalized crop rectangle.
+    const auto crop = NormalizeCrop(framebuffer.crop_rect, framebuffer);
 
+    // Get the screen properties.
+    const f32 screen_width = static_cast<f32>(screen_info.width);
+    const f32 screen_height = static_cast<f32>(screen_info.height);
+
+    // Apply the crop.
+    const f32 left = crop.left / screen_width;
+    const f32 top = crop.top / screen_height;
+    const f32 right = crop.right / screen_width;
+    const f32 bottom = crop.bottom / screen_height;
+
+    // Map the coordinates to the screen.
     const auto& screen = layout.screen;
     const auto x = static_cast<f32>(screen.left);
     const auto y = static_cast<f32>(screen.top);
     const auto w = static_cast<f32>(screen.GetWidth());
     const auto h = static_cast<f32>(screen.GetHeight());
-    data.vertices[0] = ScreenRectVertex(x, y, texcoords.top * scale_u, left_start + left * scale_v);
-    data.vertices[1] =
-        ScreenRectVertex(x + w, y, texcoords.bottom * scale_u, left_start + left * scale_v);
-    data.vertices[2] =
-        ScreenRectVertex(x, y + h, texcoords.top * scale_u, left_start + right * scale_v);
-    data.vertices[3] =
-        ScreenRectVertex(x + w, y + h, texcoords.bottom * scale_u, left_start + right * scale_v);
+
+    data.vertices[0] = ScreenRectVertex(x, y, left, top);
+    data.vertices[1] = ScreenRectVertex(x + w, y, right, top);
+    data.vertices[2] = ScreenRectVertex(x, y + h, left, bottom);
+    data.vertices[3] = ScreenRectVertex(x + w, y + h, right, bottom);
 }
 
 void BlitScreen::CreateSMAA(VkExtent2D smaa_size) {