diff --git a/Source/Core/VideoBackends/D3D/Render.cpp b/Source/Core/VideoBackends/D3D/Render.cpp
index a7518cdf64..d6a6dafd2e 100644
--- a/Source/Core/VideoBackends/D3D/Render.cpp
+++ b/Source/Core/VideoBackends/D3D/Render.cpp
@@ -622,8 +622,6 @@ void Renderer::SwapImpl(AbstractTexture* texture, const EFBRectangle& xfb_region
   g_texture_cache->OnConfigChanged(g_ActiveConfig);
   VertexShaderCache::RetreiveAsyncShaders();
 
-  SetWindowSize(xfb_texture->GetConfig().width, xfb_texture->GetConfig().height);
-
   const bool window_resized = CheckForResize();
   const bool fullscreen = D3D::GetFullscreenState();
   const bool fs_changed = m_last_fullscreen_mode != fullscreen;
diff --git a/Source/Core/VideoBackends/OGL/Render.cpp b/Source/Core/VideoBackends/OGL/Render.cpp
index 9ee96bb724..9127a6d6d9 100644
--- a/Source/Core/VideoBackends/OGL/Render.cpp
+++ b/Source/Core/VideoBackends/OGL/Render.cpp
@@ -1430,8 +1430,6 @@ void Renderer::SwapImpl(AbstractTexture* texture, const EFBRectangle& xfb_region
   }
 #endif
 
-  // Update the render window position and the backbuffer size
-  SetWindowSize(xfb_texture->GetConfig().width, xfb_texture->GetConfig().height);
   GLInterface->Update();
 
   // Was the size changed since the last frame?
diff --git a/Source/Core/VideoBackends/Vulkan/Renderer.cpp b/Source/Core/VideoBackends/Vulkan/Renderer.cpp
index a146351007..d5092e93ec 100644
--- a/Source/Core/VideoBackends/Vulkan/Renderer.cpp
+++ b/Source/Core/VideoBackends/Vulkan/Renderer.cpp
@@ -545,10 +545,6 @@ void Renderer::SwapImpl(AbstractTexture* texture, const EFBRectangle& xfb_region
   // Handle host window resizes.
   CheckForSurfaceChange();
 
-  // Update the window size based on the frame that was just rendered.
-  // Due to depending on guest state, we need to call this every frame.
-  SetWindowSize(xfb_texture->GetConfig().width, xfb_texture->GetConfig().height);
-
   // Clean up stale textures.
   TextureCache::GetInstance()->Cleanup(frameCount);
 
diff --git a/Source/Core/VideoCommon/RenderBase.cpp b/Source/Core/VideoCommon/RenderBase.cpp
index e9b0c3dff1..608d215c48 100644
--- a/Source/Core/VideoCommon/RenderBase.cpp
+++ b/Source/Core/VideoCommon/RenderBase.cpp
@@ -523,10 +523,6 @@ void Renderer::UpdateDrawRectangle()
 
 void Renderer::SetWindowSize(int width, int height)
 {
-  // Scale the window size by the EFB scale.
-  if (g_ActiveConfig.iEFBScale != EFB_SCALE_AUTO_INTEGRAL)
-    std::tie(width, height) = CalculateTargetScale(width, height);
-
   std::tie(width, height) = CalculateOutputDimensions(width, height);
 
   // Track the last values of width/height to avoid sending a window resize event every frame.
@@ -643,11 +639,12 @@ void Renderer::Swap(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight, const
 
     if (xfb_entry && xfb_entry->id != m_last_xfb_id)
     {
+      const TextureConfig& texture_config = xfb_entry->texture->GetConfig();
       m_last_xfb_texture = xfb_entry->texture.get();
       m_last_xfb_id = xfb_entry->id;
       m_last_xfb_ticks = ticks;
 
-      auto xfb_rect = xfb_entry->texture->GetConfig().GetRect();
+      auto xfb_rect = texture_config.GetRect();
       xfb_rect.right -= EFBToScaledX(fbStride - fbWidth);
 
       m_last_xfb_region = xfb_rect;
@@ -655,6 +652,10 @@ void Renderer::Swap(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight, const
       // TODO: merge more generic parts into VideoCommon
       g_renderer->SwapImpl(xfb_entry->texture.get(), xfb_rect, ticks, xfb_entry->gamma);
 
+      // Update the window size based on the frame that was just rendered.
+      // Due to depending on guest state, we need to call this every frame.
+      SetWindowSize(texture_config.width, texture_config.height);
+
       m_fps_counter.Update();
 
       if (IsFrameDumping())