diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 6ab68892c0..d94dbf8735 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -156,12 +156,10 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
       staging_pool(device, memory_allocator, scheduler), descriptor_pool(device, scheduler),
       update_descriptor_queue(device, scheduler),
       blit_image(device, scheduler, state_tracker, descriptor_pool),
-      astc_decoder_pass(device, scheduler, descriptor_pool, staging_pool, update_descriptor_queue,
-                        memory_allocator),
       render_pass_cache(device), texture_cache_runtime{device,           scheduler,
                                                        memory_allocator, staging_pool,
-                                                       blit_image,       astc_decoder_pass,
-                                                       render_pass_cache},
+                                                       blit_image,       render_pass_cache,
+                                                       descriptor_pool,  update_descriptor_queue},
       texture_cache(texture_cache_runtime, *this),
       buffer_cache_runtime(device, memory_allocator, scheduler, staging_pool,
                            update_descriptor_queue, descriptor_pool),
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index e2fdc7611d..b0bc306f5a 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -155,7 +155,6 @@ private:
     DescriptorPool descriptor_pool;
     UpdateDescriptorQueue update_descriptor_queue;
     BlitImageHelper blit_image;
-    ASTCDecoderPass astc_decoder_pass;
     RenderPassCache render_pass_cache;
 
     TextureCacheRuntime texture_cache_runtime;
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 6ad7efbdf0..853b80d8a4 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -791,12 +791,17 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched
                                          MemoryAllocator& memory_allocator_,
                                          StagingBufferPool& staging_buffer_pool_,
                                          BlitImageHelper& blit_image_helper_,
-                                         ASTCDecoderPass& astc_decoder_pass_,
-                                         RenderPassCache& render_pass_cache_)
+                                         RenderPassCache& render_pass_cache_,
+                                         DescriptorPool& descriptor_pool,
+                                         UpdateDescriptorQueue& update_descriptor_queue)
     : device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_},
       staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_},
-      astc_decoder_pass{astc_decoder_pass_}, render_pass_cache{render_pass_cache_},
-      resolution{Settings::values.resolution_info} {}
+      render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} {
+    if (Settings::values.accelerate_astc) {
+        astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
+                                  update_descriptor_queue, memory_allocator);
+    }
+}
 
 void TextureCacheRuntime::Finish() {
     scheduler.Finish();
@@ -1845,7 +1850,7 @@ void TextureCacheRuntime::AccelerateImageUpload(
     Image& image, const StagingBufferRef& map,
     std::span<const VideoCommon::SwizzleParameters> swizzles) {
     if (IsPixelFormatASTC(image.info.format)) {
-        return astc_decoder_pass.Assemble(image, map, swizzles);
+        return astc_decoder_pass->Assemble(image, map, swizzles);
     }
     ASSERT(false);
 }
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 0b7ac0df16..7ec0df1340 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -6,6 +6,7 @@
 #include <span>
 
 #include "shader_recompiler/shader_info.h"
+#include "video_core/renderer_vulkan/vk_compute_pass.h"
 #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
 #include "video_core/texture_cache/image_view_base.h"
 #include "video_core/texture_cache/texture_cache_base.h"
@@ -25,14 +26,15 @@ using VideoCommon::RenderTargets;
 using VideoCommon::SlotVector;
 using VideoCore::Surface::PixelFormat;
 
-class ASTCDecoderPass;
 class BlitImageHelper;
+class DescriptorPool;
 class Device;
 class Image;
 class ImageView;
 class Framebuffer;
 class RenderPassCache;
 class StagingBufferPool;
+class UpdateDescriptorQueue;
 class Scheduler;
 
 class TextureCacheRuntime {
@@ -41,8 +43,9 @@ public:
                                  MemoryAllocator& memory_allocator_,
                                  StagingBufferPool& staging_buffer_pool_,
                                  BlitImageHelper& blit_image_helper_,
-                                 ASTCDecoderPass& astc_decoder_pass_,
-                                 RenderPassCache& render_pass_cache_);
+                                 RenderPassCache& render_pass_cache_,
+                                 DescriptorPool& descriptor_pool,
+                                 UpdateDescriptorQueue& update_descriptor_queue);
 
     void Finish();
 
@@ -97,8 +100,8 @@ public:
     MemoryAllocator& memory_allocator;
     StagingBufferPool& staging_buffer_pool;
     BlitImageHelper& blit_image_helper;
-    ASTCDecoderPass& astc_decoder_pass;
     RenderPassCache& render_pass_cache;
+    std::optional<ASTCDecoderPass> astc_decoder_pass;
     const Settings::ResolutionScalingInfo& resolution;
 
     constexpr static size_t indexing_slots = 8 * sizeof(size_t);