diff --git a/Common/Thread/Waitable.h b/Common/Thread/Waitable.h
index de33ed2049..abda42fef9 100644
--- a/Common/Thread/Waitable.h
+++ b/Common/Thread/Waitable.h
@@ -25,11 +25,11 @@ public:
 		cond_.wait(lock, [&] { return triggered_.load(); });
 	}
 
-	bool WaitFor(double budget) {
+	bool WaitFor(double budget_s) {
 		if (triggered_)
 			return true;
 
-		uint32_t us = budget > 0 ? (uint32_t)(budget * 1000000.0) : 0;
+		uint32_t us = budget_s > 0 ? (uint32_t)(budget_s * 1000000.0) : 0;
 		if (us == 0)
 			return false;
 
diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp
index ee1ef5e736..0950d00e54 100644
--- a/GPU/Common/ReplacedTexture.cpp
+++ b/GPU/Common/ReplacedTexture.cpp
@@ -176,7 +176,8 @@ bool ReplacedTexture::Poll(double budget) {
 	lastUsed_ = now;
 
 	// Let's not even start a new texture if we're already behind.
-	if (budget <= 0.0)
+	// Note that 0.0 is used as a signalling value that we don't want to wait (just handling already finished textures).
+	if (budget < 0.0)
 		return false;
 
 	_assert_(!threadWaitable_);
diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp
index 80ab65b25e..259406e7ec 100644
--- a/GPU/Common/TextureCacheCommon.cpp
+++ b/GPU/Common/TextureCacheCommon.cpp
@@ -1594,12 +1594,19 @@ ReplacedTexture *TextureCacheCommon::FindReplacement(TexCacheEntry *entry, int *
 }
 
 void TextureCacheCommon::PollReplacement(TexCacheEntry *entry, int *w, int *h, int *d) {
-	double budget = replacementFrameBudgetSeconds_ - replacementTimeThisFrame_;
+	double waitBudget = replacementFrameBudgetSeconds_ - replacementTimeThisFrame_;
 	// Note: Don't avoid the Poll call if budget is 0, we do meaningful things there.
 	// Poll also handles negative budgets.
 
 	double replaceStart = time_now_d();
-	if (entry->replacedTexture->Poll(budget)) {
+
+	// Unless the mode is set to Instant (where the user explicitly wants to wait for each texture),
+	// it's just a waste of time to wait here really. OK, we might get a texture one frame early but
+	// we wasted a lot of time waiting, likely slowing down our framerate.
+	if (g_Config.iReplacementTextureLoadSpeed != ReplacementTextureLoadSpeed::INSTANT) {
+		waitBudget = 0.0;
+	}
+	if (entry->replacedTexture->Poll(waitBudget)) {
 		if (entry->replacedTexture->State() == ReplacementState::ACTIVE) {
 			entry->replacedTexture->GetSize(0, w, h);
 			// Consider it already "scaled.".