Fix OOB in texture replacer mip mixing and non-DXT texture decode

Vuln 17: ReplacedTexture::LoadLevelData let a KTX2/DDS file at a higher
mip level resize the shared data_ vector to its own (attacker-controlled)
mip count, so data_[mipLevel + i] indexed out of bounds and the KTX2
branch resized a different element than it wrote to. Disallow mixing
image formats across mip levels, cap the container mip count, and resize
the same element that is used as the transcode destination.

Vuln 18: DecodeTextureLevel only validated the start address for
non-DXT textures, so guest-controlled w/h/bufw could drive reads past
mapped RAM. Validate the needed range like the DXT path does and clamp
the height; ReadIndexedTex now takes the clamped w/h.

GPU/Common/ReplacedTexture.cpp

@@ -321,6 +321,17 @@ ReplacedTexture::LoadLevelResult ReplacedTexture::LoadLevelData(VFSFileReference
 	std::string magic;
 	ReplacedImageType imageType = Identify(vfs_, openFile, &magic);
 
+	// Disallow mixing image formats across mip levels: a KTX2/DDS container
+	// manages its own mip chain, so mixing one in at a higher mip level
+	// would corrupt the shared level data layout.
+	if (mipLevel == 0) {
+		firstImageType_ = imageType;
+	} else if (imageType != firstImageType_) {
+		WARN_LOG(Log::TexReplacement, "Replacement mipmap %d uses image format %d, but mip 0 uses %d. Stopping.", mipLevel, (int)imageType, (int)firstImageType_);
+		vfs_->CloseFile(openFile);
+		return LoadLevelResult::DONE;
+	}
+
 	bool ddsDX10 = false;
 	int numMips = 1;
 
@@ -531,7 +542,13 @@ ReplacedTexture::LoadLevelResult ReplacedTexture::LoadLevelData(VFSFileReference
 			WARN_LOG(Log::TexReplacement, "Block compressed replacement texture '%s' not divisible by 4x4 (%dx%d). In D3D11 (only!) we will have to expand (potentially causing glitches).", filename.c_str(), level.w, level.h);
 		}
 
-		data_.resize(numMips);
+		// Cap the mip count (attacker-controlled header field) and make sure
+		// data_ is large enough for mipLevel + numMips; otherwise the loop
+		// below indexes past the end of data_.
+		numMips = std::max(1, std::min(numMips, MAX_REPLACEMENT_MIP_LEVELS - mipLevel));
+		if ((size_t)(mipLevel + numMips) > data_.size()) {
+			data_.resize(mipLevel + numMips);
+		}
 
 		basist::ktx2_transcoder_state transcodeState;  // Each thread needs one of these.
 
@@ -553,7 +570,7 @@ ReplacedTexture::LoadLevelResult ReplacedTexture::LoadLevelData(VFSFileReference
 				outputSize = levelInfo.m_orig_width * levelInfo.m_orig_height;
 				outputPitch = levelInfo.m_orig_width;
 			}
-			data_[i].resize(dataSizeBytes);
+			out.resize(dataSizeBytes);
 
 			transcodeState.clear();
 			transcoder.transcode_image_level(i, 0, 0, &out[0], (uint32_t)outputSize, transcoderFormat, 0, (uint32_t)outputPitch, level.h, -1, -1, &transcodeState);
@@ -585,7 +602,13 @@ ReplacedTexture::LoadLevelResult ReplacedTexture::LoadLevelData(VFSFileReference
 			WARN_LOG(Log::TexReplacement, "Block compressed replacement texture '%s' not divisible by 4x4 (%dx%d). In D3D11 (only!) we will have to expand (potentially causing glitches).", filename.c_str(), level.w, level.h);
 		}
 
-		data_.resize(numMips);
+		// Cap the mip count (attacker-controlled header field) and make sure
+		// data_ is large enough for mipLevel + numMips; otherwise the loop
+		// below indexes past the end of data_.
+		numMips = std::max(1, std::min(numMips, MAX_REPLACEMENT_MIP_LEVELS - mipLevel));
+		if ((size_t)(mipLevel + numMips) > data_.size()) {
+			data_.resize(mipLevel + numMips);
+		}
 
 		// A DDS File can contain multiple mipmaps.
 		levels_.reserve(numMips);

GPU/Common/ReplacedTexture.h

@@ -220,6 +220,9 @@ private:
 
 	std::vector<std::vector<uint8_t>> data_;
 	std::vector<ReplacedTextureLevel> levels_;
+	// Image format of mip level 0; mixing formats across levels is not
+	// allowed (container formats like KTX2/DDS manage their own mip chain).
+	ReplacedImageType firstImageType_ = ReplacedImageType::INVALID;
 
 	double lastUsed_ = 0.0;
 	LimitedWaitable *threadWaitable_ = nullptr;

GPU/Common/TextureCacheCommon.cpp

@@ -1957,6 +1957,21 @@ TextureAlpha TextureCacheCommon::DecodeTextureLevel(u8 *out, int outPitch, GETex
 	const u8 *texptr = Memory::GetPointer(texaddr);
 	const uint32_t byteSize = (textureBitsPerPixel[format] * bufw * h) / 8;
 
+	// Validate the texture data fits in mapped RAM, like the DXT path does.
+	// texaddr/bufw/w/h are all guest-controlled via the GE display list.
+	const int bpp = textureBitsPerPixel[format];
+	const uint32_t bytesPerRow = (bpp * bufw) / 8;
+	// Swizzled textures are read in 8-row blocks, rounding the height up.
+	const uint32_t rows = swizzled ? ((h + 7) & ~7) : h;
+	const uint32_t neededBytes = bytesPerRow * rows;
+	if (bytesPerRow > 0 && !Memory::IsValidRange(texaddr, neededBytes)) {
+		ERROR_LOG_REPORT(Log::G3D, "Texture extends beyond valid RAM: %08x + %d x %d", texaddr, bufw, h);
+		uint32_t limited = Memory::ClampValidSizeAt(texaddr, neededBytes);
+		h = limited / bytesPerRow;
+		if (swizzled)
+			h &= ~7;
+	}
+
 	char buf[128];
 	size_t len = snprintf(buf, sizeof(buf), "Tex_%08x_%dx%d_%s", texaddr, w, h, GeTextureFormatToString(format, clutformat));
 	NotifyMemInfo(MemBlockFlags::TEXTURE, texaddr, byteSize, buf, len);
@@ -2065,13 +2080,13 @@ TextureAlpha TextureCacheCommon::DecodeTextureLevel(u8 *out, int outPitch, GETex
 			// We can't know anything about alpha.
 			return TextureAlpha::Any;
 		}
-		return ReadIndexedTex(out, outPitch, level, texptr, 1, bufw, reverseColors, expandTo32bit);
+		return ReadIndexedTex(out, outPitch, w, h, level, texptr, 1, bufw, reverseColors, expandTo32bit);
 
 	case GE_TFMT_CLUT16:
-		return ReadIndexedTex(out, outPitch, level, texptr, 2, bufw, reverseColors, expandTo32bit);
+		return ReadIndexedTex(out, outPitch, w, h, level, texptr, 2, bufw, reverseColors, expandTo32bit);
 
 	case GE_TFMT_CLUT32:
-		return ReadIndexedTex(out, outPitch, level, texptr, 4, bufw, reverseColors, expandTo32bit);
+		return ReadIndexedTex(out, outPitch, w, h, level, texptr, 4, bufw, reverseColors, expandTo32bit);
 
 	case GE_TFMT_4444:
 	case GE_TFMT_5551:
@@ -2189,10 +2204,7 @@ TextureAlpha TextureCacheCommon::DecodeTextureLevel(u8 *out, int outPitch, GETex
 	return AlphaSumIsFull(alphaSum, fullAlphaMask) ? TextureAlpha::Solid : TextureAlpha::Any;
 }
 
-TextureAlpha TextureCacheCommon::ReadIndexedTex(u8 *out, int outPitch, int level, const u8 *texptr, int bytesPerIndex, int bufw, bool reverseColors, bool expandTo32Bit) {
-	int w = gstate.getTextureWidth(level);
-	int h = gstate.getTextureHeight(level);
-
+TextureAlpha TextureCacheCommon::ReadIndexedTex(u8 *out, int outPitch, int w, int h, int level, const u8 *texptr, int bytesPerIndex, int bufw, bool reverseColors, bool expandTo32Bit) {
 	if (gstate.isTextureSwizzled()) {
 		tmpTexBuf32_.resize(bufw * ((h + 7) & ~7));
 		UnswizzleFromMem(tmpTexBuf32_.data(), bufw * bytesPerIndex, texptr, bufw, h, bytesPerIndex);

GPU/Common/TextureCacheCommon.h

@@ -425,7 +425,7 @@ protected:
 
 	TextureAlpha DecodeTextureLevel(u8 *out, int outPitch, GETextureFormat format, GEPaletteFormat clutformat, uint32_t texaddr, int level, int bufw, TexDecodeFlags flags);
 	static void UnswizzleFromMem(u32 *dest, u32 destPitch, const u8 *texptr, u32 bufw, u32 height, u32 bytesPerPixel);
-	TextureAlpha ReadIndexedTex(u8 *out, int outPitch, int level, const u8 *texptr, int bytesPerIndex, int bufw, bool reverseColors, bool expandTo32Bit);
+	TextureAlpha ReadIndexedTex(u8 *out, int outPitch, int w, int h, int level, const u8 *texptr, int bytesPerIndex, int bufw, bool reverseColors, bool expandTo32Bit);
 	ReplacedTexture *FindReplacement(TexCacheEntry *entry, int *w, int *h, int *d);
 	void PollReplacement(TexCacheEntry *entry, int *w, int *h, int *d);