Merge pull request #22325 from hrydgard/vertex-decoder-jit-match

Vertex decoder: New test, make the JITs match the C++ decoder closely

.github/workflows/build.yml

@@ -285,14 +285,14 @@ jobs:
           extra: loongarch64
           cc: gcc
           cxx: g++
-          args: ./b.sh --loongarch64 PPSSPPHeadless
+          args: ./b.sh --loongarch64 --unittest PPSSPPHeadless PPSSPPUnitTest
           id: loongarch64
 
         - os: ubuntu-26.04
           extra: riscv64
           cc: gcc
           cxx: g++
-          args: ./b.sh --riscv64 PPSSPPHeadless
+          args: ./b.sh --riscv64 --unittest PPSSPPHeadless PPSSPPUnitTest
           id: riscv64
 
         - os: macos-latest
@@ -347,7 +347,7 @@ jobs:
       if: matrix.extra == 'loongarch64' || matrix.extra == 'riscv64'
       run: |
         sudo apt-get update -y -qq
-        sudo apt-get install -y gcc-14-${{ matrix.extra }}-linux-gnu g++-14-${{ matrix.extra }}-linux-gnu binutils-${{ matrix.extra }}-linux-gnu libgl-dev libglu1-mesa-dev
+        sudo apt-get install -y gcc-14-${{ matrix.extra }}-linux-gnu g++-14-${{ matrix.extra }}-linux-gnu binutils-${{ matrix.extra }}-linux-gnu libgl-dev libglu1-mesa-dev qemu-user
         sudo cmake/scripts/setup-cross.sh ${{ matrix.extra }}
 
     - name: Install iOS dependencies
@@ -400,6 +400,30 @@ jobs:
         fi
         ${{ matrix.args }}
 
+    # These targets have no runner of their own, so the unit tests run under emulation. It's the
+    # only coverage the loongarch64 and riscv64 JITs get - a compile is not much of a test for a
+    # code generator.
+    - name: Execute unit tests under qemu
+      if: matrix.extra == 'loongarch64' || matrix.extra == 'riscv64'
+      run: |
+        qemu-${{ matrix.extra }} -L /usr/${{ matrix.extra }}-linux-gnu \
+          build-${{ matrix.extra }}/PPSSPPUnitTest all
+
+    # Only the IR JIT - it's the sole native backend these two have, and the only thing here that
+    # isn't shared code. The interpreters are portable C++ that the x86-64 and arm64 runners
+    # already cover on all four backends, and every run costs emulated wall clock.
+    - name: Execute headless tests under qemu
+      if: matrix.extra == 'loongarch64' || matrix.extra == 'riscv64'
+      run: |
+        # test.py takes the newest build*/PPSSPPHeadless, so hand it one that runs the cross build
+        # under emulation. The wall clock is raised because everything is slower under qemu.
+        mkdir -p build-qemu
+        printf '#!/bin/bash\nexec qemu-%s -L /usr/%s-linux-gnu "$(dirname "$0")/../build-%s/PPSSPPHeadless" "$@"\n' \
+          '${{ matrix.extra }}' '${{ matrix.extra }}' '${{ matrix.extra }}' > build-qemu/PPSSPPHeadless
+        chmod +x build-qemu/PPSSPPHeadless
+        python3 test.py -g --graphics=software --cpu=jit-ir --timeout=60 \
+          --known-failures=${{ matrix.extra }}
+
     - name: Package build
       if: matrix.extra == 'test' || matrix.id == 'ios'
       run: |

Common/LoongArchCPUDetect.cpp

@@ -170,21 +170,25 @@ void CPUInfo::Detect()
 	LOONGARCH_PTW       = ExtensionSupported(hwcap, 13);
 
 #ifdef USE_CPU_FEATURES
+	// Only ever add to what the hwcaps said. GetLoongArchInfo reads /proc/cpuinfo and nothing
+	// else, so anywhere that isn't the real thing - under qemu-user, where /proc/cpuinfo belongs
+	// to the host - it reports no features at all, and letting it assign would turn off LSX and
+	// with it every vector path in the JIT.
 	cpu_features::LoongArchInfo info = cpu_features::GetLoongArchInfo();
 	LOONGARCH_CPUCFG    = true;
-	LOONGARCH_LAM       = info.features.LAM;
+	LOONGARCH_LAM       = LOONGARCH_LAM      || info.features.LAM;
-	LOONGARCH_UAL       = info.features.UAL;
+	LOONGARCH_UAL       = LOONGARCH_UAL      || info.features.UAL;
-	LOONGARCH_FPU       = info.features.FPU;
+	LOONGARCH_FPU       = LOONGARCH_FPU      || info.features.FPU;
-	LOONGARCH_LSX       = info.features.LSX;
+	LOONGARCH_LSX       = LOONGARCH_LSX      || info.features.LSX;
-	LOONGARCH_LASX      = info.features.LASX;
+	LOONGARCH_LASX      = LOONGARCH_LASX     || info.features.LASX;
-	LOONGARCH_CRC32     = info.features.CRC32;
+	LOONGARCH_CRC32     = LOONGARCH_CRC32    || info.features.CRC32;
-	LOONGARCH_COMPLEX   = info.features.COMPLEX;
+	LOONGARCH_COMPLEX   = LOONGARCH_COMPLEX  || info.features.COMPLEX;
-	LOONGARCH_CRYPTO    = info.features.CRYPTO;
+	LOONGARCH_CRYPTO    = LOONGARCH_CRYPTO   || info.features.CRYPTO;
-	LOONGARCH_LVZ       = info.features.LVZ;
+	LOONGARCH_LVZ       = LOONGARCH_LVZ      || info.features.LVZ;
-	LOONGARCH_LBT_X86   = info.features.LBT_X86;
+	LOONGARCH_LBT_X86   = LOONGARCH_LBT_X86  || info.features.LBT_X86;
-	LOONGARCH_LBT_ARM   = info.features.LBT_ARM;
+	LOONGARCH_LBT_ARM   = LOONGARCH_LBT_ARM  || info.features.LBT_ARM;
-	LOONGARCH_LBT_MIPS  = info.features.LBT_MIPS;
+	LOONGARCH_LBT_MIPS  = LOONGARCH_LBT_MIPS || info.features.LBT_MIPS;
-	LOONGARCH_PTW       = info.features.PTW;
+	LOONGARCH_PTW       = LOONGARCH_PTW      || info.features.PTW;
 #endif
 }
 

Core/MIPS/LoongArch64/LoongArch64RegCache.cpp

@@ -35,7 +35,10 @@ LoongArch64RegCache::LoongArch64RegCache(MIPSComp::JitOptions *jo)
 	config_.totalNativeRegs = NUM_LAGPR + NUM_LAFPR;
 	// F regs are used for both FPU and Vec, so we don't need VREGs.
 	config_.mapUseVRegs = false;
-	config_.mapFPUSIMD = true;
+	// Every compiler in this backend picks its path from cpu_info.LOONGARCH_LSX, so the mapping
+	// has to agree with it. Claiming SIMD here while the scalar paths run leaves them addressing
+	// lanes with F(reg + n) against a mapping that has no per-lane registers.
+	config_.mapFPUSIMD = cpu_info.LOONGARCH_LSX;
 }
 
 void LoongArch64RegCache::Init(LoongArch64Emitter *emitter) {

Core/MIPS/MIPS.cpp

@@ -15,6 +15,7 @@
 // Official git repository and contact information can be found at
 // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
 
+#include <cfenv>
 #include <cmath>
 #include <limits>
 #include <mutex>
@@ -105,6 +106,13 @@ void ApplyHostRoundingMode(const MIPSState *mips) {
 		}
 
 		ARM64WriteFPCR(fpcr);
+#else
+		// No control register access written for this architecture, so go through the standard
+		// call. Rounding is all it can do: there is no portable flush-to-zero, and neither
+		// riscv64 nor loongarch64 has one in the base ISA either, so denormals stay as they are.
+		static const int roundLookup[4] = { FE_TONEAREST, FE_TOWARDZERO, FE_UPWARD, FE_DOWNWARD };
+		fesetround(roundLookup[rmode]);
+		(void)ftz;
 #endif
 	}
 }
@@ -121,6 +129,8 @@ void RestoreHostRoundingMode() {
 	fpcr &= ~(7 << 22);    // Clear bits [23:22] for rounding, 24 for FTZ
 	// Write back the modified FPCR
 	ARM64WriteFPCR(fpcr);
+#else
+	fesetround(FE_TONEAREST);
 #endif
 }
 

GPU/Common/VertexDecoderCommon.cpp

@@ -352,32 +352,48 @@ void VertexDecoder::Step_TcFloatThrough(const VertexDecoder *dec, const u8 *ptr,
 	gstate_c.vertBounds.maxV = std::max(gstate_c.vertBounds.maxV, (u16)uvdata[1]);
 }
 
+// The arm64 JIT and the NEON handwritten decoders fuse the UV prescale (FMLA), the x86 ones don't
+// (MULPS + ADDPS). Spell out which one happens here instead of leaving it to the compiler's
+// contraction setting: clang contracts this by default and MSVC doesn't, so relying on it makes the
+// steps disagree with the JIT on Windows on ARM only.
+static inline float PrescaleUV(float value, float scale, float offset) {
+#if PPSSPP_ARCH(ARM64_NEON) || PPSSPP_ARCH(RISCV64) || PPSSPP_ARCH(LOONGARCH64)
+	// The riscv64 and loongarch64 JITs fuse this too - and on those the compiler would contract
+	// the plain expression below into an FMA anyway, so say so rather than leaving it to chance.
+	return fmaf(value, scale, offset);
+#else
+	// Safe as long as x86 stays on the SSE2 baseline, which has nothing to contract into. A build
+	// targeting FMA would need this spelled out too, the other way around from the arm64 one.
+	return value * scale + offset;
+#endif
+}
+
 void VertexDecoder::Step_TcU8Prescale(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
 	float *uv = (float *)(decoded + dec->decFmt.uvoff);
 	const u8 *uvdata = (const u8 *)(ptr + dec->tcoff);
-	uv[0] = (float)uvdata[0] * (1.f / 128.f) * dec->prescaleUV_->uScale + dec->prescaleUV_->uOff;
+	uv[0] = PrescaleUV((float)uvdata[0] * (1.f / 128.f), dec->prescaleUV_->uScale, dec->prescaleUV_->uOff);
-	uv[1] = (float)uvdata[1] * (1.f / 128.f) * dec->prescaleUV_->vScale + dec->prescaleUV_->vOff;
+	uv[1] = PrescaleUV((float)uvdata[1] * (1.f / 128.f), dec->prescaleUV_->vScale, dec->prescaleUV_->vOff);
 }
 
 void VertexDecoder::Step_TcU16Prescale(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
 	float *uv = (float *)(decoded + dec->decFmt.uvoff);
 	const u16_le *uvdata = (const u16_le *)(ptr + dec->tcoff);
-	uv[0] = (float)uvdata[0] * (1.f / 32768.f) * dec->prescaleUV_->uScale + dec->prescaleUV_->uOff;
+	uv[0] = PrescaleUV((float)uvdata[0] * (1.f / 32768.f), dec->prescaleUV_->uScale, dec->prescaleUV_->uOff);
-	uv[1] = (float)uvdata[1] * (1.f / 32768.f) * dec->prescaleUV_->vScale + dec->prescaleUV_->vOff;
+	uv[1] = PrescaleUV((float)uvdata[1] * (1.f / 32768.f), dec->prescaleUV_->vScale, dec->prescaleUV_->vOff);
 }
 
 void VertexDecoder::Step_TcU16DoublePrescale(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
 	float *uv = (float *)(decoded + dec->decFmt.uvoff);
 	const u16_le *uvdata = (const u16_le *)(ptr + dec->tcoff);
-	uv[0] = (float)uvdata[0] * (1.f / 16384.f) * dec->prescaleUV_->uScale + dec->prescaleUV_->uOff;
+	uv[0] = PrescaleUV((float)uvdata[0] * (1.f / 16384.f), dec->prescaleUV_->uScale, dec->prescaleUV_->uOff);
-	uv[1] = (float)uvdata[1] * (1.f / 16384.f) * dec->prescaleUV_->vScale + dec->prescaleUV_->vOff;
+	uv[1] = PrescaleUV((float)uvdata[1] * (1.f / 16384.f), dec->prescaleUV_->vScale, dec->prescaleUV_->vOff);
 }
 
 void VertexDecoder::Step_TcFloatPrescale(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
 	float *uv = (float *)(decoded + dec->decFmt.uvoff);
 	const float_le *uvdata = (const float_le *)(ptr + dec->tcoff);
-	uv[0] = uvdata[0] * dec->prescaleUV_->uScale + dec->prescaleUV_->uOff;
+	uv[0] = PrescaleUV(uvdata[0], dec->prescaleUV_->uScale, dec->prescaleUV_->uOff);
-	uv[1] = uvdata[1] * dec->prescaleUV_->vScale + dec->prescaleUV_->vOff;
+	uv[1] = PrescaleUV(uvdata[1], dec->prescaleUV_->vScale, dec->prescaleUV_->vOff);
 }
 
 void VertexDecoder::Step_TcU8MorphToFloat(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
@@ -745,9 +761,7 @@ void VertexDecoder::Step_NormalS16Morph(const VertexDecoder *dec, const u8 *ptr,
 			acc[j] += sv[j] * multiplier;
 	}
 	float *normal = (float *)(decoded + dec->decFmt.nrmoff);
-	normal[0] = acc[0] * (1.0f / 32768.0f);
+	memcpy(normal, acc, sizeof(float) * 3);
-	normal[1] = acc[1] * (1.0f / 32768.0f);
-	normal[2] = acc[2] * (1.0f / 32768.0f);
 }
 
 void VertexDecoder::Step_NormalFloatMorph(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
@@ -835,6 +849,9 @@ void VertexDecoder::Step_PosS16(const VertexDecoder *dec, const u8 *ptr, u8 *dec
 
 void VertexDecoder::Step_PosFloat(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
 	Vec4F32 v = Vec4F32::Load((const float *)(ptr + dec->posoff));
+	// NaN and infinity only have to come out finite, so that the viewport scale can zero them later
+	// like the PSP does (0 * NaN == 0 there). The platforms differ in how (SSE clamps, NEON zeroes),
+	// and so do the JITs, which is fine.
 	v.CleanNaNInfs().Store((float *)(decoded + dec->decFmt.posoff));
 }
 
@@ -888,7 +905,9 @@ void VertexDecoder::Step_PosFloatThrough(const VertexDecoder *dec, const u8 *ptr
 	float *v = (float *)(decoded + dec->decFmt.posoff);
 	const float *fv = (const float *)(ptr + dec->posoff);
 	memcpy(v, fv, 8);
-	v[2] = fv[2] > 65535.0f ? 65535.0f : (fv[2] < 0.0f ? 0.0f : fv[2]);
+	// Depth is an integer in through mode: truncate, and clamp to 16 bits (NaN becomes 0).
+	const float z = fv[2];
+	v[2] = z >= 65535.0f ? 65535.0f : (z > 0.0f ? (float)(int)z : 0.0f);
 }
 
 void VertexDecoder::Step_PosS8Morph(const VertexDecoder *dec, const u8 *ptr, u8 *decoded) {
@@ -1191,9 +1210,8 @@ void VertexDecoder::SetVertexType(u32 fmt, const VertexDecoderOptions &options,
 		DEBUG_LOG(Log::G3D, "VTYPE: THRU=%i TC=%i COL=%i POS=%i NRM=%i WT=%i NW=%i IDX=%i MC=%i", (int)throughmode, tc, col, pos, nrm, weighttype, nweights, idx, morphcount);
 	}
 
-
+	skinInDecode = weighttype != 0;
 	if (weighttype) { // && nweights?
-		skinInDecode = true;
 		weightoff = size;
 		//size = align(size, wtalign[weighttype]);	unnecessary
 		size += wtsize[weighttype] * nweights;

GPU/Common/VertexDecoderHandwritten.cpp

@@ -1,5 +1,6 @@
 #include "Common/CommonTypes.h"
 #include "Common/Data/Convert/ColorConv.h"
+#include "Core/HDRemaster.h"
 #include "GPU/Common/VertexDecoderCommon.h"
 #include "GPU/GPUState.h"
 
@@ -61,41 +62,61 @@ void VtxDec_Tu16_C8888_Pfloat(const u8 *srcp, u8 *dstp, int count, const UVScale
 	};
 	const GOWVTX *src = (const GOWVTX *)srcp;
 	OutVTX *dst = (OutVTX *)dstp;
-	float uscale = uvScaleOffset->uScale * (1.0f / 32768.0f);
+	// The HD Remasters double the texture coordinates, see Step_TcU16DoublePrescale.
-	float vscale = uvScaleOffset->vScale * (1.0f / 32768);
+	const float uvDiv = g_DoubleTextureCoordinates ? (1.0f / 16384.0f) : (1.0f / 32768.0f);
+	float uscale = uvScaleOffset->uScale * uvDiv;
+	float vscale = uvScaleOffset->vScale * uvDiv;
 	float uoff = uvScaleOffset->uOff;
 	float voff = uvScaleOffset->vOff;
 
 	u32 alpha = 0xFFFFFFFF;
 
+	// The position is loaded together with the color, as lanes 1-3. A NaN or infinite coordinate has
+	// to come out finite, so that the viewport scale can zero it later like the PSP does. Which finite
+	// value doesn't matter, so any lane with an all-ones exponent is zeroed. Lane 0 (the color) is
+	// left alone: its mask is 0, which never equals the exponent.
+	alignas(16) static const u32 expMask[4] = { 0, 0x7F800000, 0x7F800000, 0x7F800000 };
+
 #if PPSSPP_ARCH(SSE2)
 	__m128 uvOff = _mm_setr_ps(uoff, voff, uoff, voff);
 	__m128 uvScale = _mm_setr_ps(uscale, vscale, uscale, vscale);
 	__m128i alphaMask = _mm_set1_epi32(0xFFFFFFFF);
+	const __m128i posExpMask = _mm_load_si128((const __m128i *)expMask);
+	const __m128i expAllOnes = _mm_set1_epi32(0x7F800000);
 	for (int i = 0; i < count; i++) {
 		__m128i uv = _mm_set1_epi32(src[i].packed_uv);
 		__m128 fuv = _mm_cvtepi32_ps(_mm_unpacklo_epi16(uv, _mm_setzero_si128()));
 		__m128 finalUV = _mm_add_ps(_mm_mul_ps(fuv, uvScale), uvOff);
 		u32 normal = src[i].packed_normal;
 		__m128i colpos = _mm_loadu_si128((const __m128i *)&src[i].col);
+		alphaMask = _mm_and_si128(alphaMask, colpos);
+		colpos = _mm_andnot_si128(_mm_cmpeq_epi32(_mm_and_si128(colpos, posExpMask), expAllOnes), colpos);
 		_mm_store_sd((double *)&dst[i].u, _mm_castps_pd(finalUV));
 		dst[i].packed_normal = normal;
 		_mm_storeu_si128((__m128i *)&dst[i].col, colpos);
-		alphaMask = _mm_and_si128(alphaMask, colpos);
 	}
 	alpha = _mm_cvtsi128_si32(alphaMask);
 
 #elif PPSSPP_ARCH(ARM_NEON)
-	float32x2_t uvScale = vmul_f32(vld1_f32(&uvScaleOffset->uScale), vdup_n_f32(1.0f / 32768.0f));
+	const float scaleArr[2] = { uscale, vscale };
+	float32x2_t uvScale = vld1_f32(scaleArr);
 	float32x2_t uvOff = vld1_f32(&uvScaleOffset->uOff);
 	uint32x4_t alphaMask = vdupq_n_u32(0xFFFFFFFF);
+	const uint32x4_t posExpMask = vld1q_u32(expMask);
+	const uint32x4_t expAllOnes = vdupq_n_u32(0x7F800000);
 	for (int i = 0; i < count; i++) {
 		uint16x4_t uv = vld1_u16(&src[i].u);  // TODO: We only need the first two lanes, maybe there's a better way?
 		uint32x2_t fuv = vget_low_u32(vmovl_u16(uv));  // Only using the first two lanes
+#if PPSSPP_ARCH(ARM64_NEON)
+		// Fused, like the arm64 JIT and the steps as the compiler builds them.
+		float32x2_t finalUV = vfma_f32(uvOff, vcvt_f32_u32(fuv), uvScale);
+#else
 		float32x2_t finalUV = vadd_f32(vmul_f32(vcvt_f32_u32(fuv), uvScale), uvOff);
+#endif
 		u32 normal = src[i].packed_normal;
 		uint32x4_t colpos = vld1q_u32((const u32 *)&src[i].col);
 		alphaMask = vandq_u32(alphaMask, colpos);
+		colpos = vbicq_u32(colpos, vceqq_u32(vandq_u32(colpos, posExpMask), expAllOnes));
 		vst1_f32(&dst[i].u, finalUV);
 		dst[i].packed_normal = normal;
 		vst1q_u32(&dst[i].col, colpos);
@@ -246,7 +267,11 @@ void VtxDec_Tu8_C5551_Ps16(const u8 *srcp, u8 *dstp, int numVerts, const UVScale
 		uint8x8_t uv8 = vreinterpret_u8_u64(uv8_one);
 		uint16x4_t uv16 = vget_low_u16(vmovl_u8(uv8));
 		uint32x4_t uv32 = vmovl_u16(uv16);
+#if PPSSPP_ARCH(ARM64_NEON)
+		float32x4_t uvf = vfmaq_f32(uvOffset, vcvtq_f32_u32(uv32), uvScale);
+#else
 		float32x4_t uvf = vaddq_f32(vmulq_f32(vcvtq_f32_u32(uv32), uvScale), uvOffset);
+#endif
 
 		alpha &= col0;
 
@@ -258,7 +283,7 @@ void VtxDec_Tu8_C5551_Ps16(const u8 *srcp, u8 *dstp, int numVerts, const UVScale
 		int32x2_t a_shifted = vshr_n_s32(vreinterpret_s32_u32(vshl_n_u32(vand_u32(col, amask), 16)), 7);
 		uint32x2_t a = vreinterpret_u32_s32(a_shifted);
 		col = vorr_u32(vorr_u32(r, g), b);
-		col = vorr_u32(col, vand_u32(vshl_n_u32(col, 5), lowbits));
+		col = vorr_u32(col, vand_u32(vshr_n_u32(col, 5), lowbits));
 		col = vorr_u32(col, a);
 
 		// TODO: Mix into fewer stores.

GPU/Common/VertexDecoderLoongArch64.cpp

@@ -281,10 +281,12 @@ JittedVertexDecoder VertexDecoderJitCache::Compile(const VertexDecoder &dec, int
 
     if (updateTexBounds) {
         LI(tempReg1, &gstate_c.vertBounds.minU);
-        LD_H(boundsMinUReg, tempReg1, offsetof(KnownVertexBounds, minU));
+        // Unsigned: the bounds are u16, and minU/minV start at 0xFFFF, which a signed load
-        LD_H(boundsMaxUReg, tempReg1, offsetof(KnownVertexBounds, maxU));
+        // would turn into -1 and no texcoord would ever be below it.
-        LD_H(boundsMinVReg, tempReg1, offsetof(KnownVertexBounds, minV));
+        LD_HU(boundsMinUReg, tempReg1, offsetof(KnownVertexBounds, minU));
-        LD_H(boundsMaxVReg, tempReg1, offsetof(KnownVertexBounds, maxV));
+        LD_HU(boundsMaxUReg, tempReg1, offsetof(KnownVertexBounds, maxU));
+        LD_HU(boundsMinVReg, tempReg1, offsetof(KnownVertexBounds, minV));
+        LD_HU(boundsMaxVReg, tempReg1, offsetof(KnownVertexBounds, maxV));
     }
 
     const u8 *loopStart = GetCodePtr();
@@ -657,41 +659,33 @@ void VertexDecoderJitCache::Jit_Color8888Morph() {
 	Jit_WriteMorphColor(dec_->decFmt.c0off);
 }
 
+// The packed color morph formats follow the steps channel by channel. The accumulator has to be
+// an LSX scratch register, not an F register: F4-F7 alias V4-V7, which hold the skin matrix for
+// the whole vertex.
 void VertexDecoderJitCache::Jit_Color4444Morph() {
+	const LoongArch64Reg accReg = lsxScratchReg;
+	const LoongArch64Reg weightReg = lsxScratchReg2;
+	const LoongArch64Reg valueReg = lsxScratchReg3;
+	const LoongArch64Reg scaleReg = lsxScratchReg4;
+	static const int shift[4] = { 0, 4, 8, 12 };
+	static const int width[4] = { 4, 4, 4, 4 };
+
 	LI(tempReg1, &gstate_c.morphWeights[0]);
-	VXOR_V(lsxScratchReg4, lsxScratchReg4, lsxScratchReg4);
+	VXOR_V(accReg, accReg, accReg);
+	LI(scratchReg, 255.0f / 15.0f);
+	VREPLGR2VR_W(scaleReg, scratchReg);
 
-	LI(tempReg2, 0xf00ff00f); // color 4444 mask
+	for (int n = 0; n < dec_->morphcount; n++) {
-	VREPLGR2VR_W(V8, tempReg2);
+		VLDREPL_W(weightReg, tempReg1, n * sizeof(float));
-	LI(tempReg3, 255.0f / 15.0f); // by color 4444
+		LD_HU(tempReg2, srcReg, dec_->onesize_ * n + dec_->coloff);
-	VREPLGR2VR_W(V9, tempReg2);
+		for (int j = 0; j < 4; j++) {
-
+			BSTRPICK_D(tempReg3, tempReg2, shift[j] + width[j] - 1, shift[j]);
-	bool first = true;
+			VINSGR2VR_W(valueReg, tempReg3, j);
-	for (int n = 0; n < dec_->morphcount; ++n) {
-		const LoongArch64Reg reg = first ? lsxScratchReg : lsxScratchReg2;
-		FLD_S((LoongArch64Reg)(DecodeReg(reg) + F0), srcReg, dec_->onesize_ * n + dec_->coloff);
-		VILVL_B(reg, reg, reg);
-		VAND_V(reg, reg, V8);
-		VEXTRINS_W(lsxScratchReg3, reg, 0);
-		VSLLI_H(lsxScratchReg3, lsxScratchReg3, 4);
-		VOR_V(reg, reg,lsxScratchReg3);
-		VSRLI_W(reg, reg, 4);
-
-		VILVL_B(reg, lsxScratchReg4, reg);
-		VILVL_H(reg, lsxScratchReg4, reg);
-
-		VFFINT_S_W(reg, reg);
-		VFMUL_S(reg, reg, V9);
-
-		// And now the weight.
-		VLDREPL_W(lsxScratchReg3, tempReg1, n * sizeof(float));
-		VFMUL_S(reg, reg, lsxScratchReg3);
-
-		if (!first) {
-			VFADD_S(lsxScratchReg, lsxScratchReg,lsxScratchReg2);
-		} else {
-			first = false;
 		}
+		VFFINT_S_W(valueReg, valueReg);
+		// col[j] += w * value * scale - the weight first, then the scale, as the steps do.
+		VFMUL_S(valueReg, valueReg, weightReg);
+		VFMADD_S(accReg, valueReg, scaleReg, accReg);
 	}
 
 	Jit_WriteMorphColor(dec_->decFmt.c0off);
@@ -702,46 +696,29 @@ alignas(16) static const u32 color565Mask[4] = { 0x0000f800, 0x000007e0, 0x00000
 alignas(16) static const float byColor565[4] = { 255.0f / 31.0f, 255.0f / 63.0f, 255.0f / 31.0f, 255.0f / 1.0f, };
 
 void VertexDecoderJitCache::Jit_Color565Morph() {
+	const LoongArch64Reg accReg = lsxScratchReg;
+	const LoongArch64Reg weightReg = lsxScratchReg2;
+	const LoongArch64Reg valueReg = lsxScratchReg3;
+	const LoongArch64Reg scaleReg = lsxScratchReg4;
+	static const int shift[4] = { 0, 5, 11, 0 };
+	static const int width[4] = { 5, 6, 5, 1 };
+
 	LI(tempReg1, &gstate_c.morphWeights[0]);
-	LI(tempReg2, &color565Mask[0]);
+	VXOR_V(accReg, accReg, accReg);
-	VLD(V8, tempReg2, 0);
 	LI(tempReg2, &byColor565[0]);
-	VLD(V9, tempReg2, 0);
+	VLD(scaleReg, tempReg2, 0);
 
-	bool first = true;
+	for (int n = 0; n < dec_->morphcount; n++) {
-	for (int n = 0; n < dec_->morphcount; ++n) {
+		VLDREPL_W(weightReg, tempReg1, n * sizeof(float));
-		const LoongArch64Reg reg = first ? lsxScratchReg : lsxScratchReg3;
+		LD_HU(tempReg2, srcReg, dec_->onesize_ * n + dec_->coloff);
-		// Spread it out into each lane.  We end up with it reversed (R high, A low.)
+		for (int j = 0; j < 3; j++) {
-		// Below, we shift out each lane from low to high and reverse them.
+			BSTRPICK_D(tempReg3, tempReg2, shift[j] + width[j] - 1, shift[j]);
-		VLDREPL_W(lsxScratchReg2, srcReg, dec_->onesize_ * n + dec_->coloff);
+			VINSGR2VR_W(valueReg, tempReg3, j);
-		VAND_V(lsxScratchReg2, lsxScratchReg2, V8);
-
-		// Alpha handled in Jit_WriteMorphColor.
-
-		// Blue first.
-		VEXTRINS_W(reg, lsxScratchReg2, 0);
-		VSRLI_W(reg, reg, 6);
-		VSHUF4I_W(reg, reg, 3 << 6);
-
-		// Green, let's shift it into the right lane first.
-		VEXTRINS_W(reg, lsxScratchReg2, 1);
-		VSRLI_W(reg, reg, 5);
-		VSHUF4I_W(reg, reg, (3 << 6 | 2 << 4));
-
-		// Last one, red.
-		VEXTRINS_W(reg, lsxScratchReg2, 2);
-		VFFINT_S_W(reg, reg);
-		VFMUL_S(reg, reg, V9);
-
-		// And now the weight.
-		VLDREPL_W(lsxScratchReg2, tempReg1, n * sizeof(float));
-		VFMUL_S(reg, reg, lsxScratchReg2);
-
-		if (!first) {
-			VFADD_S(lsxScratchReg, lsxScratchReg, lsxScratchReg3);
-		} else {
-			first = false;
 		}
+		VFFINT_S_W(valueReg, valueReg);
+		// col[j] += w * value * scale - the weight first, then the scale, as the steps do.
+		VFMUL_S(valueReg, valueReg, weightReg);
+		VFMADD_S(accReg, valueReg, scaleReg, accReg);
 	}
 
 	Jit_WriteMorphColor(dec_->decFmt.c0off, false);
@@ -752,59 +729,44 @@ alignas(16) static const u32 color5551Mask[4] = { 0x00008000, 0x00007c00, 0x0000
 alignas(16) static const float byColor5551[4] = { 255.0f / 31.0f, 255.0f / 31.0f, 255.0f / 31.0f, 255.0f / 1.0f, };
 
 void VertexDecoderJitCache::Jit_Color5551Morph() {
+	const LoongArch64Reg accReg = lsxScratchReg;
+	const LoongArch64Reg weightReg = lsxScratchReg2;
+	const LoongArch64Reg valueReg = lsxScratchReg3;
+	const LoongArch64Reg scaleReg = lsxScratchReg4;
+	static const int shift[4] = { 0, 5, 10, 15 };
+	static const int width[4] = { 5, 5, 5, 1 };
+
 	LI(tempReg1, &gstate_c.morphWeights[0]);
-	LI(tempReg2, &color5551Mask[0]);
+	VXOR_V(accReg, accReg, accReg);
-	VLD(V8, tempReg2, 0);
 	LI(tempReg2, &byColor5551[0]);
-	VLD(V9, tempReg2, 0);
+	VLD(scaleReg, tempReg2, 0);
 
-	bool first = true;
+	for (int n = 0; n < dec_->morphcount; n++) {
-	for (int n = 0; n < dec_->morphcount; ++n) {
+		VLDREPL_W(weightReg, tempReg1, n * sizeof(float));
-		const LoongArch64Reg reg = first ? lsxScratchReg : lsxScratchReg3;
+		LD_HU(tempReg2, srcReg, dec_->onesize_ * n + dec_->coloff);
-		// Spread it out into each lane.
+		for (int j = 0; j < 4; j++) {
-		VLDREPL_W(lsxScratchReg2, srcReg, dec_->onesize_ * n + dec_->coloff);
+			BSTRPICK_D(tempReg3, tempReg2, shift[j] + width[j] - 1, shift[j]);
-		VAND_V(lsxScratchReg2, lsxScratchReg2, V8);
+			VINSGR2VR_W(valueReg, tempReg3, j);
-
-		// Alpha first.
-		VEXTRINS_W(reg, lsxScratchReg2, 0);
-		VSRLI_W(reg, reg, 5);
-		VSHUF4I_W(reg, reg, 0);
-
-		// Blue, let's shift it into the right lane first.
-		VEXTRINS_W(reg, lsxScratchReg2, 1);
-		VSRLI_W(reg, reg, 5);
-		VSHUF4I_W(reg, reg, 3 << 6);
-
-		// Green.
-		VEXTRINS_W(reg, lsxScratchReg2, 2);
-		VSRLI_W(reg, reg, 5);
-		VSHUF4I_W(reg, reg, (3 << 6 | 2 << 4));
-
-		// Last one, red.
-		VEXTRINS_W(reg, lsxScratchReg2, 3);
-		VFFINT_S_W(reg, reg);
-		VFMUL_S(reg, reg, V9);
-
-		// And now the weight.
-		VLDREPL_W(lsxScratchReg2, tempReg1, n * sizeof(float));
-		VFMUL_S(reg, reg, lsxScratchReg2);
-
-		if (!first) {
-			VFADD_S(lsxScratchReg, lsxScratchReg, lsxScratchReg3);
-		} else {
-			first = false;
 		}
+		VFFINT_S_W(valueReg, valueReg);
+		// col[j] += w * value * scale - the weight first, then the scale, as the steps do.
+		VFMUL_S(valueReg, valueReg, weightReg);
+		VFMADD_S(accReg, valueReg, scaleReg, accReg);
 	}
 
 	Jit_WriteMorphColor(dec_->decFmt.c0off);
 }
 
 void VertexDecoderJitCache::Jit_WriteMorphColor(int outOff, bool checkAlpha) {
-	// Pack back into a u32, with saturation.
+	// Pack back into a u32, with saturation. Truncate towards zero like the (int) cast in the
-	VFTINT_W_S(lsxScratchReg, lsxScratchReg);
+	// steps, and narrow signed->signed then signed->unsigned: the logical narrowing shifts read a
-	VSSRLNI_H_W(lsxScratchReg, lsxScratchReg, 0);
+	// negative channel as a huge unsigned value and saturate it to 255 instead of clamping to 0.
-	VSSRLNI_BU_H(lsxScratchReg, lsxScratchReg, 0);
+	VFTINTRZ_W_S(lsxScratchReg, lsxScratchReg);
-	VPICKVE2GR_W(tempReg1, lsxScratchReg, 0);
+	VSSRANI_H_W(lsxScratchReg, lsxScratchReg, 0);
+	VSSRANI_BU_H(lsxScratchReg, lsxScratchReg, 0);
+	// Unsigned: the full-alpha check below compares this against 0xFF000000, and a sign-extended
+	// color with alpha >= 0x80 would look larger than any of it.
+	VPICKVE2GR_WU(tempReg1, lsxScratchReg, 0);
 
 	// TODO: Could be optimize with a SLLI on fullAlphaReg
 	SLLI_D(tempReg2, fullAlphaReg, 24);
@@ -988,14 +950,17 @@ void VertexDecoderJitCache::Jit_PosS16() {
 }
 
 void VertexDecoderJitCache::Jit_PosFloat() {
-	// Just copy 12 bytes, play with over read/write later.
+	// Step_PosFloat cleans out infinities and NaNs. Do it on the bits rather than with FMIN/FMAX
-	// TODO: This should clean out inf and NaN values.
+	// so it doesn't depend on how the FPU treats a NaN operand: anything with all exponent bits
-	LD_W(tempReg1, srcReg, dec_->posoff + 0);
+	// set becomes zero, which is finite and multiplies to zero, as the callers expect.
-	LD_W(tempReg2, srcReg, dec_->posoff + 4);
+	LI(scratchReg, 0x7F800000);
-	LD_W(tempReg3, srcReg, dec_->posoff + 8);
+	for (int i = 0; i < 3; i++) {
-	ST_W(tempReg1, dstReg, dec_->decFmt.posoff + 0);
+		LD_W(tempReg1, srcReg, dec_->posoff + i * 4);
-	ST_W(tempReg2, dstReg, dec_->decFmt.posoff + 4);
+		BSTRPICK_D(tempReg2, tempReg1, 30, 0);  // Drop the sign bit.
-	ST_W(tempReg3, dstReg, dec_->decFmt.posoff + 8);
+		SLTU(tempReg3, tempReg2, scratchReg);   // 1 if finite.
+		MASKEQZ(tempReg1, tempReg1, tempReg3);  // Zero it if not.
+		ST_W(tempReg1, dstReg, dec_->decFmt.posoff + i * 4);
+	}
 }
 
 void VertexDecoderJitCache::Jit_PosS8Through() {
@@ -1037,6 +1002,9 @@ void VertexDecoderJitCache::Jit_PosFloatThrough() {
 	MOVGR2FR_W(fpScratchReg2, scratchReg);
 	FMAX_S(fpSrc[2], fpSrc[2], fpScratchReg);
 	FMIN_S(fpSrc[2], fpSrc[2], fpScratchReg2);
+	// Depth is an integer in through mode - truncate, like Step_PosFloatThrough.
+	FTINTRZ_W_S(fpSrc[2], fpSrc[2]);
+	FFINT_S_W(fpSrc[2], fpSrc[2]);
 	FST_S(fpSrc[2], dstReg, dec_->decFmt.posoff + 8);
 }
 

GPU/Common/VertexDecoderRiscV.cpp

@@ -97,13 +97,16 @@ static float skinMatrix[12];
 static uint32_t GetMorphValueUsage(uint32_t vtype) {
 	uint32_t morphFlags = 0;
 	switch (vtype & GE_VTYPE_TC_MASK) {
-	case GE_VTYPE_TC_8BIT: morphFlags |= 1 << (int)MorphValuesIndex::BY_128; break;
+	// The prescale decoders bake by128 into the prescale and use the raw weight instead, so
-	case GE_VTYPE_TC_16BIT: morphFlags |= 1 << (int)MorphValuesIndex::BY_32768; break;
+	// both forms have to be available - which one runs isn't known from the vertex type alone.
+	case GE_VTYPE_TC_8BIT: morphFlags |= (1 << (int)MorphValuesIndex::BY_128) | (1 << (int)MorphValuesIndex::AS_FLOAT); break;
+	case GE_VTYPE_TC_16BIT: morphFlags |= (1 << (int)MorphValuesIndex::BY_32768) | (1 << (int)MorphValuesIndex::AS_FLOAT); break;
 	case GE_VTYPE_TC_FLOAT: morphFlags |= 1 << (int)MorphValuesIndex::AS_FLOAT; break;
 	}
 	switch (vtype & GE_VTYPE_COL_MASK) {
 	case GE_VTYPE_COL_565: morphFlags |= (1 << (int)MorphValuesIndex::COLOR_5) | (1 << (int)MorphValuesIndex::COLOR_6); break;
-	case GE_VTYPE_COL_5551: morphFlags |= 1 << (int)MorphValuesIndex::COLOR_5; break;
+	// 5551 alpha is accumulated with the raw weight, not the color-scaled one.
+	case GE_VTYPE_COL_5551: morphFlags |= (1 << (int)MorphValuesIndex::COLOR_5) | (1 << (int)MorphValuesIndex::AS_FLOAT); break;
 	case GE_VTYPE_COL_4444: morphFlags |= 1 << (int)MorphValuesIndex::COLOR_4; break;
 	case GE_VTYPE_COL_8888: morphFlags |= 1 << (int)MorphValuesIndex::AS_FLOAT; break;
 	}
@@ -300,10 +303,11 @@ JittedVertexDecoder VertexDecoderJitCache::Compile(const VertexDecoder &dec, int
 	if (dec.tc && dec.throughmode) {
 		// TODO: Smarter, only when doing bounds.
 		LI(tempReg1, &gstate_c.vertBounds.minU);
-		LH(boundsMinUReg, tempReg1, offsetof(KnownVertexBounds, minU));
+		// Unsigned - these are u16, and minU/minV start at 0xFFFF.
-		LH(boundsMaxUReg, tempReg1, offsetof(KnownVertexBounds, maxU));
+		LHU(boundsMinUReg, tempReg1, offsetof(KnownVertexBounds, minU));
-		LH(boundsMinVReg, tempReg1, offsetof(KnownVertexBounds, minV));
+		LHU(boundsMaxUReg, tempReg1, offsetof(KnownVertexBounds, maxU));
-		LH(boundsMaxVReg, tempReg1, offsetof(KnownVertexBounds, maxV));
+		LHU(boundsMinVReg, tempReg1, offsetof(KnownVertexBounds, minV));
+		LHU(boundsMaxVReg, tempReg1, offsetof(KnownVertexBounds, maxV));
 	}
 
 	const u8 *loopStart = GetCodePtr();
@@ -501,7 +505,9 @@ void VertexDecoderJitCache::Jit_TcU16ThroughToFloat() {
 		MAXU(boundsMaxVReg, boundsMaxVReg, tempReg2);
 	} else {
 		auto updateSide = [&](RiscVReg src, bool greater, RiscVReg dst) {
-			FixupBranch skip = BLT(greater ? dst : src, greater ? src : dst);
+			// Skip when dst is already the more extreme of the two. Unsigned: these are u16
+			// texcoords, and a value above 32767 compares negative as a signed word.
+			FixupBranch skip = greater ? BGEU(dst, src) : BGEU(src, dst);
 			MV(dst, src);
 			SetJumpTarget(skip);
 		};
@@ -558,15 +564,13 @@ void VertexDecoderJitCache::Jit_TcFloatPrescale() {
 }
 
 void VertexDecoderJitCache::Jit_TcU8MorphToFloat() {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_128 * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	LBU(tempReg1, srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	LBU(tempReg2, srcReg, dec_->tcoff + 1);
+	// lets a -0 term through where the steps give +0.
-	FCVT(FConv::S, FConv::WU, fpSrc[0], tempReg1, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FCVT(FConv::S, FConv::WU, fpSrc[1], tempReg2, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_128 * 8 + n) * 4);
 		LBU(tempReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		LBU(tempReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 1);
@@ -581,15 +585,13 @@ void VertexDecoderJitCache::Jit_TcU8MorphToFloat() {
 }
 
 void VertexDecoderJitCache::Jit_TcU16MorphToFloat() {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_32768 * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	LHU(tempReg1, srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	LHU(tempReg2, srcReg, dec_->tcoff + 2);
+	// lets a -0 term through where the steps give +0.
-	FCVT(FConv::S, FConv::WU, fpSrc[0], tempReg1, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FCVT(FConv::S, FConv::WU, fpSrc[1], tempReg2, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_32768 * 8 + n) * 4);
 		LHU(tempReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		LHU(tempReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 2);
@@ -604,13 +606,13 @@ void VertexDecoderJitCache::Jit_TcU16MorphToFloat() {
 }
 
 void VertexDecoderJitCache::Jit_TcFloatMorph() {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	FL(32, fpSrc[0], srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	FL(32, fpSrc[1], srcReg, dec_->tcoff + 4);
+	// lets a -0 term through where the steps give +0.
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + n) * 4);
 		FL(32, fpScratchReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		FL(32, fpScratchReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 4);
@@ -624,15 +626,13 @@ void VertexDecoderJitCache::Jit_TcFloatMorph() {
 
 void VertexDecoderJitCache::Jit_TcU8PrescaleMorph() {
 	// We use AS_FLOAT since by128 is already baked into precale.
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	LBU(tempReg1, srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	LBU(tempReg2, srcReg, dec_->tcoff + 1);
+	// lets a -0 term through where the steps give +0.
-	FCVT(FConv::S, FConv::WU, fpSrc[0], tempReg1, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FCVT(FConv::S, FConv::WU, fpSrc[1], tempReg2, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + n) * 4);
 		LBU(tempReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		LBU(tempReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 1);
@@ -650,15 +650,13 @@ void VertexDecoderJitCache::Jit_TcU8PrescaleMorph() {
 
 void VertexDecoderJitCache::Jit_TcU16PrescaleMorph() {
 	// We use AS_FLOAT since by32768 is already baked into precale.
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	LHU(tempReg1, srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	LHU(tempReg2, srcReg, dec_->tcoff + 2);
+	// lets a -0 term through where the steps give +0.
-	FCVT(FConv::S, FConv::WU, fpSrc[0], tempReg1, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FCVT(FConv::S, FConv::WU, fpSrc[1], tempReg2, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + n) * 4);
 		LHU(tempReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		LHU(tempReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 2);
@@ -675,13 +673,13 @@ void VertexDecoderJitCache::Jit_TcU16PrescaleMorph() {
 }
 
 void VertexDecoderJitCache::Jit_TcFloatPrescaleMorph() {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + 0) * 4);
+	// Accumulate like the Step_ functions, which the compiler contracts into fused
-	FL(32, fpSrc[0], srcReg, dec_->tcoff + 0);
+	// multiply-adds. Starting from the first product instead of +0.0 rounds differently and
-	FL(32, fpSrc[1], srcReg, dec_->tcoff + 4);
+	// lets a -0 term through where the steps give +0.
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
+	for (int j = 0; j < 2; j++)
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + n) * 4);
 		FL(32, fpScratchReg1, srcReg, dec_->onesize_ * n + dec_->tcoff + 0);
 		FL(32, fpScratchReg2, srcReg, dec_->onesize_ * n + dec_->tcoff + 4);
@@ -784,13 +782,16 @@ void VertexDecoderJitCache::Jit_PosS16() {
 }
 
 void VertexDecoderJitCache::Jit_PosFloat() {
-	// Just copy 12 bytes, play with over read/write later.
+	// Clamp to +-FLT_MAX, which turns infinities finite and, since FMIN/FMAX return the non-NaN
-	LW(tempReg1, srcReg, dec_->posoff + 0);
+	// operand, NaN into FLT_MAX. Step_PosFloat does the same via CleanNaNInfs.
-	LW(tempReg2, srcReg, dec_->posoff + 4);
+	QuickFLI(32, fpScratchReg1, FLT_MAX, scratchReg);
-	LW(tempReg3, srcReg, dec_->posoff + 8);
+	FNEG(32, fpScratchReg2, fpScratchReg1);
-	SW(tempReg1, dstReg, dec_->decFmt.posoff + 0);
+	for (int i = 0; i < 3; i++) {
-	SW(tempReg2, dstReg, dec_->decFmt.posoff + 4);
+		FL(32, fpSrc[i], srcReg, dec_->posoff + i * 4);
-	SW(tempReg3, dstReg, dec_->decFmt.posoff + 8);
+		FMIN(32, fpSrc[i], fpSrc[i], fpScratchReg1);
+		FMAX(32, fpSrc[i], fpSrc[i], fpScratchReg2);
+		FS(32, fpSrc[i], dstReg, dec_->decFmt.posoff + i * 4);
+	}
 }
 
 void VertexDecoderJitCache::Jit_PosS8Skin() {
@@ -843,6 +844,9 @@ void VertexDecoderJitCache::Jit_PosFloatThrough() {
 	FMV(FMv::W, FMv::X, fpScratchReg1, R_ZERO);
 	FMAX(32, fpSrc[2], fpSrc[2], fpScratchReg1);
 	FMIN(32, fpSrc[2], fpSrc[2], const65535Reg);
+	// Depth is an integer in through mode - truncate, like Step_PosFloatThrough.
+	FCVT(FConv::W, FConv::S, scratchReg, fpSrc[2], Round::TOZERO);
+	FCVT(FConv::S, FConv::W, fpSrc[2], scratchReg);
 	FS(32, fpSrc[2], dstReg, dec_->decFmt.posoff + 8);
 }
 
@@ -1253,28 +1257,22 @@ void VertexDecoderJitCache::Jit_AnyU16ToFloat(int srcoff, u32 bits) {
 }
 
 void VertexDecoderJitCache::Jit_AnyS8Morph(int srcoff, int dstoff) {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_128 * 8 + 0) * 4);
+	// Accumulate exactly like the Step_ functions, which the compiler contracts into fused
-	LB(tempReg1, srcReg, srcoff + 0);
+	// multiply-adds - so use FMADD here too, and start from +0.0 rather than from the first
-	LB(tempReg2, srcReg, srcoff + 1);
+	// product, which would round differently and let a -0 term through where the steps give +0.
-	LB(tempReg3, srcReg, srcoff + 2);
+	for (int j = 0; j < 3; j++)
-	FCVT(FConv::S, FConv::W, fpSrc[0], tempReg1, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FCVT(FConv::S, FConv::W, fpSrc[1], tempReg2, Round::TOZERO);
-	FCVT(FConv::S, FConv::W, fpSrc[2], tempReg3, Round::TOZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[2], fpSrc[2], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	const RiscVReg fpTemp[3] = { fpScratchReg1, fpScratchReg2, fpScratchReg3 };
+	const RiscVReg tempRegs[3] = { tempReg1, tempReg2, tempReg3 };
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_128 * 8 + n) * 4);
-		LB(tempReg1, srcReg, dec_->onesize_ * n + srcoff + 0);
+		for (int j = 0; j < 3; j++)
-		LB(tempReg2, srcReg, dec_->onesize_ * n + srcoff + 1);
+			LB(tempRegs[j], srcReg, dec_->onesize_ * n + srcoff + j);
-		LB(tempReg3, srcReg, dec_->onesize_ * n + srcoff + 2);
+		for (int j = 0; j < 3; j++)
-		FCVT(FConv::S, FConv::W, fpScratchReg1, tempReg1, Round::TOZERO);
+			FCVT(FConv::S, FConv::W, fpTemp[j], tempRegs[j]);
-		FCVT(FConv::S, FConv::W, fpScratchReg2, tempReg2, Round::TOZERO);
+		for (int j = 0; j < 3; j++)
-		FCVT(FConv::S, FConv::W, fpScratchReg3, tempReg3, Round::TOZERO);
+			FMADD(32, fpSrc[j], fpTemp[j], fpScratchReg4, fpSrc[j]);
-		FMADD(32, fpSrc[0], fpScratchReg1, fpScratchReg4, fpSrc[0]);
-		FMADD(32, fpSrc[1], fpScratchReg2, fpScratchReg4, fpSrc[1]);
-		FMADD(32, fpSrc[2], fpScratchReg3, fpScratchReg4, fpSrc[2]);
 	}
 
 	if (dstoff >= 0) {
@@ -1285,28 +1283,22 @@ void VertexDecoderJitCache::Jit_AnyS8Morph(int srcoff, int dstoff) {
 }
 
 void VertexDecoderJitCache::Jit_AnyS16Morph(int srcoff, int dstoff) {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_32768 * 8 + 0) * 4);
+	// Accumulate exactly like the Step_ functions, which the compiler contracts into fused
-	LH(tempReg1, srcReg, srcoff + 0);
+	// multiply-adds - so use FMADD here too, and start from +0.0 rather than from the first
-	LH(tempReg2, srcReg, srcoff + 2);
+	// product, which would round differently and let a -0 term through where the steps give +0.
-	LH(tempReg3, srcReg, srcoff + 4);
+	for (int j = 0; j < 3; j++)
-	FCVT(FConv::S, FConv::W, fpSrc[0], tempReg1, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FCVT(FConv::S, FConv::W, fpSrc[1], tempReg2, Round::TOZERO);
-	FCVT(FConv::S, FConv::W, fpSrc[2], tempReg3, Round::TOZERO);
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[2], fpSrc[2], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	const RiscVReg fpTemp[3] = { fpScratchReg1, fpScratchReg2, fpScratchReg3 };
+	const RiscVReg tempRegs[3] = { tempReg1, tempReg2, tempReg3 };
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::BY_32768 * 8 + n) * 4);
-		LH(tempReg1, srcReg, dec_->onesize_ * n + srcoff + 0);
+		for (int j = 0; j < 3; j++)
-		LH(tempReg2, srcReg, dec_->onesize_ * n + srcoff + 2);
+			LH(tempRegs[j], srcReg, dec_->onesize_ * n + srcoff + j * 2);
-		LH(tempReg3, srcReg, dec_->onesize_ * n + srcoff + 4);
+		for (int j = 0; j < 3; j++)
-		FCVT(FConv::S, FConv::W, fpScratchReg1, tempReg1, Round::TOZERO);
+			FCVT(FConv::S, FConv::W, fpTemp[j], tempRegs[j]);
-		FCVT(FConv::S, FConv::W, fpScratchReg2, tempReg2, Round::TOZERO);
+		for (int j = 0; j < 3; j++)
-		FCVT(FConv::S, FConv::W, fpScratchReg3, tempReg3, Round::TOZERO);
+			FMADD(32, fpSrc[j], fpTemp[j], fpScratchReg4, fpSrc[j]);
-		FMADD(32, fpSrc[0], fpScratchReg1, fpScratchReg4, fpSrc[0]);
-		FMADD(32, fpSrc[1], fpScratchReg2, fpScratchReg4, fpSrc[1]);
-		FMADD(32, fpSrc[2], fpScratchReg3, fpScratchReg4, fpSrc[2]);
 	}
 
 	if (dstoff >= 0) {
@@ -1317,22 +1309,19 @@ void VertexDecoderJitCache::Jit_AnyS16Morph(int srcoff, int dstoff) {
 }
 
 void VertexDecoderJitCache::Jit_AnyFloatMorph(int srcoff, int dstoff) {
-	FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + 0) * 4);
+	// Accumulate exactly like the Step_ functions, which the compiler contracts into fused
-	FL(32, fpSrc[0], srcReg, srcoff + 0);
+	// multiply-adds - so use FMADD here too, and start from +0.0 rather than from the first
-	FL(32, fpSrc[1], srcReg, srcoff + 4);
+	// product, which would round differently and let a -0 term through where the steps give +0.
-	FL(32, fpSrc[2], srcReg, srcoff + 8);
+	for (int j = 0; j < 3; j++)
-	FMUL(32, fpSrc[0], fpSrc[0], fpScratchReg4, Round::TOZERO);
+		FMV(FMv::W, FMv::X, fpSrc[j], R_ZERO);
-	FMUL(32, fpSrc[1], fpSrc[1], fpScratchReg4, Round::TOZERO);
-	FMUL(32, fpSrc[2], fpSrc[2], fpScratchReg4, Round::TOZERO);
 
-	for (int n = 1; n < dec_->morphcount; n++) {
+	const RiscVReg fpTemp[3] = { fpScratchReg1, fpScratchReg2, fpScratchReg3 };
+	for (int n = 0; n < dec_->morphcount; n++) {
 		FL(32, fpScratchReg4, morphBaseReg, ((int)MorphValuesIndex::AS_FLOAT * 8 + n) * 4);
-		FL(32, fpScratchReg1, srcReg, dec_->onesize_ * n + srcoff + 0);
+		for (int j = 0; j < 3; j++)
-		FL(32, fpScratchReg2, srcReg, dec_->onesize_ * n + srcoff + 4);
+			FL(32, fpTemp[j], srcReg, dec_->onesize_ * n + srcoff + j * 4);
-		FL(32, fpScratchReg3, srcReg, dec_->onesize_ * n + srcoff + 8);
+		for (int j = 0; j < 3; j++)
-		FMADD(32, fpSrc[0], fpScratchReg1, fpScratchReg4, fpSrc[0]);
+			FMADD(32, fpSrc[j], fpTemp[j], fpScratchReg4, fpSrc[j]);
-		FMADD(32, fpSrc[1], fpScratchReg2, fpScratchReg4, fpSrc[1]);
-		FMADD(32, fpSrc[2], fpScratchReg3, fpScratchReg4, fpSrc[2]);
 	}
 
 	if (dstoff >= 0) {

GPU/Common/VertexDecoderX86.cpp

@@ -691,6 +691,10 @@ void VertexDecoderJitCache::Jit_TcAnyMorph(int bits) {
 			first = false;
 		}
 	}
+
+	// The steps sum onto +0, which turns a sum of -0s into +0. Adding +0 at the end does the same.
+	XORPS(fpScratchReg2, R(fpScratchReg2));
+	ADDPS(fpScratchReg, R(fpScratchReg2));
 }
 
 void VertexDecoderJitCache::Jit_TcU8MorphToFloat() {
@@ -770,10 +774,11 @@ void VertexDecoderJitCache::Jit_TcU16ThroughToFloat() {
 		SetJumpTarget(skip);
 	};
 	// TODO: Can this actually be fast?  Hmm, floats aren't better.
-	updateSide(tempReg1, CC_GE, offsetof(KnownVertexBounds, minU));
+	// The bounds are unsigned, so unsigned conditions.
-	updateSide(tempReg1, CC_LE, offsetof(KnownVertexBounds, maxU));
+	updateSide(tempReg1, CC_AE, offsetof(KnownVertexBounds, minU));
-	updateSide(tempReg2, CC_GE, offsetof(KnownVertexBounds, minV));
+	updateSide(tempReg1, CC_BE, offsetof(KnownVertexBounds, maxU));
-	updateSide(tempReg2, CC_LE, offsetof(KnownVertexBounds, maxV));
+	updateSide(tempReg2, CC_AE, offsetof(KnownVertexBounds, minV));
+	updateSide(tempReg2, CC_BE, offsetof(KnownVertexBounds, maxV));
 }
 
 void VertexDecoderJitCache::Jit_TcFloatThrough() {
@@ -994,12 +999,12 @@ void VertexDecoderJitCache::Jit_Color4444Morph() {
 		}
 
 		CVTDQ2PS(reg, R(reg));
-		MULPS(reg, R(XMM6));
 
-		// And now the weight.
+		// The weight goes on before the scale, same order as the steps (it rounds differently).
 		MOVSS(fpScratchReg3, MDisp(tempReg1, n * sizeof(float)));
 		SHUFPS(fpScratchReg3, R(fpScratchReg3), _MM_SHUFFLE(0, 0, 0, 0));
 		MULPS(reg, R(fpScratchReg3));
+		MULPS(reg, R(XMM6));
 
 		if (!first) {
 			ADDPS(fpScratchReg, R(fpScratchReg2));
@@ -1049,12 +1054,12 @@ void VertexDecoderJitCache::Jit_Color565Morph() {
 		MOVSS(reg, R(fpScratchReg2));
 
 		CVTDQ2PS(reg, R(reg));
-		MULPS(reg, R(XMM6));
 
-		// And now the weight.
+		// The weight goes on before the scale, same order as the steps (it rounds differently).
 		MOVSS(fpScratchReg2, MDisp(tempReg1, n * sizeof(float)));
 		SHUFPS(fpScratchReg2, R(fpScratchReg2), _MM_SHUFFLE(0, 0, 0, 0));
 		MULPS(reg, R(fpScratchReg2));
+		MULPS(reg, R(XMM6));
 
 		if (!first) {
 			ADDPS(fpScratchReg, R(fpScratchReg3));
@@ -1107,12 +1112,12 @@ void VertexDecoderJitCache::Jit_Color5551Morph() {
 		MOVSS(reg, R(fpScratchReg2));
 
 		CVTDQ2PS(reg, R(reg));
-		MULPS(reg, R(XMM6));
 
-		// And now the weight.
+		// The weight goes on before the scale, same order as the steps (it rounds differently).
 		MOVSS(fpScratchReg2, MDisp(tempReg1, n * sizeof(float)));
 		SHUFPS(fpScratchReg2, R(fpScratchReg2), _MM_SHUFFLE(0, 0, 0, 0));
 		MULPS(reg, R(fpScratchReg2));
+		MULPS(reg, R(XMM6));
 
 		if (!first) {
 			ADDPS(fpScratchReg, R(fpScratchReg3));
@@ -1125,8 +1130,8 @@ void VertexDecoderJitCache::Jit_Color5551Morph() {
 }
 
 void VertexDecoderJitCache::Jit_WriteMorphColor(int outOff, bool checkAlpha) {
-	// Pack back into a u32, with saturation.
+	// Pack back into a u32, with saturation. Truncate like the other color paths.
-	CVTPS2DQ(fpScratchReg, R(fpScratchReg));
+	CVTTPS2DQ(fpScratchReg, R(fpScratchReg));
 	PACKSSDW(fpScratchReg, R(fpScratchReg));
 	PACKUSWB(fpScratchReg, R(fpScratchReg));
 	MOVD_xmm(R(tempReg1), fpScratchReg);
@@ -1467,6 +1472,9 @@ void VertexDecoderJitCache::Jit_AnyS8Morph(int srcoff, int dstoff) {
 		}
 	}
 
+	// The steps sum onto +0, which turns a sum of -0s into +0. Adding +0 at the end does the same.
+	XORPS(fpScratchReg2, R(fpScratchReg2));
+	ADDPS(fpScratchReg, R(fpScratchReg2));
 	MOVUPS(MDisp(dstReg, dstoff), fpScratchReg);
 }
 
@@ -1506,6 +1514,9 @@ void VertexDecoderJitCache::Jit_AnyS16Morph(int srcoff, int dstoff) {
 		}
 	}
 
+	// The steps sum onto +0, which turns a sum of -0s into +0. Adding +0 at the end does the same.
+	XORPS(fpScratchReg2, R(fpScratchReg2));
+	ADDPS(fpScratchReg, R(fpScratchReg2));
 	MOVUPS(MDisp(dstReg, dstoff), fpScratchReg);
 }
 
@@ -1527,6 +1538,9 @@ void VertexDecoderJitCache::Jit_AnyFloatMorph(int srcoff, int dstoff) {
 		}
 	}
 
+	// The steps sum onto +0, which turns a sum of -0s into +0. Adding +0 at the end does the same.
+	XORPS(fpScratchReg2, R(fpScratchReg2));
+	ADDPS(fpScratchReg, R(fpScratchReg2));
 	MOVUPS(MDisp(dstReg, dstoff), fpScratchReg);
 }
 

docs/building.md

@@ -146,6 +146,15 @@ we have: MSVC links an `inline` function defined in a .cpp anyway, clang correct
 build and pass on Windows and fail to link only on Android CI, with an undefined symbol pointing at a header
 line. Fix it by dropping the bogus `inline` from the definition, not by avoiding the call.
 
+The compilers also disagree about floating point contraction, which matters for any test asserting that a JIT
+is bit-identical to its C++ reference. Clang folds `a * b + c` into a single fused multiply-add by default;
+MSVC never does, under `/fp:precise`, in Debug or Release. So on arm64, where the JITs emit `FMLA`, a
+reference written as `a * b + c` matches on Mac, Linux and Android and is off by one ULP on Windows on ARM.
+Don't leave it to the compiler: write `fmaf(a, b, c)` when the fused result is wanted (MSVC compiles it to a
+single `fmadd`), and `a * b + c` when it isn't. `PrescaleUV` in `GPU/Common/VertexDecoderCommon.cpp` picks per
+architecture, matching what each JIT does. x86 doesn't have the problem, since the SSE2 baseline has no FMA
+instruction to contract into.
+
 pspautotests are a large set of tests of the PSP OS's API surface, and thus tests our HLE implementation.
 
 **To check for regressions, run them exactly the way CI does** (see `.github/workflows/build.yml`):

test.py

@@ -436,6 +436,30 @@ tests_good = [
 
 # Broken tests
 # -b flag runs these.
+
+# Tests that don't pass yet on an architecture we can only reach through emulation. Pass
+# --known-failures=<arch> to drop them from the run, so CI can still catch anything *new* breaking
+# while these stay outstanding. Keep a reason next to each one, and delete entries as they're fixed
+# rather than letting the list rot.
+known_failures = {
+  "riscv64": [
+    # No flush-to-zero: the ISA has no control for it, so a denormal result survives where the
+    # PSP would have flushed it. Everything else in this test passes.
+    "cpu/fpu/fpu",
+    # The software renderer's output differs from the reference by the same amount on both of
+    # these architectures, despite them using completely different SIMD paths. Unexplained.
+    "gpu/clipping/homogeneous",
+    "gpu/commands/cull",
+    "gpu/primitives/triangles",
+  ],
+  "loongarch64": [
+    "cpu/fpu/fpu",
+    "gpu/clipping/homogeneous",
+    "gpu/commands/cull",
+    "gpu/primitives/triangles",
+  ],
+}
+
 tests_next = [
 # These are the next tests up for fixing. These run by default.
   "cpu/fpu/fcr",
@@ -634,10 +658,17 @@ def main():
   tests = []
   args = []
   teamcity = False
+  skip_arch = None
   for arg in sys.argv[1:]:
     if arg == '--teamcity':
       args.append(arg)
       teamcity = True
+    elif arg.startswith('--known-failures='):
+      # Ours, not headless's - don't pass it through.
+      skip_arch = arg[len('--known-failures='):]
+      if skip_arch not in known_failures:
+        print("Unknown architecture for --known-failures: " + skip_arch)
+        sys.exit(1)
     elif arg[0] == '-':
       args.append(arg)
     else:
@@ -657,6 +688,12 @@ def main():
   elif '-m' in args:
     tests = [i for i in tests_next + tests_good if i.startswith(tests[0])]
 
+  if skip_arch:
+    skipped = [t for t in tests if t in known_failures[skip_arch]]
+    tests = [t for t in tests if t not in known_failures[skip_arch]]
+    if skipped:
+      print("Skipping %d known failures on %s: %s" % (len(skipped), skip_arch, ", ".join(skipped)))
+
   returncode = run_tests(tests, args)
   if teamcity:
     return 0

unittest/TestVertexJit.cpp

@@ -16,11 +16,18 @@
 // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
 
 #include <math.h>
+#include <cmath>
+#include <cstring>
+#include <map>
+#include <string>
 
 #include "Common/CommonTypes.h"
+#include "Common/MemoryUtil.h"
+#include "Common/StringUtils.h"
 #include "Common/TimeUtil.h"
 #include "Core/Config.h"
 #include "Core/ConfigValues.h"
+#include "Core/HDRemaster.h"
 #include "GPU/Common/VertexDecoderCommon.h"
 #include "GPU/ge_constants.h"
 #include "GPU/GPUState.h"
@@ -628,6 +635,443 @@ static bool TestVertexFloatSkin() {
 
 // TODO: Morph (col, pos, nrm), weights (no skin), morph + weights?
 
+// Everything below checks the JIT (and the handwritten SIMD decoders) against the step functions,
+// across the whole space of vertex formats. The two must agree bit for bit - not just closely -
+// since tests and frame dumps are recorded with one and games run with the other.
+
+namespace {
+
+struct JitMatchRng {
+	uint32_t state = 0x12345678;
+	uint32_t Next() {
+		state ^= state << 13;
+		state ^= state >> 17;
+		state ^= state << 5;
+		return state;
+	}
+	float Float(float lo, float hi) {
+		return lo + (hi - lo) * (float)(Next() & 0xFFFFFF) / (float)0xFFFFFF;
+	}
+	// The GE loads morph weights, bone matrices and the UV scale as 24-bit floats.
+	float Float24(float lo, float hi) {
+		float f = Float(lo, hi);
+		uint32_t bits;
+		memcpy(&bits, &f, 4);
+		bits &= 0xFFFFFF00;
+		memcpy(&f, &bits, 4);
+		return f;
+	}
+	// Vertex data floats: mostly arbitrary, with some exact values mixed in.
+	float VertexFloat() {
+		static const float nice[] = { 0.0f, 1.0f, -1.0f, 0.5f, -0.5f, 2.0f, 127.0f / 128.0f, 256.0f };
+		if ((Next() & 3) == 0) {
+			return nice[Next() % ARRAY_SIZE(nice)];
+		}
+		return Float(-8.0f, 8.0f);
+	}
+};
+
+// Output size of the formats the decoder can produce.
+// Accumulating steps (morph in particular) are compiled into fused multiply-adds on some
+// architectures and not others, and the prescale paths differ in where the scale lands, so the
+// last bit of a decoded value is not something every JIT can be expected to reproduce exactly.
+// Allow a couple of ULPs there - every real bug this test has caught was orders of magnitude
+// larger, so nothing interesting slips through.
+static bool NearlyEqualFloat(float a, float b, int terms) {
+	if (a == b) {
+		return true;
+	}
+	if (std::isnan(a) || std::isnan(b)) {
+		return false;
+	}
+	// Relative to the larger magnitude, with a floor of 1 - a prescaled texcoord is a small
+	// difference of larger terms, so the error is best judged against what went into it.
+	// Relative to the larger magnitude, with a floor of 1 - a prescaled texcoord is a small
+	// difference of larger terms, so the error is best judged against what went into it. Allow
+	// one rounding per accumulated term, since morph sums several.
+	const float scale = std::max(1.0f, std::max(fabsf(a), fabsf(b)));
+	return fabsf(a - b) <= scale * 1e-6f * (float)std::max(1, terms);
+}
+
+int DecodedComponentSize(u8 fmt) {
+	switch (fmt) {
+	case DEC_FLOAT_2: return 8;
+	case DEC_FLOAT_3: return 12;
+	case DEC_S8_3: return 3;
+	case DEC_S16_3: return 6;
+	case DEC_U8_4: return 4;
+	default: return -1;
+	}
+}
+
+// With specialPos, some plain float positions get a NaN or infinity, and specialPos[v] says which.
+void FillVertexData(JitMatchRng &rng, const VertexDecoder &dec, u8 *src, int count, bool *specialPos) {
+	static const int wtSize[] = { 0, 1, 2, 4 };
+	static const int tcSize[] = { 0, 1, 2, 4 };
+	static const int nrmPosSize[] = { 0, 1, 2, 4 };
+	auto fillScalars = [&](u8 *p, int n, int elemSize) {
+		for (int i = 0; i < n; i++) {
+			if (elemSize == 4) {
+				float_le f = rng.VertexFloat();
+				memcpy(p + i * 4, &f, 4);
+			} else if (elemSize == 2) {
+				u16_le v = (u16)rng.Next();
+				memcpy(p + i * 2, &v, 2);
+			} else {
+				p[i] = (u8)rng.Next();
+			}
+		}
+	};
+
+	for (int v = 0; v < count; v++) {
+		for (int m = 0; m < dec.morphcount; m++) {
+			u8 *p = src + v * dec.size + m * dec.onesize_;
+			if (dec.weighttype) {
+				int sz = wtSize[dec.weighttype];
+				for (int w = 0; w < dec.nweights; w++) {
+					// Weights are normally 0..1ish; keep floats there so skinning stays sane.
+					if (sz == 4) {
+						float_le f = rng.Float(0.0f, 1.5f);
+						memcpy(p + dec.weightoff + w * 4, &f, 4);
+					} else {
+						fillScalars(p + dec.weightoff + w * sz, 1, sz);
+					}
+				}
+			}
+			if (dec.tc) {
+				fillScalars(p + dec.tcoff, 2, tcSize[dec.tc]);
+			}
+			if (dec.col) {
+				fillScalars(p + dec.coloff, dec.col == (GE_VTYPE_COL_8888 >> GE_VTYPE_COL_SHIFT) ? 4 : 2, 1);
+			}
+			if (dec.nrm) {
+				fillScalars(p + dec.nrmoff, 3, nrmPosSize[dec.nrm]);
+			}
+			if (dec.pos) {
+				fillScalars(p + dec.posoff, 3, nrmPosSize[dec.pos]);
+			}
+		}
+		if (specialPos) {
+			specialPos[v] = (rng.Next() & 7) == 0;
+			if (specialPos[v]) {
+				static const float specials[] = { NAN, INFINITY, -INFINITY };
+				float_le f = specials[rng.Next() % 3];
+				memcpy(src + v * dec.size + dec.posoff + (rng.Next() % 3) * 4, &f, 4);
+			}
+		}
+	}
+}
+
+// Skinning is the one place the JITs may round differently: arm64 accumulates the bone matrices
+// with fused multiply-adds, the steps multiply and add separately. The difference comes from
+// rounding the intermediate terms, which can be far larger than the result when they cancel, so
+// the allowed error scales with the sum of the terms' magnitudes rather than with the result.
+void SkinTermMagnitudes(const VertexDecoder &dec, const u8 *vtx, bool pos, float out[3]) {
+	auto readScalar = [](const u8 *p, int type, int i) -> float {
+		switch (type) {
+		case 1: return fabsf((float)(s8)p[i] * (1.0f / 128.0f));
+		case 2: { s16_le s; memcpy(&s, p + i * 2, 2); return fabsf((float)(s16)s * (1.0f / 32768.0f)); }
+		default: { float_le f; memcpy(&f, p + i * 4, 4); return fabsf((float)f); }
+		}
+	};
+	auto readWeight = [](const u8 *p, int type, int i) -> float {
+		switch (type) {
+		case 1: return p[i] * (1.0f / 128.0f);
+		case 2: { u16_le w; memcpy(&w, p + i * 2, 2); return (u16)w * (1.0f / 32768.0f); }
+		default: { float_le f; memcpy(&f, p + i * 4, 4); return fabsf((float)f); }
+		}
+	};
+
+	// Weights come from the first morph frame only, same as the steps.
+	float absMatrix[12]{};
+	for (int j = 0; j < dec.nweights; j++) {
+		const float w = readWeight(vtx + dec.weightoff, dec.weighttype, j);
+		for (int k = 0; k < 12; k++) {
+			absMatrix[k] += w * fabsf(gstate.boneMatrix[j * 12 + k]);
+		}
+	}
+
+	const int type = pos ? dec.pos : dec.nrm;
+	const int off = pos ? dec.posoff : dec.nrmoff;
+	float absVec[3]{};
+	for (int m = 0; m < dec.morphcount; m++) {
+		const float mw = dec.morphcount > 1 ? fabsf(gstate_c.morphWeights[m]) : 1.0f;
+		for (int k = 0; k < 3; k++) {
+			absVec[k] += mw * readScalar(vtx + m * dec.onesize_ + off, type, k);
+		}
+	}
+
+	for (int i = 0; i < 3; i++) {
+		out[i] = absVec[0] * absMatrix[i] + absVec[1] * absMatrix[3 + i] + absVec[2] * absMatrix[6 + i] + (pos ? absMatrix[9 + i] : 0.0f);
+	}
+}
+
+struct JitMismatch {
+	int formats = 0;
+	int verts = 0;
+	std::string example;
+};
+
+}  // namespace
+
+[[maybe_unused]] static bool TestVertexJitMatchesSteps() {
+	// static, or MSVC treats these as captured references and won't use VERTS as an array bound.
+	static constexpr int VERTS = 32;
+	static constexpr int BUF_SIZE = 64 * 1024;
+	// Decode may overrun by a vertex plus 16 bytes, see DecodeVerts.
+	u8 *src = (u8 *)AllocateAlignedMemory(BUF_SIZE, 16);
+	u8 *refOut = (u8 *)AllocateAlignedMemory(BUF_SIZE, 16);
+	u8 *jitOut = (u8 *)AllocateAlignedMemory(BUF_SIZE, 16);
+
+	VertexDecoderJitCache *cache = new VertexDecoderJitCache();
+	JitMatchRng rng;
+	std::map<std::string, JitMismatch> mismatches;
+	int formatsTested = 0;
+	int formatsJitted = 0;
+	const bool savedDoubleTexCoords = g_DoubleTextureCoordinates;
+
+	auto testFormat = [&](u32 vtype, int uvGenMode, bool doubleTexCoords, bool expand8BitNormals) {
+		g_DoubleTextureCoordinates = doubleTexCoords;
+		VertexDecoderOptions opts{};
+		opts.expand8BitNormalsToFloat = expand8BitNormals;
+		const u32 vertTypeID = GetVertTypeID(vtype, uvGenMode);
+
+		VertexDecoder ref{};
+		ref.SetVertexType(vertTypeID, opts, nullptr);
+		// Without a cache, SetVertexType still installs the handwritten decoders for a couple of
+		// formats. The reference has to be the steps.
+		ref.jitted_ = nullptr;
+
+		if (cache->GetSpaceLeft() < 16384) {
+			cache->Clear();
+		}
+		VertexDecoder jit{};
+		jit.SetVertexType(vertTypeID, opts, cache);
+		formatsTested++;
+		if (!jit.jitted_) {
+			return;
+		}
+		formatsJitted++;
+
+		// Which step writes each component: weights (if any) first, then tc, col, nrm, pos.
+		int stepIndex = ref.weighttype ? 1 : 0;
+		StepFunction tcStep = ref.tc ? ref.steps_[stepIndex++] : nullptr;
+		StepFunction colStep = ref.col ? ref.steps_[stepIndex++] : nullptr;
+		StepFunction nrmStep = ref.nrm ? ref.steps_[stepIndex++] : nullptr;
+		StepFunction posStep = ref.steps_[stepIndex];
+
+		for (int i = 0; i < 8; i++) {
+			gstate_c.morphWeights[i] = rng.Float24(-0.5f, 1.5f);
+		}
+		for (int i = 0; i < 8 * 12; i++) {
+			gstate.boneMatrix[i] = rng.Float24(-2.0f, 2.0f);
+		}
+		const UVScale uvScale{ rng.Float24(-2.0f, 2.0f), rng.Float24(-2.0f, 2.0f), rng.Float24(-1.0f, 1.0f), rng.Float24(-1.0f, 1.0f) };
+
+		const int srcBytes = ref.VertexSize() * VERTS;
+		_assert_(srcBytes + 256 <= BUF_SIZE && ref.decFmt.stride * (VERTS + 1) + 16 <= BUF_SIZE);
+		// Only plain float positions are cleaned of NaN and infinity (see Step_PosFloat).
+		bool specialPos[VERTS]{};
+		FillVertexData(rng, ref, src, VERTS, posStep == &VertexDecoder::Step_PosFloat ? specialPos : nullptr);
+
+		const KnownVertexBounds initialBounds{ 0xFFFF, 0xFFFF, 0, 0 };
+		memset(refOut, 0, BUF_SIZE);
+		gstate_c.vertexFullAlpha = true;
+		gstate_c.vertBounds = initialBounds;
+		// Vary the count a little, so the decoders' leftover-vertex paths get used too.
+		const int numVerts = VERTS - (formatsTested & 3);
+		ref.DecodeVerts(refOut, src, &uvScale, numVerts);
+		const bool refFullAlpha = gstate_c.vertexFullAlpha;
+		const KnownVertexBounds refBounds = gstate_c.vertBounds;
+
+		memset(jitOut, 0, BUF_SIZE);
+		gstate_c.vertexFullAlpha = true;
+		gstate_c.vertBounds = initialBounds;
+		jit.DecodeVerts(jitOut, src, &uvScale, numVerts);
+		const bool jitFullAlpha = gstate_c.vertexFullAlpha;
+		const KnownVertexBounds jitBounds = gstate_c.vertBounds;
+
+		char fmtDesc[256]{};
+		ref.ToString(fmtDesc, sizeof(fmtDesc), true);
+		const char *kind = cache->IsInSpace((const u8 *)jit.jitted_) ? "jit" : "handwritten";
+
+		auto record = [&](const char *what, StepFunction step, int badVerts, const std::string &detail) {
+			std::string key = StringFromFormat("%-4s %-11s %s", what, kind, step ? GetStepFunctionName(step) : "-");
+			JitMismatch &m = mismatches[key];
+			if (m.formats == 0) {
+				m.example = StringFromFormat("%08x %s morph=%d uvgen=%d dbl=%d expand8=%d: %s", vertTypeID, fmtDesc, ref.morphcount, uvGenMode, (int)doubleTexCoords, (int)expand8BitNormals, detail.c_str());
+			}
+			m.formats++;
+			m.verts += badVerts;
+		};
+
+		auto compareComponent = [&](const char *what, StepFunction step, u8 fmt, int off, bool skinnedPos = false, bool skinnedNrm = false) {
+			if (fmt == DEC_NONE) {
+				return;
+			}
+			const bool skinned = skinnedPos || skinnedNrm;
+			const int sz = DecodedComponentSize(fmt);
+			if (sz < 0) {
+				record(what, step, 0, StringFromFormat("unexpected decoded format %d", fmt));
+				return;
+			}
+			int badVerts = 0;
+			std::string detail;
+			for (int v = 0; v < numVerts; v++) {
+				const u8 *r = refOut + v * ref.decFmt.stride + off;
+				const u8 *j = jitOut + v * ref.decFmt.stride + off;
+				if (specialPos[v] && step == posStep) {
+					// NaN and infinity only have to come out finite. How is up to the platform.
+					bool finite = true;
+					for (int c = 0; c < 3; c++) {
+						float fr, fj;
+						memcpy(&fr, r + c * 4, 4);
+						memcpy(&fj, j + c * 4, 4);
+						finite = finite && std::isfinite(fr) && std::isfinite(fj);
+					}
+					if (!finite && badVerts++ == 0) {
+						float fj[3];
+						memcpy(fj, j, 12);
+						detail = StringFromFormat("vert %d: NaN/inf input gave %g %g %g", v, fj[0], fj[1], fj[2]);
+					}
+					continue;
+				}
+				if (memcmp(r, j, sz) == 0) {
+					continue;
+				}
+				// Tolerate the last bit, see NearlyEqualFloat.
+				if (fmt == DEC_FLOAT_2 || fmt == DEC_FLOAT_3) {
+					bool close = true;
+					for (int c = 0; c < sz / 4; c++) {
+						float fr, fj;
+						memcpy(&fr, r + c * 4, 4);
+						memcpy(&fj, j + c * 4, 4);
+						close = close && NearlyEqualFloat(fr, fj, ref.morphcount);
+					}
+					if (close) {
+						continue;
+					}
+				} else if (fmt == DEC_U8_4) {
+					// A one-bit rounding difference upstream lands as +-1 on a channel.
+					bool close = true;
+					for (int c = 0; c < 4; c++) {
+						close = close && std::abs((int)r[c] - (int)j[c]) <= 1;
+					}
+					if (close) {
+						continue;
+					}
+				}
+				if (skinned && fmt == DEC_FLOAT_3) {
+					float terms[3];
+					SkinTermMagnitudes(ref, src + v * ref.VertexSize(), skinnedPos, terms);
+					bool close = true;
+					for (int c = 0; c < 3; c++) {
+						float fr, fj;
+						memcpy(&fr, r + c * 4, 4);
+						memcpy(&fj, j + c * 4, 4);
+						// About 32 ULPs of the largest term, to cover a rounding per accumulated bone.
+						if (!(fabsf(fr - fj) <= terms[c] * (1.0f / (1 << 19)))) {
+							close = false;
+						}
+					}
+					if (close) {
+						continue;
+					}
+				}
+				if (badVerts++ == 0) {
+					detail = StringFromFormat("vert %d: steps", v);
+					const bool isFloat = fmt == DEC_FLOAT_2 || fmt == DEC_FLOAT_3;
+					for (int pass = 0; pass < 2; pass++) {
+						const u8 *p = pass == 0 ? r : j;
+						if (pass == 1) {
+							detail += " vs jit";
+						}
+						for (int c = 0; c < (isFloat ? sz / 4 : sz); c++) {
+							if (isFloat) {
+								float f;
+								memcpy(&f, p + c * 4, 4);
+								detail += StringFromFormat(" %.9g", f);
+							} else if (fmt == DEC_S16_3) {
+								if (c < 3) {
+									s16 s;
+									memcpy(&s, p + c * 2, 2);
+									detail += StringFromFormat(" %d", s);
+								}
+							} else {
+								detail += StringFromFormat(" %d", fmt == DEC_S8_3 ? (int)(s8)p[c] : (int)p[c]);
+							}
+						}
+					}
+				}
+			}
+			if (badVerts) {
+				record(what, step, badVerts, detail);
+			}
+		};
+
+		compareComponent("uv", tcStep, ref.decFmt.uvfmt, ref.decFmt.uvoff);
+		compareComponent("col", colStep, ref.decFmt.c0fmt, ref.decFmt.c0off);
+		compareComponent("nrm", nrmStep, ref.decFmt.nrmfmt, ref.decFmt.nrmoff, false, ref.skinInDecode);
+		compareComponent("pos", posStep, DEC_FLOAT_3, ref.decFmt.posoff, ref.skinInDecode && !ref.throughmode, false);
+		if (refFullAlpha != jitFullAlpha) {
+			record("fullAlpha", colStep, 0, StringFromFormat("steps %d vs jit %d", (int)refFullAlpha, (int)jitFullAlpha));
+		}
+		// TODO: Only the steps track bounds for float UVs in through mode. Undecided which way to go.
+		if (tcStep != &VertexDecoder::Step_TcFloatThrough && memcmp(&refBounds, &jitBounds, sizeof(refBounds)) != 0) {
+			record("bounds", tcStep, 0, StringFromFormat("steps %d,%d-%d,%d vs jit %d,%d-%d,%d",
+				refBounds.minU, refBounds.minV, refBounds.maxU, refBounds.maxV, jitBounds.minU, jitBounds.minV, jitBounds.maxU, jitBounds.maxV));
+		}
+	};
+
+	static const int colFormats[] = { 0, 4, 5, 6, 7 };
+	for (int through = 0; through <= 1; through++) {
+		for (int tc = 0; tc < 4; tc++) {
+			for (int col : colFormats) {
+				for (int nrm = 0; nrm < 4; nrm++) {
+					for (int pos = 1; pos < 4; pos++) {
+						for (int wt = 0; wt < 4; wt++) {
+							for (int nweights = 1; nweights <= (wt ? 8 : 1); nweights++) {
+								for (int morph = 1; morph <= 8; morph++) {
+									u32 vtype = (tc << GE_VTYPE_TC_SHIFT) | (col << GE_VTYPE_COL_SHIFT) | (nrm << GE_VTYPE_NRM_SHIFT) |
+										(pos << GE_VTYPE_POS_SHIFT) | (wt << GE_VTYPE_WEIGHT_SHIFT) |
+										((nweights - 1) << GE_VTYPE_WEIGHTCOUNT_SHIFT) | ((morph - 1) << GE_VTYPE_MORPHCOUNT_SHIFT) |
+										(through ? GE_VTYPE_THROUGH : 0);
+									// The UV gen mode and double texcoords only change anything with texcoords, and
+									// only the TEXTURE_COORDS / TEXTURE_MATRIX split matters (the other two modes
+									// decode like these). The expand option only applies to plain 8-bit normals.
+									const int uvGenModes = (tc && !through) ? 2 : 1;
+									const int doubleModes = tc ? 2 : 1;
+									const int expandModes = (nrm == 1 && morph == 1 && !wt) ? 2 : 1;
+									for (int uvGen = 0; uvGen < uvGenModes; uvGen++) {
+										for (int dbl = 0; dbl < doubleModes; dbl++) {
+											for (int expand = 0; expand < expandModes; expand++) {
+												testFormat(vtype, uvGen == 0 ? GE_TEXMAP_TEXTURE_COORDS : GE_TEXMAP_TEXTURE_MATRIX, dbl != 0, expand != 0);
+											}
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	g_DoubleTextureCoordinates = savedDoubleTexCoords;
+	delete cache;
+	FreeAlignedMemory(src);
+	FreeAlignedMemory(refOut);
+	FreeAlignedMemory(jitOut);
+
+	printf("VertexJitMatchesSteps: %d formats, %d jitted, %d mismatch kinds\n", formatsTested, formatsJitted, (int)mismatches.size());
+	for (auto &iter : mismatches) {
+		printf("  %s: %d formats, %d verts\n    e.g. %s\n", iter.first.c_str(), iter.second.formats, iter.second.verts, iter.second.example.c_str());
+	}
+	return mismatches.empty();
+}
+
+
 typedef bool (*VertexTestFunc)();
 
 static VertexTestFunc vertdecTestFuncs[] = {
@@ -647,6 +1091,8 @@ static VertexTestFunc vertdecTestFuncs[] = {
 	&TestVertex8Skin,
 	&TestVertex16Skin,
 	&TestVertexFloatSkin,
+
+	&TestVertexJitMatchesSteps,
 };
 
 bool TestVertexJit() {