SoftGPU: Quantize fog per vertex, like the GE

The GE converts the fog factor to 8 bits per vertex, min(floor(256 * f), 255),
and interpolates that linearly in screen space. Clamping after interpolation
instead was up to 127 levels off on triangles that cross the fog range.
Measured with gpu/probe; points and wacky values still match exactly.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

GPU/Common/TransformCommon.h

@@ -160,3 +160,19 @@ struct SimpleVertex {
 	Vec3Packedf nrm;
 	Vec3Packedf pos;
 };
+
+// The GE converts the fog factor to 8 bits per vertex, min(floor(256 * f), 255), and interpolates that
+// linearly in screen space. NaN and inf go by sign, like other values (measured with gpu/probe).
+inline int GEFogFactor(float f) {
+	uint32_t bits;
+	memcpy(&bits, &f, sizeof(bits));
+	const uint32_t exp = bits >> 23;
+	if ((bits & 0x80000000) != 0 || exp <= 126 - 8) {
+		return 0;
+	}
+	if (exp > 126) {
+		return 255;
+	}
+	const uint32_t mantissa = (bits & 0x007FFFFF) | 0x00800000;
+	return mantissa >> (16 + 126 - exp);
+}

GPU/Software/Rasterizer.cpp

@@ -188,7 +188,7 @@ static inline void CalculateRasterStateFlags(RasterizerState *state, const Verte
 		if (alpha != 0xFF)
 			state->flags |= RasterizerStateFlags::VERTEX_ALPHA_NON_FULL;
 	}
-	if (!(v0.fogdepth >= 1.0f))
+	if (!(v0.fogdepth >= 255.0f / 256.0f))
 		state->flags |= RasterizerStateFlags::VERTEX_HAS_FOG;
 }
 
@@ -1008,7 +1008,7 @@ void DrawTriangleSlice(
 	const bool flatColorAll = !state.shadeGouraud;
 	const bool flatColor0 = flatColorAll || (v0.color0 == v1.color0 && v0.color0 == v2.color0);
 	const bool flatColor1 = flatColorAll || (v0.color1 == v1.color1 && v0.color1 == v2.color1);
-	const bool noFog = clearMode || !pixelID.applyFog || (v0.fogdepth >= 1.0f && v1.fogdepth >= 1.0f && v2.fogdepth >= 1.0f);
+	const bool noFog = clearMode || !pixelID.applyFog || (v0.fogdepth >= 255.0f / 256.0f && v1.fogdepth >= 255.0f / 256.0f && v2.fogdepth >= 255.0f / 256.0f);
 
 	if (pixelID.applyDepthRange && flatZ) {
 		if (v0.screenpos.z < pixelID.cached.minz || v0.screenpos.z > pixelID.cached.maxz)

GPU/Software/TransformUnit.cpp

@@ -523,7 +523,7 @@ ClipVertexData TransformUnit::ReadVertex(const VertexReader &vreader, const Tran
 		}
 
 		if (state.enableFog) {
-			vertex.v.fogdepth = Dot43(state.posToFog, pos);
+			vertex.v.fogdepth = GEFogFactor(Dot43(state.posToFog, pos)) * (1.0f / 256.0f);
 		} else {
 			vertex.v.fogdepth = 1.0f;
 		}