Shade mapping: Use the light vector as lighting sees it

Environment map S and T are (N.L + 1) / 2 with L the light's vector as
lighting uses it: from the vertex to the light for point and spot lights,
a zero vector staying zero, and the half vector for a light that does
specular. Whether lighting or the light is enabled still doesn't matter
(gpu/lighting/shademap).

The vertex shader ID now carries the type and computation of the shade
mapping lights (the ubershader reads them from u_lightControl), so both
shader caches get a new version.

Fixes the hair shine in iDOLM@STER SP (#12376).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

GPU/Common/ShaderId.cpp

@@ -116,8 +116,22 @@ void ComputeVertexShaderID(VShaderID *id_out, u32 vertType, bool useHWTransform,
 			id.SetBits(VS_BIT_LS1, 2, gstate.getUVLS1());
 		}
 
+		if (doShadeMapping) {
+			// Shade mapping depends on the type of its lights and whether they do specular, even when
+			// they're off. The ubershader reads that from u_lightControl instead.
+			if (gstate_c.Use(GPU_USE_LIGHT_UBERSHADER)) {
+				id.SetBit(VS_BIT_LIGHT_UBERSHADER);
+			} else {
+				const int shadeLights[2] = { gstate.getUVLS0(), gstate.getUVLS1() };
+				for (int l : shadeLights) {
+					id.SetBits(VS_BIT_LIGHT0_COMP + 4 * l, 2, gstate.getLightComputation(l));
+					id.SetBits(VS_BIT_LIGHT0_TYPE + 4 * l, 2, gstate.getLightType(l));
+				}
+			}
+		}
+
 		if (gstate.isLightingEnabled()) {
-			// doShadeMapping is stored as UVGenMode, and light type doesn't matter for shade mapping.
+			// doShadeMapping is stored as UVGenMode.
 			id.SetBit(VS_BIT_LIGHTING_ENABLE);
 			if (gstate_c.Use(GPU_USE_LIGHT_UBERSHADER)) {
 				id.SetBit(VS_BIT_LIGHT_UBERSHADER);

GPU/Common/SoftwareTransformCommon.cpp

@@ -333,23 +333,11 @@ SoftwareTransformAction RunSoftwareTransform(SoftwareTransformParams &params, in
 			case GE_TEXMAP_ENVIRONMENT_MAP:
 				// Shade mapping - use two light sources to generate U and V.
 				{
-					auto getLPosFloat = [&](int l, int i) {
-						return getFloat24(gstate.lpos[l * 3 + i]);
-					};
-					auto getLPos = [&](int l) {
-						return Vec3f(getLPosFloat(l, 0), getLPosFloat(l, 1), getLPosFloat(l, 2));
-					};
-					auto calcShadingLPos = [&](int l) {
-						Vec3f pos = getLPos(l);
-						return pos.NormalizedOr001(cpu_info.bSSE4_1);
-					};
-
 					// Might not have lighting enabled, so don't use lighter.
-					Vec3f lightpos0 = calcShadingLPos(gstate.getUVLS0());
-					Vec3f lightpos1 = calcShadingLPos(gstate.getUVLS1());
-
-					uv[0] = (1.0f + Dot(lightpos0, worldnormal))/2.0f;
-					uv[1] = (1.0f + Dot(lightpos1, worldnormal))/2.0f;
+					const Vec3f viewDir = PSPViewDirection(gstate.viewMatrix);
+					const Vec3f worldpos(out[0], out[1], out[2]);
+					uv[0] = PSPShadeMapCoord(gstate.getUVLS0(), worldpos, worldnormal, viewDir);
+					uv[1] = PSPShadeMapCoord(gstate.getUVLS1(), worldpos, worldnormal, viewDir);
 					uv[2] = 1.0f;
 				}
 				break;

GPU/Common/TransformCommon.h

@@ -23,6 +23,7 @@
 #include "Common/CommonTypes.h"
 #include "GPU/Math3D.h"
 #include "GPU/GPU.h"
+#include "GPU/GPUState.h"
 
 struct Color4 {
 	float r, g, b, a;
@@ -90,6 +91,26 @@ inline Vec3f PSPViewDirection(const float viewMatrix[12]) {
 	return Vec3f(viewMatrix[2], viewMatrix[5], viewMatrix[8]).NormalizedOr001(false);
 }
 
+inline Vec3f NormalizedOr000(const Vec3f &v) {
+	float len2 = v.Length2();
+	return len2 > 0.0f ? v * (1.0f / sqrtf(len2)) : Vec3f(0.0f, 0.0f, 0.0f);
+}
+
+// Shade mapping (environment map UV gen) coordinate from light l: (N.L + 1) / 2, with L the light's
+// direction as lighting sees it (a zero vector stays zero), or the half vector if the light does
+// specular. Lighting and light enables don't matter (gpu/lighting/shademap).
+inline float PSPShadeMapCoord(int l, const Vec3f &worldpos, const Vec3f &worldnormal, const Vec3f &viewDir) {
+	Vec3f L(getFloat24(gstate.lpos[l * 3]), getFloat24(gstate.lpos[l * 3 + 1]), getFloat24(gstate.lpos[l * 3 + 2]));
+	if (gstate.getLightType(l) != GE_LIGHTTYPE_DIRECTIONAL) {
+		L -= worldpos;
+	}
+	L = NormalizedOr000(L);
+	if (gstate.isUsingSpecularLight(l)) {
+		L = NormalizedOr000(L + viewDir);
+	}
+	return (Dot(L, worldnormal) + 1.0f) * 0.5f;
+}
+
 // Convenient way to do precomputation to save the parts of the lighting calculation
 // that's common between the many vertices of a draw call.
 class Lighter {

GPU/Common/VertexShaderGenerator.cpp

@@ -135,7 +135,9 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 	int matUpdate = id.Bits(VS_BIT_MATERIAL_UPDATE, 3);
 
 	bool lightUberShader = id.Bit(VS_BIT_LIGHT_UBERSHADER) && enableLighting;  // checking lighting here for the shader test's benefit, in reality if ubershader is set, lighting is set.
-	if (lightUberShader && !compat.bitwiseOps) {
+	// With the ubershader, shade mapping reads its lights' type and computation from u_lightControl.
+	bool shadeUberShader = id.Bit(VS_BIT_LIGHT_UBERSHADER) && doShadeMapping;
+	if ((lightUberShader || shadeUberShader) && !compat.bitwiseOps) {
 		*errorString = "Light ubershader requires bitwise ops in shader language";
 		return false;
 	}
@@ -349,7 +351,7 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 			WRITE(p, "uniform vec4 u_uvscaleoffset;\n");
 			*uniformMask |= DIRTY_UVSCALEOFFSET;
 
-			if (lightUberShader) {
+			if (lightUberShader || shadeUberShader) {
 				p.C("uniform uint u_lightControl;\n");
 				*uniformMask |= DIRTY_LIGHT_CONTROL;
 			}
@@ -428,6 +430,10 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 		WRITE(p, "   float len2 = dot(v, v);\n");
 		WRITE(p, "   return len2 == 0.0 ? vec3(0.0, 0.0, 1.0) : (v * inversesqrt(len2));\n");
 		WRITE(p, "}\n");
+		WRITE(p, "vec3 normalizeOr000(vec3 v) {\n");
+		WRITE(p, "   float len2 = dot(v, v);\n");
+		WRITE(p, "   return len2 == 0.0 ? splat3(0.0) : (v * inversesqrt(len2));\n");
+		WRITE(p, "}\n");
 		// The GE's pow for lighting: exp2(e * log2(x)), with log2 and exp2 each a straight line
 		// between powers of two. Continuous, so floor() landing on the wrong side of a power of
 		// two is harmless.
@@ -500,7 +506,7 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 		} else {
 			WRITE(p, "  mediump vec3 worldnormal = normalizeOr001(mul(vec4(0.0, 0.0, %s1.0, 0.0), u_world).xyz);\n", flipNormal ? "-" : "");
 		}
-		if (enableLighting) {
+		if (enableLighting || doShadeMapping) {
 			// The viewer is at infinity along view space +z: in world space, the view matrix's third column.
 			if (compat.shaderLanguage == HLSL_D3D11) {
 				WRITE(p, "  mediump vec3 viewDir = normalizeOr001(vec3(u_view[0].z, u_view[1].z, u_view[2].z));\n");
@@ -869,9 +875,31 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 						snprintf(ls0Str, sizeof(ls0Str), "%d", ls0);
 						snprintf(ls1Str, sizeof(ls1Str), "%d", ls1);
 					}
-					std::string lightFactor0 = StringFromFormat("(length(u_lightpos%s) == 0.0 ? worldnormal.z : dot(normalize(u_lightpos%s), worldnormal))", ls0Str, ls0Str);
-					std::string lightFactor1 = StringFromFormat("(length(u_lightpos%s) == 0.0 ? worldnormal.z : dot(normalize(u_lightpos%s), worldnormal))", ls1Str, ls1Str);
-					WRITE(p, "  %sv_texcoord = vec3(u_uvscaleoffset.xy * vec2(1.0 + %s, 1.0 + %s) * 0.5, 1.0);\n", compat.vsOutPrefix, lightFactor0.c_str(), lightFactor1.c_str());
+					// N.L with L the light vector as lighting sees it (zero stays zero), or the half vector
+					// if the light does specular. Whether lighting or the light is on doesn't matter.
+					auto shadeLight = [&](int ls, const char *lsStr, const char *name) {
+						if (shadeUberShader) {
+							p.F("  vec3 %s = u_lightpos%s;\n", name, lsStr);
+							p.F("  if (((u_lightControl >> 0x%02xu) & 0x3u) != 0x0u) %s = u_lightpos%s - worldpos;\n", 4 + 4 * ls + 2, name, lsStr);
+							p.F("  %s = normalizeOr000(%s);\n", name, name);
+							p.F("  if (((u_lightControl >> 0x%02xu) & 0x3u) == 0x1u) %s = normalizeOr000(%s + viewDir);\n", 4 + 4 * ls, name, name);
+							return;
+						}
+						GELightType type = static_cast<GELightType>(id.Bits(VS_BIT_LIGHT0_TYPE + 4 * ls, 2));
+						GELightComputation comp = static_cast<GELightComputation>(id.Bits(VS_BIT_LIGHT0_COMP + 4 * ls, 2));
+						if (type == GE_LIGHTTYPE_DIRECTIONAL) {
+							// Prenormalized.
+							p.F("  vec3 %s = u_lightpos%s;\n", name, lsStr);
+						} else {
+							p.F("  vec3 %s = normalizeOr000(u_lightpos%s - worldpos);\n", name, lsStr);
+						}
+						if (comp == GE_LIGHTCOMP_BOTH) {
+							p.F("  %s = normalizeOr000(%s + viewDir);\n", name, name);
+						}
+					};
+					shadeLight(ls0, ls0Str, "shadeL0");
+					shadeLight(ls1, ls1Str, "shadeL1");
+					WRITE(p, "  %sv_texcoord = vec3(u_uvscaleoffset.xy * vec2(1.0 + dot(shadeL0, worldnormal), 1.0 + dot(shadeL1, worldnormal)) * 0.5, 1.0);\n", compat.vsOutPrefix);
 				}
 				break;
 

GPU/GLES/ShaderManagerGLES.cpp

@@ -856,7 +856,7 @@ enum class CacheDetectFlags {
 };
 
 #define CACHE_HEADER_MAGIC 0x83277592
-#define CACHE_VERSION 43
+#define CACHE_VERSION 44
 
 struct CacheHeader {
 	uint32_t magic;

GPU/Software/Lighting.cpp

@@ -194,20 +194,11 @@ void ComputeState(State *state, bool hasColor0) {
 	state->usesWorldNormal = gstate.getUVGenMode() == GE_TEXMAP_ENVIRONMENT_MAP || anyDiffuse || anySpecular;
 }
 
-static inline float GenerateLightCoord(VertexData &vertex, const WorldCoords &worldnormal, int light) {
-	// TODO: Should specular lighting should affect this, too?  Doesn't in GLES.
-	Vec3<float> L = GetLightVec(gstate.lpos, light);
-	// In other words, L.Length2() == 0.0f means Dot({0, 0, 1}, worldnormal).
-	float diffuse_factor = Dot(L.NormalizedOr001(cpu_info.bSSE4_1), worldnormal);
-
-	return (diffuse_factor + 1.0f) / 2.0f;
-}
-
-void GenerateLightST(VertexData &vertex, const WorldCoords &worldnormal) {
+void GenerateLightST(VertexData &vertex, const WorldCoords &worldpos, const WorldCoords &worldnormal, const Vec3f &viewDir) {
 	// Always calculate texture coords from lighting results if environment mapping is active
 	// This should be done even if lighting is disabled altogether.
-	vertex.texturecoords.s() = GenerateLightCoord(vertex, worldnormal, gstate.getUVLS0());
-	vertex.texturecoords.t() = GenerateLightCoord(vertex, worldnormal, gstate.getUVLS1());
+	vertex.texturecoords.s() = PSPShadeMapCoord(gstate.getUVLS0(), worldpos, worldnormal, viewDir);
+	vertex.texturecoords.t() = PSPShadeMapCoord(gstate.getUVLS1(), worldpos, worldnormal, viewDir);
 }
 
 #if defined(_M_SSE)

GPU/Software/Lighting.h

@@ -68,7 +68,7 @@ struct State {
 
 void ComputeState(State *state, bool hasColor0);
 
-void GenerateLightST(VertexData &vertex, const WorldCoords &worldnormal);
+void GenerateLightST(VertexData &vertex, const WorldCoords &worldpos, const WorldCoords &worldnormal, const Vec3f &viewDir);
 void Process(VertexData &vertex, const WorldCoords &worldpos, const WorldCoords &worldnormal, const State &state);
 
 }

GPU/Software/SoftGpu.cpp

@@ -111,7 +111,7 @@ const SoftwareCommandTableEntry softgpuCommandTable[] = {
 	{ GE_CMD_FOGENABLE, 0, SoftDirty::PIXEL_BASIC | SoftDirty::PIXEL_CACHED | SoftDirty::TRANSFORM_BASIC | SoftDirty::TRANSFORM_FOG | SoftDirty::TRANSFORM_MATRIX },
 	{ GE_CMD_TEXMODE, 0, SoftDirty::SAMPLER_BASIC | SoftDirty::SAMPLER_TEXLIST | SoftDirty::RAST_TEX },
 	// Currently this doesn't affect any state, but maybe it should.
-	{ GE_CMD_TEXSHADELS },
+	{ GE_CMD_TEXSHADELS, 0, SoftDirty::TRANSFORM_BASIC },
 	{ GE_CMD_SHADEMODE, 0, SoftDirty::RAST_BASIC },
 	{ GE_CMD_TEXFUNC, 0, SoftDirty::SAMPLER_BASIC },
 	{ GE_CMD_COLORTEST, 0, SoftDirty::PIXEL_BASIC | SoftDirty::PIXEL_CACHED },

GPU/Software/TransformUnit.cpp

@@ -260,6 +260,12 @@ void ComputeTransformState(TransformState *state, const VertexReader &vreader) {
 		} else {
 			state->lightingState.usesWorldNormal = state->uvGenMode == GE_TEXMAP_ENVIRONMENT_MAP;
 		}
+		if (state->uvGenMode == GE_TEXMAP_ENVIRONMENT_MAP) {
+			// Shade mapping uses the light vector as lighting sees it, which depends on position for other lights.
+			if (!gstate.isDirectionalLight(gstate.getUVLS0()) || !gstate.isDirectionalLight(gstate.getUVLS1())) {
+				canSkipWorldPos = false;
+			}
+		}
 		state->lightingState.viewDir = PSPViewDirection(gstate.viewMatrix);
 
 		float world[16];
@@ -448,7 +454,7 @@ ClipVertexData TransformUnit::ReadVertex(const VertexReader &vreader, const Tran
 			Vec3<float> stq = Vec3ByMatrix43(source, gstate.tgenMatrix);
 			vertex.v.texturecoords = Vec3Packedf(stq.x, stq.y, stq.z);
 		} else if (state.uvGenMode == GE_TEXMAP_ENVIRONMENT_MAP) {
-			Lighting::GenerateLightST(vertex.v, worldnormal);
+			Lighting::GenerateLightST(vertex.v, worldpos, worldnormal, state.lightingState.viewDir);
 		}
 
 		PROFILE_THIS_SCOPE("light");

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -369,7 +369,7 @@ enum class VulkanCacheDetectFlags {
 };
 
 #define CACHE_HEADER_MAGIC 0xff51f420 
-#define CACHE_VERSION 60
+#define CACHE_VERSION 61
 
 struct VulkanCacheHeader {
 	uint32_t magic;