Merge pull request #11902 from unknownbrackets/gpu-stencil

GPU: Respect stencil write mask on clear

GPU/Common/GPUStateUtils.cpp

@@ -1343,7 +1343,7 @@ void ConvertStencilFuncState(GenericStencilFuncState &state) {
 		return;
 
 	// The PSP's mask is reversed (bits not to write.)
-	state.writeMask = (~gstate.pmska) & 0xFF;
+	state.writeMask = (~gstate.getStencilWriteMask()) & 0xFF;
 
 	state.sFail = gstate.getStencilOpSFail();
 	state.zFail = gstate.getStencilOpZFail();

GPU/Common/SoftwareTransformCommon.cpp

@@ -457,7 +457,9 @@ void SoftwareTransform(
 		// If alpha is not allowed to be separate, it must match for both depth/stencil and color.  Vulkan requires this.
 		bool alphaMatchesColor = gstate.isClearModeColorMask() == gstate.isClearModeAlphaMask();
 		bool depthMatchesStencil = gstate.isClearModeAlphaMask() == gstate.isClearModeDepthMask();
-		if (params->allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil)) {
+		bool matchingComponents = params->allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil);
+		bool stencilNotMasked = !gstate.isClearModeAlphaMask() || gstate.getStencilWriteMask() == 0x00;
+		if (matchingComponents && stencilNotMasked) {
 			result->color = transformed[1].color0_32;
 			// Need to rescale from a [0, 1] float.  This is the final transformed value.
 			result->depth = ToScaledDepthFromIntegerScale((int)(transformed[1].z * 65535.0f));

GPU/D3D11/StateMappingD3D11.cpp

@@ -330,7 +330,8 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 				// A normal clear will be 2 points, the second point has the color.
 				// We override this value in the pipeline from software transform for clear rectangles.
 				dynState_.stencilRef = 0xFF;
-				keys_.depthStencil.stencilWriteMask = 0xFF;
+				// But we still apply the stencil write mask.
+				keys_.depthStencil.stencilWriteMask = (~gstate.getStencilWriteMask()) & 0xFF;
 			} else {
 				keys_.depthStencil.stencilTestEnable = false;
 				dynState_.useStencil = false;

GPU/Directx9/StateMappingDX9.cpp

@@ -233,7 +233,7 @@ void DrawEngineDX9::ApplyDrawState(int prim) {
 				dxstate.stencilTest.enable();
 				dxstate.stencilOp.set(D3DSTENCILOP_REPLACE, D3DSTENCILOP_REPLACE, D3DSTENCILOP_REPLACE);
 				dxstate.stencilFunc.set(D3DCMP_ALWAYS, 255, 0xFF);
-				dxstate.stencilMask.set(0xFF);
+				dxstate.stencilMask.set((~gstate.getStencilWriteMask()) & 0xFF);
 			} else {
 				dxstate.stencilTest.disable();
 			}

GPU/GLES/StateMappingGLES.cpp

@@ -277,7 +277,7 @@ void DrawEngineGLES::ApplyDrawState(int prim) {
 				framebufferManager_->SetDepthUpdated();
 			}
 			renderManager->SetStencilFunc(gstate.isClearModeAlphaMask(), GL_ALWAYS, 0xFF, 0xFF);
-			renderManager->SetStencilOp(0xFF, GL_REPLACE, GL_REPLACE, GL_REPLACE);
+			renderManager->SetStencilOp((~gstate.getStencilWriteMask()) & 0xFF, GL_REPLACE, GL_REPLACE, GL_REPLACE);
 			renderManager->SetDepth(true, gstate.isClearModeDepthMask() ? true : false, GL_ALWAYS);
 		} else {
 			// Depth Test

GPU/Software/Rasterizer.cpp

@@ -562,7 +562,8 @@ static inline Vec4<int> GetTextureFunctionOutput(const Vec4<int>& prim_color, co
 		const __m128 b = _mm_mul_ps(p, t);
 		if (gstate.isColorDoublingEnabled()) {
 			// We double right here, only for modulate.  Other tex funcs do not color double.
-			out_rgb.ivec = _mm_cvtps_epi32(_mm_mul_ps(b, _mm_set_ps1(2.0f / 255.0f)));
+			const __m128 doubleColor = _mm_setr_ps(2.0f / 255.0f, 2.0f / 255.0f, 2.0f / 255.0f, 1.0f / 255.0f);
+			out_rgb.ivec = _mm_cvtps_epi32(_mm_mul_ps(b, doubleColor));
 		} else {
 			out_rgb.ivec = _mm_cvtps_epi32(_mm_mul_ps(b, _mm_set_ps1(1.0f / 255.0f)));
 		}

GPU/Vulkan/StateMappingVulkan.cpp

@@ -294,7 +294,8 @@ void DrawEngineVulkan::ConvertStateToVulkanKey(FramebufferManagerVulkan &fbManag
 				// A normal clear will be 2 points, the second point has the color.
 				// We override this value in the pipeline from software transform for clear rectangles.
 				dynState.stencilRef = 0xFF;
-				dynState.stencilWriteMask = 0xFF;
+				// But we still apply the stencil write mask.
+				dynState.stencilWriteMask = (~gstate.getStencilWriteMask()) & 0xFF;
 			} else {
 				key.stencilTestEnable = false;
 				key.stencilCompareOp = VK_COMPARE_OP_ALWAYS;