From eaf55c467ba948fe9dc04bfde5066ea379a8f6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 24 Sep 2026 13:56:24 -0600 Subject: [PATCH] IR: Fold ClampToZero into the 31-bit packs Vec4ClampToZero and Vec2ClampToZero only ever fed Vec4Pack31To8 and Vec2Pack31To16, for vi2uc and vi2us. The packs now clamp negative lanes to zero themselves, which saves an op and a vector temp, and lets x64 clamp with PACKUSWB's saturation after an arithmetic shift. While at it, RISC-V compiles Vec2Unpack16To31, Vec2Pack31To16 and Vec4Pack32To8, and LoongArch Vec2Unpack16To31, Vec2Pack31To16 and the non-LSX Vec4Pack32To8, all of which went to the IR interpreter. LoongArch's Vec2Pack32To16 and Vec2Unpack16To32 now take their scalar path with LSX too, instead of falling back. Co-Authored-By: Claude Opus 5.5 (1M context) --- Core/MIPS/ARM64/Arm64IRCompVec.cpp | 45 +++------ Core/MIPS/ARM64/Arm64IRJit.h | 1 - Core/MIPS/IR/IRCompVFPU.cpp | 12 +-- Core/MIPS/IR/IRInst.cpp | 2 - Core/MIPS/IR/IRInst.h | 3 +- Core/MIPS/IR/IRInterpreter.cpp | 65 ++++--------- Core/MIPS/IR/IRNativeCommon.cpp | 5 - Core/MIPS/IR/IRNativeCommon.h | 1 - Core/MIPS/IR/IRPassSimplify.cpp | 2 - Core/MIPS/LoongArch64/LoongArch64CompVec.cpp | 95 ++++++++++--------- Core/MIPS/LoongArch64/LoongArch64Jit.h | 1 - Core/MIPS/RiscV/RiscVCompVec.cpp | 96 +++++++++++++------- Core/MIPS/RiscV/RiscVJit.h | 1 - Core/MIPS/x86/X64IRCompVec.cpp | 45 +++------ Core/MIPS/x86/X64IRJit.h | 1 - 15 files changed, 162 insertions(+), 213 deletions(-) diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 08665051cb..a681eb9e47 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -576,28 +576,6 @@ void Arm64JitBackend::CompIR_VecAssign(IRInst inst) { } } -void Arm64JitBackend::CompIR_VecClamp(IRInst inst) { - CONDITIONAL_DISABLE; - - switch (inst.op) { - case IROp::Vec4ClampToZero: - regs_.Map(inst); - fp_.MOVI(32, EncodeRegToQuad(SCRATCHF1), 0); - fp_.SMAX(32, regs_.FQ(inst.dest), regs_.FQ(inst.src1), EncodeRegToQuad(SCRATCHF1)); - break; - - case IROp::Vec2ClampToZero: - regs_.Map(inst); - fp_.MOVI(32, EncodeRegToDouble(SCRATCHF1), 0); - fp_.SMAX(32, regs_.FD(inst.dest), regs_.FD(inst.src1), EncodeRegToDouble(SCRATCHF1)); - break; - - default: - INVALIDOP; - break; - } -} - void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) { CONDITIONAL_DISABLE; @@ -647,16 +625,21 @@ void Arm64JitBackend::CompIR_VecPack(IRInst inst) { break; case IROp::Vec2Pack31To16: - // Same as Vec2Pack32To16, but we shift left 1 first to nuke the sign bit. + // Same as Vec2Pack32To16, but negative lanes clamp to zero and we shift left 1 first to + // nuke the sign bit. if (Overlap(inst.dest, 1, inst.src1, 2)) { regs_.MapVec2(inst.src1, MIPSMap::DIRTY); - fp_.SHL(32, EncodeRegToDouble(SCRATCHF1), regs_.FD(inst.src1), 1); + } else { + regs_.Map(inst); + } + fp_.MOVI(32, EncodeRegToDouble(SCRATCHF2), 0); + fp_.SMAX(32, EncodeRegToDouble(SCRATCHF1), regs_.FD(inst.src1), EncodeRegToDouble(SCRATCHF2)); + fp_.SHL(32, EncodeRegToDouble(SCRATCHF1), EncodeRegToDouble(SCRATCHF1), 1); + if (Overlap(inst.dest, 1, inst.src1, 2)) { fp_.UZP2(16, EncodeRegToDouble(SCRATCHF1), EncodeRegToDouble(SCRATCHF1), EncodeRegToDouble(SCRATCHF1)); fp_.INS(32, regs_.FD(inst.dest & ~1), inst.dest & 1, EncodeRegToDouble(SCRATCHF1), 0); } else { - regs_.Map(inst); - fp_.SHL(32, regs_.FD(inst.dest), regs_.FD(inst.src1), 1); - fp_.UZP2(16, regs_.FD(inst.dest), regs_.FD(inst.dest), regs_.FD(inst.dest)); + fp_.UZP2(16, regs_.FD(inst.dest), EncodeRegToDouble(SCRATCHF1), EncodeRegToDouble(SCRATCHF1)); } break; @@ -679,9 +662,11 @@ void Arm64JitBackend::CompIR_VecPack(IRInst inst) { regs_.Map(inst); } - // Viewed as 8-bit lanes, after a shift by 23: AxxxBxxxCxxxDxxx. - // So: UZP1 -> AxBxCxDx -> UZP1 again -> ABCD - fp_.USHR(32, EncodeRegToQuad(SCRATCHF1), regs_.FQ(inst.src1), 23); + // Negative lanes clamp to zero. Then, viewed as 8-bit lanes after a shift by 23: + // AxxxBxxxCxxxDxxx. So: UZP1 -> AxBxCxDx -> UZP1 again -> ABCD + fp_.MOVI(32, EncodeRegToQuad(SCRATCHF2), 0); + fp_.SMAX(32, EncodeRegToQuad(SCRATCHF1), regs_.FQ(inst.src1), EncodeRegToQuad(SCRATCHF2)); + fp_.USHR(32, EncodeRegToQuad(SCRATCHF1), EncodeRegToQuad(SCRATCHF1), 23); fp_.UZP1(8, EncodeRegToQuad(SCRATCHF1), EncodeRegToQuad(SCRATCHF1), EncodeRegToQuad(SCRATCHF1)); // Second one directly to dest, if we can. if (Overlap(inst.dest, 1, inst.src1, 4)) { diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index 66053f42ae..a087638bbe 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -108,7 +108,6 @@ private: void CompIR_Transfer(IRInst inst) override; void CompIR_VecArith(IRInst inst) override; void CompIR_VecAssign(IRInst inst) override; - void CompIR_VecClamp(IRInst inst) override; void CompIR_VecHoriz(IRInst inst) override; void CompIR_VecLoad(IRInst inst) override; void CompIR_VecPack(IRInst inst) override; diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index d6edc866b2..fba0410aa4 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -1890,9 +1890,8 @@ namespace MIPSComp { if (bits == 8) { if (unsignedOp) { //vi2uc - // Output is only one register. - ir.Write(IROp::Vec4ClampToZero, IRVTEMP_0, srcregs[0]); - ir.Write(IROp::Vec4Pack31To8, tempregs[0], IRVTEMP_0); + // Output is only one register. The pack clamps negative lanes to zero. + ir.Write(IROp::Vec4Pack31To8, tempregs[0], srcregs[0]); } else { //vi2c ir.Write(IROp::Vec4Pack32To8, tempregs[0], srcregs[0]); } @@ -1900,11 +1899,10 @@ namespace MIPSComp { // bits == 16 if (unsignedOp) { //vi2us // Output is only one register. - ir.Write(IROp::Vec2ClampToZero, IRVTEMP_0, srcregs[0]); - ir.Write(IROp::Vec2Pack31To16, tempregs[0], IRVTEMP_0); + // The pack clamps negative lanes to zero. + ir.Write(IROp::Vec2Pack31To16, tempregs[0], srcregs[0]); if (outsize == V_Pair) { - ir.Write(IROp::Vec2ClampToZero, IRVTEMP_0 + 2, srcregs[2]); - ir.Write(IROp::Vec2Pack31To16, tempregs[1], IRVTEMP_0 + 2); + ir.Write(IROp::Vec2Pack31To16, tempregs[1], srcregs[2]); } } else { //vi2s ir.Write(IROp::Vec2Pack32To16, tempregs[0], srcregs[0]); diff --git a/Core/MIPS/IR/IRInst.cpp b/Core/MIPS/IR/IRInst.cpp index c0941c5aae..5eb7c2d3a5 100644 --- a/Core/MIPS/IR/IRInst.cpp +++ b/Core/MIPS/IR/IRInst.cpp @@ -160,8 +160,6 @@ static const IRMeta irMeta[] = { { IROp::Vec4Unpack8To32, "Vec4Unpack8To32", "VF" }, { IROp::Vec4DuplicateUpperBitsAndShift1, "Vec4DuplicateUpperBitsAndShift1", "VV" }, - { IROp::Vec4ClampToZero, "Vec4ClampToZero", "VV" }, - { IROp::Vec2ClampToZero, "Vec2ClampToZero", "22" }, { IROp::Vec4Pack32To8, "Vec4Pack32To8", "FV" }, { IROp::Vec4Pack31To8, "Vec4Pack31To8", "FV" }, { IROp::Vec2Pack32To16, "Vec2Pack32To16", "F2" }, diff --git a/Core/MIPS/IR/IRInst.h b/Core/MIPS/IR/IRInst.h index 83a1bc9af6..d7dd909edf 100644 --- a/Core/MIPS/IR/IRInst.h +++ b/Core/MIPS/IR/IRInst.h @@ -187,8 +187,7 @@ enum class IROp : uint8_t { Vec2Unpack16To32, Vec4Unpack8To32, Vec4DuplicateUpperBitsAndShift1, // Bizarro vuc2i behaviour, in an instruction. Split? - Vec4ClampToZero, - Vec2ClampToZero, + // vi2x. The 31 ones take bits 30 and down, after clamping negative lanes to zero (vi2uc, vi2us). Vec4Pack31To8, Vec4Pack32To8, Vec2Pack31To16, diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp index 2828a59402..858c7824f5 100644 --- a/Core/MIPS/IR/IRInterpreter.cpp +++ b/Core/MIPS/IR/IRInterpreter.cpp @@ -438,9 +438,10 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { case IROp::Vec2Pack31To16: { - // Used in Tekken 6 - u32 val = (mips->fi[inst->src1] >> 15) & 0xFFFF; - mips->fi[inst->dest] = val | ((mips->fi[(u32)inst->src1 + 1] << 1) & 0xFFFF0000); + // Used in Tekken 6. Negative lanes clamp to zero. + const u32 s0 = (s32)mips->fi[inst->src1] < 0 ? 0 : mips->fi[inst->src1]; + const u32 s1 = (s32)mips->fi[(u32)inst->src1 + 1] < 0 ? 0 : mips->fi[(u32)inst->src1 + 1]; + mips->fi[inst->dest] = ((s0 >> 15) & 0xFFFF) | ((s1 << 1) & 0xFFFF0000); break; } @@ -484,63 +485,29 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { { // Used in Tekken 6, Gran Turismo + // Bits 30-23 of each lane, with negative lanes clamped to zero. #if PPSSPP_ARCH(SSE2) __m128i src = _mm_loadu_si128((__m128i *) & mips->fi[inst->src1]); - // Shift each 32-bit lane left by 1, then take the top byte - that is, (v >> 23) & 0xFF. - // Shifting right by 24 first would drop bit 23. - src = _mm_srli_epi32(_mm_slli_epi32(src, 1), 24); - // Pack 32-bit lanes to 16-bit, then 16-bit to 8-bit - // This moves our target bytes to the bottom of the XMM register + // An arithmetic shift leaves 0-255 for positive lanes and negative values for negative + // ones, which the unsigned saturation in the second pack turns into 0. + src = _mm_srai_epi32(src, 23); src = _mm_packs_epi32(src, src); src = _mm_packus_epi16(src, src); - // Extract the lower 32 bits (which now contains our 4 bytes) mips->fi[inst->dest] = (u32)_mm_cvtsi128_si32(src); #elif PPSSPP_ARCH(ARM_NEON) - uint32x4_t value = vld1q_u32(&mips->fi[inst->src1]); - value = vshlq_n_u32(value, 1); - uint16x4_t halved = vshrn_n_u32(value, 16); + int32x4_t value = vmaxq_s32(vld1q_s32((const int32_t *)&mips->fi[inst->src1]), vdupq_n_s32(0)); + uint32x4_t shifted = vshlq_n_u32(vreinterpretq_u32_s32(value), 1); + uint16x4_t halved = vshrn_n_u32(shifted, 16); uint8x8_t halvedAgain = vshrn_n_u16(vcombine_u16(halved, vdup_n_u16(0)), 8); mips->fi[inst->dest] = vget_lane_u32(vreinterpret_u32_u8(halvedAgain), 0); #else - u32 val = (mips->fi[(u32)inst->src1] >> 23) & 0xFF; - val |= (mips->fi[(u32)inst->src1 + 1] >> 15) & 0xFF00; - val |= (mips->fi[(u32)inst->src1 + 2] >> 7) & 0xFF0000; - val |= (mips->fi[(u32)inst->src1 + 3] << 1) & 0xFF000000; - mips->fi[(u32)inst->dest] = val; -#endif - break; - } - - case IROp::Vec2ClampToZero: - { - const u32 temp0 = mips->fi[(u32)inst->src1]; - const u32 temp1 = mips->fi[(u32)inst->src1 + 1]; - mips->fi[(u32)inst->dest] = (int)temp0 >= 0 ? temp0 : 0; - mips->fi[(u32)inst->dest + 1] = (int)temp1 >= 0 ? temp1 : 0; - break; - } - - case IROp::Vec4ClampToZero: - { -#if PPSSPP_ARCH(SSE2) - // Trickery: Expand the sign bit, and use andnot to zero negative values. - __m128i val = _mm_load_si128((const __m128i *)&mips->fi[inst->src1]); - __m128i mask = _mm_srai_epi32(val, 31); - val = _mm_andnot_si128(mask, val); - _mm_store_si128((__m128i *)&mips->fi[inst->dest], val); -#elif PPSSPP_ARCH(ARM_NEON) - // On ARM we use a compare. On ARM64 we could also do a shift like on x86. - int32x4_t val = vld1q_s32((const int32_t *)&mips->fi[inst->src1]); - uint32x4_t mask = vcgtq_s32(val, vdupq_n_s32(-1)); // val > -1 → keeps >= 0 - val = vandq_s32(val, vreinterpretq_s32_u32(mask)); // zero out negative lanes - vst1q_s32((int32_t *)&mips->fi[inst->dest], val); -#else - const int src1 = inst->src1; - const int dest = inst->dest; + u32 val = 0; for (int i = 0; i < 4; i++) { - u32 val = mips->fi[src1 + i]; - mips->fi[dest + i] = (int)val >= 0 ? val : 0; + const u32 lane = mips->fi[(u32)inst->src1 + i]; + if ((s32)lane > 0) + val |= ((lane >> 23) & 0xFF) << (8 * i); } + mips->fi[(u32)inst->dest] = val; #endif break; } diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index 1e8944b139..142816c29b 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -422,11 +422,6 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { CompIR_VecPack(inst); break; - case IROp::Vec4ClampToZero: - case IROp::Vec2ClampToZero: - CompIR_VecClamp(inst); - break; - case IROp::FSin: case IROp::FCos: case IROp::FRSqrt: diff --git a/Core/MIPS/IR/IRNativeCommon.h b/Core/MIPS/IR/IRNativeCommon.h index f5ce920521..c9e3b1f540 100644 --- a/Core/MIPS/IR/IRNativeCommon.h +++ b/Core/MIPS/IR/IRNativeCommon.h @@ -125,7 +125,6 @@ protected: virtual void CompIR_Transfer(IRInst inst) = 0; virtual void CompIR_VecArith(IRInst inst) = 0; virtual void CompIR_VecAssign(IRInst inst) = 0; - virtual void CompIR_VecClamp(IRInst inst) = 0; virtual void CompIR_VecHoriz(IRInst inst) = 0; virtual void CompIR_VecLoad(IRInst inst) = 0; virtual void CompIR_VecPack(IRInst inst) = 0; diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index b9486aadc9..2434a4e4d5 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -833,8 +833,6 @@ bool PropagateConstants(const IRWriter &in, IRWriter &out, const IROptions &opts case IROp::Vec4Unpack8To32: case IROp::Vec2Unpack16To32: case IROp::Vec4DuplicateUpperBitsAndShift1: - case IROp::Vec2ClampToZero: - case IROp::Vec4ClampToZero: out.Write(inst); break; diff --git a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp index 88f29b487f..079cfd71ea 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp @@ -398,10 +398,36 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { switch (inst.op) { case IROp::Vec2Unpack16To31: - case IROp::Vec2Pack31To16: - CompIR_Generic(inst); + // Like Vec2Unpack16To32, shifted down one more. + regs_.Map(inst); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src1)); + SLLI_W(SCRATCH1, SCRATCH2, 16); + SRLI_W(SCRATCH1, SCRATCH1, 1); + MOVGR2FR_W(regs_.F(inst.dest), SCRATCH1); + SRLI_W(SCRATCH1, SCRATCH2, 16); + SLLI_W(SCRATCH1, SCRATCH1, 15); + MOVGR2FR_W(regs_.F(inst.dest + 1), SCRATCH1); break; + case IROp::Vec2Pack31To16: + { + // Bits 30-15 of each lane, with negative lanes clamped to zero. + regs_.Map(inst); + LoongArch64Reg maskReg = regs_.GetAndLockTempGPR(); + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1)); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src1 + 1)); + SRAI_W(maskReg, SCRATCH1, 31); + ANDN(SCRATCH1, SCRATCH1, maskReg); + SRAI_W(maskReg, SCRATCH2, 31); + ANDN(SCRATCH2, SCRATCH2, maskReg); + SRLI_D(SCRATCH1, SCRATCH1, 15); + SRLI_D(SCRATCH2, SCRATCH2, 15); + SLLI_D(SCRATCH2, SCRATCH2, 16); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + MOVGR2FR_W(regs_.F(inst.dest), SCRATCH1); + break; + } + case IROp::Vec4Pack32To8: if (cpu_info.LOONGARCH_LSX) { if (Overlap(inst.dest, 1, inst.src1, 4)) @@ -412,7 +438,19 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { VPICKEV_B(EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1)); VPICKEV_B(regs_.V(inst.dest), EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1)); } else { - CompIR_Generic(inst); + // The top byte of each lane. Every lane is read before dest is written. + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i)); + SRLI_W(SCRATCH1, SCRATCH1, 24); + if (i == 0) { + MOVE(SCRATCH2, SCRATCH1); + } else { + SLLI_D(SCRATCH1, SCRATCH1, 8 * i); + OR(SCRATCH2, SCRATCH2, SCRATCH1); + } + } + MOVGR2FR_W(regs_.F(inst.dest), SCRATCH2); } break; @@ -443,10 +481,6 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { case IROp::Vec2Unpack16To32: // TODO: This works for now, but may need to handle aliasing for vectors. - if (cpu_info.LOONGARCH_LSX) { - CompIR_Generic(inst); - break; - } regs_.Map(inst); MOVFR2GR_S(SCRATCH2, regs_.F(inst.src1)); SLLI_D(SCRATCH1, SCRATCH2, 16); @@ -479,23 +513,30 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { case IROp::Vec4Pack31To8: // TODO: This works for now, but may need to handle aliasing for vectors. + // Bits 30-23 of each lane, with negative lanes clamped to zero. if (cpu_info.LOONGARCH_LSX) { if (Overlap(inst.dest, 1, inst.src1, 4)) DISABLE; regs_.Map(inst); - VSRLI_W(EncodeRegToV(SCRATCHF1), regs_.V(inst.src1), 23); + VREPLGR2VR_D(EncodeRegToV(SCRATCHF1), R_ZERO); + VMAX_W(EncodeRegToV(SCRATCHF1), regs_.V(inst.src1), EncodeRegToV(SCRATCHF1)); + VSRLI_W(EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1), 23); VPICKEV_B(EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1)); VPICKEV_B(regs_.V(inst.dest), EncodeRegToV(SCRATCHF1), EncodeRegToV(SCRATCHF1)); } else { + // Every lane is read before dest is written. regs_.Map(inst); + LoongArch64Reg maskReg = regs_.GetAndLockTempGPR(); for (int i = 0; i < 4; ++i) { MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i)); + SRAI_W(maskReg, SCRATCH1, 31); + ANDN(SCRATCH1, SCRATCH1, maskReg); + // At most 0x7FFFFFFF, so this leaves 0-255. SRLI_D(SCRATCH1, SCRATCH1, 23); if (i == 0) { - ANDI(SCRATCH2, SCRATCH1, 0xFF); + MOVE(SCRATCH2, SCRATCH1); } else { - ANDI(SCRATCH1, SCRATCH1, 0xFF); SLLI_D(SCRATCH1, SCRATCH1, 8 * i); OR(SCRATCH2, SCRATCH2, SCRATCH1); } @@ -506,10 +547,6 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { case IROp::Vec2Pack32To16: // TODO: This works for now, but may need to handle aliasing for vectors. - if (cpu_info.LOONGARCH_LSX) { - CompIR_Generic(inst); - break; - } regs_.Map(inst); MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1)); MOVFR2GR_S(SCRATCH2, regs_.F(inst.src1 + 1)); @@ -531,34 +568,4 @@ void LoongArch64JitBackend::CompIR_VecPack(IRInst inst) { } } -void LoongArch64JitBackend::CompIR_VecClamp(IRInst inst) { - CONDITIONAL_DISABLE; - - switch (inst.op) { - case IROp::Vec4ClampToZero: - regs_.Map(inst); - if (cpu_info.LOONGARCH_LSX) { - VREPLGR2VR_D(EncodeRegToV(SCRATCHF1), R_ZERO); - VMAX_W(regs_.V(inst.dest), regs_.V(inst.src1), EncodeRegToV(SCRATCHF1)); - } else { - for (int i = 0; i < 4; i++) { - MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i)); - SRAI_W(SCRATCH2, SCRATCH1, 31); - ORN(SCRATCH2, R_ZERO, SCRATCH2); - AND(SCRATCH1, SCRATCH1, SCRATCH2); - MOVGR2FR_W(regs_.F(inst.dest + i), SCRATCH1); - } - } - break; - - case IROp::Vec2ClampToZero: - CompIR_Generic(inst); - break; - - default: - INVALIDOP; - break; - } -} - } // namespace MIPSComp diff --git a/Core/MIPS/LoongArch64/LoongArch64Jit.h b/Core/MIPS/LoongArch64/LoongArch64Jit.h index ed93a2848c..3253e05663 100644 --- a/Core/MIPS/LoongArch64/LoongArch64Jit.h +++ b/Core/MIPS/LoongArch64/LoongArch64Jit.h @@ -98,7 +98,6 @@ private: void CompIR_Transfer(IRInst inst) override; void CompIR_VecArith(IRInst inst) override; void CompIR_VecAssign(IRInst inst) override; - void CompIR_VecClamp(IRInst inst) override; void CompIR_VecHoriz(IRInst inst) override; void CompIR_VecLoad(IRInst inst) override; void CompIR_VecPack(IRInst inst) override; diff --git a/Core/MIPS/RiscV/RiscVCompVec.cpp b/Core/MIPS/RiscV/RiscVCompVec.cpp index 8118fc7195..51141203e4 100644 --- a/Core/MIPS/RiscV/RiscVCompVec.cpp +++ b/Core/MIPS/RiscV/RiscVCompVec.cpp @@ -321,13 +321,63 @@ void RiscVJitBackend::CompIR_VecHoriz(IRInst inst) { void RiscVJitBackend::CompIR_VecPack(IRInst inst) { CONDITIONAL_DISABLE; + // Clamps a sign extended value in reg to zero if negative. Without Zbb, maskReg is a temp. + auto clampToZero = [&](RiscVReg reg, RiscVReg maskReg) { + if (cpu_info.RiscV_Zbb) { + MAX(reg, reg, R_ZERO); + } else { + SRAI(maskReg, reg, XLEN - 1); + NOT(maskReg, maskReg); + AND(reg, reg, maskReg); + } + }; + switch (inst.op) { case IROp::Vec2Unpack16To31: - case IROp::Vec4Pack32To8: - case IROp::Vec2Pack31To16: - CompIR_Generic(inst); + // Like Vec2Unpack16To32, shifted down one more. + regs_.Map(inst); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src1)); + SLLI(SCRATCH1, SCRATCH2, 16); + SRLIW(SCRATCH1, SCRATCH1, 1); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + SRLIW(SCRATCH1, SCRATCH2, 16); + SLLI(SCRATCH1, SCRATCH1, 15); + FMV(FMv::W, FMv::X, regs_.F(inst.dest + 1), SCRATCH1); break; + case IROp::Vec4Pack32To8: + // The top byte of each lane. Every lane is read before dest is written. + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i)); + SRLIW(SCRATCH1, SCRATCH1, 24); + if (i == 0) { + MV(SCRATCH2, SCRATCH1); + } else { + SLLI(SCRATCH1, SCRATCH1, 8 * i); + OR(SCRATCH2, SCRATCH2, SCRATCH1); + } + } + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH2); + break; + + case IROp::Vec2Pack31To16: + { + // Bits 30-15 of each lane, with negative lanes clamped to zero. + regs_.Map(inst); + RiscVReg maskReg = cpu_info.RiscV_Zbb ? INVALID_REG : regs_.GetAndLockTempGPR(); + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1)); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src1 + 1)); + clampToZero(SCRATCH1, maskReg); + clampToZero(SCRATCH2, maskReg); + SRLI(SCRATCH1, SCRATCH1, 15); + SRLI(SCRATCH2, SCRATCH2, 15); + SLLI(SCRATCH2, SCRATCH2, 16); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + break; + } + case IROp::Vec4Unpack8To32: // TODO: This works for now, but may need to handle aliasing for vectors. regs_.Map(inst); @@ -369,15 +419,19 @@ void RiscVJitBackend::CompIR_VecPack(IRInst inst) { break; case IROp::Vec4Pack31To8: - // TODO: This works for now, but may need to handle aliasing for vectors. + { + // Bits 30-23 of each lane, with negative lanes clamped to zero. Every lane is read before + // dest is written. regs_.Map(inst); + RiscVReg maskReg = cpu_info.RiscV_Zbb ? INVALID_REG : regs_.GetAndLockTempGPR(); for (int i = 0; i < 4; ++i) { FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i)); + clampToZero(SCRATCH1, maskReg); + // At most 0x7FFFFFFF, so this leaves 0-255. SRLI(SCRATCH1, SCRATCH1, 23); if (i == 0) { - ANDI(SCRATCH2, SCRATCH1, 0xFF); + MV(SCRATCH2, SCRATCH1); } else { - ANDI(SCRATCH1, SCRATCH1, 0xFF); SLLI(SCRATCH1, SCRATCH1, 8 * i); OR(SCRATCH2, SCRATCH2, SCRATCH1); } @@ -385,6 +439,7 @@ void RiscVJitBackend::CompIR_VecPack(IRInst inst) { FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH2); break; + } case IROp::Vec2Pack32To16: // TODO: This works for now, but may need to handle aliasing for vectors. @@ -409,33 +464,4 @@ void RiscVJitBackend::CompIR_VecPack(IRInst inst) { } } -void RiscVJitBackend::CompIR_VecClamp(IRInst inst) { - CONDITIONAL_DISABLE; - - switch (inst.op) { - case IROp::Vec4ClampToZero: - regs_.Map(inst); - for (int i = 0; i < 4; i++) { - FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i)); - SRAIW(SCRATCH2, SCRATCH1, 31); - if (cpu_info.RiscV_Zbb) { - ANDN(SCRATCH1, SCRATCH1, SCRATCH2); - } else { - NOT(SCRATCH2, SCRATCH2); - AND(SCRATCH1, SCRATCH1, SCRATCH2); - } - FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1); - } - break; - - case IROp::Vec2ClampToZero: - CompIR_Generic(inst); - break; - - default: - INVALIDOP; - break; - } -} - } // namespace MIPSComp diff --git a/Core/MIPS/RiscV/RiscVJit.h b/Core/MIPS/RiscV/RiscVJit.h index ba4eb383ef..8c680003c3 100644 --- a/Core/MIPS/RiscV/RiscVJit.h +++ b/Core/MIPS/RiscV/RiscVJit.h @@ -99,7 +99,6 @@ private: void CompIR_Transfer(IRInst inst) override; void CompIR_VecArith(IRInst inst) override; void CompIR_VecAssign(IRInst inst) override; - void CompIR_VecClamp(IRInst inst) override; void CompIR_VecHoriz(IRInst inst) override; void CompIR_VecLoad(IRInst inst) override; void CompIR_VecPack(IRInst inst) override; diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index c448fa5b3a..aea78af280 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -242,32 +242,6 @@ void X64JitBackend::CompIR_VecAssign(IRInst inst) { } } -void X64JitBackend::CompIR_VecClamp(IRInst inst) { - CONDITIONAL_DISABLE; - - switch (inst.op) { - case IROp::Vec4ClampToZero: - case IROp::Vec2ClampToZero: - { - const int lanes = inst.op == IROp::Vec4ClampToZero ? 4 : 2; - if (inst.dest != inst.src1 && Overlap(inst.dest, lanes, inst.src1, lanes)) { - DISABLE; - } - // Expand the sign bit to a mask, then clear the negative lanes. - X64Reg tempReg = regs_.MapWithFPRTemp(inst); - MOVDQA(tempReg, regs_.F(inst.src1)); - PSRAD(tempReg, 31); - PANDN(tempReg, regs_.F(inst.src1)); - MOVDQA(regs_.FX(inst.dest), R(tempReg)); - break; - } - - default: - INVALIDOP; - break; - } -} - void X64JitBackend::CompIR_VecHoriz(IRInst inst) { CONDITIONAL_DISABLE; @@ -317,12 +291,15 @@ void X64JitBackend::CompIR_VecPack(IRInst inst) { if (Overlap(inst.dest, 1, inst.src1, 4)) { DISABLE; } - // The top byte of each lane (for 31, the byte below the sign), packed into one lane. + // The top byte of each lane, packed into one lane. For 31, the byte below the sign, with + // negative lanes clamped to zero: the arithmetic shift makes them negative, and PACKUSWB + // saturates them to 0. X64Reg tempReg = regs_.MapWithFPRTemp(inst); MOVDQA(tempReg, regs_.F(inst.src1)); if (inst.op == IROp::Vec4Pack31To8) - PSLLD(tempReg, 1); - PSRLD(tempReg, 24); + PSRAD(tempReg, 23); + else + PSRLD(tempReg, 24); PACKSSDW(tempReg, R(tempReg)); PACKUSWB(tempReg, R(tempReg)); MOVDQA(regs_.FX(inst.dest), R(tempReg)); @@ -335,12 +312,16 @@ void X64JitBackend::CompIR_VecPack(IRInst inst) { if (Overlap(inst.dest, 1, inst.src1, 2)) { DISABLE; } - // The top 16 bits of each lane (for 31, the 16 below the sign), packed into one lane. The - // arithmetic shift keeps them in PACKSSDW's range, so the bits come through unchanged. + // The top 16 bits of each lane (for 31, the 16 below the sign, with negative lanes clamped + // to zero), packed into one lane. The arithmetic shift keeps them in PACKSSDW's range, so + // the bits come through unchanged. X64Reg tempReg = regs_.MapWithFPRTemp(inst); MOVDQA(tempReg, regs_.F(inst.src1)); - if (inst.op == IROp::Vec2Pack31To16) + if (inst.op == IROp::Vec2Pack31To16) { + PSRAD(tempReg, 31); + PANDN(tempReg, regs_.F(inst.src1)); PSLLD(tempReg, 1); + } PSRAD(tempReg, 16); PACKSSDW(tempReg, R(tempReg)); MOVDQA(regs_.FX(inst.dest), R(tempReg)); diff --git a/Core/MIPS/x86/X64IRJit.h b/Core/MIPS/x86/X64IRJit.h index fe44eed3ad..edd86594e0 100644 --- a/Core/MIPS/x86/X64IRJit.h +++ b/Core/MIPS/x86/X64IRJit.h @@ -115,7 +115,6 @@ private: void CompIR_Transfer(IRInst inst) override; void CompIR_VecArith(IRInst inst) override; void CompIR_VecAssign(IRInst inst) override; - void CompIR_VecClamp(IRInst inst) override; void CompIR_VecHoriz(IRInst inst) override; void CompIR_VecLoad(IRInst inst) override; void CompIR_VecPack(IRInst inst) override;