From 85b80bc9e5786169f6e477fee1a78669968116f6 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 3 Sep 2023 17:37:33 -0700 Subject: [PATCH] arm64jit: Implement load/store in IR. --- Common/Arm64Emitter.h | 17 ++ Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp | 288 ++++++++++++++++++++++- Core/MIPS/ARM64/Arm64IRJit.h | 10 + Core/MIPS/RiscV/RiscVCompLoadStore.cpp | 4 + Core/MIPS/x86/X64IRCompLoadStore.cpp | 9 +- 5 files changed, 319 insertions(+), 9 deletions(-) diff --git a/Common/Arm64Emitter.h b/Common/Arm64Emitter.h index 36e969dd2b..6dbfa3deed 100644 --- a/Common/Arm64Emitter.h +++ b/Common/Arm64Emitter.h @@ -290,6 +290,23 @@ public: } m_shifttype = ST_LSL; } + ArithOption(ARM64Reg Rd, bool index, bool signExtend) { + if (index) + m_shift = 4; + else + m_shift = 0; + + m_destReg = Rd; + m_type = TYPE_EXTENDEDREG; + if (Is64Bit(Rd)) { + m_width = WIDTH_64BIT; + m_extend = EXTEND_UXTX; + } else { + m_width = WIDTH_32BIT; + m_extend = signExtend ? EXTEND_SXTW : EXTEND_UXTW; + } + m_shifttype = ST_LSL; + } ArithOption(ARM64Reg Rd, ShiftType shift_type, u32 shift) { m_destReg = Rd; diff --git a/Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp b/Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp index 29167b7dd8..42a966d437 100644 --- a/Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp @@ -38,20 +38,173 @@ namespace MIPSComp { using namespace Arm64Gen; using namespace Arm64IRJitConstants; +static int IROpToByteWidth(IROp op) { + switch (op) { + case IROp::Load8: + case IROp::Load8Ext: + case IROp::Store8: + return 1; + + case IROp::Load16: + case IROp::Load16Ext: + case IROp::Store16: + return 2; + + case IROp::Load32: + case IROp::Load32Linked: + case IROp::Load32Left: + case IROp::Load32Right: + case IROp::LoadFloat: + case IROp::Store32: + case IROp::Store32Conditional: + case IROp::Store32Left: + case IROp::Store32Right: + case IROp::StoreFloat: + return 4; + + case IROp::LoadVec4: + case IROp::StoreVec4: + return 16; + + default: + _assert_msg_(false, "Unexpected op: %s", GetIRMeta(op) ? GetIRMeta(op)->name : "?"); + return -1; + } +} + +Arm64JitBackend::LoadStoreArg Arm64JitBackend::PrepareSrc1Address(IRInst inst) { + const IRMeta *m = GetIRMeta(inst.op); + + bool src1IsPointer = regs_.IsGPRMappedAsPointer(inst.src1); + bool readsFromSrc1 = inst.src1 == inst.src3 && (m->flags & (IRFLAG_SRC3 | IRFLAG_SRC3DST)) != 0; + // If it's about to be clobbered, don't waste time pointerifying. Use displacement. + bool clobbersSrc1 = !readsFromSrc1 && regs_.IsGPRClobbered(inst.src1); + + int32_t imm = (int32_t)inst.constant; + LoadStoreArg addrArg; + if (inst.src1 == MIPS_REG_ZERO) { + // The constant gets applied later. + addrArg.base = MEMBASEREG; +#ifdef MASKED_PSP_MEMORY + imm &= Memory::MEMVIEW32_MASK; +#endif + } else if (!jo.enablePointerify && readsFromSrc1) { +#ifndef MASKED_PSP_MEMORY + if (imm == 0) { + addrArg.base = MEMBASEREG; + addrArg.regOffset = regs_.MapGPR(inst.src1); + addrArg.useRegisterOffset = true; + addrArg.signExtendRegOffset = false; + } +#endif + + // Since we can't modify src1, let's just use a temp reg while copying. + if (!addrArg.useRegisterOffset) { + ADDI2R(SCRATCH1, regs_.MapGPR(inst.src1), (s64)imm, SCRATCH2); +#ifdef MASKED_PSP_MEMORY + ANDI2R(SCRATCH1, SCRATCH1, Memory::MEMVIEW32_MASK, SCRATCH2); +#endif + + addrArg.base = MEMBASEREG; + addrArg.regOffset = SCRATCH1; + addrArg.useRegisterOffset = true; + addrArg.signExtendRegOffset = false; + } + } else if ((jo.cachePointers && !clobbersSrc1) || src1IsPointer) { + // The offset gets set later. + addrArg.base = regs_.MapGPRAsPointer(inst.src1); + } else { + ADDI2R(SCRATCH1, regs_.MapGPR(inst.src1), (s64)imm, SCRATCH2); +#ifdef MASKED_PSP_MEMORY + ANDI2R(SCRATCH1, SCRATCH1, Memory::MEMVIEW32_MASK, SCRATCH2); +#endif + + addrArg.base = MEMBASEREG; + addrArg.regOffset = SCRATCH1; + addrArg.useRegisterOffset = true; + addrArg.signExtendRegOffset = false; + } + + // That's src1 taken care of, and possibly imm. + // If useRegisterOffset is false, imm still needs to be accounted for. + if (!addrArg.useRegisterOffset && imm != 0) { +#ifdef MASKED_PSP_MEMORY + // In case we have an address + offset reg. + if (imm > 0) + imm &= Memory::MEMVIEW32_MASK; +#endif + + int scale = IROpToByteWidth(inst.op); + if (imm > 0 && (imm & (scale - 1)) == 0 && imm <= 0xFFF * scale) { + // Okay great, use the LDR/STR form. + addrArg.immOffset = imm; + addrArg.useUnscaled = false; + } else if (imm >= -256 && imm < 256) { + // An unscaled offset (LDUR/STUR) should work fine for this range. + addrArg.immOffset = imm; + addrArg.useUnscaled = true; + } else { + // No luck, we'll need to load into a register. + MOVI2R(SCRATCH1, (s64)imm); + addrArg.regOffset = SCRATCH1; + addrArg.useRegisterOffset = true; + addrArg.signExtendRegOffset = true; + } + } + + return addrArg; +} + void Arm64JitBackend::CompIR_CondStore(IRInst inst) { CONDITIONAL_DISABLE; if (inst.op != IROp::Store32Conditional) INVALIDOP; - CompIR_Generic(inst); + regs_.SpillLockGPR(IRREG_LLBIT, inst.src3, inst.src1); + LoadStoreArg addrArg = PrepareSrc1Address(inst); + ARM64Reg valueReg = regs_.MapGPR(inst.src3, MIPSMap::INIT); + + regs_.MapGPR(IRREG_LLBIT, MIPSMap::INIT); + + // TODO: Safe memory? Or enough to have crash handler + validate? + + FixupBranch condFailed = CBZ(regs_.R(IRREG_LLBIT)); + + if (addrArg.useRegisterOffset) { + STR(valueReg, addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + STUR(valueReg, addrArg.base, addrArg.immOffset); + } else { + STR(INDEX_UNSIGNED, valueReg, addrArg.base, addrArg.immOffset); + } + + if (inst.dest != MIPS_REG_ZERO) { + MOVI2R(regs_.R(inst.dest), 1); + FixupBranch finish = B(); + + SetJumpTarget(condFailed); + MOVI2R(regs_.R(inst.dest), 0); + SetJumpTarget(finish); + } else { + SetJumpTarget(condFailed); + } } void Arm64JitBackend::CompIR_FLoad(IRInst inst) { CONDITIONAL_DISABLE; + LoadStoreArg addrArg = PrepareSrc1Address(inst); + switch (inst.op) { case IROp::LoadFloat: - CompIR_Generic(inst); + regs_.MapFPR(inst.dest, MIPSMap::NOINIT); + if (addrArg.useRegisterOffset) { + fp_.LDR(32, regs_.F(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + fp_.LDUR(32, regs_.F(inst.dest), addrArg.base, addrArg.immOffset); + } else { + fp_.LDR(32, INDEX_UNSIGNED, regs_.F(inst.dest), addrArg.base, addrArg.immOffset); + } break; default: @@ -63,9 +216,18 @@ void Arm64JitBackend::CompIR_FLoad(IRInst inst) { void Arm64JitBackend::CompIR_FStore(IRInst inst) { CONDITIONAL_DISABLE; + LoadStoreArg addrArg = PrepareSrc1Address(inst); + switch (inst.op) { case IROp::StoreFloat: - CompIR_Generic(inst); + regs_.MapFPR(inst.src3); + if (addrArg.useRegisterOffset) { + fp_.STR(32, regs_.F(inst.src3), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + fp_.STUR(32, regs_.F(inst.src3), addrArg.base, addrArg.immOffset); + } else { + fp_.STR(32, INDEX_UNSIGNED, regs_.F(inst.src3), addrArg.base, addrArg.immOffset); + } break; default: @@ -77,14 +239,75 @@ void Arm64JitBackend::CompIR_FStore(IRInst inst) { void Arm64JitBackend::CompIR_Load(IRInst inst) { CONDITIONAL_DISABLE; + regs_.SpillLockGPR(inst.dest, inst.src1); + LoadStoreArg addrArg = PrepareSrc1Address(inst); + // With NOINIT, MapReg won't subtract MEMBASEREG even if dest == src1. + regs_.MapGPR(inst.dest, MIPSMap::NOINIT); + + // TODO: Safe memory? Or enough to have crash handler + validate? + switch (inst.op) { case IROp::Load8: + if (addrArg.useRegisterOffset) { + LDRB(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDURB(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDRB(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + break; + case IROp::Load8Ext: + if (addrArg.useRegisterOffset) { + LDRSB(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDURSB(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDRSB(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + break; + case IROp::Load16: + if (addrArg.useRegisterOffset) { + LDRH(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDURH(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDRH(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + break; + case IROp::Load16Ext: + if (addrArg.useRegisterOffset) { + LDRSH(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDURSH(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDRSH(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + break; + case IROp::Load32: + if (addrArg.useRegisterOffset) { + LDR(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDUR(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDR(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + break; + case IROp::Load32Linked: - CompIR_Generic(inst); + if (inst.dest != MIPS_REG_ZERO) { + if (addrArg.useRegisterOffset) { + LDR(regs_.R(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + LDUR(regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } else { + LDR(INDEX_UNSIGNED, regs_.R(inst.dest), addrArg.base, addrArg.immOffset); + } + } + regs_.SetGPRImm(IRREG_LLBIT, 1); break; default: @@ -112,11 +335,44 @@ void Arm64JitBackend::CompIR_LoadShift(IRInst inst) { void Arm64JitBackend::CompIR_Store(IRInst inst) { CONDITIONAL_DISABLE; + regs_.SpillLockGPR(inst.src3, inst.src1); + LoadStoreArg addrArg = PrepareSrc1Address(inst); + + ARM64Reg valueReg = regs_.TryMapTempImm(inst.src3); + if (valueReg == INVALID_REG) + valueReg = regs_.MapGPR(inst.src3); + + // TODO: Safe memory? Or enough to have crash handler + validate? + switch (inst.op) { case IROp::Store8: + if (addrArg.useRegisterOffset) { + STRB(valueReg, addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + STURB(valueReg, addrArg.base, addrArg.immOffset); + } else { + STRB(INDEX_UNSIGNED, valueReg, addrArg.base, addrArg.immOffset); + } + break; + case IROp::Store16: + if (addrArg.useRegisterOffset) { + STRH(valueReg, addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + STURH(valueReg, addrArg.base, addrArg.immOffset); + } else { + STRH(INDEX_UNSIGNED, valueReg, addrArg.base, addrArg.immOffset); + } + break; + case IROp::Store32: - CompIR_Generic(inst); + if (addrArg.useRegisterOffset) { + STR(valueReg, addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + STUR(valueReg, addrArg.base, addrArg.immOffset); + } else { + STR(INDEX_UNSIGNED, valueReg, addrArg.base, addrArg.immOffset); + } break; default: @@ -144,9 +400,18 @@ void Arm64JitBackend::CompIR_StoreShift(IRInst inst) { void Arm64JitBackend::CompIR_VecLoad(IRInst inst) { CONDITIONAL_DISABLE; + LoadStoreArg addrArg = PrepareSrc1Address(inst); + switch (inst.op) { case IROp::LoadVec4: - CompIR_Generic(inst); + regs_.MapVec4(inst.dest, MIPSMap::NOINIT); + if (addrArg.useRegisterOffset) { + fp_.LDR(128, regs_.FQ(inst.dest), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + fp_.LDUR(128, regs_.FQ(inst.dest), addrArg.base, addrArg.immOffset); + } else { + fp_.LDR(128, INDEX_UNSIGNED, regs_.FQ(inst.dest), addrArg.base, addrArg.immOffset); + } break; default: @@ -158,9 +423,18 @@ void Arm64JitBackend::CompIR_VecLoad(IRInst inst) { void Arm64JitBackend::CompIR_VecStore(IRInst inst) { CONDITIONAL_DISABLE; + LoadStoreArg addrArg = PrepareSrc1Address(inst); + switch (inst.op) { case IROp::StoreVec4: - CompIR_Generic(inst); + regs_.MapVec4(inst.src3); + if (addrArg.useRegisterOffset) { + fp_.STR(128, regs_.FQ(inst.src3), addrArg.base, ArithOption(addrArg.regOffset, false, addrArg.signExtendRegOffset)); + } else if (addrArg.useUnscaled) { + fp_.STUR(128, regs_.FQ(inst.src3), addrArg.base, addrArg.immOffset); + } else { + fp_.STR(128, INDEX_UNSIGNED, regs_.FQ(inst.src3), addrArg.base, addrArg.immOffset); + } break; default: diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index fa8bfd89f2..fa2428504c 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -110,6 +110,16 @@ private: void CompIR_VecStore(IRInst inst) override; void CompIR_ValidateAddress(IRInst inst) override; + struct LoadStoreArg { + Arm64Gen::ARM64Reg base = Arm64Gen::INVALID_REG; + Arm64Gen::ARM64Reg regOffset = Arm64Gen::INVALID_REG; + int immOffset = 0; + bool useUnscaled = false; + bool useRegisterOffset = false; + bool signExtendRegOffset = false; + }; + LoadStoreArg PrepareSrc1Address(IRInst inst); + JitOptions &jo; Arm64IRRegCache regs_; Arm64Gen::ARM64FloatEmitter fp_; diff --git a/Core/MIPS/RiscV/RiscVCompLoadStore.cpp b/Core/MIPS/RiscV/RiscVCompLoadStore.cpp index 08f5cb88e2..80b149ca02 100644 --- a/Core/MIPS/RiscV/RiscVCompLoadStore.cpp +++ b/Core/MIPS/RiscV/RiscVCompLoadStore.cpp @@ -55,6 +55,10 @@ void RiscVJitBackend::SetScratch1ToSrc1Address(IRReg src1) { int32_t RiscVJitBackend::AdjustForAddressOffset(RiscVGen::RiscVReg *reg, int32_t constant, int32_t range) { if (constant < -2048 || constant + range > 2047) { +#ifdef MASKED_PSP_MEMORY + if (constant > 0) + constant &= Memory::MEMVIEW32_MASK; +#endif LI(SCRATCH2, constant); ADD(SCRATCH1, *reg, SCRATCH2); *reg = SCRATCH1; diff --git a/Core/MIPS/x86/X64IRCompLoadStore.cpp b/Core/MIPS/x86/X64IRCompLoadStore.cpp index 14c12c2b8a..d033832bf3 100644 --- a/Core/MIPS/x86/X64IRCompLoadStore.cpp +++ b/Core/MIPS/x86/X64IRCompLoadStore.cpp @@ -45,6 +45,11 @@ Gen::OpArg X64JitBackend::PrepareSrc1Address(IRInst inst) { // If it's about to be clobbered, don't waste time pointerifying. Use displacement. bool clobbersSrc1 = !readsFromSrc1 && regs_.IsGPRClobbered(inst.src1); +#ifdef MASKED_PSP_MEMORY + if (inst.constant > 0) + inst.constant &= Memory::MEMVIEW32_MASK; +#endif + OpArg addrArg; if (inst.src1 == MIPS_REG_ZERO) { #ifdef MASKED_PSP_MEMORY @@ -130,7 +135,7 @@ void X64JitBackend::CompIR_FStore(IRInst inst) { switch (inst.op) { case IROp::StoreFloat: regs_.MapFPR(inst.src3); - MOVSS(addrArg, regs_.FX(inst.dest)); + MOVSS(addrArg, regs_.FX(inst.src3)); break; default: @@ -287,7 +292,7 @@ void X64JitBackend::CompIR_VecStore(IRInst inst) { switch (inst.op) { case IROp::StoreVec4: regs_.MapVec4(inst.src3); - MOVUPS(addrArg, regs_.FX(inst.dest)); + MOVUPS(addrArg, regs_.FX(inst.src3)); break; default: