Merge pull request #22357 from hrydgard/ir-interpreter-opts

IR interpreter optimizations

Core/MIPS/ARM64/Arm64IRCompLoadStore.cpp

@@ -200,9 +200,70 @@ void Arm64JitBackend::CompIR_CondStore(IRInst inst) {
 	}
 }
 
+bool Arm64JitBackend::TryCompileLoadStorePair(IRInst inst) {
+	if (compilingIndex_ + 1 >= compilingCount_ || inst.src1 == MIPS_REG_ZERO)
+		return false;
+	const IRInst next = compilingInsts_[compilingIndex_ + 1];
+	if (next.op != inst.op || next.src1 != inst.src1)
+		return false;
+	const int32_t offset = (int32_t)inst.constant;
+	const int32_t nextOffset = (int32_t)next.constant;
+	if (nextOffset != offset + 4 && nextOffset != offset - 4)
+		return false;
+	// The pair's scaled 7-bit signed offset.
+	const int32_t low = std::min(offset, nextOffset);
+	if ((low & 3) != 0 || low < -256 || low > 252)
+		return false;
+	// The base must be usable as a pointer.
+	if (!regs_.IsGPRMappedAsPointer(inst.src1) && (!jo.cachePointers || regs_.IsGPRClobbered(inst.src1)))
+		return false;
+
+	const bool isLoad = inst.op == IROp::Load32 || inst.op == IROp::LoadFloat;
+	const bool isFloat = inst.op == IROp::LoadFloat || inst.op == IROp::StoreFloat;
+	// A load may not change the base, and an LDP can't write one register twice.
+	if (isLoad && (next.dest == inst.dest || (!isFloat && (inst.dest == inst.src1 || next.dest == inst.src1))))
+		return false;
+	// Nor can a stored value be the base, which is mapped as a pointer.
+	if (!isLoad && !isFloat && (inst.src3 == inst.src1 || next.src3 == inst.src1))
+		return false;
+
+	const IRReg regLow = offset < nextOffset ? inst.dest : next.dest;
+	const IRReg regHigh = offset < nextOffset ? next.dest : inst.dest;
+	if (isFloat) {
+		regs_.SpillLockFPR(regLow, regHigh);
+		regs_.SpillLockGPR(inst.src1);
+	} else {
+		regs_.SpillLockGPR(inst.src1, regLow, regHigh);
+	}
+	ARM64Reg base = regs_.MapGPRAsPointer(inst.src1);
+
+	if (isFloat) {
+		MIPSMap flags = isLoad ? MIPSMap::NOINIT : MIPSMap::INIT;
+		regs_.MapFPR(regLow, flags);
+		regs_.MapFPR(regHigh, flags);
+		if (isLoad)
+			fp_.LDP(32, INDEX_SIGNED, regs_.F(regLow), regs_.F(regHigh), base, low);
+		else
+			fp_.STP(32, INDEX_SIGNED, regs_.F(regLow), regs_.F(regHigh), base, low);
+	} else if (isLoad) {
+		regs_.MapGPR(regLow, MIPSMap::NOINIT);
+		regs_.MapGPR(regHigh, MIPSMap::NOINIT);
+		LDP(INDEX_SIGNED, regs_.R(regLow), regs_.R(regHigh), base, low);
+	} else {
+		ARM64Reg valueLow = regs_.MapGPR(regLow);
+		ARM64Reg valueHigh = regs_.MapGPR(regHigh);
+		STP(INDEX_SIGNED, valueLow, valueHigh, base, low);
+	}
+
+	skipNextInst_ = true;
+	return true;
+}
+
 void Arm64JitBackend::CompIR_FLoad(IRInst inst) {
 	CONDITIONAL_DISABLE;
 
+	if (inst.op == IROp::LoadFloat && TryCompileLoadStorePair(inst))
+		return;
 	LoadStoreArg addrArg = PrepareSrc1Address(inst);
 
 	switch (inst.op) {
@@ -226,6 +287,8 @@ void Arm64JitBackend::CompIR_FLoad(IRInst inst) {
 void Arm64JitBackend::CompIR_FStore(IRInst inst) {
 	CONDITIONAL_DISABLE;
 
+	if (inst.op == IROp::StoreFloat && TryCompileLoadStorePair(inst))
+		return;
 	LoadStoreArg addrArg = PrepareSrc1Address(inst);
 
 	switch (inst.op) {
@@ -249,6 +312,8 @@ void Arm64JitBackend::CompIR_FStore(IRInst inst) {
 void Arm64JitBackend::CompIR_Load(IRInst inst) {
 	CONDITIONAL_DISABLE;
 
+	if (inst.op == IROp::Load32 && TryCompileLoadStorePair(inst))
+		return;
 	regs_.SpillLockGPR(inst.dest, inst.src1);
 	LoadStoreArg addrArg = PrepareSrc1Address(inst);
 	// With NOINIT, MapReg won't subtract MEMBASEREG even if dest == src1.
@@ -345,6 +410,8 @@ void Arm64JitBackend::CompIR_LoadShift(IRInst inst) {
 void Arm64JitBackend::CompIR_Store(IRInst inst) {
 	CONDITIONAL_DISABLE;
 
+	if (inst.op == IROp::Store32 && TryCompileLoadStorePair(inst))
+		return;
 	regs_.SpillLockGPR(inst.src3, inst.src1);
 	LoadStoreArg addrArg = PrepareSrc1Address(inst);
 

Core/MIPS/ARM64/Arm64IRJit.cpp

@@ -99,12 +99,21 @@ bool Arm64JitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) {
 	std::vector<const u8 *> addresses;
 	addresses.reserve(block->GetNumIRInstructions());
 	const IRInst *instructions = irBlockCache->GetBlockInstructionPtr(*block);
+	compilingInsts_ = instructions;
+	compilingCount_ = block->GetNumIRInstructions();
 	for (int i = 0; i < block->GetNumIRInstructions(); ++i) {
 		const IRInst &inst = instructions[i];
 		regs_.SetIRIndex(i);
+		compilingIndex_ = i;
 		addresses.push_back(GetCodePtr());
 
 		CompileIRInst(inst);
+		if (skipNextInst_) {
+			// It was compiled together with this one.
+			skipNextInst_ = false;
+			addresses.push_back(GetCodePtr());
+			i++;
+		}
 
 		if (jo.Disabled(JitDisable::REGALLOC_GPR) || jo.Disabled(JitDisable::REGALLOC_FPR))
 			regs_.FlushAll(jo.Disabled(JitDisable::REGALLOC_GPR), jo.Disabled(JitDisable::REGALLOC_FPR));

Core/MIPS/ARM64/Arm64IRJit.h

@@ -123,6 +123,9 @@ private:
 		bool signExtendRegOffset = false;
 	};
 	LoadStoreArg PrepareSrc1Address(IRInst inst);
+	// Compiles inst together with the next instruction as one LDP/STP when they access adjacent
+	// words through the same base. The next instruction is then skipped.
+	bool TryCompileLoadStorePair(IRInst inst);
 
 	JitOptions &jo;
 	Arm64IRRegCache regs_;
@@ -148,6 +151,11 @@ private:
 
 	int jitStartOffset_ = 0;
 	int compilingBlockNum_ = -1;
+	// The block's instructions while compiling, for looking at the next one.
+	const IRInst *compilingInsts_ = nullptr;
+	int compilingIndex_ = 0;
+	int compilingCount_ = 0;
+	bool skipNextInst_ = false;
 	int logBlocks_ = 0;
 	// Only useful in breakpoints, where it's set immediately prior.
 	uint32_t lastConstPC_ = 0;

Core/MIPS/IR/IRInst.cpp

@@ -131,6 +131,12 @@ static const IRMeta irMeta[] = {
 	{ IROp::FMovToGPR, "FMovToGPR", "GF" },
 	{ IROp::OptFMovToGPRShr8, "OptFMovToGPRShr8", "GF" },
 	{ IROp::OptFCvtSWFromGPR, "OptFCvtSWFromGPR", "FG" },
+	{ IROp::OptExitToConstIfEqElse, "OptExitIfEqElse", "CGG", IRFLAG_EXIT },
+	{ IROp::OptExitToConstIfNeqElse, "OptExitIfNeqElse", "CGG", IRFLAG_EXIT },
+	{ IROp::OptExitToConstIfGtZElse, "OptExitIfGtZElse", "CG", IRFLAG_EXIT },
+	{ IROp::OptExitToConstIfGeZElse, "OptExitIfGeZElse", "CG", IRFLAG_EXIT },
+	{ IROp::OptExitToConstIfLtZElse, "OptExitIfLtZElse", "CG", IRFLAG_EXIT },
+	{ IROp::OptExitToConstIfLeZElse, "OptExitIfLeZElse", "CG", IRFLAG_EXIT },
 	{ IROp::FpCondFromReg, "FpCondFromReg", "_G" },
 	{ IROp::FpCondToReg, "FpCondToReg", "G" },
 	{ IROp::FpCtrlFromReg, "FpCtrlFromReg", "_G" },

Core/MIPS/IR/IRInst.h

@@ -141,6 +141,14 @@ enum class IROp : uint8_t {
 	OptFCvtSWFromGPR,
 	FMovToGPR,
 	OptFMovToGPRShr8,
+	// A conditional exit merged with the ExitToConst after it, which stays behind as the target
+	// to take otherwise (in its constant).
+	OptExitToConstIfEqElse,
+	OptExitToConstIfNeqElse,
+	OptExitToConstIfGtZElse,
+	OptExitToConstIfGeZElse,
+	OptExitToConstIfLtZElse,
+	OptExitToConstIfLeZElse,
 
 	FSat0_1,
 	FSatMinus1_1,

Core/MIPS/IR/IRPassSimplify.cpp

@@ -1955,8 +1955,9 @@ bool ApplyMemoryValidation(const IRWriter &in, IRWriter &out, const IROptions &o
 
 bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts) {
 	CONDITIONAL_DISABLE;
-	// Only do this when using a SIMD backend.
-	if (!opts.preferVec4) {
+	// Only do this when using a SIMD backend. The interpreter has no flushes to avoid, and would
+	// only get more instructions to dispatch.
+	if (!opts.preferVec4 || opts.optimizeForInterpreter) {
 		DISABLE;
 	}
 
@@ -2406,6 +2407,19 @@ bool OptimizeForInterpreter(const IRWriter &in, IRWriter &out, const IROptions &
 			}
 			out.Write(inst);
 			break;
+		case IROp::ExitToConstIfEq:
+		case IROp::ExitToConstIfNeq:
+		case IROp::ExitToConstIfGtZ:
+		case IROp::ExitToConstIfGeZ:
+		case IROp::ExitToConstIfLtZ:
+		case IROp::ExitToConstIfLeZ:
+			// With an ExitToConst right after, one op can pick either target. The ExitToConst stays
+			// as the second target's holder.
+			static_assert((int)IROp::OptExitToConstIfLeZElse - (int)IROp::OptExitToConstIfEqElse == (int)IROp::ExitToConstIfLeZ - (int)IROp::ExitToConstIfEq, "Else exits must match the exits' order");
+			if (!last && in.GetInstructions()[i + 1].op == IROp::ExitToConst)
+				inst.op = (IROp)((int)IROp::OptExitToConstIfEqElse + ((int)inst.op - (int)IROp::ExitToConstIfEq));
+			out.Write(inst);
+			break;
 		default:
 			out.Write(inst);
 			break;

unittest/TestIRPassSimplify.cpp

@@ -510,6 +510,19 @@ static const IRVerification tests[] = {
 		},
 		{ &OptimizeLoadsAfterStores },
 	},
+	{
+		"InterpreterExitElse",
+		{
+			{ IROp::ExitToConstIfLtZ, { 0 }, MIPS_REG_A0, 0, 0x08804000 },
+			{ IROp::ExitToConst, { 0 }, 0, 0, 0x08804100 },
+		},
+		{
+			{ IROp::Downcount, { 0 }, 0, 0, 0 },
+			{ IROp::OptExitToConstIfLtZElse, { 0 }, MIPS_REG_A0, 0, 0x08804000 },
+			{ IROp::ExitToConst, { 0 }, 0, 0, 0x08804100 },
+		},
+		{ &OptimizeForInterpreter },
+	},
 };
 
 bool TestIRPassSimplify() {