Merge pull request #22252 from hrydgard/scemp4-real-prx

Run the real libmp4.prx/mp4msv.prx instead of our sceMp4 HLE, optionally. Fixes Speedball

AGENTS.md

@@ -28,12 +28,16 @@ for it:
 4. **Don't write code on `master`.** When asked to make a code change while on `master`, create an
    appropriately named branch first (`git checkout -b some-descriptive-name`) and do the work there.
    If you're already on a topic branch, just keep working on it.
-5. **Most files in this repo are CRLF** - `.vcxproj`, `.vcxproj.filters`, `android/jni/Android.mk`,
-   `libretro/Makefile.common`, `AGENTS.md`, and much of the source. If you patch one with a script, read *and*
-   write with `newline=''`; reading with Python's default universal-newline translation and writing with
-   `newline=''` silently converts the whole file, turning a two-line addition into a 5000-line diff. Check
-   `git diff --stat` before committing - a whole-file rewrite is obvious there and invisible in the editor.
-   Prefer the Edit tool, which does exact string replacement and can't do this.
+5. **Never assume a file's line endings - preserve whatever is on disk.** Which ending a file has
+   depends on where it was checked out: on Windows everything is auto-checked-out as CRLF, while a
+   Linux checkout leaves files as they are stored, so the same file (`.vcxproj`, `.vcxproj.filters`,
+   `android/jni/Android.mk`, `libretro/Makefile.common`, this file, much of the source) is CRLF in one
+   working copy and LF in another. Don't hardcode either, and don't "fix" a file's endings to match
+   what a doc claims. If you patch one with a script, read *and* write with `newline=''`, which keeps
+   whatever was there; reading with Python's default universal-newline translation and writing with
+   `newline=''` silently converts the whole file, turning a two-line addition into a 5000-line diff.
+   Check `git diff --stat` before committing - a whole-file rewrite is obvious there and invisible in
+   the editor. Prefer the Edit tool, which does exact string replacement and can't do this.
 6. **Don't feed Python to `bash -c` via a heredoc when the code contains backslashes.** The Git Bash / MinGW
    layer strips one level of backslash escaping on the way in, *even with a quoted delimiter* (`<<'PY'`), which
    normally suppresses all substitution. So the script Python receives is not the one you wrote:
@@ -61,6 +65,13 @@ for it:
 
 1. For HLE, CPU, GPU, timing, threading, and memory changes, call out regression risks explicitly.
 2. Consider savestate compatibility when changing serialized state.
+3. **Never insert an entry into the middle of an `HLEFunction` array.** A savestate stores the
+   syscall opcode, which encodes the entry's *index* in that array - so inserting anywhere but the
+   end silently repoints every later entry, and old savestates start calling the wrong function.
+   This applies to adding a *single* function to an *existing* module, which is when it is easiest
+   to forget: put it last in the array even when alphabetical or NID order would put it elsewhere,
+   and even when the array is otherwise tidily sorted. The same rule governs the order of
+   `Register_*()` calls in `Core/HLE/HLETables.cpp` - new modules go at the very end.
 
 ## Build and validation
 
@@ -174,7 +185,8 @@ things silently if ignored:
   never inserted alphabetically among the existing `Register_*()` calls.
 - **New entries in an existing module's function table go at the very end of that array too** - a
   savestate captures the syscall opcode encoding the entry's array index, so shifting later entries
-  makes old savestates call the wrong function.
+  makes old savestates call the wrong function. See Core Safety Checks above: this holds for any
+  edit to any `HLEFunction` array, not just when adding a module.
 
 Also: a new `.cpp`/`.c` file has to be added to **seven** build files (CMake, Core.vcxproj + filters,
 the two UWP projects, `android/jni/Android.mk`, `libretro/Makefile.common`); headers to the first five.

Core/CmdLine.cpp

@@ -217,6 +217,7 @@ static const CommandLineParam g_autoParams[] = {
 	{POFF(debuggerPort), CmdParamType::Int, "debugger", '\0', "Enable the WebSocket debugger on this port (0 = pick automatically); see docs/WebSocketDebugger.md"},
 	{POFF(autoSaveLoadSymbols), CmdParamType::Bool, "auto-save-load-symbols", '\0', "Auto save/load per-module and per-game symbol files (see bAutoSaveLoadSymbols)", CmdLineMode::Both},
 	{POFF(bootVSH), CmdParamType::Bool, "vsh", '\0', "Boot the VSH (requires files dumped from a PSP in the flash0 directory)"},
+	{POFF(disableHLE), CmdParamType::Int, "disable-hle", '\0', "Bitmask of libraries to run the real firmware module for instead of our HLE", CmdLineMode::Both},
 	{POFF(memReadAction), CmdParamType::Enum, "memread", '\0', "Set the action for memory read exceptions", CmdLineMode::Both, g_ExceptionActionValues, ARRAY_SIZE(g_ExceptionActionValues)},
 	{POFF(memWriteAction), CmdParamType::Enum, "memwrite", '\0', "Set the action for memory write exceptions", CmdLineMode::Both, g_ExceptionActionValues, ARRAY_SIZE(g_ExceptionActionValues)},
 	{POFF(breakAction), CmdParamType::Enum, "break", '\0', "Set the action for break exceptions", CmdLineMode::Both, g_ExceptionActionValues, ARRAY_SIZE(g_ExceptionActionValues)},
@@ -544,6 +545,12 @@ void CommandLineOptions::ApplyToConfig() const {
 		g_Config.DoNotSaveSetting(&g_Config.bAutoSaveLoadSymbols);
 	}
 
+	if (disableHLE.has_value()) {
+		// DoNotSaveSetting so a per-game config can't quietly put the HLE back.
+		g_Config.iDisableHLE = disableHLE.value();
+		g_Config.DoNotSaveSetting(&g_Config.iDisableHLE);
+	}
+
 	if (logLevel.has_value()) {
 		g_logManager.SetAllLogLevels(logLevel.value());
 	}

Core/CmdLine.h

@@ -78,6 +78,10 @@ struct CommandLineOptions {
 	std::optional<std::string> unpackUpdaterModel;
 	std::optional<std::string> unpackUpdaterFilter;
 
+	// Bitmask of DisableHLEFlags: run the real firmware module instead of our HLE for those
+	// libraries. Needs a firmware dump under the NAND directory.
+	std::optional<int> disableHLE;
+
 	std::optional<int> memReadAction;
 	std::optional<int> memWriteAction;
 	std::optional<int> breakAction;

Core/ConfigValues.h

@@ -164,7 +164,9 @@ enum class DisableHLEFlags : int {
 	sceMp3 = (1 << 5),
 	sceParseHttp = (1 << 6),
 	sceCcc = (1 << 7),  // character conversion library.
-	Count = 8,
+	// Swaps in flash0:/kd/libmp4.prx and mp4msv.prx, which then decode through our sceAudiocodec.
+	sceMp4 = (1 << 8),
+	Count = 9,
 	// TODO: Some of the networking libraries may be interesting candidates, like HTTP.
 };
 ENUM_CLASS_BITOPS(DisableHLEFlags);

Core/HLE/HLE.cpp

@@ -22,6 +22,9 @@
 
 #include "Common/Math/CrossSIMD.h"
 
+#include "Common/File/FileUtil.h"
+#include "Common/Data/Text/I18n.h"
+#include "Common/System/OSD.h"
 #include "Common/Profiler/Profiler.h"
 
 #include "Common/Log.h"
@@ -153,6 +156,9 @@ static const HLEModuleMeta g_moduleMeta[] = {
 	{"scePsmfPlayer", "scePsmfPlayer", DisableHLEFlags::scePsmfPlayer},
 	{"sceSAScore", "sceSasCore"},
 	{"sceCcc_Library", "sceCcc", DisableHLEFlags::sceCcc},
+	// libmp4.prx needs 41 functions from mp4msv.prx, so the two only make sense swapped together.
+	{"sceMp4_library", "sceMp4", DisableHLEFlags::sceMp4},
+	{"mp4msv_module", "mp4msv", DisableHLEFlags::sceMp4},
 	{"SceParseHTTPheader_Library", "sceParseHttp", DisableHLEFlags::sceParseHttp},
 	{"SceParseURI_Library"},
 	// Guessing these names
@@ -198,6 +204,12 @@ DisableHLEFlags AlwaysDisableHLEFlags() {
 }
 
 // Process compat flags.
+// Flags the user asked for that we can't honour this boot, because the firmware modules they
+// need aren't in the NAND directory. Subtracted in GetDisableHLEFlags so that a missing dump
+// leaves the HLE in place rather than handing the game unresolved imports, which is much worse
+// than our stubs. Recomputed per boot, since the dump can appear between runs.
+static DisableHLEFlags g_unavailableDisableFlags = (DisableHLEFlags)0;
+
 static DisableHLEFlags GetDisableHLEFlags() {
 	DisableHLEFlags flags = (DisableHLEFlags)g_Config.iDisableHLE | AlwaysDisableHLEFlags();
 	if (PSP_CoreParameter().compat.flags().DisableHLESceFont) {
@@ -208,6 +220,8 @@ static DisableHLEFlags GetDisableHLEFlags() {
 	}
 
 	flags &= ~(DisableHLEFlags)g_Config.iForceEnableHLE;
+	// Anything whose firmware module isn't actually present stays HLE'd.
+	flags &= ~g_unavailableDisableFlags;
 	return flags;
 }
 
@@ -262,7 +276,25 @@ static void hleDelayResultFinish(u64 userdata, int cycleslate) {
 		WARN_LOG(Log::HLE, "Someone else woke up HLE-blocked thread %d?", threadID);
 }
 
+// Which firmware files a flag needs before it can be honoured.
+static void CheckDisableHLEAvailability() {
+	g_unavailableDisableFlags = (DisableHLEFlags)0;
+
+	if ((DisableHLEFlags)g_Config.iDisableHLE & DisableHLEFlags::sceMp4) {
+		const Path kd = g_Config.nandRootDirectory / "flash0" / "kd";
+		if (!File::Exists(kd / "libmp4.prx") || !File::Exists(kd / "mp4msv.prx")) {
+			g_unavailableDisableFlags |= DisableHLEFlags::sceMp4;
+			ERROR_LOG(Log::HLE, "Asked to run the real sceMp4, but %s doesn't have libmp4.prx and "
+				"mp4msv.prx - keeping the HLE.", kd.c_str());
+			auto sy = GetI18NCategory(I18NCat::SYSTEM);
+			g_OSD.Show(OSDType::MESSAGE_WARNING,
+				sy->T("Real sceMp4 needs a firmware dump in the NAND folder - using HLE instead"), 6.0f);
+		}
+	}
+}
+
 void HLEInit() {
+	CheckDisableHLEAvailability();
 	RegisterAllModules();
 	g_stackSize = 0;
 	delayedResultEvent = CoreTiming::RegisterEvent("HLEDelayedResult", hleDelayResultFinish);

Core/HLE/sceKernelModule.cpp

@@ -2713,6 +2713,34 @@ struct GetModuleIdByAddressArg
 	SceUID result;
 };
 
+// ModuleMgrForUser_D2FBC957. Looks up the gp value of whichever module contains an address, which
+// is how a library that is handed function pointers from another module can call them: MIPS code
+// needs the callee's gp in place. libmp4.prx uses it on the three callbacks it is given.
+// Named after what it does; the official name isn't known.
+static u32 sceKernelGetModuleGPByAddress(u32 addr, u32 gpPtr) {
+	if (!Memory::IsValidAddress(gpPtr)) {
+		return hleLogError(Log::sceModule, SCE_KERNEL_ERROR_ILLEGAL_ADDR, "bad gp pointer");
+	}
+
+	u32 gp = 0;
+	bool found = false;
+	kernelObjects.Iterate<PSPModule>([&](int id, PSPModule *module) -> bool {
+		const u32 start = module->memoryBlockAddr, size = module->memoryBlockSize;
+		if (start != 0 && start <= addr && start + size > addr) {
+			gp = module->nm.gp_value;
+			found = true;
+			return false;
+		}
+		return true;
+	});
+
+	if (!found) {
+		return hleLogError(Log::sceModule, SCE_KERNEL_ERROR_UNKNOWN_MODULE, "no module at %08x", addr);
+	}
+	Memory::WriteUnchecked_U32(gp, gpPtr);
+	return hleLogDebug(Log::sceModule, 0, "gp=%08x", gp);
+}
+
 static u32 sceKernelGetModuleIdByAddress(u32 moduleAddr)
 {
 	GetModuleIdByAddressArg state;
@@ -3028,6 +3056,7 @@ const HLEFunction ModuleMgrForUser[] = {
 	{0XF2D8D1B4, &WrapU_CUU<sceKernelLoadModuleNpDrm>,                  "sceKernelLoadModuleNpDrm",                'x', "sxx"    },
 	{0XE4C4211C, nullptr,                                               "sceKernelLoadModuleWithBlockOffset",      '?', ""       },
 	{0XFBE27467, nullptr,                                               "sceKernelLoadModuleByIDWithBlockOffset",  '?', ""       },
+	{0XD2FBC957, &WrapU_UU<sceKernelGetModuleGPByAddress>,              "sceKernelGetModuleGPByAddress",          'x', "xx"     },
 };
 
 const HLEFunction ModuleMgrForKernel[] = {

Core/HLE/sceUtility.cpp

@@ -85,6 +85,49 @@ static void NotifyLoadStatusAvcodec(int state, u32 loadAddr, u32 totalSize) {
 	JpegNotifyLoadStatus(state);
 }
 
+// The MP4 libraries are a good candidate for running the real thing: libmp4.prx needs only two
+// functions from sceAudiocodec (Init and Decode) plus ordinary kernel calls, and mp4msv.prx - the
+// 41 functions libmp4 leans on - imports nothing at all. So with a firmware dump present, the pair
+// can be loaded for real and left to decode through our sceAudiocodec HLE.
+static SceUID g_mp4RealModules[2] = { 0, 0 };
+
+static void NotifyLoadStatusMp4(int state, u32 loadAddr, u32 totalSize) {
+	if (!((DisableHLEFlags)g_Config.iDisableHLE & DisableHLEFlags::sceMp4)) {
+		return;
+	}
+
+	if (state == 1) {
+		// mp4msv first - libmp4 imports from it, and an import can only resolve to a module that
+		// is already loaded.
+		static const char *const paths[2] = {
+			"flash0:/kd/mp4msv.prx",
+			"flash0:/kd/libmp4.prx",
+		};
+		for (int i = 0; i < 2; i++) {
+			if (g_mp4RealModules[i]) {
+				continue;
+			}
+			std::string error;
+			SceUID id = KernelLoadModule(paths[i], &error);
+			if (id < 0) {
+				ERROR_LOG(Log::sceUtility, "sceMp4 HLE is disabled, but %s wouldn't load (%s) - "
+					"the game will get unresolved imports", paths[i], error.c_str());
+				return;
+			}
+			int result = __KernelStartModule(id, 0, 0, 0, nullptr, nullptr);
+			if (result < 0) {
+				ERROR_LOG(Log::sceUtility, "Failed to start %s (%08x)", paths[i], result);
+				return;
+			}
+			g_mp4RealModules[i] = id;
+			INFO_LOG(Log::sceUtility, "Loaded the real %s", paths[i]);
+		}
+	} else if (state == -1) {
+		g_mp4RealModules[0] = 0;
+		g_mp4RealModules[1] = 0;
+	}
+}
+
 static void NotifyLoadStatusAtrac(int state, u32 loadAddr, u32 totalSize) {
 	if (state == 1) {
 		// If HLE of sceAtrac is disabled, things will break!
@@ -139,7 +182,7 @@ static const ModuleLoadInfo moduleLoadInfo[] = {
 	ModuleLoadInfo(0x305, 0x0000a300, "av_vaudio"),
 	ModuleLoadInfo(0x306, 0x00004000, "av_aac"),
 	ModuleLoadInfo(0x307, 0x00000000, "av_g729"),
-	ModuleLoadInfo(0x308, 0x0003c000, "av_mp4", mp4ModuleDeps),
+	ModuleLoadInfo(0x308, 0x0003c000, "av_mp4", mp4ModuleDeps, &NotifyLoadStatusMp4),
 	ModuleLoadInfo(0x3fe, 0x00000000, "me_stuff"),
 	ModuleLoadInfo(0x3ff, 0x00000000, "me_core"),  // ME Core?
 	ModuleLoadInfo(0x400, 0x0000c000, "np_common"),

headless/Headless.cpp

@@ -727,13 +727,18 @@ int main(int argc, const char* argv[]) {
 	g_Config.iReverbVolume = VOLUMEHI_FULL;
 	g_Config.internalDataDirectory.clear();
 	g_Config.bUseOldAtrac = oldAtrac;
-	g_Config.iForceEnableHLE = 0xFFFFFFFF;  // Run all modules as HLE. We don't have anything to load in this context.
 	g_Config.bSkipDeadbeefFilling = false;
 
 	// ApplyToConfig() has the final say, applied after RestoreDefaults() and the headless
 	// overrides above, so a matching command line flag always wins.
 	cmdLineOptions.ApplyToConfig();
 
+	// Run all modules as HLE - a headless run normally has no firmware to load them from. An
+	// explicit --disable-hle means the caller does have a dump and wants the real thing, so leave
+	// the modules they asked for alone.
+	g_Config.iForceEnableHLE = 0xFFFFFFFF & ~g_Config.iDisableHLE;
+
+
 	// This looks contradictory to the above. But, this preserves the old test behavior which apparently ran the JIT for the CPU
 	// but ended up running software vertex decoding due to the setting in g_Config. Yeah, it's a mess.
 	CPUCore cpuCore = CPUCore::JIT;