Merge pull request #19672 from hrydgard/file-system-perf-part-3

File system perf, part 3

Common/File/DirListing.cpp

@@ -43,7 +43,12 @@
 #endif // HAVE_LIBNX
 
 // NOTE: There's another one in FileUtil.cpp.
+#ifdef _WIN32
 constexpr bool SIMULATE_SLOW_IO = false;
+#else
+constexpr bool SIMULATE_SLOW_IO = false;
+#endif
+constexpr bool LOG_IO = false;
 
 namespace File {
 
@@ -57,8 +62,10 @@ static uint64_t FiletimeToStatTime(FILETIME ft) {
 #endif
 
 bool GetFileInfo(const Path &path, FileInfo * fileInfo) {
-	if (SIMULATE_SLOW_IO) {
+	if (LOG_IO) {
 		INFO_LOG(Log::System, "GetFileInfo %s", path.c_str());
+	}
+	if (SIMULATE_SLOW_IO) {
 		sleep_ms(300, "slow-io-sim");
 	}
 
@@ -184,8 +191,10 @@ std::vector<File::FileInfo> ApplyFilter(std::vector<File::FileInfo> files, const
 }
 
 bool GetFilesInDir(const Path &directory, std::vector<FileInfo> *files, const char *filter, int flags, std::string_view prefix) {
-	if (SIMULATE_SLOW_IO) {
+	if (LOG_IO) {
 		INFO_LOG(Log::System, "GetFilesInDir %s (ext %s, prefix %.*s)", directory.c_str(), filter, (int)prefix.size(), prefix.data());
+	}
+	if (SIMULATE_SLOW_IO) {
 		sleep_ms(300, "slow-io-sim");
 	}
 
@@ -267,10 +276,12 @@ bool GetFilesInDir(const Path &directory, std::vector<FileInfo> *files, const ch
 			continue;
 		}
 
+		/*
 		if (SIMULATE_SLOW_IO) {
 			INFO_LOG(Log::System, "GetFilesInDir item %s", virtualName.c_str());
 			sleep_ms(50, "slow-io-sim");
 		}
+		*/
 
 		FileInfo info;
 		info.name = virtualName;
@@ -345,6 +356,9 @@ bool GetFilesInDir(const Path &directory, std::vector<FileInfo> *files, const ch
 	closedir(dirp);
 #endif
 	std::sort(files->begin(), files->end());
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "GetFilesInDir: Found %d files", (int)files->size());
+	}
 	return true;
 }
 

Common/File/DiskFree.h

@@ -5,5 +5,6 @@
 #include "Common/File/Path.h"
 
 // If this fails, false is returned and space is negative.
-// Try to avoid calling this from the main thread, if possible. Can be SLOW.
+// Try to avoid calling this from the main thread, if possible. Not super fast,
+// but is also not allowed to do things like scan the entire disk.
 bool free_disk_space(const Path &path, int64_t &space);

Common/File/FileUtil.cpp

@@ -91,7 +91,12 @@
 #include <sys/stat.h>
 
 // NOTE: There's another one in DirListing.cpp.
+#ifdef _WIN32
 constexpr bool SIMULATE_SLOW_IO = false;
+#else
+constexpr bool SIMULATE_SLOW_IO = false;
+#endif
+constexpr bool LOG_IO = false;
 
 #ifndef S_ISDIR
 #define S_ISDIR(m)  (((m)&S_IFMT) == S_IFDIR)
@@ -115,8 +120,10 @@ constexpr bool SIMULATE_SLOW_IO = false;
 namespace File {
 
 FILE *OpenCFile(const Path &path, const char *mode) {
-	if (SIMULATE_SLOW_IO) {
+	if (LOG_IO) {
 		INFO_LOG(Log::System, "OpenCFile %s, %s", path.c_str(), mode);
+	}
+	if (SIMULATE_SLOW_IO) {
 		sleep_ms(300, "slow-io-sim");
 	}
 	switch (path.Type()) {
@@ -217,8 +224,10 @@ static std::string OpenFlagToString(OpenFlag flags) {
 }
 
 int OpenFD(const Path &path, OpenFlag flags) {
-	if (SIMULATE_SLOW_IO) {
+	if (LOG_IO) {
 		INFO_LOG(Log::System, "OpenFD %s, %d", path.c_str(), flags);
+	}
+	if (SIMULATE_SLOW_IO) {
 		sleep_ms(300, "slow-io-sim");
 	}
 
@@ -315,8 +324,10 @@ static bool ResolvePathVista(const std::wstring &path, wchar_t *buf, DWORD bufSi
 #endif
 
 std::string ResolvePath(const std::string &path) {
-	if (SIMULATE_SLOW_IO) {
+	if (LOG_IO) {
 		INFO_LOG(Log::System, "ResolvePath %s", path.c_str());
+	}
+	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
 	}
 
@@ -408,9 +419,11 @@ bool ExistsInDir(const Path &path, const std::string &filename) {
 }
 
 bool Exists(const Path &path) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "Exists %s", path.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(200, "slow-io-sim");
-		INFO_LOG(Log::System, "Exists %s", path.c_str());
 	}
 
 	if (path.Type() == PathType::CONTENT_URI) {
@@ -445,9 +458,11 @@ bool Exists(const Path &path) {
 
 // Returns true if filename exists and is a directory
 bool IsDirectory(const Path &path) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "IsDirectory %s", path.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
-		INFO_LOG(Log::System, "IsDirectory %s", path.c_str());
 	}
 
 	switch (path.Type()) {
@@ -660,9 +675,11 @@ bool CreateFullPath(const Path &path) {
 
 // renames file srcFilename to destFilename, returns true on success
 bool Rename(const Path &srcFilename, const Path &destFilename) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "Rename %s -> %s", srcFilename.c_str(), destFilename.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
-		INFO_LOG(Log::System, "Rename %s -> %s", srcFilename.c_str(), destFilename.c_str());
 	}
 
 	if (srcFilename.Type() != destFilename.Type()) {
@@ -713,9 +730,11 @@ bool Rename(const Path &srcFilename, const Path &destFilename) {
 
 // copies file srcFilename to destFilename, returns true on success
 bool Copy(const Path &srcFilename, const Path &destFilename) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "Copy %s -> %s", srcFilename.c_str(), destFilename.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
-		INFO_LOG(Log::System, "Copy %s -> %s", srcFilename.c_str(), destFilename.c_str());
 	}
 	switch (srcFilename.Type()) {
 	case PathType::NATIVE:
@@ -857,9 +876,11 @@ bool MoveIfFast(const Path &srcFilename, const Path &destFilename) {
 // Returns the size of file (64bit)
 // TODO: Add a way to return an error.
 uint64_t GetFileSize(const Path &filename) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "GetFileSize %s", filename.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
-		INFO_LOG(Log::System, "GetFileSize %s", filename.c_str());
 	}
 	switch (filename.Type()) {
 	case PathType::NATIVE:
@@ -967,9 +988,11 @@ bool CreateEmptyFile(const Path &filename) {
 // Deletes an empty directory, returns true on success
 // WARNING: On Android with content URIs, it will delete recursively!
 bool DeleteDir(const Path &path) {
+	if (LOG_IO) {
+		INFO_LOG(Log::System, "DeleteDir %s", path.c_str());
+	}
 	if (SIMULATE_SLOW_IO) {
 		sleep_ms(100, "slow-io-sim");
-		INFO_LOG(Log::System, "DeleteDir %s", path.c_str());
 	}
 	switch (path.Type()) {
 	case PathType::NATIVE:

Common/Thread/ThreadManager.cpp

@@ -130,10 +130,10 @@ bool ThreadManager::TeardownTask(Task *task, bool enqueue) {
 
 static void WorkerThreadFunc(GlobalThreadContext *global, TaskThreadContext *thread) {
 	if (thread->type == TaskType::CPU_COMPUTE) {
-		snprintf(thread->name, sizeof(thread->name), "PoolWorker %d", thread->index);
+		snprintf(thread->name, sizeof(thread->name), "PoolW %d", thread->index);
 	} else {
 		_assert_(thread->type == TaskType::IO_BLOCKING);
-		snprintf(thread->name, sizeof(thread->name), "PoolWorkerIO %d", thread->index);
+		snprintf(thread->name, sizeof(thread->name), "PoolW IO %d", thread->index);
 	}
 	SetCurrentThreadName(thread->name);
 

Core/Dialog/SavedataParam.cpp

@@ -327,7 +327,7 @@ bool SavedataParam::Delete(SceUtilitySavedataParam* param, int saveId) {
 	}
 
 	// Sanity check, preventing full delete of savedata/ in MGS PW demo (!)
-	if (!strlen(param->gameName) && param->mode != SCE_UTILITY_SAVEDATA_TYPE_LISTALLDELETE) {
+	if (!strnlen(param->gameName, sizeof(param->gameName)) && param->mode != SCE_UTILITY_SAVEDATA_TYPE_LISTALLDELETE) {
 		ERROR_LOG(Log::sceUtility, "Bad param with gameName empty - cannot delete save directory");
 		return false;
 	}
@@ -1119,17 +1119,15 @@ inline std::string FmtPspTime(const ScePspDateTime &dt) {
 	return StringFromFormat("%04d-%02d-%02d %02d:%02d:%02d.%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond);
 }
 
-int SavedataParam::GetSizes(SceUtilitySavedataParam *param)
-{
+int SavedataParam::GetSizes(SceUtilitySavedataParam *param) {
 	if (!param) {
 		return SCE_UTILITY_SAVEDATA_ERROR_SIZES_NO_DATA;
 	}
 
 	int ret = 0;
-
 	if (param->msFree.IsValid())
 	{
-		const u64 freeBytes = MemoryStick_FreeSpace();
+		const u64 freeBytes = MemoryStick_FreeSpace(GetGameName(param));
 		param->msFree->clusterSize = (u32)MemoryStick_SectorSize();
 		param->msFree->freeClusters = (u32)(freeBytes / MemoryStick_SectorSize());
 		param->msFree->freeSpaceKB = (u32)(freeBytes / 0x400);
@@ -1230,6 +1228,8 @@ bool SavedataParam::GetList(SceUtilitySavedataParam *param)
 
 		std::vector<PSPFileInfo> validDir;
 		std::vector<PSPFileInfo> sfoFiles;
+
+		// TODO: Here we can filter by prefix - only the savename in param is likely to be a regex.
 		std::vector<PSPFileInfo> allDir = pspFileSystem.GetDirListing(savePath);
 
 		std::string searchString = GetGameName(param) + GetSaveName(param);
@@ -1419,7 +1419,7 @@ bool SavedataParam::GetSize(SceUtilitySavedataParam *param) {
 
 	if (param->sizeInfo.IsValid()) {
 		auto listing = pspFileSystem.GetDirListing(saveDir, &exists);
-		const u64 freeBytes = MemoryStick_FreeSpace();
+		const u64 freeBytes = MemoryStick_FreeSpace(GetGameName(param));
 
 		s64 overwriteBytes = 0;
 		s64 writeBytes = 0;
@@ -1511,6 +1511,11 @@ int SavedataParam::SetPspParam(SceUtilitySavedataParam *param)
 		return 0;
 	}
 
+	std::string gameName = GetGameName(param);
+	if (!gameName.empty()) {
+		MemoryStick_NotifyGameName(gameName);
+	}
+
 	if (param->mode == SCE_UTILITY_SAVEDATA_TYPE_LISTALLDELETE) {
 		Clear();
 		int realCount = 0;

Core/FileSystems/DirectoryFileSystem.cpp

@@ -366,12 +366,15 @@ size_t DirectoryFileHandle::Write(const u8* pointer, s64 size)
 		g_OSD.Show(OSDType::MESSAGE_ERROR, err->T("Disk full while writing data"), 0.0f, "diskfull");
 		// We only return an error when the disk is actually full.
 		// When writing this would cause the disk to be full, so it wasn't written, we return 0.
-		if (MemoryStick_FreeSpace() == 0) {
-			// Sign extend on 64-bit.
-			return (size_t)(s64)(s32)SCE_KERNEL_ERROR_ERRNO_DEVICE_NO_FREE_SPACE;
+		Path saveFolder = GetSysDirectory(DIRECTORY_SAVEDATA);
+		int64_t space;
+		if (free_disk_space(saveFolder, space)) {
+			if (space < size) {
+				// Sign extend to a 64-bit value.
+				return (size_t)(s64)(s32)SCE_KERNEL_ERROR_ERRNO_DEVICE_NO_FREE_SPACE;
+			}
 		}
 	}
-
 	return bytesWritten;
 }
 

Core/FileSystems/MetaFileSystem.cpp

@@ -682,3 +682,9 @@ int64_t MetaFileSystem::ComputeRecursiveDirectorySize(const std::string &filenam
 		return false;
 	}
 }
+
+bool MetaFileSystem::ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) {
+	// Shouldn't be called. Can't recurse MetaFileSystem.
+	_dbg_assert_(false);
+	return false;
+}

Core/FileSystems/MetaFileSystem.h

@@ -144,16 +144,7 @@ public:
 
 	int64_t ComputeRecursiveDirectorySize(const std::string &dirPath);
 
-	// Shouldn't ever be called, but meh.
-	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override {
-		int64_t sizeTemp = ComputeRecursiveDirectorySize(path);
-		if (sizeTemp >= 0) {
-			*size = sizeTemp;
-			return true;
-		} else {
-			return false;
-		}
-	}
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override;
 
 	void Describe(char *buf, size_t size) const override { snprintf(buf, size, "Meta"); }
 

Core/HW/MemoryStick.cpp

@@ -23,18 +23,23 @@
 #include "Common/Serialize/Serializer.h"
 #include "Common/Serialize/SerializeFuncs.h"
 #include "Common/Thread/ThreadUtil.h"
+#include "Common/File/DiskFree.h"
+#include "Common/File/FileUtil.h"
+#include "Common/File/DirListing.h"
 #include "Core/Config.h"
 #include "Core/CoreTiming.h"
 #include "Core/Compatibility.h"
 #include "Core/FileSystems/MetaFileSystem.h"
 #include "Core/HW/MemoryStick.h"
 #include "Core/System.h"
+#include "Common/CommonTypes.h"
+#include "Common/Thread/Promise.h"
 
 // MS and FatMS states.
 static MemStickState memStickState;
 static MemStickFatState memStickFatState;
 static bool memStickNeedsAssign = false;
-static u64 memStickInsertedAt = 0;
+static uint64_t memStickInsertedAt = 0;
 static uint64_t memstickInitialFree = 0;
 static uint64_t memstickCurrentUse = 0;
 static bool memstickCurrentUseValid = false;
@@ -46,13 +51,10 @@ enum FreeCalcStatus {
 	CLEANED_UP,
 };
 
-static std::thread freeCalcThread;
-static std::condition_variable freeCalcCond;
-static std::mutex freeCalcMutex;
-static FreeCalcStatus freeCalcStatus = FreeCalcStatus::NONE;
+static const uint64_t normalMemstickSize = 9ULL * 1024 * 1024 * 1024;
+static const uint64_t smallMemstickSize = 1ULL * 1024 * 1024 * 1024;
 
-static const u64 normalMemstickSize = 9ULL * 1024 * 1024 * 1024;
-static const u64 smallMemstickSize = 1ULL * 1024 * 1024 * 1024;
+static Promise<uint64_t> *g_initialMemstickSizePromise = nullptr;
 
 void MemoryStick_DoState(PointerWrap &p) {
 	auto s = p.Section("MemoryStick", 1, 5);
@@ -95,36 +97,20 @@ u64 MemoryStick_SectorSize() {
 	return 32 * 1024; // 32KB
 }
 
-static void MemoryStick_CalcInitialFree() {
-	std::unique_lock<std::mutex> guard(freeCalcMutex);
-	freeCalcStatus = FreeCalcStatus::RUNNING;
-	freeCalcThread = std::thread([] {
-		SetCurrentThreadName("CalcInitialFree");
-
-		AndroidJNIThreadContext jniContext;
-
-		memstickInitialFree = pspFileSystem.FreeDiskSpace("ms0:/") + pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/");
-
-		std::unique_lock<std::mutex> guard(freeCalcMutex);
-		freeCalcStatus = FreeCalcStatus::DONE;
-		freeCalcCond.notify_all();
-	});
-}
-
-static void MemoryStick_WaitInitialFree() {
-	std::unique_lock<std::mutex> guard(freeCalcMutex);
-	while (freeCalcStatus == FreeCalcStatus::RUNNING) {
-		freeCalcCond.wait(guard);
+static uint64_t ComputeSizeOfSavedataForGame(const Path &saveFolder, const std::string_view gameID) {
+	uint64_t space = 0;
+	std::vector<File::FileInfo> subDirs;
+	File::GetFilesInDir(saveFolder, &subDirs, nullptr, 0, gameID);  // gameID as directory prefix.
+	for (auto &dir : subDirs) {
+		if (!dir.isDirectory)
+			continue;
+		space += File::ComputeRecursiveDirectorySize(saveFolder / dir.name);
 	}
-	if (freeCalcStatus == FreeCalcStatus::DONE)
-		freeCalcThread.join();
-	freeCalcStatus = FreeCalcStatus::CLEANED_UP;
+	return space;
 }
 
-u64 MemoryStick_FreeSpace() {
-	NOTICE_LOG(Log::IO, "Calculated free disk space");
-
-	MemoryStick_WaitInitialFree();
+u64 MemoryStick_FreeSpace(std::string gameID) {
+	INFO_LOG(Log::IO, "Calculating free disk space (%s)", gameID.c_str());
 
 	const CompatFlags &flags = PSP_CoreParameter().compat.flags();
 	u64 realFreeSpace = pspFileSystem.FreeDiskSpace("ms0:/");
@@ -134,9 +120,10 @@ u64 MemoryStick_FreeSpace() {
 	// We have a compat setting to make it even smaller for Harry Potter : Goblet of Fire, see #13266.
 	const u64 memStickSize = flags.ReportSmallMemstick ? smallMemstickSize : (u64)g_Config.iMemStickSizeGB * 1024 * 1024 * 1024;
 
-	// Assume the memory stick is only used to store savedata.
+	// Assume the memory stick is only used to store savedata, for the current game only.
 	if (!memstickCurrentUseValid) {
-		memstickCurrentUse = pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/");
+		Path saveFolder = GetSysDirectory(DIRECTORY_SAVEDATA);
+		memstickCurrentUse = ComputeSizeOfSavedataForGame(saveFolder, gameID);
 		memstickCurrentUseValid = true;
 	}
 
@@ -149,7 +136,10 @@ u64 MemoryStick_FreeSpace() {
 		simulatedFreeSpace = smallMemstickSize / 2;  // just pick a value.
 	}
 	if (flags.MemstickFixedFree) {
+		const u64 memstickInitialFree = g_initialMemstickSizePromise->BlockUntilReady();
+		_dbg_assert_(g_initialMemstickSizePromise);
 		// Assassin's Creed: Bloodlines fails to save if free space changes incorrectly during game.
+		// See issue #12761
 		realFreeSpace = 0;
 		if (memstickCurrentUse <= memstickInitialFree) {
 			realFreeSpace = memstickInitialFree - memstickCurrentUse;
@@ -184,6 +174,30 @@ void MemoryStick_SetState(MemStickState state) {
 	}
 }
 
+void MemoryStick_NotifyGameName(std::string gameID) {
+	const CompatFlags &flags = PSP_CoreParameter().compat.flags();
+	if (!flags.MemstickFixedFree) {
+		return;
+	}
+
+	// See issue #12761
+	if (!g_initialMemstickSizePromise) {
+		g_initialMemstickSizePromise = Promise<uint64_t>::Spawn(&g_threadManager, [gameID]() -> uint64_t {
+			INFO_LOG(Log::System, "Calculating initial savedata size for %s...", gameID.c_str());
+			// NOTE: We only really need to sum up the diskspace for subfolders related to this game, and add it to the actual free space,
+			// to obtain the free space with the save data removed.
+			// We previously went through the meta file system here, but the memstick is always a directory so no need.
+			Path saveFolder = GetSysDirectory(DIRECTORY_SAVEDATA);
+			int64_t freeSpace = 0;
+			free_disk_space(saveFolder, freeSpace);
+
+			// Only add the space of the folders that the game itself touches.
+			freeSpace += ComputeSizeOfSavedataForGame(saveFolder, gameID);
+			return freeSpace;
+		}, TaskType::IO_BLOCKING);
+	}
+}
+
 void MemoryStick_Init() {
 	if (g_Config.bMemStickInserted) {
 		memStickState = PSP_MEMORYSTICK_STATE_INSERTED;
@@ -194,9 +208,12 @@ void MemoryStick_Init() {
 	}
 
 	memStickNeedsAssign = false;
-	MemoryStick_CalcInitialFree();
 }
 
 void MemoryStick_Shutdown() {
-	MemoryStick_WaitInitialFree();
+	if (g_initialMemstickSizePromise) {
+		g_initialMemstickSizePromise->BlockUntilReady();
+	}
+	delete g_initialMemstickSizePromise;
+	g_initialMemstickSizePromise = nullptr;
 }

Core/HW/MemoryStick.h

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <string>
+
 #include "Common/CommonTypes.h"
 
 class PointerWrap;
@@ -42,6 +44,9 @@ enum MemStickDriverState {
 
 void MemoryStick_Init();
 void MemoryStick_Shutdown();
+
+void MemoryStick_NotifyGameName(std::string gameName);
+
 void MemoryStick_DoState(PointerWrap &p);
 MemStickState MemoryStick_State();
 MemStickFatState MemoryStick_FatState();
@@ -50,5 +55,5 @@ void MemoryStick_SetState(MemStickState state);
 void MemoryStick_SetFatState(MemStickFatState state);
 
 u64 MemoryStick_SectorSize();
-u64 MemoryStick_FreeSpace();
+u64 MemoryStick_FreeSpace(std::string gameID);
 void MemoryStick_NotifyWrite();

Core/PSPLoaders.cpp

@@ -441,13 +441,11 @@ bool Load_PSP_ELF_PBP(FileLoader *fileLoader, std::string *error_string) {
 
 	std::string homebrewName = PSP_CoreParameter().fileToStart.ToVisualString();
 	std::size_t lslash = homebrewName.find_last_of('/');
-#if PPSSPP_PLATFORM(UWP)
-	if (lslash == homebrewName.npos) {
-		lslash = homebrewName.find_last_of("\\");
-	}
-#endif
+	std::size_t rslash = homebrewName.find_last_of('\\');
 	if (lslash != homebrewName.npos)
 		homebrewName = homebrewName.substr(lslash + 1);
+	if (rslash != homebrewName.npos)
+		homebrewName = homebrewName.substr(rslash + 1);
 	std::string homebrewTitle = g_paramSFO.GetValueString("TITLE");
 	if (homebrewTitle.empty())
 		homebrewTitle = homebrewName;

assets/compat.ini

@@ -1135,6 +1135,7 @@ UCJS10039 = true
 UCJS18012 = true
 
 [MemstickFixedFree]
+# Assassin's Creed : Bloodlines - issue #12761
 ULJM05571 = true
 ULES01367 = true
 NPEH00029 = true