Interrupts: Refuse sub-interrupt handlers where the firmware does

Only the GE and vblank interrupts take user sub-interrupt handlers, and
vblank only in slots 0-15, with some of the rest already held by the
kernel. The errors follow interruptman.prx's checks, and which interrupts
have handlers at all is read back from pspautotests intr/registersub and
intr/releasesub, which now pass.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

Core/HLE/sceKernelInterrupt.cpp

@@ -512,13 +512,64 @@ int __ReleaseSubIntrHandler(int intrNumber, int subIntrNumber) {
 	return 0;
 }
 
+// What a user mode caller finds on each interrupt, as interruptman.prx checks it: whether a driver
+// has installed a handler for the interrupt at all, whether it has sub-interrupt slots, and whether
+// user handlers are allowed in them. This is system state rather than a rule, read back from
+// intr/registersub and intr/releasesub on a 6.61 PSP (running PSPLink, whose USB drivers may count.)
+enum class IntrUserAccess : u8 {
+	NO_HANDLER,   // SCE_KERNEL_ERROR_NOTFOUND_HANDLER
+	NO_SUBS,      // SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, the sub number is always out of range
+	KERNEL_SUBS,  // SCE_KERNEL_ERROR_ILLEGAL_INTRCODE to register, empty slots to release
+	USER,
+};
+
+static IntrUserAccess GetIntrUserAccess(u32 intrNumber) {
+	switch (intrNumber) {
+	case PSP_GE_INTR:
+	case PSP_VBLANK_INTR:
+		return IntrUserAccess::USER;
+	case 4: case 6: case 8: case 21:
+		return IntrUserAccess::KERNEL_SUBS;
+	case 7: case 10: case 12: case 15: case 16: case 17: case 18: case 19: case 20: case 22:
+	case 23: case 24: case 26: case 31: case 36: case 50: case 56: case 57: case 58: case 59:
+	case 60: case 61: case 65:
+		return IntrUserAccess::NO_SUBS;
+	default:
+		return IntrUserAccess::NO_HANDLER;
+	}
+}
+
+// The vblank slots a user handler may take (the rest are the kernel's), and those the display
+// driver already holds.
+static bool IsUserVblankSubIntr(u32 subIntrNumber) {
+	return subIntrNumber < 16;
+}
+static bool IsKernelHeldVblankSubIntr(u32 subIntrNumber) {
+	return (subIntrNumber >= 18 && subIntrNumber <= 20) || (subIntrNumber >= 24 && subIntrNumber <= 26);
+}
+
 u32 sceKernelRegisterSubIntrHandler(u32 intrNumber, u32 subIntrNumber, u32 handler, u32 handlerArg) {
 	if (intrNumber >= PSP_NUMBER_INTERRUPTS) {
 		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "invalid interrupt");
 	}
+	switch (GetIntrUserAccess(intrNumber)) {
+	case IntrUserAccess::NO_HANDLER:
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_NOTFOUND_HANDLER, "no handler for this interrupt");
+	case IntrUserAccess::NO_SUBS:
+	case IntrUserAccess::KERNEL_SUBS:
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "no user subinterrupts");
+	case IntrUserAccess::USER:
+		break;
+	}
 	if (subIntrNumber >= PSP_NUMBER_SUBINTERRUPTS) {
 		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "invalid subinterrupt");
 	}
+	if (intrNumber == PSP_VBLANK_INTR) {
+		if (IsKernelHeldVblankSubIntr(subIntrNumber))
+			return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_FOUND_HANDLER, "held by the kernel");
+		if (!IsUserVblankSubIntr(subIntrNumber))
+			return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "kernel only subinterrupt");
+	}
 
 	u32 error;
 	SubIntrHandler *subIntrHandler = __RegisterSubIntrHandler(intrNumber, subIntrNumber, handler, handlerArg, error);
@@ -539,9 +590,17 @@ u32 sceKernelReleaseSubIntrHandler(u32 intrNumber, u32 subIntrNumber) {
 	if (intrNumber >= PSP_NUMBER_INTERRUPTS) {
 		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "invalid interrupt");
 	}
-	if (subIntrNumber >= PSP_NUMBER_SUBINTERRUPTS) {
+	const IntrUserAccess access = GetIntrUserAccess(intrNumber);
+	if (access == IntrUserAccess::NO_HANDLER) {
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_NOTFOUND_HANDLER, "no handler for this interrupt");
+	}
+	if (access == IntrUserAccess::NO_SUBS || subIntrNumber >= PSP_NUMBER_SUBINTERRUPTS) {
 		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE, "invalid subinterrupt");
 	}
+	// User code can't have put anything in the kernel's slots, and can't release what's there.
+	if (access == IntrUserAccess::KERNEL_SUBS || (intrNumber == PSP_VBLANK_INTR && !IsUserVblankSubIntr(subIntrNumber))) {
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_NOTFOUND_HANDLER, "not a user subinterrupt");
+	}
 
 	u32 error = __ReleaseSubIntrHandler(intrNumber, subIntrNumber);
 	return hleLogDebugOrError(Log::sceIntc, error);