VMManager: Fix deadlock when asking to disable hardcore in fullscreen

pcsx2-gsrunner/Main.cpp

@@ -871,7 +871,7 @@ static void CPUThreadMain(VMBootParameters* params, std::atomic<int>* ret)
 		VMManager::ApplySettings();
 		GSDumpReplayer::SetIsDumpRunner(true);
 
-		if (VMManager::Initialize(*params))
+		if (VMManager::Initialize(*params) == VMBootResult::StartupSuccess)
 		{
 			// run until end
 			GSDumpReplayer::SetLoopCount(s_loop_count);

pcsx2-qt/QtHost.cpp

@@ -240,8 +240,6 @@ void EmuThread::startVM(std::shared_ptr<VMBootParameters> boot_params)
 		return;
 	}
 
-	pxAssertRel(!VMManager::HasValidVM(), "VM is shut down");
-
 	// Determine whether to start fullscreen or not.
 	m_is_rendering_to_main = shouldRenderToMain();
 	if (boot_params->fullscreen.has_value())
@@ -249,22 +247,35 @@ void EmuThread::startVM(std::shared_ptr<VMBootParameters> boot_params)
 	else
 		m_is_fullscreen = Host::GetBaseBoolSettingValue("UI", "StartFullscreen", false);
 
-	if (!VMManager::Initialize(*boot_params))
+	auto hardcore_disable_callback = [](std::string reason, VMBootRestartCallback restart_callback) {
-		return;
+		QtHost::RunOnUIThread([reason = std::move(reason), restart_callback = std::move(restart_callback)]() {
+			QString title(Achievements::GetHardcoreModeDisableTitle());
+			QString text(QString::fromStdString(Achievements::GetHardcoreModeDisableText(reason.c_str())));
+			if (g_main_window->confirmMessage(title, text))
+				Host::RunOnCPUThread(restart_callback);
+		});
+	};
 
-	if (!Host::GetBoolSettingValue("UI", "StartPaused", false))
+	auto done_callback = [](VMBootResult result) {
-	{
+		if (result != VMBootResult::StartupSuccess)
-		// This will come back and call OnVMResumed().
+			return;
-		VMManager::SetState(VMState::Running);
-	}
-	else
-	{
-		// When starting paused, redraw the window, so there's at least something there.
-		redrawDisplayWindow();
-		Host::OnVMPaused();
-	}
 
-	m_event_loop->quit();
+		if (!Host::GetBoolSettingValue("UI", "StartPaused", false))
+		{
+			// This will come back and call OnVMResumed().
+			VMManager::SetState(VMState::Running);
+		}
+		else
+		{
+			// When starting paused, redraw the window, so there's at least something there.
+			g_emu_thread->redrawDisplayWindow();
+			Host::OnVMPaused();
+		}
+
+		g_emu_thread->getEventLoop()->quit();
+	};
+
+	VMManager::InitializeAsync(*boot_params, std::move(hardcore_disable_callback), std::move(done_callback));
 }
 
 void EmuThread::resetVM()

pcsx2/Achievements.cpp

@@ -1903,62 +1903,19 @@ bool Achievements::ConfirmSystemReset()
 	return true;
 }
 
-bool Achievements::ConfirmHardcoreModeDisable(const char* trigger)
+const char* Achievements::GetHardcoreModeDisableTitle()
 {
-#ifdef ENABLE_RAINTEGRATION
+	return TRANSLATE("Achievements", "Confirm Hardcore Mode");
-	if (IsUsingRAIntegration())
-		return (RA_WarnDisableHardcore(trigger) != 0);
-#endif
-
-	// I really hope this doesn't deadlock :/
-	const bool confirmed = Host::ConfirmMessage(TRANSLATE("Achievements", "Confirm Hardcore Mode"),
-		fmt::format(TRANSLATE_FS("Achievements", "{0} cannot be performed while hardcore mode is active. Do you "
-												 "want to disable hardcore mode? {0} will be cancelled if you select No."),
-			trigger));
-	if (!confirmed)
-		return false;
-
-	DisableHardcoreMode();
-	return true;
 }
 
-void Achievements::ConfirmHardcoreModeDisableAsync(const char* trigger, std::function<void(bool)> callback)
+std::string Achievements::GetHardcoreModeDisableText(const char* reason)
 {
-#ifdef ENABLE_RAINTEGRATION
+	return fmt::format(
-	if (IsUsingRAIntegration())
+		TRANSLATE_FS("Achievements",
-	{
+			"{0} cannot be performed while hardcore mode is active. "
-		const bool result = (RA_WarnDisableHardcore(trigger) != 0);
+			"Do you want to disable hardcore mode? "
-		callback(result);
+			"{0} will be cancelled if you select No."),
-		return;
+		reason);
-	}
-#endif
-
-	MTGS::RunOnGSThread([trigger = TinyString(trigger), callback = std::move(callback)]() {
-		if (!FullscreenUI::Initialize())
-		{
-			Host::AddOSDMessage(fmt::format(TRANSLATE_FS("Cannot {} while hardcore mode is active.", trigger)),
-				Host::OSD_WARNING_DURATION);
-			callback(false);
-			return;
-		}
-
-		auto real_callback = [callback = std::move(callback)](bool res) mutable {
-			// don't run the callback in the middle of rendering the UI
-			Host::RunOnCPUThread([callback = std::move(callback), res]() {
-				if (res)
-					DisableHardcoreMode();
-				callback(res);
-			});
-		};
-
-		ImGuiFullscreen::OpenConfirmMessageDialog(
-			TRANSLATE_STR("Achievements", "Confirm Hardcore Mode"),
-			fmt::format(TRANSLATE_FS("Achievements", "{0} cannot be performed while hardcore mode is active. Do you "
-													 "want to disable hardcore mode? {0} will be cancelled if you select No."),
-				trigger),
-			std::move(real_callback), true, fmt::format(ICON_FA_CHECK " {}", TRANSLATE_SV("Achievements", "Yes")),
-			fmt::format(ICON_FA_XMARK " {}", TRANSLATE_SV("Achievements", "No")));
-	});
 }
 
 void Achievements::ClearUIState()

pcsx2/Achievements.h

@@ -73,9 +73,13 @@ namespace Achievements
 	/// Forces hardcore mode off until next reset.
 	void DisableHardcoreMode();
 
-	/// Prompts the user to disable hardcore mode, if they agree, returns true.
+	/// Returns the translated title to display for the message box asking the
-	bool ConfirmHardcoreModeDisable(const char* trigger);
+	// user to disable hardcore mode.
-	void ConfirmHardcoreModeDisableAsync(const char* trigger, std::function<void(bool)> callback);
+	const char* GetHardcoreModeDisableTitle();
+
+	/// Returns the translated text to display in the message box asking the
+	/// user to disable hardcore mode.
+	std::string GetHardcoreModeDisableText(const char* reason);
 
 	/// Returns true if hardcore mode is active, and functionality should be restricted.
 	bool IsHardcoreModeActive();

pcsx2/ImGui/FullscreenUI.cpp

@@ -489,6 +489,7 @@ namespace FullscreenUI
 	static ImGuiFullscreen::FileSelectorFilters GetDiscImageFilters();
 	static ImGuiFullscreen::FileSelectorFilters GetAudioFileFilters();
 	static ImGuiFullscreen::FileSelectorFilters GetImageFileFilters();
+	static void DoVMInitialize(const VMBootParameters& boot_params, bool switch_to_landing_on_failure);
 	static void DoStartPath(
 		const std::string& path, std::optional<s32> state_index = std::nullopt, std::optional<bool> fast_boot = std::nullopt);
 	static void DoStartFile();
@@ -1436,6 +1437,40 @@ ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetImageFileFilters()
 	return {"*.png", "*.jpg", "*.jpeg", "*.bmp"};
 }
 
+void FullscreenUI::DoVMInitialize(const VMBootParameters& boot_params, bool switch_to_landing_on_failure)
+{
+	auto hardcore_disable_callback = [switch_to_landing_on_failure](
+											   std::string reason, VMBootRestartCallback restart_callback) {
+		MTGS::RunOnGSThread([reason = std::move(reason),
+								restart_callback = std::move(restart_callback),
+								switch_to_landing_on_failure]() {
+			const auto callback = [restart_callback = std::move(restart_callback),
+									  switch_to_landing_on_failure](bool confirmed) {
+				if (confirmed)
+					Host::RunOnCPUThread(restart_callback);
+				else if (switch_to_landing_on_failure)
+					SwitchToLanding();
+			};
+
+			ImGuiFullscreen::OpenConfirmMessageDialog(
+				Achievements::GetHardcoreModeDisableTitle(),
+				Achievements::GetHardcoreModeDisableText(reason.c_str()),
+				std::move(callback), true,
+				fmt::format(ICON_FA_CHECK " {}", TRANSLATE_SV("Achievements", "Yes")),
+				fmt::format(ICON_FA_XMARK " {}", TRANSLATE_SV("Achievements", "No")));
+		});
+	};
+
+	auto done_callback = [switch_to_landing_on_failure](VMBootResult result) {
+		if (result == VMBootResult::StartupSuccess)
+			VMManager::SetState(VMState::Running);
+		else if (switch_to_landing_on_failure)
+			MTGS::RunOnGSThread(SwitchToLanding);
+	};
+
+	VMManager::InitializeAsync(boot_params, std::move(hardcore_disable_callback), std::move(done_callback));
+}
+
 void FullscreenUI::DoStartPath(const std::string& path, std::optional<s32> state_index, std::optional<bool> fast_boot)
 {
 	VMBootParameters params;
@@ -1445,11 +1480,7 @@ void FullscreenUI::DoStartPath(const std::string& path, std::optional<s32> state
 
 	// switch to nothing, we'll get brought back if init fails
 	Host::RunOnCPUThread([params = std::move(params)]() {
-		if (VMManager::HasValidVM())
+		DoVMInitialize(std::move(params), false);
-			return;
-
-		if (VMManager::Initialize(params))
-			VMManager::SetState(VMState::Running);
 	});
 }
 
@@ -1472,10 +1503,7 @@ void FullscreenUI::DoStartBIOS()
 			return;
 
 		VMBootParameters params;
-		if (VMManager::Initialize(params))
+		DoVMInitialize(std::move(params), true);
-			VMManager::SetState(VMState::Running);
-		else
-			SwitchToLanding();
 	});
 
 	// switch to nothing, we'll get brought back if init fails
@@ -1491,10 +1519,7 @@ void FullscreenUI::DoStartDisc(const std::string& drive)
 		VMBootParameters params;
 		params.filename = std::move(drive);
 		params.source_type = CDVD_SourceType::Disc;
-		if (VMManager::Initialize(params))
+		DoVMInitialize(std::move(params), true);
-			VMManager::SetState(VMState::Running);
-		else
-			SwitchToLanding();
 	});
 }
 
@@ -7628,8 +7653,7 @@ void FullscreenUI::DoLoadState(std::string path)
 			VMBootParameters params;
 			params.filename = std::move(boot_path);
 			params.save_state = std::move(path);
-			if (VMManager::Initialize(params))
+			DoVMInitialize(params, false);
-				VMManager::SetState(VMState::Running);
 		}
 	});
 }

pcsx2/VMManager.cpp

@@ -1267,10 +1267,42 @@ void VMManager::PrecacheCDVDFile()
 	}
 }
 
-bool VMManager::Initialize(const VMBootParameters& boot_params)
+void VMManager::InitializeAsync(
+	const VMBootParameters& boot_params,
+	VMBootHardcoreDisableCallback hardcore_disable_callback,
+	VMBootDoneCallback done_callback)
+{
+	VMBootResult result = VMManager::Initialize(boot_params);
+
+	if (result == VMBootResult::PromptDisableHardcoreMode)
+	{
+		std::string reason;
+		if (DebugInterface::getPauseOnEntry())
+			reason = TRANSLATE_STR("VMManager", "Boot and Debug");
+		else
+			reason = TRANSLATE_STR("VMManager", "Resuming state");
+
+		hardcore_disable_callback(reason,
+			[boot_params, done_callback = std::move(done_callback)]() {
+				VMBootParameters new_boot_params = std::move(boot_params);
+				new_boot_params.disable_achievements_hardcore_mode = true;
+				done_callback(VMManager::Initialize(new_boot_params));
+			});
+
+		return;
+	}
+
+	done_callback(result);
+}
+
+VMBootResult VMManager::Initialize(const VMBootParameters& boot_params)
 {
 	const Common::Timer init_timer;
-	pxAssertRel(s_state.load(std::memory_order_acquire) == VMState::Shutdown, "VM is shutdown");
+	if (s_state.load(std::memory_order_acquire) != VMState::Shutdown)
+	{
+		Host::ReportErrorAsync("Error", "The virtual machine is already running.");
+		return VMBootResult::StartupFailure;
+	}
 
 	// cancel any game list scanning, we need to use CDVD!
 	// TODO: we can get rid of this once, we make CDVD not use globals...
@@ -1313,14 +1345,14 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		if (boot_params.filename.empty())
 		{
 			Host::ReportErrorAsync("Error", "Cannot load an indexed save state without a boot filename.");
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 
 		state_to_load = GetSaveStateFileName(boot_params.filename.c_str(), boot_params.state_index.value());
 		if (state_to_load.empty())
 		{
 			Host::ReportErrorAsync("Error", "Could not resolve path indexed save state load.");
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 	}
 
@@ -1328,7 +1360,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!cdvdLock(&cdvd_lock_error))
 	{
 		Host::ReportErrorAsync("Startup Error", cdvd_lock_error.GetDescription());
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 
 	ScopedGuard unlock_cdvd = &cdvdUnlock;
@@ -1341,7 +1373,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		{
 			Host::ReportErrorAsync(
 				"Error", fmt::format("Requested filename '{}' does not exist.", boot_params.filename));
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 
 		// Use specified source type.
@@ -1352,7 +1384,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	{
 		// Automatic type detection of boot parameter based on filename.
 		if (!AutoDetectSource(boot_params.filename))
-			return false;
+			return VMBootResult::StartupFailure;
 	}
 
 	ScopedGuard close_cdvd_files(&CDVDsys_ClearFiles);
@@ -1372,7 +1404,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 						"PCSX2 will be able to run once you've placed your BIOS image inside the folder named \"bios\" within the data directory "
 						"(Tools Menu -> Open Data Directory)."),
 					PCSX2_DOCUMENTATION_BIOS_URL_SHORTENED));
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 
 		// Must happen after BIOS load, depends on BIOS version.
@@ -1386,7 +1418,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		Host::ReportErrorAsync("Startup Error", fmt::format("Failed to open CDVD '{}': {}.",
 													Path::GetFileName(CDVDsys_GetFile(CDVDsys_GetSourceType())),
 													error.GetDescription()));
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_cdvd(&DoCDVDclose);
 
@@ -1407,7 +1439,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		if (!FileSystem::FileExists(s_elf_override.c_str()))
 		{
 			Host::ReportErrorAsync("Error", fmt::format("Requested boot ELF '{}' does not exist.", s_elf_override));
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 
 		Hle_SetHostRoot(s_elf_override.c_str());
@@ -1429,42 +1461,9 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		Achievements::DisableHardcoreMode();
 	else
 		Achievements::ResetHardcoreMode(true);
-	if (Achievements::IsHardcoreModeActive())
-	{
-		auto confirm_hc_mode_disable = [&boot_params](const char* trigger) mutable {
-			if (FullscreenUI::IsInitialized())
-			{
-				s_elf_override = {};
 
-				VMBootParameters new_boot_params = boot_params;
+	if (Achievements::IsHardcoreModeActive() && (!state_to_load.empty() || DebugInterface::getPauseOnEntry()))
-				new_boot_params.disable_achievements_hardcore_mode = true;
+		return VMBootResult::PromptDisableHardcoreMode;
-
-				Achievements::ConfirmHardcoreModeDisableAsync(trigger,
-					[new_boot_params = std::move(new_boot_params)](bool approved) mutable {
-						if (approved && Initialize(new_boot_params))
-							SetState(VMState::Running);
-					});
-
-				return false;
-			}
-			else if (!Achievements::ConfirmHardcoreModeDisable(trigger))
-			{
-				return false;
-			}
-			return true;
-		};
-
-		if (!state_to_load.empty())
-		{
-			if (!confirm_hc_mode_disable(TRANSLATE("VMManager", "Resuming state")))
-				return false;
-		}
-		if (DebugInterface::getPauseOnEntry())
-		{
-			if (!confirm_hc_mode_disable(TRANSLATE("VMManager", "Boot and Debug")))
-				return false;
-		}
-	}
 
 	s_limiter_mode = LimiterModeType::Nominal;
 	s_target_speed = GetTargetSpeedForLimiterMode(s_limiter_mode);
@@ -1486,7 +1485,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	{
 		// we assume GS is going to report its own error
 		Console.WriteLn("Failed to open GS.");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 
 	ScopedGuard close_gs = []() {
@@ -1498,7 +1497,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!SPU2::Open())
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize SPU2.");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_spu2(&SPU2::Close);
 
@@ -1507,7 +1506,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!Pad::Initialize())
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize PAD");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_pad = &Pad::Shutdown;
 
@@ -1515,7 +1514,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!g_Sio2.Initialize())
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize SIO2");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_sio2 = []() {
 		g_Sio2.Shutdown();
@@ -1525,7 +1524,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!g_Sio0.Initialize())
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize SIO0");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_sio0 = []() {
 		g_Sio0.Shutdown();
@@ -1535,7 +1534,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (DEV9init() != 0 || DEV9open() != 0)
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize DEV9.");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_dev9 = []() {
 		DEV9close();
@@ -1546,7 +1545,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (!USBopen())
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize USB.");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_usb = []() { USBclose(); };
 
@@ -1554,7 +1553,7 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 	if (FWopen() != 0)
 	{
 		Host::ReportErrorAsync("Startup Error", "Failed to initialize FW.");
-		return false;
+		return VMBootResult::StartupFailure;
 	}
 	ScopedGuard close_fw = []() { FWclose(); };
 
@@ -1594,12 +1593,12 @@ bool VMManager::Initialize(const VMBootParameters& boot_params)
 		{
 			Host::ReportErrorAsync(TRANSLATE_SV("VMManager", "Failed to load save state."), state_error.GetDescription());
 			Shutdown(false);
-			return false;
+			return VMBootResult::StartupFailure;
 		}
 	}
 
 	PerformanceMetrics::Clear();
-	return true;
+	return VMBootResult::StartupSuccess;
 }
 
 void VMManager::Shutdown(bool save_resume_state)

pcsx2/VMManager.h

@@ -39,6 +39,30 @@ struct VMBootParameters
 	bool disable_achievements_hardcore_mode = false;
 };
 
+enum class VMBootResult
+{
+	/// The boot succeeded.
+	StartupSuccess,
+	/// The boot failed and an error should be displayed in the UI.
+	StartupFailure,
+	/// The boot failed because the user needs to be prompted to disable
+	/// hardcore mode. If the user agrees, VMManager::Initialize should be
+	/// called again with disable_achievements_hardcore_mode set to true.
+	PromptDisableHardcoreMode
+};
+
+/// Callback used to restart the VM boot process after the user has consented
+/// to hardcore mode being disabled.
+using VMBootRestartCallback = std::function<void()>;
+
+/// Callback used when the VM boot process has been interrupted because the user
+/// needs to be prompted to disable hardcore mode.
+using VMBootHardcoreDisableCallback = std::function<void(std::string reason, VMBootRestartCallback restart_callback)>;
+
+/// Callback used when the VM boot process has finished. This may be called
+/// asynchronously after the user has been prompted to disable hardcore mode.
+using VMBootDoneCallback = std::function<void(VMBootResult result)>;
+
 namespace VMManager
 {
 	/// The number of usable save state slots.
@@ -83,8 +107,16 @@ namespace VMManager
 	/// Returns the path to the ELF which is currently running. Only safe to read on the EE thread.
 	const std::string& GetCurrentELF();
 
-	/// Initializes all system components.
+	/// Initializes all system components. May restart itself asynchronously
-	bool Initialize(const VMBootParameters& boot_params);
+	/// using the provided hardcore_disable_callback function. Will call the
+	/// done_callback function on either success or failure.
+	void InitializeAsync(
+		const VMBootParameters& boot_params,
+		VMBootHardcoreDisableCallback hardcore_disable_callback,
+		VMBootDoneCallback done_callback);
+
+	/// Initializes all system components. Will not attempt to restart itself.
+	VMBootResult Initialize(const VMBootParameters& boot_params);
 
 	/// Destroys all system components.
 	void Shutdown(bool save_resume_state);