diff --git a/src/yuzu/configuration/configure_input.cpp b/src/yuzu/configuration/configure_input.cpp
index 567a36d9bc..422022d02b 100644
--- a/src/yuzu/configuration/configure_input.cpp
+++ b/src/yuzu/configuration/configure_input.cpp
@@ -190,12 +190,16 @@ void ConfigureInput::ApplyConfiguration() {
     // This emulates a delay between disconnecting and reconnecting controllers as some games
     // do not respond to a change in controller type if it was instantaneous.
     using namespace std::chrono_literals;
-    std::this_thread::sleep_for(60ms);
+    std::this_thread::sleep_for(150ms);
 
     for (auto* controller : player_controllers) {
         controller->TryConnectSelectedController();
     }
 
+    // This emulates a delay between disconnecting and reconnecting controllers as some games
+    // do not respond to a change in controller type if it was instantaneous.
+    std::this_thread::sleep_for(150ms);
+
     advanced->ApplyConfiguration();
 
     const bool pre_docked_mode = Settings::values.use_docked_mode.GetValue();
diff --git a/src/yuzu/configuration/configure_input_player.cpp b/src/yuzu/configuration/configure_input_player.cpp
index 13f0351d4c..fbe36046b3 100644
--- a/src/yuzu/configuration/configure_input_player.cpp
+++ b/src/yuzu/configuration/configure_input_player.cpp
@@ -579,11 +579,11 @@ void ConfigureInputPlayer::ApplyConfiguration() {
     // Apply configuration for handheld
     if (player_index == 0) {
         auto& handheld = Settings::values.players.GetValue()[HANDHELD_INDEX];
+        const auto handheld_connected = handheld.connected;
         if (player.controller_type == Settings::ControllerType::Handheld) {
             handheld = player;
         }
-        handheld.connected = ui->groupConnectedController->isChecked() &&
-                             player.controller_type == Settings::ControllerType::Handheld;
+        handheld.connected = handheld_connected;
     }
 }
 
@@ -595,6 +595,18 @@ void ConfigureInputPlayer::TryConnectSelectedController() {
     const auto player_connected = ui->groupConnectedController->isChecked() &&
                                   controller_type != Settings::ControllerType::Handheld;
 
+    // Connect Handheld depending on Player 1's controller configuration.
+    if (player_index == 0 && controller_type == Settings::ControllerType::Handheld) {
+        auto& handheld = Settings::values.players.GetValue()[HANDHELD_INDEX];
+        const auto handheld_connected = ui->groupConnectedController->isChecked() &&
+                                        controller_type == Settings::ControllerType::Handheld;
+        // Connect only if handheld is going from disconnected to connected
+        if (!handheld.connected && handheld_connected) {
+            UpdateController(controller_type, HANDHELD_INDEX, true);
+        }
+        handheld.connected = handheld_connected;
+    }
+
     if (player.controller_type == controller_type && player.connected == player_connected) {
         // Set vibration devices in the event that the input device has changed.
         ConfigureVibration::SetVibrationDevices(player_index);
@@ -606,22 +618,11 @@ void ConfigureInputPlayer::TryConnectSelectedController() {
 
     ConfigureVibration::SetVibrationDevices(player_index);
 
-    // Connect/Disconnect Handheld depending on Player 1's controller configuration.
-    if (player_index == 0) {
-        auto& handheld = Settings::values.players.GetValue()[HANDHELD_INDEX];
-        if (controller_type == Settings::ControllerType::Handheld) {
-            handheld = player;
-        }
-        handheld.connected = ui->groupConnectedController->isChecked() &&
-                             controller_type == Settings::ControllerType::Handheld;
-        UpdateController(Settings::ControllerType::Handheld, HANDHELD_INDEX, handheld.connected);
-    }
-
     if (!player.connected) {
         return;
     }
 
-    UpdateController(controller_type, player_index, player_connected);
+    UpdateController(controller_type, player_index, true);
 }
 
 void ConfigureInputPlayer::TryDisconnectSelectedController() {
@@ -632,11 +633,28 @@ void ConfigureInputPlayer::TryDisconnectSelectedController() {
     const auto player_connected = ui->groupConnectedController->isChecked() &&
                                   controller_type != Settings::ControllerType::Handheld;
 
+    // Disconnect Handheld depending on Player 1's controller configuration.
+    if (player_index == 0 && player.controller_type == Settings::ControllerType::Handheld) {
+        const auto& handheld = Settings::values.players.GetValue()[HANDHELD_INDEX];
+        const auto handheld_connected = ui->groupConnectedController->isChecked() &&
+                                        controller_type == Settings::ControllerType::Handheld;
+        // Disconnect only if handheld is going from connected to disconnected
+        if (handheld.connected && !handheld_connected) {
+            UpdateController(controller_type, HANDHELD_INDEX, false);
+        }
+        return;
+    }
+
     // Do not do anything if the controller configuration has not changed.
     if (player.controller_type == controller_type && player.connected == player_connected) {
         return;
     }
 
+    // Do not disconnect if the controller is already disconnected
+    if (!player.connected) {
+        return;
+    }
+
     // Disconnect the controller first.
     UpdateController(controller_type, player_index, false);
 }