diff --git a/Assets/Mirror/Core/NetworkConnectionToClient.cs b/Assets/Mirror/Core/NetworkConnectionToClient.cs
index d80749c13e..df82ef9110 100644
--- a/Assets/Mirror/Core/NetworkConnectionToClient.cs
+++ b/Assets/Mirror/Core/NetworkConnectionToClient.cs
@@ -217,7 +217,7 @@ internal void DestroyOwnedObjects()
                 {
                     // disown scene objects, destroy instantiated objects.
                     if (netIdentity.sceneId != 0)
-                        NetworkServer.RemovePlayerForConnection(this, RemovePlayerOptions.KeepActive);
+                        netIdentity.RemoveClientOwner();
                     else
                         NetworkServer.Destroy(netIdentity.gameObject);
                 }
diff --git a/Assets/Mirror/Core/NetworkIdentity.cs b/Assets/Mirror/Core/NetworkIdentity.cs
index 2efc692149..8b3893b634 100644
--- a/Assets/Mirror/Core/NetworkIdentity.cs
+++ b/Assets/Mirror/Core/NetworkIdentity.cs
@@ -1591,6 +1591,18 @@ internal void SetClientOwner(NetworkConnectionToClient conn)
             connectionToClient = conn;
         }
 
+        // Do not merge with RemoveClientAuthority
+        // This is called from NetworkConnectionToClient::DestroyOwnedObjects
+        // when a player disconnects to quietly remove ownerhip of scene objects
+        internal void RemoveClientOwner()
+        {
+            if (connectionToClient != null)
+            {
+                clientAuthorityCallback?.Invoke(connectionToClient, this, false);
+                connectionToClient = null;
+            }
+        }
+
         /// <summary>Removes ownership for an object.</summary>
         // Applies to objects that had authority set by AssignClientAuthority,
         // or NetworkServer.Spawn with a NetworkConnection parameter included.
@@ -1611,9 +1623,8 @@ public void RemoveClientAuthority()
 
             if (connectionToClient != null)
             {
-                clientAuthorityCallback?.Invoke(connectionToClient, this, false);
                 NetworkConnectionToClient previousOwner = connectionToClient;
-                connectionToClient = null;
+                RemoveClientOwner();
                 NetworkServer.SendChangeOwnerMessage(this, previousOwner);
             }
         }