diff --git a/packages/core/src/components/Handle/handler.ts b/packages/core/src/components/Handle/handler.ts
index 6c484edb..9b8855a3 100644
--- a/packages/core/src/components/Handle/handler.ts
+++ b/packages/core/src/components/Handle/handler.ts
@@ -136,7 +136,7 @@ export function handlePointerDown({
       connectionStatus: getConnectionStatus(!!prevClosestHandle, result.isValid),
     });
 
-    if (!prevClosestHandle) {
+    if (!prevClosestHandle && !result.isValid) {
       return resetRecentHandle(prevActiveHandle);
     }
 
diff --git a/packages/core/src/components/Handle/utils.ts b/packages/core/src/components/Handle/utils.ts
index 89e58c49..58263781 100644
--- a/packages/core/src/components/Handle/utils.ts
+++ b/packages/core/src/components/Handle/utils.ts
@@ -74,14 +74,6 @@ export function isValidHandle(
   isValidConnection: ValidConnectionFunc,
   doc: Document | ShadowRoot
 ) {
-  if (!handle) {
-    return {
-      handleDomNode: null,
-      isValid: false,
-      connection: nullConnection,
-    };
-  }
-
   const isTarget = fromType === 'target';
   const handleDomNode = doc.querySelector(
     `.react-flow__handle[data-id="${handle?.nodeId}-${handle?.id}-${handle?.type}"]`
@@ -102,9 +94,9 @@ export function isValidHandle(
     const handleId = handleToCheck.getAttribute('data-handleid');
 
     const connection: Connection = {
-      source: isTarget ? handle.nodeId : fromNodeId,
+      source: isTarget ? handleNodeId : fromNodeId,
       sourceHandle: isTarget ? handleId : fromHandleId,
-      target: isTarget ? fromNodeId : handle.nodeId,
+      target: isTarget ? fromNodeId : handleNodeId,
       targetHandle: isTarget ? fromHandleId : handleId,
     };