diff --git a/packages/core/src/components/Handle/handler.ts b/packages/core/src/components/Handle/handler.ts
index 9b8855a3..a0c2e935 100644
--- a/packages/core/src/components/Handle/handler.ts
+++ b/packages/core/src/components/Handle/handler.ts
@@ -154,8 +154,10 @@ export function handlePointerDown({
   }
 
   function onPointerUp(event: MouseEvent | TouchEvent) {
-    if (connection && isValid) {
-      onConnect?.(connection);
+    if (prevClosestHandle) {
+      if (connection && isValid) {
+        onConnect?.(connection);
+      }
     }
 
     // it's important to get a fresh reference from the store here