diff --git a/packages/system/src/xyhandle/XYHandle.ts b/packages/system/src/xyhandle/XYHandle.ts
index 77b6da50..e971a22b 100644
--- a/packages/system/src/xyhandle/XYHandle.ts
+++ b/packages/system/src/xyhandle/XYHandle.ts
@@ -282,16 +282,12 @@ function isValidHandle(
 
     result.isValid = isValid && isValidConnection(connection);
 
-    if (handleLookup) {
-      const toHandle = handleLookup.find(
-        (h) => h.id === handleId && h.nodeId === handleNodeId && h.type === handleType
-      );
+    const toHandle = handleLookup?.get(`${handleNodeId}-${handleType}-${handleId}`);
 
-      if (toHandle) {
-        result.toHandle = {
-          ...toHandle,
-        };
-      }
+    if (toHandle) {
+      result.toHandle = {
+        ...toHandle,
+      };
     }
   }
 
diff --git a/packages/system/src/xyhandle/types.ts b/packages/system/src/xyhandle/types.ts
index 299fbc8f..ac1b6e23 100644
--- a/packages/system/src/xyhandle/types.ts
+++ b/packages/system/src/xyhandle/types.ts
@@ -48,7 +48,7 @@ export type IsValidParams = {
   doc: Document | ShadowRoot;
   lib: string;
   flowId: string | null;
-  handleLookup?: Handle[];
+  handleLookup?: Map<string, Handle>;
 };
 
 export type XYHandleInstance = {
diff --git a/packages/system/src/xyhandle/utils.ts b/packages/system/src/xyhandle/utils.ts
index 409d148c..04099bcf 100644
--- a/packages/system/src/xyhandle/utils.ts
+++ b/packages/system/src/xyhandle/utils.ts
@@ -29,11 +29,15 @@ function getHandles(
   return [handles, excludedHandle];
 }
 
-export function getClosestHandle(pos: XYPosition, connectionRadius: number, handles: Handle[]): Handle | null {
+export function getClosestHandle(
+  pos: XYPosition,
+  connectionRadius: number,
+  handleLookup: Map<string, Handle>
+): Handle | null {
   let closestHandles: Handle[] = [];
   let minDistance = Infinity;
 
-  for (const handle of handles) {
+  for (const handle of handleLookup.values()) {
     const distance = Math.sqrt(Math.pow(handle.x - pos.x, 2) + Math.pow(handle.y - pos.y, 2));
     if (distance <= connectionRadius) {
       if (distance < minDistance) {
@@ -68,21 +72,38 @@ export function getHandleLookup({
   nodeId,
   handleId,
   handleType,
-}: GetHandleLookupParams): [Handle[], Handle] {
-  const connectionHandles: Handle[] = [];
+}: GetHandleLookupParams): [Map<string, Handle>, Handle] {
+  const connectionHandles: Map<string, Handle> = new Map();
   const currentHandle = { nodeId, handleId, handleType };
-  let excludedHandle: Handle | null = null;
+  let matchingHandle: Handle | null = null;
 
   for (const node of nodeLookup.values()) {
     if (node.internals.handleBounds) {
       const [sourceHandles, excludedSource] = getHandles(node, node.internals.handleBounds, 'source', currentHandle);
       const [targetHandles, excludedTarget] = getHandles(node, node.internals.handleBounds, 'target', currentHandle);
-      excludedHandle = excludedHandle ? excludedHandle : excludedSource ?? excludedTarget;
-      connectionHandles.push(...sourceHandles, ...targetHandles);
+
+      matchingHandle = matchingHandle ? matchingHandle : excludedSource ?? excludedTarget;
+
+      [...sourceHandles, ...targetHandles].forEach((handle) =>
+        connectionHandles.set(`${handle.nodeId}-${handle.type}-${handle.id}`, handle)
+      );
     }
   }
 
-  return [connectionHandles, excludedHandle!];
+  // if the user only works with handles that are type="source" + connectionMode="loose"
+  // it happens that we can't find a matching handle. The reason for this is, that the
+  // edge don't know about the handles and always assumes that there is source and a target.
+  // In this case we need to find the matching handle by switching the handleType
+  if (!matchingHandle) {
+    const node = nodeLookup.get(nodeId);
+    if (node?.internals.handleBounds) {
+      currentHandle.handleType = handleType === 'source' ? 'target' : 'source';
+      const [, excluded] = getHandles(node, node.internals.handleBounds, currentHandle.handleType, currentHandle);
+      matchingHandle = excluded;
+    }
+  }
+
+  return [connectionHandles, matchingHandle!];
 }
 
 export function getHandleType(