diff --git a/.changeset/perfect-spiders-bathe.md b/.changeset/perfect-spiders-bathe.md
new file mode 100644
index 00000000..79db30b8
--- /dev/null
+++ b/.changeset/perfect-spiders-bathe.md
@@ -0,0 +1,6 @@
+---
+'@xyflow/react': patch
+'@xyflow/svelte': patch
+---
+
+fix(pane) only capture pointer after a valid selection has started, fixes #4492
diff --git a/packages/react/src/container/Pane/index.tsx b/packages/react/src/container/Pane/index.tsx
index 3fd1d079..9e35829c 100644
--- a/packages/react/src/container/Pane/index.tsx
+++ b/packages/react/src/container/Pane/index.tsx
@@ -120,7 +120,6 @@ export function Pane({
   const onPointerDown = (event: ReactPointerEvent): void => {
     const { resetSelectedElements, domNode, edgeLookup } = store.getState();
     containerBounds.current = domNode?.getBoundingClientRect();
-    (event.target as Element)?.setPointerCapture?.(event.pointerId);
 
     if (
       !elementsSelectable ||
@@ -132,6 +131,8 @@ export function Pane({
       return;
     }
 
+    (event.target as Element)?.setPointerCapture?.(event.pointerId);
+
     selectionStarted.current = true;
     selectionInProgress.current = false;
     edgeIdLookup.current = new Map();
diff --git a/packages/svelte/src/lib/container/Pane/Pane.svelte b/packages/svelte/src/lib/container/Pane/Pane.svelte
index 0856381a..8b413389 100644
--- a/packages/svelte/src/lib/container/Pane/Pane.svelte
+++ b/packages/svelte/src/lib/container/Pane/Pane.svelte
@@ -92,7 +92,6 @@
 
   function onPointerDown(event: PointerEvent) {
     containerBounds = container.getBoundingClientRect();
-    (event.target as Element)?.setPointerCapture?.(event.pointerId);
 
     if (
       !elementsSelectable ||
@@ -104,6 +103,8 @@
       return;
     }
 
+    (event.target as Element)?.setPointerCapture?.(event.pointerId);
+
     const { x, y } = getEventPosition(event, containerBounds);
 
     unselectNodesAndEdges();