diff --git a/.changeset/honest-deers-thank.md b/.changeset/honest-deers-thank.md
new file mode 100644
index 00000000..54b18fb3
--- /dev/null
+++ b/.changeset/honest-deers-thank.md
@@ -0,0 +1,5 @@
+---
+"@vue-flow/core": patch
+---
+
+Only capture pointer when a valid selection has started
diff --git a/packages/core/src/container/Pane/Pane.vue b/packages/core/src/container/Pane/Pane.vue
index 4661d97a..52d2db5b 100644
--- a/packages/core/src/container/Pane/Pane.vue
+++ b/packages/core/src/container/Pane/Pane.vue
@@ -115,7 +115,6 @@ function onWheel(event: WheelEvent) {
 
 function onPointerDown(event: PointerEvent) {
   containerBounds.value = vueFlowRef.value?.getBoundingClientRect()
-  ;(event.target as Element)?.setPointerCapture?.(event.pointerId)
 
   if (
     !elementsSelectable.value ||
@@ -127,6 +126,8 @@ function onPointerDown(event: PointerEvent) {
     return
   }
 
+  ;(event.target as Element)?.setPointerCapture?.(event.pointerId)
+
   const { x, y } = getMousePosition(event, containerBounds.value)
 
   selectionStarted = true