diff --git a/packages/core/src/components/Handle/handler.ts b/packages/core/src/components/Handle/handler.ts
index 86aab175..40be2090 100644
--- a/packages/core/src/components/Handle/handler.ts
+++ b/packages/core/src/components/Handle/handler.ts
@@ -1,7 +1,7 @@
 import type { MouseEvent as ReactMouseEvent } from 'react';
 import { StoreApi } from 'zustand';
 
-import { getHostForElement, calcAutoPanVelocity } from '../../utils';
+import { getHostForElement, calcAutoPan } from '../../utils';
 import type { OnConnect, HandleType, ReactFlowState } from '../../types';
 import { pointToRendererPoint, rendererPointToPoint } from '../../utils/graph';
 import {
@@ -60,6 +60,7 @@ export function handleMouseDown({
   const containerBounds = domNode.getBoundingClientRect();
   let prevActiveHandle: Element;
   let connectionPosition = getConnectionPosition(event, containerBounds);
+  let autoPanStarted = false;
 
   const handleLookup = getHandleLookup({
     nodes: getNodes(),
@@ -73,15 +74,12 @@ export function handleMouseDown({
     if (!autoPanOnConnect) {
       return;
     }
-    const xMovement = calcAutoPanVelocity(connectionPosition.x, 35, containerBounds.width - 35) * 20;
-    const yMovement = calcAutoPanVelocity(connectionPosition.y, 35, containerBounds.height - 35) * 20;
+    const [xMovement, yMovement] = calcAutoPan(connectionPosition, containerBounds);
 
     panBy({ x: xMovement, y: yMovement });
     autoPanId = requestAnimationFrame(autoPan);
   };
 
-  autoPan();
-
   setState({
     connectionPosition,
     connectionNodeId: nodeId,
@@ -95,13 +93,17 @@ export function handleMouseDown({
     const { transform } = getState();
 
     connectionPosition = getConnectionPosition(event, containerBounds);
-
     prevClosestHandle = getClosestHandle(
       pointToRendererPoint(connectionPosition, transform, false, [1, 1]),
       connectionRadius,
       handleLookup
     );
 
+    if (!autoPanStarted) {
+      autoPan();
+      autoPanStarted = true;
+    }
+
     setState({
       connectionPosition: prevClosestHandle
         ? rendererPointToPoint(
@@ -138,6 +140,7 @@ export function handleMouseDown({
 
   function onMouseUp(event: MouseEvent) {
     cancelAnimationFrame(autoPanId);
+    autoPanStarted = false;
 
     if (prevClosestHandle) {
       const { connection, isValid } = isValidHandle(
diff --git a/packages/core/src/hooks/useDrag/index.ts b/packages/core/src/hooks/useDrag/index.ts
index 49cd7afe..c9d0ab3d 100644
--- a/packages/core/src/hooks/useDrag/index.ts
+++ b/packages/core/src/hooks/useDrag/index.ts
@@ -7,7 +7,7 @@ import { useStoreApi } from '../../hooks/useStore';
 import { getDragItems, getEventHandlerParams, hasSelector, calcNextPosition } from './utils';
 import { handleNodeClick } from '../../components/Nodes/utils';
 import useGetPointerPosition from '../useGetPointerPosition';
-import { calcAutoPanVelocity } from '../../utils';
+import { calcAutoPan } from '../../utils';
 import type { NodeDragItem, Node, SelectionDragHandler, UseDragEvent, XYPosition } from '../../types';
 
 export type UseDragData = { dx: number; dy: number };
@@ -110,8 +110,7 @@ function useDrag({
           return;
         }
 
-        const xMovement = calcAutoPanVelocity(mousePosition.current.x, 35, containerBounds.current.width - 35) * 20;
-        const yMovement = calcAutoPanVelocity(mousePosition.current.y, 35, containerBounds.current.height - 35) * 20;
+        const [xMovement, yMovement] = calcAutoPan(mousePosition.current, containerBounds.current);
 
         if (xMovement !== 0 || yMovement !== 0) {
           const { transform, panBy } = store.getState();
diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts
index 2364f352..30aa3789 100644
--- a/packages/core/src/utils/index.ts
+++ b/packages/core/src/utils/index.ts
@@ -16,16 +16,23 @@ export const clampPosition = (position: XYPosition = { x: 0, y: 0 }, extent: Coo
 
 // returns a number between 0 and 1 that represents the velocity of the movement
 // when the mouse is close to the edge of the canvas
-export const calcAutoPanVelocity = (value: number, min: number, max: number): number => {
+const calcAutoPanVelocity = (value: number, min: number, max: number): number => {
   if (value < min) {
-    return clamp(Math.abs(value - min), 1, 100) / 100;
+    return clamp(Math.abs(value - min), 1, 50) / 50;
   } else if (value > max) {
-    return -clamp(Math.abs(value - max), 1, 100) / 100;
+    return -clamp(Math.abs(value - max), 1, 50) / 50;
   }
 
   return 0;
 };
 
+export const calcAutoPan = (pos: XYPosition, bounds: Dimensions): number[] => {
+  const xMovement = calcAutoPanVelocity(pos.x, 35, bounds.width - 35) * 20;
+  const yMovement = calcAutoPanVelocity(pos.y, 35, bounds.height - 35) * 20;
+
+  return [xMovement, yMovement];
+};
+
 export const getHostForElement = (element: HTMLElement): Document | ShadowRoot =>
   (element.getRootNode?.() as Document | ShadowRoot) || window?.document;