@@ -21,7 +21,6 @@
 ### Bug Fixes
 
 -   `useConstrainedTabbing`: Retain focus when the constrained area has no tabbable elements ([#84344](https://github.com/WordPress/gutenberg/pull/84344)).
--   `useDropZone`: Start a drag on the first `dragover` when the document got no `dragenter`, which Chrome can skip in an iframe, so that `onDragEnd` is still called ([#84500](https://github.com/WordPress/gutenberg/pull/84500)).
 
 ### Internal
 
 
@@ -108,12 +108,6 @@ export default function useDropZone( {
 			}
 
 			function onDragOver( event: DragEvent ): void {
-				// Chrome can send `dragover` to an iframe without a `dragenter`
-				// first, after a previous drag left the iframe over an element
-				// of the parent document. Start the drag here too, so that
-				// its end is still handled.
-				maybeDragStart( event );
-
 				if ( ! event.defaultPrevented && _onDragOver ) {
 					onDragOverEvent( event );
 				}
@@ -167,6 +161,15 @@ export default function useDropZone( {
 					return;
 				}
 
+				// A mouse move with a button held is still part of the drag,
+				// in browsers that send them while dragging.
+				if (
+					event.type === 'mousemove' &&
+					( event as MouseEvent ).buttons !== 0
+				) {
+					return;
+				}
+
 				isDragging = false;
 
 				ownerDocument.removeEventListener( 'dragend', maybeDragEnd );
@@ -185,6 +188,12 @@ export default function useDropZone( {
 			// The `dragstart` event doesn't fire if the drag started outside
 			// the document.
 			ownerDocument.addEventListener( 'dragenter', maybeDragStart );
+			// Chrome can send `dragover` to an iframe without a `dragenter`
+			// first, after a previous drag left the iframe over an element of
+			// the parent document. Start the drag on the first `dragover` too,
+			// so that its end is still handled. It's a capture listener, so
+			// that the drag starts before the drop zone's own `dragover`.
+			ownerDocument.addEventListener( 'dragover', maybeDragStart, true );
 
 			return () => {
 				element.removeAttribute( 'data-is-drop-zone' );
@@ -198,6 +207,11 @@ export default function useDropZone( {
 					'dragenter',
 					maybeDragStart
 				);
+				ownerDocument.removeEventListener(
+					'dragover',
+					maybeDragStart,
+					true
+				);
 			};
 		},
 		[ isDisabled, dropZoneElement ] // Refresh when the passed in dropZoneElement changes.
 
@@ -51,28 +51,70 @@ describe( 'useDropZone', () => {
 			'data-is-drop-zone'
 		);
 	} );
+	const ComponentWithDragCallbacks = ( {
+		onDragStart,
+		onDragEnd,
+	}: {
+		onDragStart: () => void;
+		onDragEnd: () => void;
+	} ) => {
+		const dropZoneRef = useDropZone( { onDragStart, onDragEnd } );
 
-	it( 'ends a drag that reaches the drop zone without a `dragenter` on the document', () => {
+		return (
+			<div role="main">
+				<div role="region" ref={ dropZoneRef }>
+					<div>Drop Zone</div>
+				</div>
+			</div>
+		);
+	};
+
+	function renderWithDragCallbacks() {
 		const onDragStart = vi.fn();
 		const onDragEnd = vi.fn();
-		const Component = () => {
-			const dropZoneRef = useDropZone( { onDragStart, onDragEnd } );
+		const { rerender } = render(
+			<ComponentWithDragCallbacks
+				onDragStart={ onDragStart }
+				onDragEnd={ onDragEnd }
+			/>
+		);
+		// Ensure `useEffect` has run.
+		rerender(
+			<ComponentWithDragCallbacks
+				onDragStart={ onDragStart }
+				onDragEnd={ onDragEnd }
+			/>
+		);
+		return { onDragStart, onDragEnd };
+	}
 
-			return (
-				<div role="region" ref={ dropZoneRef }>
-					Drop Zone
-				</div>
-			);
-		};
-		render( <Component /> );
+	it( 'starts a drag on the first dragover in the document, without a dragenter', () => {
+		const { onDragStart } = renderWithDragCallbacks();
+
+		fireEvent.dragOver( document.body );
+		fireEvent.dragOver( document.body );
+
+		expect( onDragStart ).toHaveBeenCalledTimes( 1 );
+	} );
+
+	it( 'ends a drag that reaches the drop zone without a dragenter on the document', () => {
+		const { onDragStart, onDragEnd } = renderWithDragCallbacks();
 
-		// Chrome can send `dragover` to an iframe without a `dragenter`
-		// first, after a previous drag left the iframe over an element of
-		// the parent document.
 		fireEvent.dragOver( screen.getByRole( 'region' ) );
 		expect( onDragStart ).toHaveBeenCalledTimes( 1 );
 
 		fireEvent.dragEnd( document );
 		expect( onDragEnd ).toHaveBeenCalledTimes( 1 );
 	} );
+
+	it( 'does not end a drag on a mousemove while a button is held', () => {
+		const { onDragEnd } = renderWithDragCallbacks();
+
+		fireEvent.dragEnter( document.body );
+		fireEvent.mouseMove( document, { buttons: 1 } );
+		expect( onDragEnd ).not.toHaveBeenCalled();
+
+		fireEvent.mouseMove( document, { buttons: 0 } );
+		expect( onDragEnd ).toHaveBeenCalledTimes( 1 );
+	} );
 } );