@@ -8,6 +8,7 @@ import classnames from 'classnames';
  */
 import { __ } from '@wordpress/i18n';
 import { useCallback, useState, useRef } from '@wordpress/element';
+import { useSelect } from '@wordpress/data';
 import {
 	Button,
 	ButtonGroup,
@@ -23,6 +24,7 @@ import {
 	InspectorControls,
 	InspectorAdvancedControls,
 	RichText,
+	store as blockEditorStore,
 	useBlockProps,
 	__experimentalLinkControl as LinkControl,
 	__experimentalUseEditorFeature as useEditorFeature,
@@ -100,7 +102,7 @@ function WidthPanel( { selectedWidth, setAttributes } ) {
 }
 
 function URLPicker( {
-	isSelected,
+	mayOpen,
 	url,
 	setAttributes,
 	opensInNewTab,
@@ -109,7 +111,6 @@ function URLPicker( {
 } ) {
 	const [ isURLPickerOpen, setIsURLPickerOpen ] = useState( false );
 	const urlIsSet = !! url;
-	const urlIsSetandSelected = urlIsSet && isSelected;
 	const openLinkControl = () => {
 		setIsURLPickerOpen( true );
 		return false; // prevents default behaviour for event
@@ -122,7 +123,7 @@ function URLPicker( {
 		} );
 		setIsURLPickerOpen( false );
 	};
-	const linkControl = ( isURLPickerOpen || urlIsSetandSelected ) && (
+	const linkControl = ( isURLPickerOpen || urlIsSet ) && mayOpen && (
 		<Popover
 			position="bottom center"
 			onClose={ () => setIsURLPickerOpen( false ) }
@@ -156,7 +157,7 @@ function URLPicker( {
 						onClick={ openLinkControl }
 					/>
 				) }
-				{ urlIsSetandSelected && (
+				{ urlIsSet && (
 					<ToolbarButton
 						name="link"
 						icon={ linkOff }
@@ -167,7 +168,7 @@ function URLPicker( {
 					/>
 				) }
 			</BlockControls>
-			{ isSelected && (
+			{ mayOpen && (
 				<KeyboardShortcuts
 					bindGlobal
 					shortcuts={ {
@@ -186,6 +187,7 @@ function ButtonEdit( props ) {
 		attributes,
 		setAttributes,
 		className,
+		clientId,
 		isSelected,
 		onReplace,
 		mergeBlocks,
@@ -231,6 +233,14 @@ function ButtonEdit( props ) {
 		setAttributes( { text: newText.replace( /<\/?a[^>]*>/g, '' ) } );
 	};
 
+	const isVisualEditMode = useSelect( ( select ) => {
+		const store = select( blockEditorStore );
+		return (
+			'visual' === store.getBlockMode( clientId ) &&
+			! store.isNavigationMode()
+		);
+	}, [] );
+
 	const colorProps = getColorAndStyleProps( attributes, colors, true );
 	const ref = useRef();
 	const blockProps = useBlockProps( { ref } );
@@ -277,7 +287,7 @@ function ButtonEdit( props ) {
 			<URLPicker
 				url={ url }
 				setAttributes={ setAttributes }
-				isSelected={ isSelected }
+				mayOpen={ isVisualEditMode && isSelected }
 				opensInNewTab={ linkTarget === '_blank' }
 				onToggleOpenInNewTab={ onToggleOpenInNewTab }
 				anchorRef={ ref }