diff --git a/packages/block-library/src/navigation-link/edit.js b/packages/block-library/src/navigation-link/edit.js
index 125dd90ee87bec..9e2e171975570f 100644
--- a/packages/block-library/src/navigation-link/edit.js
+++ b/packages/block-library/src/navigation-link/edit.js
@@ -261,6 +261,7 @@ export default function NavigationLinkEdit( {
 	const isDraggingWithin = useIsDraggingWithin( listItemRef );
 	const itemLabelPlaceholder = __( 'Add label…' );
 	const ref = useRef();
+	const linkUIref = useRef();
 	const prevUrl = usePrevious( url );
 
 	// Change the label using inspector causes rich text to change focus on firefox.
@@ -570,14 +571,27 @@ export default function NavigationLinkEdit( {
 					) }
 					{ isLinkOpen && (
 						<LinkUI
+							ref={ linkUIref }
 							clientId={ clientId }
 							link={ attributes }
 							onClose={ () => {
 								// If there is no link then remove the auto-inserted block.
 								// This avoids empty blocks which can provided a poor UX.
 								if ( ! url ) {
-									// Select the previous block to keep focus nearby
-									selectPreviousBlock( clientId, true );
+									// Fixes https://github.com/WordPress/gutenberg/issues/61361
+									// There's a chance we're closing due to the user selecting the browse all button.
+									// Only move focus if the focus is still within the popover ui. If it's not within
+									// the popover, it's because something has taken the focus from the popover, and
+									// we don't want to steal it back.
+									if (
+										linkUIref.current.contains(
+											window.document.activeElement
+										)
+									) {
+										// Select the previous block to keep focus nearby
+										selectPreviousBlock( clientId, true );
+									}
+
 									// Remove the link.
 									onReplace( [] );
 									return;
diff --git a/packages/block-library/src/navigation-link/link-ui.js b/packages/block-library/src/navigation-link/link-ui.js
index ce79af40e4708c..6619c46253546e 100644
--- a/packages/block-library/src/navigation-link/link-ui.js
+++ b/packages/block-library/src/navigation-link/link-ui.js
@@ -20,6 +20,7 @@ import {
 	useState,
 	useRef,
 	useEffect,
+	forwardRef,
 } from '@wordpress/element';
 import {
 	store as coreStore,
@@ -145,7 +146,7 @@ function LinkUIBlockInserter( { clientId, onBack, onSelectBlock } ) {
 	);
 }
 
-export function LinkUI( props ) {
+function UnforwardedLinkUI( props, ref ) {
 	const [ addingBlock, setAddingBlock ] = useState( false );
 	const [ focusAddBlockButton, setFocusAddBlockButton ] = useState( false );
 	const { saveEntityRecord } = useDispatch( coreStore );
@@ -214,6 +215,7 @@ export function LinkUI( props ) {
 
 	return (
 		<Popover
+			ref={ ref }
 			placement="bottom"
 			onClose={ props.onClose }
 			anchor={ props.anchor }
@@ -298,6 +300,8 @@ export function LinkUI( props ) {
 	);
 }
 
+export const LinkUI = forwardRef( UnforwardedLinkUI );
+
 const LinkUITools = ( { setAddingBlock, focusAddBlockButton } ) => {
 	const blockInserterAriaRole = 'listbox';
 	const addBlockButtonRef = useRef();