diff --git a/packages/block-editor/src/components/block-toolbar/index.js b/packages/block-editor/src/components/block-toolbar/index.js
index 13dfd6b33e56a6..b14ceb50590495 100644
--- a/packages/block-editor/src/components/block-toolbar/index.js
+++ b/packages/block-editor/src/components/block-toolbar/index.js
@@ -65,6 +65,7 @@ export function PrivateBlockToolbar( {
 		shouldShowVisualToolbar,
 		showParentSelector,
 		isUsingBindings,
+		hasParentPattern,
 	} = useSelect( ( select ) => {
 		const {
 			getBlockName,
@@ -74,6 +75,7 @@ export function PrivateBlockToolbar( {
 			isBlockValid,
 			getBlockEditingMode,
 			getBlockAttributes,
+			getBlockParentsByBlockName,
 		} = select( blockEditorStore );
 		const selectedBlockClientIds = getSelectedBlockClientIds();
 		const selectedBlockClientId = selectedBlockClientIds[ 0 ];
@@ -94,6 +96,13 @@ export function PrivateBlockToolbar( {
 			( clientId ) =>
 				!! getBlockAttributes( clientId )?.metadata?.bindings
 		);
+
+		const _hasParentPattern = selectedBlockClientIds.every(
+			( clientId ) =>
+				getBlockParentsByBlockName( clientId, 'core/block', true )
+					.length > 0
+		);
+
 		return {
 			blockClientId: selectedBlockClientId,
 			blockClientIds: selectedBlockClientIds,
@@ -113,6 +122,7 @@ export function PrivateBlockToolbar( {
 				selectedBlockClientIds.length === 1 &&
 				_isDefaultEditingMode,
 			isUsingBindings: _isUsingBindings,
+			hasParentPattern: _hasParentPattern,
 		};
 	}, [] );
 
@@ -167,7 +177,7 @@ export function PrivateBlockToolbar( {
 					isDefaultEditingMode && <BlockParentSelector /> }
 				{ ( shouldShowVisualToolbar || isMultiToolbar ) &&
 					( isDefaultEditingMode ||
-						isContentOnlyEditingMode ||
+						( isContentOnlyEditingMode && ! hasParentPattern ) ||
 						isSynced ) && (
 						<div
 							ref={ nodeRef }