diff --git a/packages/block-library/src/image/edit.js b/packages/block-library/src/image/edit.js
index e08d0baff0503d..6388925cfc348a 100644
--- a/packages/block-library/src/image/edit.js
+++ b/packages/block-library/src/image/edit.js
@@ -42,6 +42,7 @@ import {
 	LINK_DESTINATION_MEDIA,
 	LINK_DESTINATION_NONE,
 	ALLOWED_MEDIA_TYPES,
+	SIZED_LAYOUTS,
 } from './constants';
 
 export const pickRelevantMediaFiles = ( image, size ) => {
@@ -113,14 +114,6 @@ export function ImageEdit( {
 	const [ temporaryURL, setTemporaryURL ] = useState( attributes.blob );
 
 	const containerRef = useRef();
-	// Only observe the max width from the parent container when the parent layout is not flex nor grid.
-	// This won't work for them because the container width changes with the image.
-	// TODO: Find a way to observe the container width for flex and grid layouts.
-	const isMaxWidthContainerWidth =
-		! parentLayout ||
-		( parentLayout.type !== 'flex' && parentLayout.type !== 'grid' );
-	const [ maxWidthObserver, maxContentWidth ] = useMaxWidthObserver();
-
 	const [ placeholderResizeListener, { width: placeholderWidth } ] =
 		useResizeObserver();
 
@@ -363,6 +356,16 @@ export function ImageEdit( {
 		ref: containerRef,
 		className: classes,
 	} );
+	const [ maxWidthObserver, maxContentWidth ] = useMaxWidthObserver( {
+		className: blockProps.className,
+		isActive:
+			isSingleSelected &&
+			// Only observe the max width from the parent container when the parent layout
+			// is not flex nor grid. This won't work for them because the container width
+			// changes with the image.
+			// TODO: Find a way to observe the container width for flex and grid layouts.
+			( ! parentLayout || ! SIZED_LAYOUTS.includes( parentLayout.type ) ),
+	} );
 
 	// Much of this description is duplicated from MediaPlaceholder.
 	const { lockUrlControls = false, lockUrlControlsMessage } = useSelect(
@@ -472,7 +475,7 @@ export function ImageEdit( {
 			{
 				// The listener cannot be placed as the first element as it will break the in-between inserter.
 				// See https://github.com/WordPress/gutenberg/blob/71134165868298fc15e22896d0c28b41b3755ff7/packages/block-editor/src/components/block-list/use-in-between-inserter.js#L120
-				isSingleSelected && isMaxWidthContainerWidth && maxWidthObserver
+				maxWidthObserver
 			}
 		</>
 	);
diff --git a/packages/block-library/src/image/use-max-width-observer.js b/packages/block-library/src/image/use-max-width-observer.js
index 684392537fac7a..f558cb5548e9f0 100644
--- a/packages/block-library/src/image/use-max-width-observer.js
+++ b/packages/block-library/src/image/use-max-width-observer.js
@@ -1,17 +1,27 @@
 /**
  * WordPress dependencies
  */
-import { useRef } from '@wordpress/element';
+import { useState } from '@wordpress/element';
 import { useResizeObserver } from '@wordpress/compose';
 
-function useMaxWidthObserver() {
-	const [ contentResizeListener, { width } ] = useResizeObserver();
-	const observerRef = useRef();
+function useMaxWidthObserver( { className, isActive } ) {
+	const [ usedMaxWidth, setUsedMaxWidth ] = useState();
+	const setResizeObserved = useResizeObserver( ( [ entry ] ) => {
+		setUsedMaxWidth( entry.contentRect.width );
+	} );
+	if ( ! isActive ) {
+		return [];
+	}
+	// The block’s alignment class names need to be applied because the max-width
+	// may vary by that. The base `wp-block` is needed for classic themes.
+	const usedClassName = className
+		.split( ' ' )
+		.filter( ( name ) => /^(wp-block|align[^\b]+)$/.test( name ) )
+		.join( ' ' );
 
 	const maxWidthObserver = (
 		<div
-			// Some themes set max-width on blocks.
-			className="wp-block"
+			className={ usedClassName }
 			aria-hidden="true"
 			style={ {
 				position: 'absolute',
@@ -20,13 +30,11 @@ function useMaxWidthObserver() {
 				height: 0,
 				margin: 0,
 			} }
-			ref={ observerRef }
-		>
-			{ contentResizeListener }
-		</div>
+			ref={ setResizeObserved }
+		/>
 	);
 
-	return [ maxWidthObserver, width ];
+	return [ maxWidthObserver, usedMaxWidth ];
 }
 
 export { useMaxWidthObserver };