diff --git a/packages/block-library/src/site-logo/edit.js b/packages/block-library/src/site-logo/edit.js
index 6d3eda830878e..d2d2327736fd7 100644
--- a/packages/block-library/src/site-logo/edit.js
+++ b/packages/block-library/src/site-logo/edit.js
@@ -33,8 +33,6 @@ import {
 	BlockControls,
 	InspectorControls,
 	MediaPlaceholder,
-	MediaUpload,
-	MediaUploadCheck,
 	MediaReplaceFlow,
 	useBlockProps,
 	store as blockEditorStore,
@@ -347,29 +345,24 @@ const SiteLogo = ( {
 
 // This is a light wrapper around MediaReplaceFlow because the block has two
 // different MediaReplaceFlows, one for the inspector and one for the toolbar.
-function SiteLogoReplaceFlow( {
-	mediaURL,
-	onRemoveLogo,
-	...mediaReplaceProps
-} ) {
+function SiteLogoReplaceFlow( { mediaURL, ...mediaReplaceProps } ) {
 	return (
 		<MediaReplaceFlow
 			{ ...mediaReplaceProps }
 			mediaURL={ mediaURL }
 			allowedTypes={ ALLOWED_MEDIA_TYPES }
 			accept={ ACCEPT_MEDIA_STRING }
-			onReset={ onRemoveLogo }
 		/>
 	);
 }
 
-const InspectorLogoPreview = ( { mediaItemData = {}, itemGroupProps } ) => {
+const InspectorLogoPreview = ( { media, itemGroupProps } ) => {
 	const {
 		alt_text: alt,
 		source_url: logoUrl,
 		slug: logoSlug,
 		media_details: logoMediaDetails,
-	} = mediaItemData;
+	} = media ?? {};
 	const logoLabel = logoMediaDetails?.sizes?.full?.file || logoSlug;
 	return (
 		<ItemGroup { ...itemGroupProps } as="span">
@@ -530,7 +523,7 @@ export default function LogoEdit( {
 		name: ! logoUrl ? __( 'Choose logo' ) : __( 'Replace' ),
 		onSelect: onSelectLogo,
 		onError: onUploadError,
-		onRemoveLogo,
+		onReset: onRemoveLogo,
 	};
 	const controls = canUserEdit && (
 		<BlockControls group="other">
@@ -603,50 +596,40 @@ export default function LogoEdit( {
 		<InspectorControls>
 			<PanelBody title={ __( 'Media' ) }>
 				<div className="block-library-site-logo__inspector-media-replace-container">
-					{ ! canUserEdit && !! logoUrl && (
+					{ ! canUserEdit ? (
 						<InspectorLogoPreview
-							mediaItemData={ mediaItemData }
+							media={ mediaItemData }
 							itemGroupProps={ {
 								isBordered: true,
 								className:
 									'block-library-site-logo__inspector-readonly-logo-preview',
 							} }
 						/>
-					) }
-					{ canUserEdit && !! logoUrl && (
-						<SiteLogoReplaceFlow
-							{ ...mediaReplaceFlowProps }
-							name={
-								<InspectorLogoPreview
-									mediaItemData={ mediaItemData }
-								/>
-							}
-							popoverProps={ {} }
-						/>
-					) }
-					{ canUserEdit && ! logoUrl && (
-						<MediaUploadCheck>
-							<MediaUpload
-								onSelect={ onInitialSelectLogo }
-								allowedTypes={ ALLOWED_MEDIA_TYPES }
-								render={ ( { open } ) => (
-									<div className="block-library-site-logo__inspector-upload-container">
-										<Button
-											__next40pxDefaultSize
-											onClick={ open }
-											variant="secondary"
-										>
-											{ isLoading ? (
-												<Spinner />
-											) : (
-												__( 'Choose logo' )
-											) }
-										</Button>
-										<DropZone onFilesDrop={ onFilesDrop } />
-									</div>
+					) : (
+						<>
+							<SiteLogoReplaceFlow
+								{ ...mediaReplaceFlowProps }
+								name={
+									!! logoUrl ? (
+										<InspectorLogoPreview
+											media={ mediaItemData }
+										/>
+									) : (
+										__( 'Choose logo' )
+									)
+								}
+								renderToggle={ ( props ) => (
+									<Button { ...props } __next40pxDefaultSize>
+										{ temporaryURL ? (
+											<Spinner />
+										) : (
+											props.children
+										) }
+									</Button>
 								) }
 							/>
-						</MediaUploadCheck>
+							<DropZone onFilesDrop={ onFilesDrop } />
+						</>
 					) }
 				</div>
 			</PanelBody>
diff --git a/packages/block-library/src/site-logo/editor.scss b/packages/block-library/src/site-logo/editor.scss
index 2f9d1917079be..da6d8d0830411 100644
--- a/packages/block-library/src/site-logo/editor.scss
+++ b/packages/block-library/src/site-logo/editor.scss
@@ -108,17 +108,16 @@
 	}
 }
 
-.block-library-site-logo__inspector-upload-container {
+.block-library-site-logo__inspector-media-replace-container {
+	// Ensure the dropzone is positioned to the size of the item.
 	position: relative;
+
 	// Since there is no option to skip rendering the drag'n'drop icon in drop
 	// zone, we hide it for now.
 	.components-drop-zone__content-icon {
 		display: none;
 	}
-}
 
-.block-library-site-logo__inspector-upload-container,
-.block-library-site-logo__inspector-media-replace-container {
 	button.components-button {
 		color: $gray-900;
 		box-shadow: inset 0 0 0 1px $gray-400;
@@ -144,9 +143,7 @@
 		text-align: start;
 		text-align-last: center;
 	}
-}
 
-.block-library-site-logo__inspector-media-replace-container {
 	.components-dropdown {
 		display: block;
 	}