AI Featured Image: support backend prompts (#37668)

* Export generic image generation method by parameters * Change generation call to a generic one so the backend can decide which one to use * Introduce activeModel cost and use it to infer the model name * changelog Committed via a GitHub action: https://github.com/Automattic/jetpack/actions/runs/9355495915 Upstream-Ref: Automattic/jetpack@7ad86f9
Automattic · Jun 3, 2024 · a9c9481 · a9c9481
1 parent 6efea7f
commit a9c9481
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 60 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 ### This is a list detailing changes for all Jetpack releases.
 
+## 13.6-a.0 - unreleased
+
+This is an alpha version! The changes listed here are not final.
+
+### Other changes <!-- Non-user-facing changes go here. This section will not be copied to readme.txt. -->
+- AI Featured Image: let the backend decide the model for the image generation.
+
 ## 13.5-beta - 2024-06-03
 ### Enhancements
 - AI Assistant: Fallback to transformation when multiple blocks are selected. [#37632]

diff --git a/_inc/blocks/editor-beta.asset.php b/_inc/blocks/editor-beta.asset.php
@@ -1 +1 @@
-<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '1b8c2e0b520ff701a1f5');
+<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '44ae0b0597d90ceb0e30');
diff --git a/_inc/blocks/editor-beta.js b/_inc/blocks/editor-beta.js
diff --git a/_inc/blocks/editor-experimental.asset.php b/_inc/blocks/editor-experimental.asset.php
@@ -1 +1 @@
-<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => 'dd31d027322e454dea88');
+<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '0c221a1d53da55dc6c31');
diff --git a/_inc/blocks/editor-experimental.js b/_inc/blocks/editor-experimental.js
diff --git a/_inc/blocks/editor-no-post-editor.asset.php b/_inc/blocks/editor-no-post-editor.asset.php
@@ -1 +1 @@
-<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets'), 'version' => 'bda03b0be6a1fa4d96a0');
+<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets'), 'version' => '943ebc51baa07de59788');
diff --git a/_inc/blocks/editor-no-post-editor.js b/_inc/blocks/editor-no-post-editor.js
diff --git a/_inc/blocks/editor.asset.php b/_inc/blocks/editor.asset.php
@@ -1 +1 @@
-<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => 'caccf1506bf2ebe6cd47');
+<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '53690cab00c7dfbbe591');
diff --git a/_inc/blocks/editor.js b/_inc/blocks/editor.js
diff --git a/extensions/plugins/ai-assistant-plugin/components/featured-image/index.tsx b/extensions/plugins/ai-assistant-plugin/components/featured-image/index.tsx
@@ -18,7 +18,6 @@ import { Icon, external } from '@wordpress/icons';
 import './style.scss';
 import UpgradePrompt from '../../../../blocks/ai-assistant/components/upgrade-prompt';
 import useAiFeature from '../../../../blocks/ai-assistant/hooks/use-ai-feature';
-import { getFeatureAvailability } from '../../../../blocks/ai-assistant/lib/utils/get-feature-availability';
 import { PLAN_TYPE_UNLIMITED, usePlanType } from '../../../../shared/use-plan-type';
 import usePostContent from '../../hooks/use-post-content';
 import useSaveToMediaLibrary from '../../hooks/use-save-to-media-library';
@@ -34,17 +33,8 @@ const FEATURED_IMAGE_UPGRADE_PROMPT_PLACEMENT = 'ai-image-generator';
 const FEATURED_IMAGE_FEATURE_NAME = 'featured-post-image';
 export const FEATURED_IMAGE_PLACEMENT_MEDIA_SOURCE_DROPDOWN = 'media-source-dropdown';
 
-/**
- * Control experimental image generation for the featured image.
- */
-const AI_ASSISTANT_EXPERIMENTAL_IMAGE_GENERATION_SUPPORT =
-	'ai-assistant-experimental-image-generation-support';
-const isAiAssistantExperimentalImageGenerationSupportEnabled = getFeatureAvailability(
-	AI_ASSISTANT_EXPERIMENTAL_IMAGE_GENERATION_SUPPORT
-);
-const IMAGE_GENERATION_MODEL = isAiAssistantExperimentalImageGenerationSupportEnabled
-	? 'stable-diffusion'
-	: 'dall-e-3';
+const IMAGE_GENERATION_MODEL_STABLE_DIFFUSION = 'stable-diffusion';
+const IMAGE_GENERATION_MODEL_DALL_E_3 = 'dall-e-3';
 /**
  * Determine the site type for tracking purposes.
  *
@@ -86,7 +76,7 @@ export default function FeaturedImage( {
 	const triggeredAutoGeneration = useRef( false );
 
 	const { enableComplementaryArea } = useDispatch( 'core/interface' );
-	const { generateImage, generateImageWithStableDiffusion } = useImageGenerator();
+	const { generateImageWithParameters } = useImageGenerator();
 	const { saveToMediaLibrary } = useSaveToMediaLibrary();
 	const { tracks } = useAnalytics();
 	const { recordEvent } = tracks;
@@ -104,9 +94,11 @@ export default function FeaturedImage( {
 		costs,
 	} = useAiFeature();
 	const planType = usePlanType( currentTier );
-	const featuredImageCost = isAiAssistantExperimentalImageGenerationSupportEnabled
-		? costs?.[ FEATURED_IMAGE_FEATURE_NAME ]?.stableDiffusion ?? 1
-		: costs?.[ FEATURED_IMAGE_FEATURE_NAME ]?.image;
+	const featuredImageCost = costs?.[ FEATURED_IMAGE_FEATURE_NAME ]?.activeModel ?? 10;
+	const featuredImageActiveModel =
+		featuredImageCost === costs?.[ FEATURED_IMAGE_FEATURE_NAME ]?.stableDiffusion
+			? IMAGE_GENERATION_MODEL_STABLE_DIFFUSION
+			: IMAGE_GENERATION_MODEL_DALL_E_3;
 	const isUnlimited = planType === PLAN_TYPE_UNLIMITED;
 	const requestsBalance = requestsLimit - requestsCount;
 	const notEnoughRequests = requestsBalance < featuredImageCost;
@@ -150,12 +142,12 @@ export default function FeaturedImage( {
 				recordEvent( 'jetpack_ai_featured_image_generation_error', {
 					placement,
 					error: data.error?.message,
-					model: IMAGE_GENERATION_MODEL,
+					model: featuredImageActiveModel,
 					site_type: SITE_TYPE,
 				} );
 			}
 		},
-		[ placement, recordEvent ]
+		[ placement, recordEvent, featuredImageActiveModel ]
 	);
 
 	const handlePreviousImage = useCallback( () => {
@@ -203,19 +195,25 @@ export default function FeaturedImage( {
 			return;
 		}
 
-		/** Decide between standard or experimental generation */
-		const generateImagePromise = isAiAssistantExperimentalImageGenerationSupportEnabled
-			? generateImageWithStableDiffusion( {
-					feature: FEATURED_IMAGE_FEATURE_NAME,
-					postContent,
-					userPrompt,
-			  } )
-			: generateImage( {
-					feature: FEATURED_IMAGE_FEATURE_NAME,
-					postContent,
-					responseFormat: 'b64_json',
-					userPrompt,
-			  } );
+		/**
+		 * Make a generic call to backend and let it decide the model.
+		 */
+		const generateImagePromise = generateImageWithParameters( {
+			feature: FEATURED_IMAGE_FEATURE_NAME,
+			size: '1792x1024', // the size, when the generation happens with DALL-E-3
+			responseFormat: 'b64_json', // the response format, when the generation happens with DALL-E-3
+			style: 'photographic', // the style of the image, when the generation happens with Stable Diffusion
+			messages: [
+				{
+					role: 'jetpack-ai',
+					context: {
+						type: 'featured-image-generation',
+						request: userPrompt ? userPrompt : null,
+						content: postContent,
+					},
+				},
+			],
+		} );
 
 		generateImagePromise
 			.then( result => {
@@ -240,8 +238,7 @@ export default function FeaturedImage( {
 	}, [
 		notEnoughRequests,
 		updateImages,
-		generateImage,
-		generateImageWithStableDiffusion,
+		generateImageWithParameters,
 		postContent,
 		userPrompt,
 		updateRequestsCount,
@@ -261,36 +258,42 @@ export default function FeaturedImage( {
 		// track the generate image event
 		recordEvent( 'jetpack_ai_featured_image_generation_generate_image', {
 			placement,
-			model: IMAGE_GENERATION_MODEL,
+			model: featuredImageActiveModel,
 			site_type: SITE_TYPE,
 		} );
 
 		toggleFeaturedImageModal();
 		processImageGeneration();
-	}, [ toggleFeaturedImageModal, processImageGeneration, recordEvent, placement ] );
+	}, [
+		toggleFeaturedImageModal,
+		processImageGeneration,
+		recordEvent,
+		placement,
+		featuredImageActiveModel,
+	] );
 
 	const handleRegenerate = useCallback( () => {
 		// track the regenerate image event
 		recordEvent( 'jetpack_ai_featured_image_generation_generate_another_image', {
 			placement,
-			model: IMAGE_GENERATION_MODEL,
+			model: featuredImageActiveModel,
 			site_type: SITE_TYPE,
 		} );
 
 		processImageGeneration();
 		setCurrent( crrt => crrt + 1 );
-	}, [ processImageGeneration, recordEvent, placement ] );
+	}, [ processImageGeneration, recordEvent, placement, featuredImageActiveModel ] );
 
 	const handleTryAgain = useCallback( () => {
 		// track the try again event
 		recordEvent( 'jetpack_ai_featured_image_generation_try_again', {
 			placement,
-			model: IMAGE_GENERATION_MODEL,
+			model: featuredImageActiveModel,
 			site_type: SITE_TYPE,
 		} );
 
 		processImageGeneration();
-	}, [ processImageGeneration, recordEvent, placement ] );
+	}, [ processImageGeneration, recordEvent, placement, featuredImageActiveModel ] );
 
 	const handleUserPromptChange = useCallback(
 		( e: React.ChangeEvent< HTMLTextAreaElement > ) => {
@@ -309,7 +312,7 @@ export default function FeaturedImage( {
 		// track the accept/use image event
 		recordEvent( 'jetpack_ai_featured_image_generation_use_image', {
 			placement,
-			model: IMAGE_GENERATION_MODEL,
+			model: featuredImageActiveModel,
 			site_type: SITE_TYPE,
 		} );
 
@@ -355,6 +358,7 @@ export default function FeaturedImage( {
 		triggerComplementaryArea,
 		handleModalClose,
 		placement,
+		featuredImageActiveModel,
 	] );
 
 	/**
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '1b8c2e0b520ff701a1f5');
		<?php return array('dependencies' => array('lodash', 'moment', 'react', 'react-dom', 'wp-a11y', 'wp-api-fetch', 'wp-blob', 'wp-block-editor', 'wp-block-serialization-default-parser', 'wp-blocks', 'wp-components', 'wp-compose', 'wp-core-data', 'wp-data', 'wp-date', 'wp-dom-ready', 'wp-edit-post', 'wp-editor', 'wp-element', 'wp-escape-html', 'wp-hooks', 'wp-html-entities', 'wp-i18n', 'wp-keycodes', 'wp-notices', 'wp-plugins', 'wp-polyfill', 'wp-primitives', 'wp-rich-text', 'wp-token-list', 'wp-url', 'wp-viewport', 'wp-widgets', 'wp-wordcount'), 'version' => '44ae0b0597d90ceb0e30');