Merge pull request #374 from dkotter/fix/alt-text-instructions

Adjust alt text generation to more closely match the W3C Alt decision tree

Co-authored-by: dkotter <dkotter@git.wordpress.org>
Co-authored-by: afercia <afercia@git.wordpress.org>

Author: 3 people

includes/Abilities/Image/Alt_Text_Generation.php

@@ -18,7 +18,7 @@
 /**
  * Alt text generation WordPress Ability.
  *
- * Uses AI vision models to generate descriptive alt text for images.
+ * Uses AI vision models to propose alt text aligned with WCAG-oriented practice.
  *
  * @since 0.3.0
  */
@@ -33,6 +33,15 @@ class Alt_Text_Generation extends Abstract_Ability {
 	 */
 	protected const MAX_ALT_TEXT_LENGTH = 125;
 
+	/**
+	 * Model output token that means the correct alternative text is empty (alt="").
+	 *
+	 * @since x.x.x
+	 *
+	 * @var string
+	 */
+	private const DECORATIVE_ALT_TOKEN = '[[DECORATIVE_ALT]]';
+
 	/**
 	 * {@inheritDoc}
 	 *
@@ -57,6 +66,10 @@ protected function input_schema(): array {
 					'sanitize_callback' => 'sanitize_textarea_field',
 					'description'       => esc_html__( 'Optional context about the image or surrounding content to improve alt text relevance.', 'ai' ),
 				),
+				'image_meta'    => array(
+					'type'        => 'string',
+					'description' => esc_html__( 'Structured metadata about how the image block is used, such as whether it is linked.', 'ai' ),
+				),
 			),
 		);
 	}
@@ -70,9 +83,13 @@ protected function output_schema(): array {
 		return array(
 			'type'       => 'object',
 			'properties' => array(
-				'alt_text' => array(
+				'alt_text'      => array(
 					'type'        => 'string',
-					'description' => esc_html__( 'Generated alt text for the image.', 'ai' ),
+					'description' => esc_html__( 'Generated alternative text for the image; may be empty when alt="" is correct.', 'ai' ),
+				),
+				'is_decorative' => array(
+					'type'        => 'boolean',
+					'description' => esc_html__( 'Whether the image was determined to be decorative.', 'ai' ),
 				),
 			),
 		);
@@ -91,6 +108,7 @@ protected function execute_callback( $input ) {
 				'attachment_id' => null,
 				'image_url'     => null,
 				'context'       => '',
+				'image_meta'    => '',
 			),
 		);
 
@@ -102,16 +120,21 @@ protected function execute_callback( $input ) {
 		}
 
 		// Generate the alt text.
-		$result = $this->generate_alt_text( $image_reference, normalize_content( $args['context'] ) );
+		$result = $this->generate_alt_text(
+			$image_reference,
+			normalize_content( $args['context'] ),
+			sanitize_textarea_field( $args['image_meta'] )
+		);
 
 		if ( is_wp_error( $result ) ) {
 			return $result;
 		}
 
-		if ( empty( $result ) ) {
-			return new WP_Error(
-				'no_results',
-				esc_html__( 'No alt text was generated.', 'ai' )
+		// Detect the decorative token from the AI response.
+		if ( 0 === strcasecmp( trim( $result ), self::DECORATIVE_ALT_TOKEN ) ) {
+			return array(
+				'alt_text'      => '',
+				'is_decorative' => true,
 			);
 		}
 
@@ -185,10 +208,11 @@ protected function get_image_reference( array $args ) {
 	 *
 	 * @param array{reference: string} $image_reference Prepared image reference containing a data URI.
 	 * @param string                   $context         Optional context to improve alt text relevance.
+	 * @param string                   $image_meta      Optional metadata about how the image block is used.
 	 * @return string|\WP_Error The generated alt text or WP_Error on failure.
 	 */
-	protected function generate_alt_text( array $image_reference, string $context = '' ) {
-		$result = wp_ai_client_prompt( $this->build_prompt( $context ) )
+	protected function generate_alt_text( array $image_reference, string $context = '', string $image_meta = '' ) {
+		$result = wp_ai_client_prompt( $this->build_prompt( $context, $image_meta ) )
 			->with_file( $image_reference['reference'] )
 			->using_system_instruction( $this->get_system_instruction( 'alt-text-system-instruction.php' ) )
 			->using_temperature( 0.3 )
@@ -368,12 +392,18 @@ protected function normalize_upload_url( string $url ): string {
 	 *
 	 * @since 0.3.0
 	 *
-	 * @param string $context Optional context about the image.
+	 * @param string $context    Optional context about the image.
+	 * @param string $image_meta Optional metadata about how the image block is used.
 	 * @return string The prompt for the AI.
 	 */
-	protected function build_prompt( string $context = '' ): string {
+	protected function build_prompt( string $context = '', string $image_meta = '' ): string {
 		$prompt = __( 'Generate alt text for this image.', 'ai' );
 
+		// If we have image block usage metadata, add it to the prompt.
+		if ( ! empty( $image_meta ) ) {
+			$prompt .= "\n\n<image-meta>" . $image_meta . '</image-meta>';
+		}
+
 		// If we have additional context, add it to the prompt.
 		if ( ! empty( $context ) ) {
 			$prompt .= "\n\n<additional-context>" . $context . '</additional-context>';

includes/Abilities/Image/alt-text-system-instruction.php

@@ -12,21 +12,35 @@
 
 // phpcs:ignore Squiz.PHP.Heredoc.NotAllowed
 return <<<'INSTRUCTION'
-You are an accessibility expert that generates alt text for images on websites.
+You are an accessibility expert that proposes alternative (alt) text for HTML images. Your output must follow the same decisions authors make with the W3C "An alt Decision Tree" (decorative vs functional vs informative vs complex images).
 
-Goal: Analyze the provided image and generate concise, descriptive alt text that accurately describes the image content for users who cannot see it. The alt text should be optimized for screen readers and accessibility compliance. If additional context is provided, use it to generate a more relevant alt text.
+Core rule: Alt text is not always a description of what the picture looks like. It must convey the information or purpose that the image serves in this specific context. If the image disappeared, what would be lost for someone who cannot see it—that is what belongs in alt text (or in empty alt when nothing should be announced).
 
-Requirements for the alt text:
+Follow this order:
 
-- Be concise: Keep it under 125 characters when possible
-- Be descriptive: Describe what is visually present in the image
-- Be objective: Describe what you see, not interpretations or assumptions
-- Avoid redundancy: Do not start with "Image of", "Picture of", or "Photo of"
-- Include relevant details: People, objects, actions, colors, and context when meaningful
-- Consider context: If context is provided, ensure the alt text is relevant to the surrounding content
-- Plain text only: No markdown, quotes, or special formatting
+1) Decorative or redundant?
+- Purely decorative (flourish, spacer, visual-only styling) OR the same information is already in adjacent text (including visible link text in the same link as the image).
+- Output: respond with exactly this token and nothing else: [[DECORATIVE_ALT]]
+- Do not describe the image for decorative/redundant cases.
 
-For images containing text, include the text in your description if it's essential to understanding the image.
+2) Functional (image is a control or the main content of a link or button)?
+- Examples: linked image with no other text in the link; icon-only button; logo linking home.
+- Output: short text that describes the action or destination (where the link goes, what happens when activated)—not the photo or illustration subject.
+- If `<image-meta>` gives a URL or destination name, base the alt on that purpose. Do not substitute a visual description of the image.
 
-Respond with only the alt text, nothing else.
+3) Informative (image adds meaning that is not covered by nearby text)?
+- Output: concise objective description of the information the image communicates (people, objects, setting, actions) relevant to context.
+- Do not start with "Image of", "Picture of", or "Photo of".
+- If the image contains essential text, include that text in the alt (or summarize if it is very long, and note that a longer text alternative may be needed elsewhere).
+- If `<additional-context>` is provided, use it to understand the purpose, subject, and relevance of the image within the article. Be sure to describe only information not already conveyed in nearby text
+
+4) Complex (chart, diagram, infographic, detailed map)?
+- Output: a short summary of the main point; do not paste entire data sets into alt text. If context implies a longer description exists or should exist on the page, you may mention that the full explanation is in surrounding content.
+
+General requirements:
+- Prefer under 125 characters when possible (except when essential text in the image requires more).
+- Plain text only: no markdown, quotes, or labels—except the exact token [[DECORATIVE_ALT]] when appropriate.
+- Use any `<image-meta>` section and `<additional-context>` to decide role and wording; they override guessing from pixels alone.
+
+Respond with only the alt text string, or exactly [[DECORATIVE_ALT]] for empty alternative text—nothing else.
 INSTRUCTION;

includes/Abilities/Review_Notes/system-instruction.php

@@ -62,7 +62,7 @@
 The review types to perform for each block are provided in <review-types> tags.
 
 **core/image**
-- accessibility: The content in <block-content> is the alt text for the image. Ensure it isn't empty and is descriptive. Flag missing or generic alt text (e.g. "image", "photo", file name)
+- accessibility: The content in <block-content> is the alt text for the image. Empty alt can be correct for decorative images; flag missing or poor alt when the image appears informative, functional (e.g. linked), or redundant with adjacent text. Flag generic alt text (e.g. "image", "photo", file name) when it fails to convey purpose or content
 - Skip readability, grammar, and seo for image blocks
 
 **core/heading**

includes/Experiments/Alt_Text_Generation/Alt_Text_Generation.php

@@ -21,7 +21,7 @@
 /**
  * Alt text generation experiment.
  *
- * Generates descriptive alt text for images using AI vision models.
+ * Generates accessible alternative text for images using AI vision models.
  *
  * @since 0.3.0
  */
@@ -48,7 +48,7 @@ public static function get_id(): string {
 	protected function load_metadata(): array {
 		return array(
 			'label'       => __( 'Alt Text Generation', 'ai' ),
-			'description' => __( 'Generates descriptive alt text for images using AI vision models.', 'ai' ),
+			'description' => __( 'Generates accessible alternative (alt) text for images using AI vision models, following common web accessibility guidance.', 'ai' ),
 			'category'    => Experiment_Category::EDITOR,
 		);
 	}

src/experiments/alt-text-generation/components/AltTextControls.tsx

@@ -5,7 +5,12 @@
 /**
  * WordPress dependencies
  */
-import { Button, TextareaControl, Spinner } from '@wordpress/components';
+import {
+	Button,
+	TextareaControl,
+	Spinner,
+	Notice,
+} from '@wordpress/components';
 import { InspectorControls } from '@wordpress/block-editor';
 import { useState } from '@wordpress/element';
 import { __ } from '@wordpress/i18n';
@@ -57,6 +62,7 @@ export function AltTextControls( {
 
 	const [ isGenerating, setIsGenerating ] = useState< boolean >( false );
 	const [ generatedAlt, setGeneratedAlt ] = useState< string | null >( null );
+	const [ isDecorative, setIsDecorative ] = useState< boolean >( false );
 
 	// Don't show controls if there's no image.
 	if ( ! attachmentId && ! imageUrl ) {
@@ -72,6 +78,7 @@ export function AltTextControls( {
 	const handleGenerate = async () => {
 		setIsGenerating( true );
 		setGeneratedAlt( null );
+		setIsDecorative( false );
 
 		// Clear any previous notices.
 		( dispatch( noticesStore ) as any ).removeNotice(
@@ -84,9 +91,24 @@ export function AltTextControls( {
 				attachmentId,
 				imageUrl,
 				content,
-				clientId
+				clientId,
+				{
+					linkDestination: attributes?.linkDestination,
+					href: attributes?.href,
+					linkTarget: attributes?.linkTarget,
+					caption:
+						typeof attributes?.caption === 'string'
+							? attributes.caption
+							: ( attributes?.caption as any )?.text,
+				}
 			);
-			setGeneratedAlt( result );
+
+			if ( result.is_decorative ) {
+				setIsDecorative( true );
+				setGeneratedAlt( '' );
+			} else {
+				setGeneratedAlt( result.alt_text );
+			}
 		} catch ( err: any ) {
 			const errorMessage =
 				err?.message ||
@@ -107,17 +129,21 @@ export function AltTextControls( {
 	 * Applies the generated alt text to the image block.
 	 */
 	const handleApply = () => {
-		if ( generatedAlt ) {
+		if ( isDecorative ) {
+			setAttributes( { alt: '' } );
+		} else if ( generatedAlt ) {
 			setAttributes( { alt: generatedAlt } );
-			setGeneratedAlt( null );
 		}
+		setGeneratedAlt( null );
+		setIsDecorative( false );
 	};
 
 	/**
 	 * Dismisses the generated alt text suggestion.
 	 */
 	const handleDismiss = () => {
 		setGeneratedAlt( null );
+		setIsDecorative( false );
 	};
 
 	return (
@@ -127,7 +153,7 @@ export function AltTextControls( {
 				style={ { padding: '0 16px' } }
 			>
 				{ /* Generated alt text preview */ }
-				{ hasGeneratedAlt && (
+				{ hasGeneratedAlt && ! isDecorative && (
 					<div style={ { marginBottom: '12px' } }>
 						<TextareaControl
 							label={ __( 'Generated Alt Text', 'ai' ) }
@@ -157,8 +183,37 @@ export function AltTextControls( {
 					</div>
 				) }
 
+				{ /* Decorative image notice */ }
+				{ isDecorative && (
+					<div style={ { marginBottom: '12px' } }>
+						<Notice status="info" isDismissible={ false }>
+							{ __(
+								'This image appears to be decorative. Applying will set an empty alt attribute, which tells screen readers to skip it.',
+								'ai'
+							) }
+						</Notice>
+						<div
+							style={ {
+								display: 'flex',
+								gap: '8px',
+								marginTop: '8px',
+							} }
+						>
+							<Button variant="primary" onClick={ handleApply }>
+								{ __( 'Apply', 'ai' ) }
+							</Button>
+							<Button
+								variant="secondary"
+								onClick={ handleDismiss }
+							>
+								{ __( 'Dismiss', 'ai' ) }
+							</Button>
+						</div>
+					</div>
+				) }
+
 				{ /* Generate button */ }
-				{ ! hasGeneratedAlt && (
+				{ ! hasGeneratedAlt && ! isDecorative && (
 					<Button
 						variant="secondary"
 						onClick={ handleGenerate }

src/experiments/alt-text-generation/types.ts

@@ -9,6 +9,7 @@ export interface AltTextGenerationAbilityInput {
 	attachment_id?: number;
 	image_url?: string;
 	context?: string;
+	image_meta?: string;
 	[ key: string ]: string | number | undefined;
 }
 

src/experiments/image-generation/functions/upload-image.ts

@@ -62,17 +62,18 @@ export async function uploadImage(
 	if ( isAltTextEnabled ) {
 		try {
 			onProgress?.( __( 'Generating alt text…', 'ai' ) );
-			params.alt_text = await generateAltText(
+			const altResult = await generateAltText(
 				undefined,
 				`data:image/png;base64,${ image.data }`
 			);
+			params.alt_text = altResult.alt_text;
 		} catch ( error ) {
 			params.alt_text = prompt;
 		}
 	}
 
 	// Set our image title to be a trimmed version of the alt text.
-	params.title = trimText( params.alt_text );
+	params.title = trimText( params.alt_text ?? '' );
 
 	onProgress?.( __( 'Importing image…', 'ai' ) );