Python: updated azure ai inference sample (microsoft#5028)

* updated azure ai inference sample

* openai multimodel fix

* update language

Author: eavanvalkenburg

python/packages/core/agent_framework/azure/__init__.py

@@ -15,6 +15,10 @@
     "AzureAISearchContextProvider": ("agent_framework_azure_ai_search", "agent-framework-azure-ai-search"),
     "AzureAISearchSettings": ("agent_framework_azure_ai_search", "agent-framework-azure-ai-search"),
     "AzureAISettings": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
+    "AzureAIInferenceEmbeddingClient": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
+    "AzureAIInferenceEmbeddingOptions": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
+    "AzureAIInferenceEmbeddingSettings": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
+    "RawAzureAIInferenceEmbeddingClient": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
     "AzureCredentialTypes": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
     "AzureTokenProvider": ("agent_framework_azure_ai", "agent-framework-azure-ai"),
     "DurableAIAgent": ("agent_framework_durabletask", "agent-framework-durabletask"),

python/packages/core/agent_framework/azure/__init__.pyi

@@ -4,9 +4,13 @@
 # Install the relevant packages for full type support.
 
 from agent_framework_azure_ai import (
+    AzureAIInferenceEmbeddingClient,
+    AzureAIInferenceEmbeddingOptions,
+    AzureAIInferenceEmbeddingSettings,
     AzureAISettings,
     AzureCredentialTypes,
     AzureTokenProvider,
+    RawAzureAIInferenceEmbeddingClient,
 )
 from agent_framework_azure_ai_search import (
     AzureAISearchContextProvider,
@@ -26,6 +30,9 @@ __all__ = [
     "AgentCallbackContext",
     "AgentFunctionApp",
     "AgentResponseCallbackProtocol",
+    "AzureAIInferenceEmbeddingClient",
+    "AzureAIInferenceEmbeddingOptions",
+    "AzureAIInferenceEmbeddingSettings",
     "AzureAISearchContextProvider",
     "AzureAISearchSettings",
     "AzureAISettings",
@@ -35,4 +42,5 @@ __all__ = [
     "DurableAIAgentClient",
     "DurableAIAgentOrchestrationContext",
     "DurableAIAgentWorker",
+    "RawAzureAIInferenceEmbeddingClient",
 ]

python/samples/02-agents/embeddings/azure_ai_inference_embeddings.py

@@ -12,7 +12,7 @@
 import pathlib
 
 from agent_framework import Content
-from agent_framework_azure_ai import AzureAIInferenceEmbeddingClient
+from agent_framework.azure import AzureAIInferenceEmbeddingClient
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -24,8 +24,12 @@
 Images are passed as ``Content`` objects created with ``Content.from_data()``.
 
 Prerequisites:
-    Set the following environment variables or add them to a .env file:
-    - AZURE_AI_INFERENCE_ENDPOINT: Your Azure AI model inference endpoint URL
+    Deploy an embedding model in Azure AI Inference that supports image inputs, such as Cohere-embed-v3-english.
+
+    The details page for that model, has a target URI and a Key, which should be set in environment variables or a .env
+    file as follows, the target URI should append the `/models` path:
+    - AZURE_AI_INFERENCE_ENDPOINT: Your Azure AI model inference endpoint URL, for instance:
+        https://<apim-instance>.azure-api.net/<foundry-instance>/models
     - AZURE_AI_INFERENCE_API_KEY: Your API key
     - AZURE_AI_INFERENCE_EMBEDDING_MODEL_ID: The text embedding model name
       (e.g. "text-embedding-3-small")
@@ -73,15 +77,17 @@ async def main() -> None:
 
 
 """
-Sample output (using Cohere-embed-v3-english):
+Sample output (using deployment: Cohere-embed-v3-english, which is Cohere's "embed-english-v3.0-image" model):
 Image embedding dimensions: 1024
-First 5 values: [0.023, -0.045, 0.067, -0.089, 0.011]
-Model: Cohere-embed-v3-english
-Usage: {'prompt_tokens': 1, 'total_tokens': 1}
+First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865]
+Model: embed-english-v3.0-image
+Usage: {'input_token_count': 1000, 'output_token_count': 0}
 
-Image+text (separate) results:
 Text embedding dimensions: 1536
+First 5 values: [-0.019439403, 0.015791258, 0.012358093, 0.0028533707, -0.01649483]
 Image embedding dimensions: 1024
+First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865]
 
 Document embedding dimensions: 1024
+First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865]
 """

python/samples/02-agents/multimodal_input/openai_chat_multimodal.py

@@ -6,14 +6,21 @@
 from pathlib import Path
 
 from agent_framework import Content, Message
-from agent_framework.foundry import FoundryChatClient
+from agent_framework.openai import OpenAIChatClient, OpenAIChatCompletionClient
 from dotenv import load_dotenv
 
 # Load environment variables from .env file
 load_dotenv()
 
 ASSETS_DIR = Path(__file__).resolve().parents[2] / "shared" / "sample_assets"
 
+"""
+Leverage multimodel capabilities of different models.
+
+Uses the OpenAIChatClient and OpenAIChatCompletionClient to demonstrate multimodal input handling with the gpt-4o and gpt-4o-audio-preview models, respectively. The sample includes demonstrations for image, audio, and PDF inputs, showcasing how to create appropriate Content objects and send them in messages to the chat clients.
+
+"""
+
 
 def load_sample_pdf() -> bytes:
     """Read the bundled sample PDF for tests."""
@@ -46,7 +53,7 @@ def create_sample_audio() -> str:
 
 async def test_image() -> None:
     """Test image analysis with OpenAI."""
-    client = FoundryChatClient(model="gpt-4o")
+    client = OpenAIChatClient(model="gpt-4o")
 
     image_uri = create_sample_image()
     message = Message(
@@ -63,7 +70,7 @@ async def test_image() -> None:
 
 async def test_audio() -> None:
     """Test audio analysis with OpenAI."""
-    client = FoundryChatClient(model="gpt-4o-audio-preview")
+    client = OpenAIChatCompletionClient(model="gpt-4o-audio-preview-2025-06-03")
 
     audio_uri = create_sample_audio()
     message = Message(
@@ -80,7 +87,7 @@ async def test_audio() -> None:
 
 async def test_pdf() -> None:
     """Test PDF document analysis with OpenAI."""
-    client = FoundryChatClient(model="gpt-4o")
+    client = OpenAIChatClient(model="gpt-4o")
 
     pdf_bytes = load_sample_pdf()
     message = Message(