Skip to content

Commit 233f129

Browse files
slister1001Copilot
slister1001
and
Copilot
authored
[Evaluation] Fix model_config 404 for Foundry-style endpoints in red team (#45502)
* Fix model_config 404 for Foundry-style endpoints Detect Foundry-style Azure endpoints (*.services.ai.azure.com) in get_chat_target() and append /openai/v1 before passing to PyRIT's OpenAIChatTarget. Without this, PyRIT constructs URLs like https://resource.services.ai.azure.com/chat/completions which returns 404. The correct URL path is /openai/v1/chat/completions. Traditional Azure OpenAI endpoints (*.openai.azure.com) are not affected. Fixes: Tests 1.10, 1.11, 1.13, 1.14 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add unit tests for Foundry endpoint URL normalization Test that *.services.ai.azure.com endpoints get /openai/v1 appended, double-append is prevented, trailing slashes are handled, and traditional *.openai.azure.com endpoints are not modified. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Format with black Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address review comments: improve endpoint normalization, consolidate CHANGELOG Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Apply black formatting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use urlparse for Foundry endpoint detection (review feedback) Replace substring matching with proper hostname parsing via urlparse to fix case-sensitivity (RFC 4343) and false-positive URL matching. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent d13f84c commit 233f129

4 files changed

Lines changed: 157 additions & 7 deletions

File tree

‎sdk/evaluation/azure-ai-evaluation/CHANGELOG.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Release History
22

3-
## 1.15.3 (Unreleased)
3+
## 1.16.0 (Unreleased)
44

55
### Bugs Fixed
6-
6+
- Fixed `NotFoundError: 404` when using `model_config` dict target with Foundry-style endpoints (`*.services.ai.azure.com`) by appending `/openai/v1` to the endpoint URL for PyRIT compatibility.
77
- Fixed red team scan status stuck at `in_progress` in results.json despite the scan completing, by treating leftover `pending` entries as `failed`.
88
- Fixed `ungrounded_attributes` risk category being silently skipped due to a cache key mismatch (`isa` vs `ungrounded_attributes`) in the Foundry execution path.
99
- Fixed RAI evaluation service errors (`ServiceInvocationException`) incorrectly inflating attack success rate by treating error responses as undetermined instead of attack success.

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
# ---------------------------------------------------------
44
# represents upcoming version
55

6-
VERSION = "1.15.3"
6+
VERSION = "1.16.0"

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_utils/strategy_utils.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import random
66
from typing import Dict, List, Union, Optional, Any, Callable, cast
7+
from urllib.parse import urlparse
78
import logging
89

910
import httpx
@@ -217,14 +218,26 @@ def _message_to_dict(message):
217218
chat_target = None
218219
if not isinstance(target, Callable):
219220
if "azure_deployment" in target and "azure_endpoint" in target: # Azure OpenAI
221+
# Fix Foundry-style endpoints for PyRIT compatibility
222+
# Foundry endpoints (*.services.ai.azure.com) need /openai/v1 appended
223+
# because PyRIT's OpenAIChatTarget passes the URL directly to AsyncOpenAI(base_url=)
224+
endpoint = target["azure_endpoint"].rstrip("/")
225+
parsed = urlparse(endpoint)
226+
hostname = (parsed.hostname or "").lower()
227+
if hostname.endswith(".services.ai.azure.com"):
228+
if endpoint.endswith("/openai"):
229+
endpoint = endpoint + "/v1"
230+
elif not endpoint.endswith("/openai/v1"):
231+
endpoint = endpoint + "/openai/v1"
232+
220233
api_key = target.get("api_key", None)
221234
# Check for credential in target dict or use passed credential parameter
222235
target_credential = target.get("credential", None) or credential
223236
if api_key:
224237
# Use API key authentication
225238
chat_target = OpenAIChatTarget(
226239
model_name=target["azure_deployment"],
227-
endpoint=target["azure_endpoint"],
240+
endpoint=endpoint,
228241
api_key=api_key,
229242
httpx_client_kwargs={"timeout": timeout},
230243
)
@@ -233,7 +246,7 @@ def _message_to_dict(message):
233246
token_provider = _create_token_provider(target_credential)
234247
chat_target = OpenAIChatTarget(
235248
model_name=target["azure_deployment"],
236-
endpoint=target["azure_endpoint"],
249+
endpoint=endpoint,
237250
api_key=token_provider, # PyRIT accepts callable that returns token
238251
httpx_client_kwargs={"timeout": timeout},
239252
)
@@ -243,7 +256,7 @@ def _message_to_dict(message):
243256

244257
chat_target = OpenAIChatTarget(
245258
model_name=target["azure_deployment"],
246-
endpoint=target["azure_endpoint"],
259+
endpoint=endpoint,
247260
api_key=get_azure_openai_auth(target["azure_endpoint"]),
248261
httpx_client_kwargs={"timeout": timeout},
249262
)

‎sdk/evaluation/azure-ai-evaluation/tests/unittests/test_redteam/test_strategy_utils.py‎

Lines changed: 138 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,140 @@ def simple_fn(query):
438438
# Verify we get a callback target
439439
assert isinstance(result, _CallbackChatTarget)
440440

441+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
442+
def test_get_chat_target_foundry_endpoint_appends_openai_v1(self, mock_openai_chat_target):
443+
"""Test that Foundry-style endpoints get /openai/v1 appended."""
444+
mock_instance = MagicMock()
445+
mock_openai_chat_target.return_value = mock_instance
446+
447+
config = {
448+
"azure_deployment": "gpt-4o",
449+
"azure_endpoint": "https://my-resource.services.ai.azure.com",
450+
"api_key": "test-key",
451+
}
452+
453+
result = get_chat_target(config)
454+
455+
call_kwargs = mock_openai_chat_target.call_args[1]
456+
assert (
457+
call_kwargs["endpoint"] == "https://my-resource.services.ai.azure.com/openai/v1"
458+
), f"Foundry endpoint should have /openai/v1 appended, got: {call_kwargs['endpoint']}"
459+
assert call_kwargs["model_name"] == "gpt-4o"
460+
461+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
462+
def test_get_chat_target_foundry_endpoint_openai_without_v1(self, mock_openai_chat_target):
463+
"""Test that Foundry endpoint ending in /openai (without /v1) gets upgraded."""
464+
mock_instance = MagicMock()
465+
mock_openai_chat_target.return_value = mock_instance
466+
467+
config = {
468+
"azure_deployment": "gpt-4o",
469+
"azure_endpoint": "https://my-resource.services.ai.azure.com/openai",
470+
"api_key": "test-key",
471+
}
472+
473+
result = get_chat_target(config)
474+
475+
call_kwargs = mock_openai_chat_target.call_args[1]
476+
assert (
477+
call_kwargs["endpoint"] == "https://my-resource.services.ai.azure.com/openai/v1"
478+
), f"Endpoint ending in /openai should be upgraded to /openai/v1, got: {call_kwargs['endpoint']}"
479+
480+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
481+
def test_get_chat_target_foundry_endpoint_no_double_append(self, mock_openai_chat_target):
482+
"""Test that already-normalized Foundry endpoints don't get /openai/v1 doubled."""
483+
mock_instance = MagicMock()
484+
mock_openai_chat_target.return_value = mock_instance
485+
486+
config = {
487+
"azure_deployment": "gpt-4o",
488+
"azure_endpoint": "https://my-resource.services.ai.azure.com/openai/v1",
489+
"api_key": "test-key",
490+
}
491+
492+
result = get_chat_target(config)
493+
494+
call_kwargs = mock_openai_chat_target.call_args[1]
495+
assert (
496+
call_kwargs["endpoint"] == "https://my-resource.services.ai.azure.com/openai/v1"
497+
), f"Already-normalized endpoint should not be modified, got: {call_kwargs['endpoint']}"
498+
499+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
500+
def test_get_chat_target_foundry_endpoint_with_trailing_slash(self, mock_openai_chat_target):
501+
"""Test that Foundry endpoint with trailing slash is handled correctly."""
502+
mock_instance = MagicMock()
503+
mock_openai_chat_target.return_value = mock_instance
504+
505+
config = {
506+
"azure_deployment": "gpt-4o",
507+
"azure_endpoint": "https://my-resource.services.ai.azure.com/",
508+
"api_key": "test-key",
509+
}
510+
511+
result = get_chat_target(config)
512+
513+
call_kwargs = mock_openai_chat_target.call_args[1]
514+
assert (
515+
call_kwargs["endpoint"] == "https://my-resource.services.ai.azure.com/openai/v1"
516+
), f"Trailing slash should be stripped before appending, got: {call_kwargs['endpoint']}"
517+
518+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
519+
def test_get_chat_target_traditional_aoai_not_modified(self, mock_openai_chat_target):
520+
"""Test that traditional Azure OpenAI endpoints are NOT modified."""
521+
mock_instance = MagicMock()
522+
mock_openai_chat_target.return_value = mock_instance
523+
524+
config = {
525+
"azure_deployment": "gpt-4o",
526+
"azure_endpoint": "https://my-resource.openai.azure.com",
527+
"api_key": "test-key",
528+
}
529+
530+
result = get_chat_target(config)
531+
532+
call_kwargs = mock_openai_chat_target.call_args[1]
533+
assert (
534+
call_kwargs["endpoint"] == "https://my-resource.openai.azure.com"
535+
), f"Traditional AOAI endpoint should not be modified, got: {call_kwargs['endpoint']}"
536+
537+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
538+
def test_get_chat_target_foundry_endpoint_case_insensitive(self, mock_openai_chat_target):
539+
"""Test that Foundry hostname detection is case-insensitive (RFC 4343)."""
540+
mock_instance = MagicMock()
541+
mock_openai_chat_target.return_value = mock_instance
542+
543+
config = {
544+
"azure_deployment": "gpt-4o",
545+
"azure_endpoint": "https://my-resource.SERVICES.AI.AZURE.COM",
546+
"api_key": "test-key",
547+
}
548+
549+
result = get_chat_target(config)
550+
551+
call_kwargs = mock_openai_chat_target.call_args[1]
552+
assert (
553+
call_kwargs["endpoint"] == "https://my-resource.SERVICES.AI.AZURE.COM/openai/v1"
554+
), f"Case-insensitive hostname should be detected, got: {call_kwargs['endpoint']}"
555+
556+
@patch("azure.ai.evaluation.red_team._utils.strategy_utils.OpenAIChatTarget")
557+
def test_get_chat_target_non_foundry_url_with_matching_substring_not_modified(self, mock_openai_chat_target):
558+
"""Test that non-Foundry URLs containing .services.ai.azure.com in the path are NOT modified."""
559+
mock_instance = MagicMock()
560+
mock_openai_chat_target.return_value = mock_instance
561+
562+
config = {
563+
"azure_deployment": "gpt-4o",
564+
"azure_endpoint": "https://my-resource.openai.azure.com",
565+
"api_key": "test-key",
566+
}
567+
568+
result = get_chat_target(config)
569+
570+
call_kwargs = mock_openai_chat_target.call_args[1]
571+
assert (
572+
call_kwargs["endpoint"] == "https://my-resource.openai.azure.com"
573+
), f"Non-Foundry endpoint should not be modified, got: {call_kwargs['endpoint']}"
574+
441575

442576
@pytest.mark.unittest
443577
class TestHttpxTimeoutConfiguration:
@@ -458,7 +592,10 @@ def test_custom_http_timeout(self, mock_openai_chat_target):
458592
call_kwargs = mock_openai_chat_target.call_args[1]
459593
assert call_kwargs["httpx_client_kwargs"] == {
460594
"timeout": httpx.Timeout(
461-
connect=DEFAULT_CONNECT_TIMEOUT, read=300, write=DEFAULT_WRITE_TIMEOUT, pool=DEFAULT_POOL_TIMEOUT
595+
connect=DEFAULT_CONNECT_TIMEOUT,
596+
read=300,
597+
write=DEFAULT_WRITE_TIMEOUT,
598+
pool=DEFAULT_POOL_TIMEOUT,
462599
)
463600
}
464601

0 commit comments

Comments
 (0)