Skip to content

Commit 1650e1d

Browse files
ezhang6811liustve
andauthored
test: add contract tests for converse API (#747)
*Issue #, if available:* *Description of changes:* Similar to aws-observability/aws-otel-js-instrumentation#450 Add converse contract test to validate upstream behavior and be notified of changes in Converse API support. The converse_stream test is not included, as botocore is unable to parse a mocked streaming response - it requires a live HTTP connection to construct an EventStream object which our infra cannot provide. This is only covered by the NodeJS contract tests. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --------- Co-authored-by: Steve Liu <liustve@amazon.com>
1 parent d19ca2e commit 1650e1d

2 files changed

Lines changed: 51 additions & 0 deletions

File tree

contract-tests/images/applications/botocore/botocore_server.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,27 @@ def _handle_bedrock_request(self) -> None:
679679
bedrock_runtime_client.invoke_model(
680680
body=request_body, modelId=model_id, accept=accept, contentType=content_type
681681
)
682+
elif self.in_path("converse/converse"):
683+
set_main_status(200)
684+
bedrock_runtime_client.meta.events.register(
685+
"before-call.bedrock-runtime.Converse",
686+
lambda **kwargs: inject_200_success(
687+
output={"message": {"role": "assistant", "content": [{"text": "Hello! How can I help?"}]}},
688+
stopReason="end_turn",
689+
usage={"inputTokens": 12, "outputTokens": 8},
690+
**kwargs,
691+
),
692+
)
693+
bedrock_runtime_client.converse(
694+
modelId="anthropic.claude-v2:1",
695+
messages=[{"role": "user", "content": [{"text": "Hello"}]}],
696+
inferenceConfig={
697+
"maxTokens": 512,
698+
"temperature": 0.7,
699+
"topP": 0.9,
700+
"stopSequences": ["Human:"],
701+
},
702+
)
682703
else:
683704
set_main_status(404)
684705

contract-tests/tests/test/amazon/botocore/botocore_test.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
_GEN_AI_RESPONSE_FINISH_REASONS: str = "gen_ai.response.finish_reasons"
5959
_GEN_AI_USAGE_INPUT_TOKENS: str = "gen_ai.usage.input_tokens"
6060
_GEN_AI_USAGE_OUTPUT_TOKENS: str = "gen_ai.usage.output_tokens"
61+
_GEN_AI_REQUEST_STOP_SEQUENCES: str = "gen_ai.request.stop_sequences"
6162
_GEN_AI_SYSTEM: str = "gen_ai.system"
6263
_GEN_AI_RUNTIME_ID: str = "gen_ai.runtime.id"
6364
_GEN_AI_BROWSER_ID: str = "gen_ai.browser.id"
@@ -663,6 +664,35 @@ def test_bedrock_runtime_invoke_model_mistral(self):
663664
span_name="chat mistral.mistral-7b-instruct-v0:2",
664665
)
665666

667+
def test_bedrock_runtime_converse(self):
668+
self.do_test_requests(
669+
"bedrock/converse/converse",
670+
"GET",
671+
200,
672+
0,
673+
0,
674+
rpc_service="Bedrock Runtime",
675+
remote_service="AWS::BedrockRuntime",
676+
remote_operation="Converse",
677+
remote_resource_type="AWS::Bedrock::Model",
678+
remote_resource_identifier="anthropic.claude-v2:1",
679+
cloudformation_primary_identifier="anthropic.claude-v2:1",
680+
request_specific_attributes={
681+
_GEN_AI_REQUEST_MODEL: "anthropic.claude-v2:1",
682+
_GEN_AI_SYSTEM: "aws.bedrock",
683+
_GEN_AI_REQUEST_MAX_TOKENS: 512,
684+
_GEN_AI_REQUEST_TEMPERATURE: 0.7,
685+
_GEN_AI_REQUEST_TOP_P: 0.9,
686+
_GEN_AI_REQUEST_STOP_SEQUENCES: ["Human:"],
687+
},
688+
response_specific_attributes={
689+
_GEN_AI_RESPONSE_FINISH_REASONS: ["end_turn"],
690+
_GEN_AI_USAGE_INPUT_TOKENS: 12,
691+
_GEN_AI_USAGE_OUTPUT_TOKENS: 8,
692+
},
693+
span_name="chat anthropic.claude-v2:1",
694+
)
695+
666696
def test_bedrock_get_guardrail(self):
667697
self.do_test_requests(
668698
"bedrock/getguardrail/get-guardrail",

0 commit comments

Comments
 (0)