Skip to content

Commit 4f7009d

Browse files
Krishanx92Krishanx92
authored
Merge pull request #767 from nimsara66/provider-template-tests
Implement integration tests for provider template
2 parents 6a16fe6 + 7ef4999 commit 4f7009d

4 files changed

Lines changed: 319 additions & 0 deletions

File tree

‎gateway/gateway-controller/default-llm-provider-templates/mistral-template.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# --------------------------------------------------------------------
2+
# Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com).
3+
#
4+
# WSO2 LLC. licenses this file to you under the Apache License,
5+
# Version 2.0 (the "License"); you may not use this file except
6+
# in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
# --------------------------------------------------------------------
18+
19+
apiVersion: gateway.api-platform.wso2.com/v1alpha1
20+
kind: LlmProviderTemplate
21+
metadata:
22+
name: mistralai
23+
spec:
24+
displayName: MistralAI
25+
promptTokens:
26+
location: payload
27+
identifier: $.usage.prompt_tokens
28+
completionTokens:
29+
location: payload
30+
identifier: $.usage.completion_tokens
31+
totalTokens:
32+
location: payload
33+
identifier: $.usage.total_tokens
34+
remainingTokens:
35+
location: header
36+
identifier: x-ratelimit-remaining-tokens
37+
requestModel:
38+
location: payload
39+
identifier: $.model
40+
responseModel:
41+
location: payload
42+
identifier: $.model

‎gateway/it/features/llm-provider-templates.feature‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
# --------------------------------------------------------------------
2+
# Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com).
3+
#
4+
# WSO2 LLC. licenses this file to you under the Apache License,
5+
# Version 2.0 (the "License"); you may not use this file except
6+
# in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
# --------------------------------------------------------------------
18+
19+
Feature: LLM Provider Template Management
20+
As an API administrator
21+
I want to manage LLM provider templates in the gateway
22+
So that I can configure token tracking and model extraction metadata for different LLM providers
23+
24+
Background:
25+
Given the gateway services are running
26+
27+
# ========================================
28+
# Scenario Group 1: Template Lifecycle (Happy Path)
29+
# ========================================
30+
31+
Scenario: Complete template lifecycle - create, retrieve, update, and delete
32+
Given I authenticate using basic auth as "admin"
33+
When I create this LLM provider template:
34+
"""
35+
apiVersion: gateway.api-platform.wso2.com/v1alpha1
36+
kind: LlmProviderTemplate
37+
metadata:
38+
name: openai-test
39+
spec:
40+
displayName: OpenAI
41+
promptTokens:
42+
location: payload
43+
identifier: $.usage.prompt_tokens
44+
completionTokens:
45+
location: payload
46+
identifier: $.usage.completion_tokens
47+
totalTokens:
48+
location: payload
49+
identifier: $.usage.total_tokens
50+
remainingTokens:
51+
location: header
52+
identifier: x-ratelimit-remaining-tokens
53+
requestModel:
54+
location: payload
55+
identifier: $.model
56+
responseModel:
57+
location: payload
58+
identifier: $.model
59+
"""
60+
Then the response status code should be 201
61+
And the response should be valid JSON
62+
And the JSON response field "status" should be "success"
63+
And the JSON response field "id" should be "openai-test"
64+
And the JSON response field "message" should be "LLM provider template created successfully"
65+
66+
Given I authenticate using basic auth as "admin"
67+
When I retrieve the LLM provider template "openai-test"
68+
Then the response status code should be 200
69+
And the response should be valid JSON
70+
And the JSON response field "status" should be "success"
71+
And the JSON response field "template.id" should be "openai-test"
72+
And the JSON response field "template.configuration.spec.displayName" should be "OpenAI"
73+
And the JSON response field "template.configuration.spec.promptTokens.location" should be "payload"
74+
And the JSON response field "template.configuration.spec.promptTokens.identifier" should be "$.usage.prompt_tokens"
75+
76+
Given I authenticate using basic auth as "admin"
77+
When I update the LLM provider template "openai-test" with:
78+
"""
79+
apiVersion: gateway.api-platform.wso2.com/v1alpha1
80+
kind: LlmProviderTemplate
81+
metadata:
82+
name: openai-test
83+
spec:
84+
displayName: OpenAI Updated
85+
promptTokens:
86+
location: payload
87+
identifier: $.usage.promptTokens
88+
completionTokens:
89+
location: payload
90+
identifier: $.usage.completion_tokens
91+
totalTokens:
92+
location: payload
93+
identifier: $.usage.total_tokens
94+
remainingTokens:
95+
location: header
96+
identifier: x-ratelimit-remaining-tokens
97+
requestModel:
98+
location: payload
99+
identifier: $.model
100+
responseModel:
101+
location: payload
102+
identifier: $.model
103+
"""
104+
Then the response status code should be 200
105+
And the response should be valid JSON
106+
And the JSON response field "status" should be "success"
107+
And the JSON response field "id" should be "openai-test"
108+
And the JSON response field "message" should be "LLM provider template updated successfully"
109+
110+
Given I authenticate using basic auth as "admin"
111+
When I retrieve the LLM provider template "openai-test"
112+
Then the response status code should be 200
113+
And the JSON response field "template.configuration.spec.displayName" should be "OpenAI Updated"
114+
And the JSON response field "template.configuration.spec.promptTokens.location" should be "payload"
115+
And the JSON response field "template.configuration.spec.promptTokens.identifier" should be "$.usage.promptTokens"
116+
117+
Given I authenticate using basic auth as "admin"
118+
When I delete the LLM provider template "openai-test"
119+
Then the response status code should be 200
120+
And the JSON response field "status" should be "success"
121+
And the JSON response field "message" should be "LLM provider template deleted successfully"
122+
123+
Given I authenticate using basic auth as "admin"
124+
When I retrieve the LLM provider template "openai-test"
125+
Then the response status code should be 404
126+
And the response should be valid JSON
127+
And the JSON response field "status" should be "error"
128+
129+
Scenario: Create template with minimal required fields
130+
Given I authenticate using basic auth as "admin"
131+
When I create this LLM provider template:
132+
"""
133+
apiVersion: gateway.api-platform.wso2.com/v1alpha1
134+
kind: LlmProviderTemplate
135+
metadata:
136+
name: minimal-template
137+
spec:
138+
displayName: Minimal Template
139+
"""
140+
Then the response status code should be 201
141+
And the response should be valid JSON
142+
And the JSON response field "id" should be "minimal-template"
143+
144+
Given I authenticate using basic auth as "admin"
145+
When I retrieve the LLM provider template "minimal-template"
146+
Then the response status code should be 200
147+
And the JSON response field "template.configuration.spec.displayName" should be "Minimal Template"
148+
149+
Given I authenticate using basic auth as "admin"
150+
When I delete the LLM provider template "minimal-template"
151+
Then the response status code should be 200
152+
153+
Scenario: List LLM provider templates returns valid JSON with OOB Templates
154+
Given I authenticate using basic auth as "admin"
155+
When I list all LLM provider templates
156+
Then the response status code should be 200
157+
And the response should be valid JSON
158+
And the JSON response field "status" should be "success"
159+
And the response should contain oob-templates

‎gateway/it/steps_llm.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com).
3+
*
4+
* WSO2 LLC. licenses this file to you under the Apache License,
5+
* Version 2.0 (the "License"); you may not use this file except
6+
* in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing,
12+
* software distributed under the License is distributed on an
13+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
* KIND, either express or implied. See the License for the
15+
* specific language governing permissions and limitations
16+
* under the License.
17+
*/
18+
19+
package it
20+
21+
import (
22+
"encoding/json"
23+
"fmt"
24+
25+
"github.com/cucumber/godog"
26+
"github.com/wso2/api-platform/gateway/it/steps"
27+
)
28+
29+
// RegisterLLMSteps registers all LLM provider template step definitions
30+
func RegisterLLMSteps(ctx *godog.ScenarioContext, state *TestState, httpSteps *steps.HTTPSteps) {
31+
ctx.Step(`^I create this LLM provider template:$`, func(body *godog.DocString) error {
32+
httpSteps.SetHeader("Content-Type", "application/yaml")
33+
return httpSteps.SendPOSTToService("gateway-controller", "/llm-provider-templates", body)
34+
})
35+
36+
ctx.Step(`^I retrieve the LLM provider template "([^"]*)"$`, func(templateID string) error {
37+
return httpSteps.SendGETToService("gateway-controller", "/llm-provider-templates/"+templateID)
38+
})
39+
40+
ctx.Step(`^I update the LLM provider template "([^"]*)" with:$`, func(templateID string, body *godog.DocString) error {
41+
httpSteps.SetHeader("Content-Type", "application/yaml")
42+
return httpSteps.SendPUTToService("gateway-controller", "/llm-provider-templates/"+templateID, body)
43+
})
44+
45+
ctx.Step(`^I delete the LLM provider template "([^"]*)"$`, func(templateID string) error {
46+
return httpSteps.SendDELETEToService("gateway-controller", "/llm-provider-templates/"+templateID)
47+
})
48+
49+
ctx.Step(`^I list all LLM provider templates$`, func() error {
50+
return httpSteps.SendGETToService("gateway-controller", "/llm-provider-templates")
51+
})
52+
53+
ctx.Step(`^I list LLM provider templates with filter "([^"]*)" as "([^"]*)"$`, func(filterKey, filterValue string) error {
54+
return httpSteps.SendGETToService("gateway-controller", "/llm-provider-templates?"+filterKey+"="+filterValue)
55+
})
56+
57+
ctx.Step(`^the response should contain oob-templates$`, func() error {
58+
// This step verifies that out-of-box templates are present in the list response
59+
// The actual assertion is done by checking the response body
60+
body := httpSteps.LastBody()
61+
if len(body) == 0 {
62+
return fmt.Errorf("expected non-empty response body for oob-templates assertion")
63+
}
64+
// The actual validation of OOB templates should be done using JSON assertions
65+
// in the feature file itself, so this step just ensures we got a response
66+
var response struct {
67+
Count int `json:"count"`
68+
Templates []struct {
69+
ID string `json:"id"`
70+
} `json:"templates"`
71+
}
72+
73+
if err := json.Unmarshal([]byte(body), &response); err != nil {
74+
return fmt.Errorf("failed to parse response JSON: %w", err)
75+
}
76+
77+
// 1️⃣ Expected OOB template IDs
78+
expectedIDs := []string{
79+
"azureai-foundry",
80+
"anthropic",
81+
"openai",
82+
"gemini",
83+
"azure-openai",
84+
"mistralai",
85+
"awsbedrock",
86+
}
87+
88+
// 2️⃣ Validate count is at least the expected set
89+
expectedCount := len(expectedIDs)
90+
if response.Count < expectedCount {
91+
return fmt.Errorf(
92+
"expected template count to be >= %d, but got %d",
93+
expectedCount,
94+
response.Count,
95+
)
96+
}
97+
98+
// 3️⃣ Collect actual template IDs
99+
actualIDs := make(map[string]bool)
100+
for _, t := range response.Templates {
101+
actualIDs[t.ID] = true
102+
}
103+
104+
// 4️⃣ Validate all expected IDs are present
105+
for _, expectedID := range expectedIDs {
106+
if !actualIDs[expectedID] {
107+
return fmt.Errorf(
108+
"expected oob-template with id '%s' was not found in response",
109+
expectedID,
110+
)
111+
}
112+
}
113+
114+
return nil
115+
})
116+
}

‎gateway/it/suite_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func TestFeatures(t *testing.T) {
8080
"features/basic-ratelimit.feature",
8181
"features/jwt-auth.feature",
8282
"features/cors.feature",
83+
"features/llm-provider-templates.feature",
8384
},
8485
TestingT: t,
8586
},
@@ -236,6 +237,7 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
236237
RegisterAPISteps(ctx, testState, httpSteps)
237238
RegisterMCPSteps(ctx, testState, httpSteps)
238239
RegisterJWTSteps(ctx, testState, httpSteps, jwtSteps)
240+
RegisterLLMSteps(ctx, testState, httpSteps)
239241
}
240242

241243
// Register common HTTP and assertion steps

0 commit comments

Comments
 (0)