Skip to content

Commit 4f47a62

Browse files
dimetronEItanyapeterj
authored
Feature/llm default headers (#406)
* fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow ui build fix python docker * fix slow build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * fix ui build * allow AzureOpenAI defaultHeaders configuration * allow AzureOpenAI defaultHeaders configuration * allow AzureOpenAI defaultHeaders configuration * allow AzureOpenAI defaultHeaders configuration * fix default headers * ++ defaultHeaders modelconfig.yaml * fix defaultHeaders for Ollama * fix defaultHeaders for Ollama * defaultHeaders modelConfig * defaultHeaders modelConfig * defaultHeaders modelConfig * defaultHeaders modelConfig * fix make make helm-cleanup --------- Co-authored-by: Eitan Yarmush <eitan.yarmush@solo.io> Co-authored-by: Peter Jausovec <peterj@users.noreply.github.com>
1 parent a04f31c commit 4f47a62

10 files changed

Lines changed: 65 additions & 16 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,3 +198,4 @@ go/bin/
198198

199199
## Helm
200200
*.tgz
201+
/tmp/

‎Makefile‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,18 @@ retag-docker-images: build
178178
docker tag $(UI_IMG) $(RETAGGED_UI_IMG)
179179
docker tag $(APP_IMG) $(RETAGGED_APP_IMG)
180180

181+
.PHONY: helm-cleanup
182+
helm-cleanup:
183+
rm -f *.tgz
184+
185+
.PHONY: helm-test
186+
helm-test: helm-version
187+
mkdir -p tmp
188+
echo $$(helm template kagent ./helm/kagent/ --namespace kagent --set providers.default=ollama | tee tmp/ollama.yaml | grep ^kind: | wc -l)
189+
echo $$(helm template kagent ./helm/kagent/ --namespace kagent --set providers.default=openAI --set providers.openAI.apiKey=your-openai-api-key | tee tmp/openAI.yaml | grep ^kind: | wc -l)
190+
echo $$(helm template kagent ./helm/kagent/ --namespace kagent --set providers.default=anthropic --set providers.anthropic.apiKey=your-anthropic-api-key | tee tmp/anthropic.yaml | grep ^kind: | wc -l)
191+
echo $$(helm template kagent ./helm/kagent/ --namespace kagent --set providers.default=azureOpenAI --set providers.azureOpenAI.apiKey=your-openai-api-key | tee tmp/azureOpenAI.yaml | grep ^kind: | wc -l)
192+
181193
.PHONY: helm-agents
182194
helm-agents:
183195
VERSION=$(VERSION) envsubst < helm/agents/k8s/Chart-template.yaml > helm/agents/k8s/Chart.yaml
@@ -198,7 +210,7 @@ helm-agents:
198210
helm package helm/agents/cilium-crd
199211

200212
.PHONY: helm-version
201-
helm-version: helm-agents
213+
helm-version: helm-cleanup helm-agents
202214
VERSION=$(VERSION) envsubst < helm/kagent-crds/Chart-template.yaml > helm/kagent-crds/Chart.yaml
203215
VERSION=$(VERSION) envsubst < helm/kagent/Chart-template.yaml > helm/kagent/Chart.yaml
204216
helm dependency update helm/kagent
@@ -258,9 +270,10 @@ helm-publish: helm-version
258270
helm push kgateway-agent-$(VERSION).tgz oci://ghcr.io/kagent-dev/kagent/agents
259271

260272
.PHONY: kagent-cli-install
261-
kagent-cli-install: build-cli-local kind-load-docker-images
273+
kagent-cli-install: build-cli-local kind-load-docker-images helm-version
262274
kagent-cli-install:
263-
KAGENT_HELM_REPO=./helm/ ./go/bin/kagent-local
275+
KAGENT_HELM_REPO=./helm/ ./go/bin/kagent-local install
276+
KAGENT_HELM_REPO=./helm/ ./go/bin/kagent-local dashboard
264277

265278
.PHONY: kagent-cli-port-forward
266279
kagent-cli-port-forward: use-kind-cluster

‎go/autogen/api/models.go‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,13 @@ type StreamOptions struct {
2626
IncludeUsage bool `json:"include_usage,omitempty"`
2727
}
2828

29+
type BaseClientConfig struct {
30+
// Base OpenAI fields
31+
DefaultHeaders map[string]string `json:"default_headers,omitempty"`
32+
}
33+
2934
type BaseOpenAIClientConfig struct {
35+
BaseClientConfig
3036
// Base OpenAI fields
3137
Model string `json:"model"`
3238
APIKey string `json:"api_key,omitempty"`
@@ -82,14 +88,14 @@ type AnthropicCreateArguments struct {
8288
}
8389

8490
type BaseAnthropicClientConfiguration struct {
85-
APIKey string `json:"api_key,omitempty"`
86-
BaseURL string `json:"base_url,omitempty"`
87-
Model string `json:"model"`
88-
ModelCapabilities *ModelInfo `json:"model_capabilities,omitempty"`
89-
ModelInfo *ModelInfo `json:"model_info,omitempty"`
90-
Timeout float64 `json:"timeout,omitempty"`
91-
MaxRetries int `json:"max_retries,omitempty"`
92-
DefaultHeaders map[string]string `json:"default_headers,omitempty"`
91+
APIKey string `json:"api_key,omitempty"`
92+
BaseURL string `json:"base_url,omitempty"`
93+
Model string `json:"model"`
94+
ModelCapabilities *ModelInfo `json:"model_capabilities,omitempty"`
95+
ModelInfo *ModelInfo `json:"model_info,omitempty"`
96+
Timeout float64 `json:"timeout,omitempty"`
97+
MaxRetries int `json:"max_retries,omitempty"`
98+
BaseClientConfig
9399
AnthropicCreateArguments
94100
}
95101

‎go/config/crd/bases/kagent.dev_modelconfigs.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ spec:
104104
- apiVersion
105105
- azureEndpoint
106106
type: object
107+
defaultHeaders:
108+
additionalProperties:
109+
type: string
110+
type: object
107111
model:
108112
type: string
109113
modelInfo:

‎go/controller/api/v1alpha1/autogenmodelconfig_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,9 @@ type ModelConfigSpec struct {
167167
// +optional
168168
APIKeySecretKey string `json:"apiKeySecretKey"`
169169

170+
// +optional
171+
DefaultHeaders map[string]string `json:"defaultHeaders,omitempty"`
172+
170173
// ModelInfo contains information about the model.
171174
// This field is required if the model is not one of the
172175
// pre-defined autogen models. That list can be found here:

‎go/controller/api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go/controller/internal/autogen/autogen_api_translator.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,7 @@ func (a *apiTranslator) translateBuiltinTool(
580580
}
581581
}
582582
if toolNeedsOpenaiApiKey(tool.Name) {
583-
if modelConfig.Spec.Provider != v1alpha1.OpenAI {
583+
if (modelConfig.Spec.Provider != v1alpha1.OpenAI) && modelConfig.Spec.Provider != v1alpha1.AzureOpenAI {
584584
return nil, fmt.Errorf("tool %s requires OpenAI API key, but model config is not OpenAI", tool.Name)
585585
}
586586
apiKey, err := a.getModelConfigApiKey(ctx, modelConfig)
@@ -825,6 +825,7 @@ func addOpenaiApiKeyToConfig(
825825

826826
// createModelClientForProvider creates a model client component based on the model provider
827827
func (a *apiTranslator) createModelClientForProvider(ctx context.Context, modelConfig *v1alpha1.ModelConfig, stream bool) (*api.Component, error) {
828+
828829
switch modelConfig.Spec.Provider {
829830
case v1alpha1.Anthropic:
830831
apiKey, err := a.getModelConfigApiKey(ctx, modelConfig)
@@ -871,7 +872,7 @@ func (a *apiTranslator) createModelClientForProvider(ctx context.Context, modelC
871872
if err != nil {
872873
return nil, fmt.Errorf("failed to convert Anthropic config: %w", err)
873874
}
874-
875+
config.DefaultHeaders = modelConfig.Spec.DefaultHeaders
875876
return &api.Component{
876877
Provider: "autogen_ext.models.anthropic.AnthropicChatCompletionClient",
877878
ComponentType: "model",
@@ -921,7 +922,7 @@ func (a *apiTranslator) createModelClientForProvider(ctx context.Context, modelC
921922
}
922923
}
923924
}
924-
925+
config.DefaultHeaders = modelConfig.Spec.DefaultHeaders
925926
return &api.Component{
926927
Provider: "autogen_ext.models.openai.AzureOpenAIChatCompletionClient",
927928
ComponentType: "model",
@@ -993,6 +994,7 @@ func (a *apiTranslator) createModelClientForProvider(ctx context.Context, modelC
993994
}
994995
}
995996

997+
config.DefaultHeaders = modelConfig.Spec.DefaultHeaders
996998
return &api.Component{
997999
Provider: "autogen_ext.models.openai.OpenAIChatCompletionClient",
9981000
ComponentType: "model",
@@ -1018,6 +1020,7 @@ func (a *apiTranslator) createModelClientForProvider(ctx context.Context, modelC
10181020
}
10191021
}
10201022

1023+
config.Headers = modelConfig.Spec.DefaultHeaders
10211024
return &api.Component{
10221025
Provider: "autogen_ext.models.ollama.OllamaChatCompletionClient",
10231026
ComponentType: "model",

‎helm/kagent-crds/templates/kagent.dev_modelconfigs.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ spec:
104104
- apiVersion
105105
- azureEndpoint
106106
type: object
107+
defaultHeaders:
108+
additionalProperties:
109+
type: string
110+
type: object
107111
model:
108112
type: string
109113
modelInfo:

‎helm/kagent/templates/modelconfig.yaml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
{{- $dot := . }}
2-
{{- $model := index $dot.Values.providers $dot.Values.providers.default }}
2+
{{- $defaultProfider := .Values.providers.default | default "openAI" }}
3+
{{- $model := index .Values.providers $defaultProfider }}
4+
{{- if hasKey .Values.providers $defaultProfider | not }}
5+
{{- fail (printf "Provider key=%s is not found under .Values.providers" $defaultProfider) }}
6+
{{- end }}
37
---
48
apiVersion: kagent.dev/v1alpha1
59
kind: ModelConfig
@@ -18,6 +22,10 @@ spec:
1822
{{- if $model.apiKeySecretKey }}
1923
apiKeySecretKey: {{.apiKeySecretKey}}
2024
{{- end }}
25+
{{- if hasKey $model "defaultHeaders" }}
26+
defaultHeaders:
27+
{{- toYaml $model.defaultHeaders | nindent 4 }}
28+
{{- end }}
2129
{{ $dot.Values.providers.default }}:
2230
{{- toYaml $model.config | nindent 4 }}
2331
{{- end}}

‎helm/kagent/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ otel:
129129
enabled: false
130130
exporter:
131131
otlp:
132-
endpoint: http://localhost:4317
132+
endpoint: http://host.docker.internal:4317
133133
timeout: 10
134134
insecure: true
135135

0 commit comments

Comments
 (0)