Skip to content

Commit 6577498

Browse files
Raderganisback
Rader
and
ganisback
authored
Update claw-eval API defaults (#1242)
Co-authored-by: ganisback <370036720@qq.com>
1 parent acf6674 commit 6577498

6 files changed

Lines changed: 64 additions & 24 deletions

File tree

‎.mockery.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ packages:
148148
config:
149149
interfaces:
150150
Deployer:
151+
config:
152+
replace-type:
153+
- opencsg.com/csghub-server/common/types.ClawEvaluationReq=commontypes:opencsg.com/csghub-server/common/types.ClawEvaluationReq
154+
- opencsg.com/csghub-server/common/types.ArgoWorkFlowRes=commontypes:opencsg.com/csghub-server/common/types.ArgoWorkFlowRes
151155
opencsg.com/csghub-server/builder/deploy/cluster:
152156
config:
153157
interfaces:

‎builder/deploy/deployer.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -933,12 +933,16 @@ func (d *deployer) SubmitClawEvaluation(ctx context.Context, req types.ClawEvalu
933933
} else {
934934
env["CLAW_EVAL_TASKS"] = types.ClawEvalTasksNormal
935935
}
936-
if req.Trials > 0 {
937-
env["CLAW_EVAL_TRIALS"] = strconv.Itoa(req.Trials)
936+
trials := req.Trials
937+
if trials <= 0 {
938+
trials = types.ClawEvalDefaultTrials
938939
}
939-
if req.Parallel > 0 {
940-
env["CLAW_EVAL_PARALLEL"] = strconv.Itoa(req.Parallel)
940+
env["CLAW_EVAL_TRIALS"] = strconv.Itoa(trials)
941+
parallel := req.Parallel
942+
if parallel <= 0 {
943+
parallel = types.ClawEvalDefaultParallel
941944
}
945+
env["CLAW_EVAL_PARALLEL"] = strconv.Itoa(parallel)
942946
if req.JudgeModel != "" {
943947
env["CLAW_EVAL_JUDGE_MODEL"] = req.JudgeModel
944948
} else if !req.NoJudge {

‎builder/deploy/deployer_test.go‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"errors"
77
"net/http"
88
"net/http/httptest"
9+
"strconv"
910
"strings"
1011
"testing"
1112
"time"
@@ -861,20 +862,24 @@ func TestDeployer_SubmitClawEvaluation(t *testing.T) {
861862
require.Equal(t, "http://aigateway.test/v1", env["CLAW_EVAL_JUDGE_BASE_URL"])
862863
require.Equal(t, "gk-judge-key", env["CLAW_EVAL_JUDGE_API_KEY"])
863864
require.Equal(t, "1-9", env["CLAW_EVAL_TASKS"])
865+
require.Equal(t, "2", env["CLAW_EVAL_TRIALS"])
866+
require.Equal(t, "3", env["CLAW_EVAL_PARALLEL"])
864867
require.Equal(t, types.ClawEvalDefaultJudgeModel, env["CLAW_EVAL_JUDGE_MODEL"])
865868
return &types.ArgoWorkFlowRes{ID: 2}, nil
866869
},
867870
)
868871
resp, err := tester.SubmitClawEvaluation(ctx, types.ClawEvaluationReq{
869-
ClusterID: "cluster-1",
870-
Model: "glm-5.1",
871-
BaseURL: "http://localhost:11435/v1",
872-
ApiKey: "sk-test",
873-
Tasks: "1-9",
874-
JudgeBaseURL: "http://aigateway.test/v1",
875-
JudgeApiKey: "gk-judge-key",
876-
TaskType: types.TaskTypeClawEval,
877-
Image: "opencsghq/claw-eval:1.0.0",
872+
ClusterID: "cluster-1",
873+
Model: "glm-5.1",
874+
BaseURL: "http://localhost:11435/v1",
875+
ApiKey: "sk-test",
876+
Tasks: "1-9",
877+
Trials: 2,
878+
Parallel: 3,
879+
JudgeBaseURL: "http://aigateway.test/v1",
880+
JudgeApiKey: "gk-judge-key",
881+
TaskType: types.TaskTypeClawEval,
882+
Image: "opencsghq/claw-eval:1.0.0",
878883
})
879884
require.NoError(t, err)
880885
require.Equal(t, &types.ArgoWorkFlowRes{ID: 2}, resp)
@@ -889,6 +894,8 @@ func TestDeployer_SubmitClawEvaluation_DefaultTasksNormal(t *testing.T) {
889894
func(ctx context.Context, awfr *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) {
890895
env := awfr.Templates[0].Env
891896
require.Equal(t, types.ClawEvalTasksNormal, env["CLAW_EVAL_TASKS"])
897+
require.Equal(t, strconv.Itoa(types.ClawEvalDefaultTrials), env["CLAW_EVAL_TRIALS"])
898+
require.Equal(t, strconv.Itoa(types.ClawEvalDefaultParallel), env["CLAW_EVAL_PARALLEL"])
892899
return &types.ArgoWorkFlowRes{ID: 3}, nil
893900
},
894901
)

‎common/types/claw_eval_summary.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import (
77
)
88

99
const (
10-
ClawEvalTasksNormal = "normal"
11-
ClawEvalDefaultJudgeModel = "qwen3.7-max"
10+
ClawEvalTasksNormal = "normal"
11+
ClawEvalDefaultJudgeModel = "qwen3.7-max"
12+
ClawEvalDefaultTrials = 3
13+
ClawEvalDefaultParallel = 4
1214
)
1315

1416
// ClawEvalSummary mirrors claw-eval batch_summary.json.

‎component/evaluation.go‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,9 @@ func (c *evaluationComponentImpl) OrgEvaluations(ctx context.Context, req *types
406406

407407
func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req types.EvaluationReq, user database.User, frame *database.RuntimeFramework) (*types.ArgoWorkFlowRes, error) {
408408
operatorUsername := req.Username
409+
if req.Model == "" {
410+
req.Model = req.ModelId
411+
}
409412
if req.TaskName == "" {
410413
return nil, fmt.Errorf("task_name is required")
411414
}
@@ -477,8 +480,27 @@ func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req
477480
req.TaskType = types.TaskTypeClawEval
478481

479482
clawReq := req.ToClawEvaluationReq()
483+
var defaultAPIKey string
484+
resolveDefaultAPIKey := func() (string, error) {
485+
if defaultAPIKey != "" {
486+
return defaultAPIKey, nil
487+
}
488+
apiKey, err := c.resolveClawEvalAIGatewayAPIKey(ctx, billingUUID, req.OwnerNamespace, operatorUsername)
489+
if err != nil {
490+
return "", err
491+
}
492+
defaultAPIKey = apiKey
493+
return defaultAPIKey, nil
494+
}
495+
if clawReq.ApiKey == "" {
496+
apiKey, err := resolveDefaultAPIKey()
497+
if err != nil {
498+
return nil, err
499+
}
500+
clawReq.ApiKey = apiKey
501+
}
480502
if !req.NoJudge {
481-
judgeAPIKey, err := c.resolveClawEvalJudgeAPIKey(ctx, billingUUID, req.OwnerNamespace, operatorUsername)
503+
judgeAPIKey, err := resolveDefaultAPIKey()
482504
if err != nil {
483505
return nil, err
484506
}
@@ -488,7 +510,7 @@ func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req
488510
return c.deployer.SubmitClawEvaluation(ctx, clawReq)
489511
}
490512

491-
func (c *evaluationComponentImpl) resolveClawEvalJudgeAPIKey(ctx context.Context, billingUUID, ownerNamespace, operatorUsername string) (string, error) {
513+
func (c *evaluationComponentImpl) resolveClawEvalAIGatewayAPIKey(ctx context.Context, billingUUID, ownerNamespace, operatorUsername string) (string, error) {
492514
token, err := c.tokenStore.FindBuiltinByNsUUID(ctx, billingUUID, string(types.AccessTokenAppAIGateway))
493515
if err == nil && token != nil && token.Token != "" {
494516
return token.Token, nil
@@ -505,10 +527,10 @@ func (c *evaluationComponentImpl) resolveClawEvalJudgeAPIKey(ctx context.Context
505527
"claw-eval",
506528
)
507529
if err != nil {
508-
return "", fmt.Errorf("failed to resolve judge api key, %w", err)
530+
return "", fmt.Errorf("failed to resolve claw-eval api key, %w", err)
509531
}
510532
if apiKey == "" {
511-
return "", fmt.Errorf("failed to resolve judge api key")
533+
return "", fmt.Errorf("failed to resolve claw-eval api key")
512534
}
513535
return apiKey, nil
514536
}

‎component/evaluation_claw_test.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import (
99
"github.com/stretchr/testify/mock"
1010
"github.com/stretchr/testify/require"
1111
mockdeploy "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/deploy"
12-
mockComps "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component"
13-
mockdb "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/database"
1412
mockrpc "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/rpc"
13+
mockdb "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/database"
14+
mockComps "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component"
1515
"opencsg.com/csghub-server/builder/deploy"
1616
"opencsg.com/csghub-server/builder/git/membership"
1717
"opencsg.com/csghub-server/builder/loki"
@@ -113,7 +113,7 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoBuiltinJudgeAPIKey(t *test
113113
TaskName: "claw-job",
114114
RuntimeFrameworkId: 1,
115115
ResourceId: 2,
116-
Model: "glm-5.1",
116+
ModelId: "glm-5.1",
117117
BaseURL: "http://localhost:11435/v1",
118118
Tasks: "T001",
119119
}
@@ -142,7 +142,8 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoBuiltinJudgeAPIKey(t *test
142142
Token: "gk-builtin-key",
143143
}, nil)
144144
mockDeployer.EXPECT().SubmitClawEvaluation(ctx, mock.MatchedBy(func(r types.ClawEvaluationReq) bool {
145-
return r.ApiKey == "" &&
145+
return r.ApiKey == "gk-builtin-key" &&
146+
r.Model == "glm-5.1" &&
146147
r.JudgeApiKey == "gk-builtin-key" &&
147148
r.JudgeBaseURL == "http://aigateway.test/v1"
148149
})).Return(&types.ArgoWorkFlowRes{ID: 2}, nil)
@@ -191,7 +192,7 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoJudgeAPIKeyFallback(t *tes
191192
ctx, "user1", "user1", string(types.AccessTokenAppAIGateway), "claw-eval",
192193
).Return("gk-fallback-key", nil)
193194
mockDeployer.EXPECT().SubmitClawEvaluation(ctx, mock.MatchedBy(func(r types.ClawEvaluationReq) bool {
194-
return r.ApiKey == "" &&
195+
return r.ApiKey == "gk-fallback-key" &&
195196
r.JudgeApiKey == "gk-fallback-key" &&
196197
r.JudgeBaseURL == "http://aigateway.test/v1"
197198
})).Return(&types.ArgoWorkFlowRes{ID: 3}, nil)

0 commit comments

Comments
 (0)