diff --git a/.mockery.yaml b/.mockery.yaml index cb028727..243c2324 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -148,6 +148,10 @@ packages: config: interfaces: Deployer: + config: + replace-type: + - opencsg.com/csghub-server/common/types.ClawEvaluationReq=commontypes:opencsg.com/csghub-server/common/types.ClawEvaluationReq + - opencsg.com/csghub-server/common/types.ArgoWorkFlowRes=commontypes:opencsg.com/csghub-server/common/types.ArgoWorkFlowRes opencsg.com/csghub-server/builder/deploy/cluster: config: interfaces: diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index edaffe5d..327818b2 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -933,12 +933,16 @@ func (d *deployer) SubmitClawEvaluation(ctx context.Context, req types.ClawEvalu } else { env["CLAW_EVAL_TASKS"] = types.ClawEvalTasksNormal } - if req.Trials > 0 { - env["CLAW_EVAL_TRIALS"] = strconv.Itoa(req.Trials) + trials := req.Trials + if trials <= 0 { + trials = types.ClawEvalDefaultTrials } - if req.Parallel > 0 { - env["CLAW_EVAL_PARALLEL"] = strconv.Itoa(req.Parallel) + env["CLAW_EVAL_TRIALS"] = strconv.Itoa(trials) + parallel := req.Parallel + if parallel <= 0 { + parallel = types.ClawEvalDefaultParallel } + env["CLAW_EVAL_PARALLEL"] = strconv.Itoa(parallel) if req.JudgeModel != "" { env["CLAW_EVAL_JUDGE_MODEL"] = req.JudgeModel } else if !req.NoJudge { diff --git a/builder/deploy/deployer_test.go b/builder/deploy/deployer_test.go index 844216c3..4de1be21 100644 --- a/builder/deploy/deployer_test.go +++ b/builder/deploy/deployer_test.go @@ -6,6 +6,7 @@ import ( "errors" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -861,20 +862,24 @@ func TestDeployer_SubmitClawEvaluation(t *testing.T) { require.Equal(t, "http://aigateway.test/v1", env["CLAW_EVAL_JUDGE_BASE_URL"]) require.Equal(t, "gk-judge-key", env["CLAW_EVAL_JUDGE_API_KEY"]) require.Equal(t, "1-9", env["CLAW_EVAL_TASKS"]) + require.Equal(t, "2", env["CLAW_EVAL_TRIALS"]) + require.Equal(t, "3", env["CLAW_EVAL_PARALLEL"]) require.Equal(t, types.ClawEvalDefaultJudgeModel, env["CLAW_EVAL_JUDGE_MODEL"]) return &types.ArgoWorkFlowRes{ID: 2}, nil }, ) resp, err := tester.SubmitClawEvaluation(ctx, types.ClawEvaluationReq{ - ClusterID: "cluster-1", - Model: "glm-5.1", - BaseURL: "http://localhost:11435/v1", - ApiKey: "sk-test", - Tasks: "1-9", - JudgeBaseURL: "http://aigateway.test/v1", - JudgeApiKey: "gk-judge-key", - TaskType: types.TaskTypeClawEval, - Image: "opencsghq/claw-eval:1.0.0", + ClusterID: "cluster-1", + Model: "glm-5.1", + BaseURL: "http://localhost:11435/v1", + ApiKey: "sk-test", + Tasks: "1-9", + Trials: 2, + Parallel: 3, + JudgeBaseURL: "http://aigateway.test/v1", + JudgeApiKey: "gk-judge-key", + TaskType: types.TaskTypeClawEval, + Image: "opencsghq/claw-eval:1.0.0", }) require.NoError(t, err) require.Equal(t, &types.ArgoWorkFlowRes{ID: 2}, resp) @@ -889,6 +894,8 @@ func TestDeployer_SubmitClawEvaluation_DefaultTasksNormal(t *testing.T) { func(ctx context.Context, awfr *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) { env := awfr.Templates[0].Env require.Equal(t, types.ClawEvalTasksNormal, env["CLAW_EVAL_TASKS"]) + require.Equal(t, strconv.Itoa(types.ClawEvalDefaultTrials), env["CLAW_EVAL_TRIALS"]) + require.Equal(t, strconv.Itoa(types.ClawEvalDefaultParallel), env["CLAW_EVAL_PARALLEL"]) return &types.ArgoWorkFlowRes{ID: 3}, nil }, ) diff --git a/common/types/claw_eval_summary.go b/common/types/claw_eval_summary.go index eb21044d..0394259e 100644 --- a/common/types/claw_eval_summary.go +++ b/common/types/claw_eval_summary.go @@ -7,8 +7,10 @@ import ( ) const ( - ClawEvalTasksNormal = "normal" - ClawEvalDefaultJudgeModel = "qwen3.7-max" + ClawEvalTasksNormal = "normal" + ClawEvalDefaultJudgeModel = "qwen3.7-max" + ClawEvalDefaultTrials = 3 + ClawEvalDefaultParallel = 4 ) // ClawEvalSummary mirrors claw-eval batch_summary.json. diff --git a/component/evaluation.go b/component/evaluation.go index 5b386d8d..fd6b8fdd 100644 --- a/component/evaluation.go +++ b/component/evaluation.go @@ -406,6 +406,9 @@ func (c *evaluationComponentImpl) OrgEvaluations(ctx context.Context, req *types func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req types.EvaluationReq, user database.User, frame *database.RuntimeFramework) (*types.ArgoWorkFlowRes, error) { operatorUsername := req.Username + if req.Model == "" { + req.Model = req.ModelId + } if req.TaskName == "" { return nil, fmt.Errorf("task_name is required") } @@ -477,8 +480,27 @@ func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req req.TaskType = types.TaskTypeClawEval clawReq := req.ToClawEvaluationReq() + var defaultAPIKey string + resolveDefaultAPIKey := func() (string, error) { + if defaultAPIKey != "" { + return defaultAPIKey, nil + } + apiKey, err := c.resolveClawEvalAIGatewayAPIKey(ctx, billingUUID, req.OwnerNamespace, operatorUsername) + if err != nil { + return "", err + } + defaultAPIKey = apiKey + return defaultAPIKey, nil + } + if clawReq.ApiKey == "" { + apiKey, err := resolveDefaultAPIKey() + if err != nil { + return nil, err + } + clawReq.ApiKey = apiKey + } if !req.NoJudge { - judgeAPIKey, err := c.resolveClawEvalJudgeAPIKey(ctx, billingUUID, req.OwnerNamespace, operatorUsername) + judgeAPIKey, err := resolveDefaultAPIKey() if err != nil { return nil, err } @@ -488,7 +510,7 @@ func (c *evaluationComponentImpl) createClawEvaluation(ctx context.Context, req return c.deployer.SubmitClawEvaluation(ctx, clawReq) } -func (c *evaluationComponentImpl) resolveClawEvalJudgeAPIKey(ctx context.Context, billingUUID, ownerNamespace, operatorUsername string) (string, error) { +func (c *evaluationComponentImpl) resolveClawEvalAIGatewayAPIKey(ctx context.Context, billingUUID, ownerNamespace, operatorUsername string) (string, error) { token, err := c.tokenStore.FindBuiltinByNsUUID(ctx, billingUUID, string(types.AccessTokenAppAIGateway)) if err == nil && token != nil && token.Token != "" { return token.Token, nil @@ -505,10 +527,10 @@ func (c *evaluationComponentImpl) resolveClawEvalJudgeAPIKey(ctx context.Context "claw-eval", ) if err != nil { - return "", fmt.Errorf("failed to resolve judge api key, %w", err) + return "", fmt.Errorf("failed to resolve claw-eval api key, %w", err) } if apiKey == "" { - return "", fmt.Errorf("failed to resolve judge api key") + return "", fmt.Errorf("failed to resolve claw-eval api key") } return apiKey, nil } diff --git a/component/evaluation_claw_test.go b/component/evaluation_claw_test.go index 267f7a68..3e5d806c 100644 --- a/component/evaluation_claw_test.go +++ b/component/evaluation_claw_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" mockdeploy "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/deploy" - mockComps "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component" - mockdb "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/database" mockrpc "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/rpc" + mockdb "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/database" + mockComps "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component" "opencsg.com/csghub-server/builder/deploy" "opencsg.com/csghub-server/builder/git/membership" "opencsg.com/csghub-server/builder/loki" @@ -113,7 +113,7 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoBuiltinJudgeAPIKey(t *test TaskName: "claw-job", RuntimeFrameworkId: 1, ResourceId: 2, - Model: "glm-5.1", + ModelId: "glm-5.1", BaseURL: "http://localhost:11435/v1", Tasks: "T001", } @@ -142,7 +142,8 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoBuiltinJudgeAPIKey(t *test Token: "gk-builtin-key", }, nil) mockDeployer.EXPECT().SubmitClawEvaluation(ctx, mock.MatchedBy(func(r types.ClawEvaluationReq) bool { - return r.ApiKey == "" && + return r.ApiKey == "gk-builtin-key" && + r.Model == "glm-5.1" && r.JudgeApiKey == "gk-builtin-key" && r.JudgeBaseURL == "http://aigateway.test/v1" })).Return(&types.ArgoWorkFlowRes{ID: 2}, nil) @@ -191,7 +192,7 @@ func TestEvaluationComponent_CreateClawEvaluation_AutoJudgeAPIKeyFallback(t *tes ctx, "user1", "user1", string(types.AccessTokenAppAIGateway), "claw-eval", ).Return("gk-fallback-key", nil) mockDeployer.EXPECT().SubmitClawEvaluation(ctx, mock.MatchedBy(func(r types.ClawEvaluationReq) bool { - return r.ApiKey == "" && + return r.ApiKey == "gk-fallback-key" && r.JudgeApiKey == "gk-fallback-key" && r.JudgeBaseURL == "http://aigateway.test/v1" })).Return(&types.ArgoWorkFlowRes{ID: 3}, nil)