Skip to content

Commit f27d2fb

Browse files
lsy357lsy357
authored
[fix][evaluation] target/evaluator err msg conv (#445)
* fix(evaluation): idl op_type * feat(evaluation): target/evaluator record err conv * fix(evaluation): event lock ttl * fix(evaluation): errhandler * fix(evaluation): err msg * fix(evaluation): ut * fix(evaluation): ut * fix(evaluation): ut
1 parent 674a4b2 commit f27d2fb

12 files changed

Lines changed: 62 additions & 29 deletions

‎backend/modules/evaluation/application/wire_gen.go‎

Lines changed: 13 additions & 13 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backend/modules/evaluation/domain/service/evaluator_impl.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/coze-dev/coze-loop/backend/infra/middleware/session"
1818
"github.com/coze-dev/coze-loop/backend/infra/mq"
1919
"github.com/coze-dev/coze-loop/backend/modules/evaluation/consts"
20+
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component"
2021
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem"
2122
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity"
2223
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo"
@@ -43,6 +44,7 @@ func NewEvaluatorServiceImpl(
4344
configer conf.IConfiger,
4445
evaluatorSourceServices map[entity.EvaluatorType]EvaluatorSourceService,
4546
plainRateLimiter repo.IPlainRateLimiter,
47+
cConfiger component.IConfiger,
4648
) EvaluatorService {
4749
onceEvaluatorService.Do(func() {
4850
singletonEvaluatorService = &EvaluatorServiceImpl{
@@ -55,6 +57,7 @@ func NewEvaluatorServiceImpl(
5557
configer: configer,
5658
evaluatorSourceServices: evaluatorSourceServices,
5759
plainRateLimiter: plainRateLimiter,
60+
cConfiger: cConfiger,
5861
}
5962
})
6063
return singletonEvaluatorService
@@ -71,6 +74,8 @@ type EvaluatorServiceImpl struct {
7174
configer conf.IConfiger
7275
evaluatorSourceServices map[entity.EvaluatorType]EvaluatorSourceService
7376
plainRateLimiter repo.IPlainRateLimiter
77+
78+
cConfiger component.IConfiger
7479
}
7580

7681
// ListEvaluator 按查询条件查询 evaluator_version
@@ -684,6 +689,12 @@ func (e *EvaluatorServiceImpl) RunEvaluator(ctx context.Context, request *entity
684689
},
685690
},
686691
}
692+
if recordDO.EvaluatorOutputData != nil &&
693+
recordDO.EvaluatorOutputData.EvaluatorRunError != nil &&
694+
recordDO.EvaluatorOutputData.EvaluatorRunError.Code != int32(errno.CustomRPCEvaluatorRunFailedCode) &&
695+
len(recordDO.EvaluatorOutputData.EvaluatorRunError.Message) > 0 {
696+
recordDO.EvaluatorOutputData.EvaluatorRunError.Message = e.cConfiger.GetErrCtrl(ctx).ConvertErrMsg(recordDO.EvaluatorOutputData.EvaluatorRunError.Message)
697+
}
687698
err = e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, recordDO)
688699
if err != nil {
689700
return nil, err

‎backend/modules/evaluation/domain/service/evaluator_impl_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
mqmocks "github.com/coze-dev/coze-loop/backend/infra/mq/mocks"
1919
"github.com/coze-dev/coze-loop/backend/modules/evaluation/consts"
2020
idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks"
21+
componentMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks"
2122
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity"
2223
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks"
2324

@@ -42,8 +43,8 @@ func TestNewEvaluatorServiceImpl(t *testing.T) {
4243
mockConfiger := confmocks.NewMockIConfiger(ctrl)
4344
mockSourceService := mocks.NewMockEvaluatorSourceService(ctrl)
4445
mockPlainLimiter := repomocks.NewMockIPlainRateLimiter(ctrl)
46+
mockErrConfiger := componentMocks.NewMockIConfiger(ctrl)
4547

46-
// 这里需要传递一个 EvaluatorSourceService 的 slice
4748
service := NewEvaluatorServiceImpl(
4849
mockIdgen,
4950
mockLimiter,
@@ -56,6 +57,7 @@ func TestNewEvaluatorServiceImpl(t *testing.T) {
5657
entity.EvaluatorTypePrompt: mockSourceService,
5758
},
5859
mockPlainLimiter,
60+
mockErrConfiger,
5961
)
6062

6163
assert.IsType(t, &EvaluatorServiceImpl{}, service)

‎backend/modules/evaluation/domain/service/expt_result_impl.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1817,10 +1817,6 @@ func (e *ExptResultBuilder) getTurnTargetOutput(ctx context.Context, itemID, tur
18171817
return &entity.TurnTargetOutput{}
18181818
}
18191819

1820-
if turnTargetOutput.EvalTargetRecord != nil && turnTargetOutput.EvalTargetRecord.EvalTargetOutputData != nil && turnTargetOutput.EvalTargetRecord.EvalTargetOutputData.EvalTargetRunError != nil {
1821-
turnTargetOutput.EvalTargetRecord.EvalTargetOutputData.EvalTargetRunError.Message = errno.ServiceInternalErrMsg
1822-
}
1823-
18241820
return turnTargetOutput
18251821
}
18261822

‎backend/modules/evaluation/domain/service/expt_run_item_event_impl.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R
207207
func (e *ExptItemEventEvalServiceImpl) HandleEventLock(next RecordEvalEndPoint) RecordEvalEndPoint {
208208
return func(ctx context.Context, event *entity.ExptItemEvalEvent) error {
209209
lockKey := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, event.EvalSetItemID)
210-
locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*10, time.Second*60*60)
210+
locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*5, time.Second*60*60)
211211
if err != nil {
212212
return err
213213
}

‎backend/modules/evaluation/domain/service/expt_run_item_impl.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,14 +169,13 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *
169169
}
170170

171171
if evalErr != nil {
172-
var rawErrMsg string
172+
var errMsg string
173173
if se, ok := errorx.FromStatusError(evalErr); ok && (se.Code() == errno.CustomEvalTargetInvokeFailCode || se.Code() == errno.CustomRPCEvaluatorRunFailedCode) {
174-
rawErrMsg = errorx.ErrorWithoutStack(evalErr)
174+
errMsg = errorx.ErrorWithoutStack(evalErr)
175175
} else {
176-
rawErrMsg = evalErr.Error()
176+
errMsg = e.Configer.GetErrCtrl(ctx).ConvertErrMsg(evalErr.Error())
177177
}
178178

179-
errMsg := e.Configer.GetErrCtrl(ctx).ConvertErrMsg(rawErrMsg)
180179
logs.CtxWarn(ctx, "[ExptTurnEval] store turn run err, before: %v, after: %v", evalErr, errMsg)
181180

182181
ei, ok := errno.ParseErrImpl(evalErr)

‎backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ func (e *ExptSchedulerImpl) makeExptRunExecLockKey(exptID, exptRunID int64) stri
163163
func (e *ExptSchedulerImpl) HandleEventLock(next SchedulerEndPoint) SchedulerEndPoint {
164164
return func(ctx context.Context, event *entity.ExptScheduleEvent) error {
165165
key := e.makeExptRunExecLockKey(event.ExptID, event.ExptRunID)
166-
locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*10, time.Second*60*5)
166+
locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*5, time.Second*60*5)
167167
if err != nil {
168168
return err
169169
}

‎backend/modules/evaluation/domain/service/target_impl.go‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ func (e *EvalTargetServiceImpl) ExecuteTarget(ctx context.Context, spaceID, targ
246246
if ok {
247247
outputData.EvalTargetRunError = &entity.EvalTargetRunError{
248248
Code: statusErr.Code(),
249-
Message: statusErr.Error(),
249+
Message: errorx.ErrorWithoutStack(err),
250250
}
251251
spanParam.ErrCode = strconv.FormatInt(int64(statusErr.Code()), 10)
252252
} else {
@@ -323,6 +323,7 @@ func (e *EvalTargetServiceImpl) ExecuteTarget(ctx context.Context, spaceID, targ
323323
UpdatedAt: gptr.Of(time.Now().UnixMilli()),
324324
},
325325
}
326+
e.convEvalTargetRunErr(ctx, record)
326327

327328
_, errCreate := e.evalTargetRepo.CreateEvalTargetRecord(ctx, record)
328329
if errCreate != nil {
@@ -548,13 +549,27 @@ func (e *EvalTargetServiceImpl) DebugTarget(ctx context.Context, param *entity.D
548549
UpdatedAt: gptr.Of(time.Now().UnixMilli()),
549550
},
550551
}
552+
e.convEvalTargetRunErr(ctx, record)
553+
551554
if _, err := e.evalTargetRepo.CreateEvalTargetRecord(ctx, record); err != nil {
552555
return nil, err
553556
}
554557

555558
return record, nil
556559
}
557560

561+
func (e *EvalTargetServiceImpl) convEvalTargetRunErr(ctx context.Context, record *entity.EvalTargetRecord) {
562+
if record == nil || record.EvalTargetOutputData == nil || record.EvalTargetOutputData.EvalTargetRunError == nil {
563+
return
564+
}
565+
if record.EvalTargetOutputData.EvalTargetRunError.Code == int32(errno.CustomEvalTargetInvokeFailCode) {
566+
return
567+
}
568+
if len(record.EvalTargetOutputData.EvalTargetRunError.Message) > 0 {
569+
record.EvalTargetOutputData.EvalTargetRunError.Message = e.configer.GetErrCtrl(ctx).ConvertErrMsg(record.EvalTargetOutputData.EvalTargetRunError.Message)
570+
}
571+
}
572+
558573
func (e *EvalTargetServiceImpl) AsyncDebugTarget(ctx context.Context, param *entity.DebugTargetParam) (record *entity.EvalTargetRecord, callee string, err error) {
559574
return e.asyncExecuteTarget(ctx, param.SpaceID, param.PatchyTarget, &entity.ExecuteTargetCtx{}, param.InputData)
560575
}
@@ -592,6 +607,8 @@ func (e *EvalTargetServiceImpl) ReportInvokeRecords(ctx context.Context, param *
592607

593608
record.EvalTargetOutputData = param.OutputData
594609
record.Status = gptr.Of(param.Status)
610+
e.convEvalTargetRunErr(ctx, record)
611+
595612
if err := e.evalTargetRepo.SaveEvalTargetRecord(ctx, record); err != nil {
596613
return err
597614
}

‎backend/modules/evaluation/domain/service/target_impl_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,8 @@ func TestEvalTargetServiceImpl_ExecuteTarget(t *testing.T) {
521521
deps.metric.EXPECT().EmitRun(evalTarget.SpaceID, gomock.Any(), gomock.Any()).Times(1)
522522
// default trajectory conf, not used in these cases (target type does not support trajectory)
523523
deps.configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).AnyTimes().Return(&entity.TargetTrajectoryConf{})
524+
// convEvalTargetRunErr (in ExecuteTarget defer) may call GetErrCtrl when record has EvalTargetRunError
525+
deps.configer.EXPECT().GetErrCtrl(gomock.Any()).AnyTimes().Return(entity.DefaultExptErrCtrl())
524526
deps.idgen.EXPECT().GenID(ctx).Return(int64(9999), nil)
525527

526528
var savedRecord *entity.EvalTargetRecord
@@ -775,6 +777,7 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords(t *testing.T) {
775777
deps.repo.EXPECT().CreateEvalTargetRecord(gomock.Any(), gomock.Any()).AnyTimes()
776778
deps.metric.EXPECT().EmitRun(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
777779
deps.configer.EXPECT().GetTargetTrajectoryConf(gomock.Any()).AnyTimes().Return(&entity.TargetTrajectoryConf{})
780+
deps.configer.EXPECT().GetErrCtrl(gomock.Any()).AnyTimes().Return(entity.DefaultExptErrCtrl())
778781

779782
param.Session = &entity.Session{UserID: "user"}
780783
param.OutputData = &entity.EvalTargetOutputData{
@@ -934,6 +937,8 @@ func TestEvalTargetServiceImpl_ReportInvokeRecords_Trajectory(t *testing.T) {
934937

935938
// main flow expectations (same as success case)
936939
deps.repo.EXPECT().GetEvalTargetRecordByIDAndSpaceID(ctx, param.SpaceID, param.RecordID).Return(record, nil)
940+
// convEvalTargetRunErr is called before SaveEvalTargetRecord when param.OutputData has EvalTargetRunError with Message
941+
deps.configer.EXPECT().GetErrCtrl(gomock.Any()).Return(&entity.ExptErrCtrl{}).AnyTimes()
937942
var saved *entity.EvalTargetRecord
938943
deps.repo.EXPECT().SaveEvalTargetRecord(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, rec *entity.EvalTargetRecord) error {
939944
saved = rec
@@ -1353,6 +1358,7 @@ func TestEvalTargetServiceImpl_DebugTarget(t *testing.T) {
13531358
prepare: func(ctx context.Context, deps *evalTargetServiceTestDeps, param *entity.DebugTargetParam) {
13541359
deps.operator.EXPECT().ValidateInput(ctx, param.SpaceID, param.PatchyTarget.EvalTargetVersion.InputSchema, param.InputData).Return(nil)
13551360
deps.operator.EXPECT().Execute(ctx, param.SpaceID, gomock.Any()).Return(nil, entity.EvalTargetRunStatusFail, errorx.NewByCode(errno.CommonInternalErrorCode))
1361+
deps.configer.EXPECT().GetErrCtrl(gomock.Any()).Return(&entity.ExptErrCtrl{}).Times(1)
13561362
deps.idgen.EXPECT().GenID(ctx).Return(int64(999), nil)
13571363
deps.repo.EXPECT().CreateEvalTargetRecord(ctx, gomock.Any()).Return(int64(999), nil)
13581364
deps.metric.EXPECT().EmitRun(param.SpaceID, gomock.Any(), gomock.Any()).Times(1)
@@ -1364,6 +1370,7 @@ func TestEvalTargetServiceImpl_DebugTarget(t *testing.T) {
13641370
prepare: func(ctx context.Context, deps *evalTargetServiceTestDeps, param *entity.DebugTargetParam) {
13651371
deps.operator.EXPECT().ValidateInput(ctx, param.SpaceID, param.PatchyTarget.EvalTargetVersion.InputSchema, param.InputData).Return(nil)
13661372
deps.operator.EXPECT().Execute(ctx, param.SpaceID, gomock.Any()).Return(nil, entity.EvalTargetRunStatusFail, errorx.New("common error"))
1373+
deps.configer.EXPECT().GetErrCtrl(gomock.Any()).Return(&entity.ExptErrCtrl{}).Times(1)
13671374
deps.idgen.EXPECT().GenID(ctx).Return(int64(999), nil)
13681375
deps.repo.EXPECT().CreateEvalTargetRecord(ctx, gomock.Any()).Return(int64(999), nil)
13691376
deps.metric.EXPECT().EmitRun(param.SpaceID, gomock.Any(), gomock.Any()).Times(1)
@@ -1482,6 +1489,7 @@ func TestEvalTargetServiceImpl_DebugTarget(t *testing.T) {
14821489
evalTargetRepo: deps.repo,
14831490
idgen: deps.idgen,
14841491
metric: deps.metric,
1492+
configer: deps.configer,
14851493
typedOperators: typedOps,
14861494
}
14871495

‎idl/thrift/coze/loop/evaluation/coze.loop.evaluation.eval_set.thrift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,7 @@ service EvaluationSetService {
411411

412412
// 数据管理
413413
BatchCreateEvaluationSetItemsResponse BatchCreateEvaluationSetItems(1: BatchCreateEvaluationSetItemsRequest req) (
414-
api.category="evaluation_set", api.post = "/api/evaluation/v1/evaluation_sets/:evaluation_set_id/items/batch_create", api.op_type = 'query', api.tag = 'volc-agentkit,open'
414+
api.category="evaluation_set", api.post = "/api/evaluation/v1/evaluation_sets/:evaluation_set_id/items/batch_create", api.op_type = 'create', api.tag = 'volc-agentkit,open'
415415
)
416416
UpdateEvaluationSetItemResponse UpdateEvaluationSetItem(1: UpdateEvaluationSetItemRequest req) (
417417
api.category="evaluation_set", api.put = "/api/evaluation/v1/evaluation_sets/:evaluation_set_id/items/:item_id", api.op_type = 'update', api.tag = 'volc-agentkit,open'

0 commit comments

Comments
 (0)