diff --git a/backend/go.sum b/backend/go.sum index a3801a02e..6285b5857 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -473,6 +473,7 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAx github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/backend/infra/lock/lock.go b/backend/infra/lock/lock.go index 56e764fa6..5084bc64f 100644 --- a/backend/infra/lock/lock.go +++ b/backend/infra/lock/lock.go @@ -30,6 +30,13 @@ type ILocker interface { // 后退出续期。调用方做写操作前应检查 ctx.Done 以确认仍持有锁，发生错误时应调用 cancel 以主动释放锁。 LockBackoffWithRenew(parent context.Context, key string, ttl time.Duration, maxHold time.Duration) (locked bool, ctx context.Context, cancel func(), err error) LockWithRenew(parent context.Context, key string, ttl time.Duration, maxHold time.Duration) (locked bool, ctx context.Context, cancel func(), err error) + + BackoffLockWithValue(ctx context.Context, key, val string, expiresIn time.Duration, backoff time.Duration) (bool, string, error) + UnlockWithValue(ctx context.Context, key, val string) (bool, error) + // UnlockForce deletes the key without comparing its value. + UnlockForce(ctx context.Context, key string) (bool, error) + // Exists returns true if the key exists. + Exists(ctx context.Context, key string) (bool, error) } func NewRedisLocker(c redis.Cmdable) ILocker { @@ -143,6 +150,35 @@ func (r *redisLocker) Unlock(key string) (bool, error) { return rt == 1, nil } +func (r *redisLocker) UnlockWithValue(ctx context.Context, key, val string) (bool, error) { + const unlockWithValueScript = `if redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]); return 1; end; return 0;` + result, err := r.c.Eval(ctx, unlockWithValueScript, []string{key}, val).Result() + if err != nil { + return false, errors.WithMessage(err, "unlock with lua script") + } + rt, ok := result.(int64) + if !ok { + return false, errors.Errorf("unknown result type %T", result) + } + return rt == 1, nil +} + +func (r *redisLocker) UnlockForce(ctx context.Context, key string) (bool, error) { + n, err := r.c.Del(ctx, key).Result() + if err != nil { + return false, errors.WithMessage(err, "unlock force del") + } + return n > 0, nil +} + +func (r *redisLocker) Exists(ctx context.Context, key string) (bool, error) { + n, err := r.c.Exists(ctx, key).Result() + if err != nil { + return false, errors.WithMessage(err, "exists") + } + return n > 0, nil +} + func (r *redisLocker) renewLock(ctx context.Context, key string, ttl time.Duration, maxHold time.Duration) { t1 := time.After(maxHold) t2 := time.NewTicker(gvalue.Max(time.Second, ttl>>2)) @@ -203,3 +239,59 @@ func (r *redisLocker) ExpireLockIn(key string, expiresIn time.Duration) (bool, e } return rt == 1, nil } + +const setNXWithGetScript = ` +local ok = redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', ARGV[2]) +if ok then + return {1, ARGV[1]} +else + local cur = redis.call('GET', KEYS[1]) + return {0, cur or ''} +end +` + +func (r *redisLocker) BackoffLockWithValue(ctx context.Context, key, val string, expiresIn time.Duration, maxWait time.Duration) (bool, string, error) { + if expiresIn < time.Second { + return false, "", fmt.Errorf("lock ttl too short") + } + + var ok bool + var lastHolder string + bf := backoff.NewExponentialBackOff() + bf.InitialInterval = 50 * time.Millisecond + bf.MaxInterval = 300 * time.Millisecond + bf.MaxElapsedTime = maxWait + + errNotLocked := errors.New("lock hold by others") + err := backoff.Retry(func() error { + result, err := r.c.Eval(ctx, setNXWithGetScript, []string{key}, val, int64(expiresIn/time.Millisecond)).Result() + if err != nil { + return errors.WithMessage(err, fmt.Sprintf("redis setnx with get fail, key: %v", key)) + } + sl, okType := result.([]interface{}) + if !okType || len(sl) != 2 { + return errors.Errorf("unexpected script result type %T or length", result) + } + locked, _ := sl[0].(int64) + if locked == 1 { + ok = true + return nil + } + switch v := sl[1].(type) { + case string: + lastHolder = v + case []byte: + lastHolder = string(v) + default: + return errors.Errorf("unexpected lua script result type %T or length, key: %v", sl[1], key) + } + return errNotLocked + }, bf) + if err != nil { + if errors.Is(err, errNotLocked) { + return false, lastHolder, nil + } + return false, "", err + } + return ok, val, nil +} diff --git a/backend/infra/lock/lock_test.go b/backend/infra/lock/lock_test.go index f53cfc0b9..ae0d052e5 100644 --- a/backend/infra/lock/lock_test.go +++ b/backend/infra/lock/lock_test.go @@ -25,6 +25,157 @@ func newIntCmdResult(val int64, err error) *redis.Cmd { return cmd } +// helper to build a redis.Cmd for Eval script result (val is typically []interface{}{locked, holder}) +func newEvalCmdResult(val interface{}, err error) *redis.Cmd { + cmd := redis.NewCmd(context.Background()) + if err != nil { + cmd.SetErr(err) + return cmd + } + cmd.SetVal(val) + return cmd +} + +func TestRedisLocker_BackoffLockWithValue(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cases := []struct { + name string + expiresIn time.Duration + maxWait time.Duration + setupMock func(m *redisMocks.MockPersistentCmdable) + wantOk bool + wantVal string + wantErr bool + }{ + { + name: "ttl_too_short_returns_error", + expiresIn: 100 * time.Millisecond, + maxWait: time.Second, + setupMock: func(m *redisMocks.MockPersistentCmdable) {}, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "success_locked_on_first_try", + expiresIn: 2 * time.Second, + maxWait: time.Second, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, []string{"key1"}, "val1", int64(2000)). + Return(newEvalCmdResult([]interface{}{int64(1), "val1"}, nil)). + Times(1) + }, + wantOk: true, + wantVal: "val1", + wantErr: false, + }, + { + name: "locked_by_others_returns_holder_string", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), "other_holder"}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "other_holder", + wantErr: false, + }, + { + name: "locked_by_others_returns_holder_bytes", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), []byte("byte_holder")}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "byte_holder", + wantErr: false, + }, + { + name: "redis_error_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult(nil, context.DeadlineExceeded)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_script_result_type_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult(interface{}("not_a_slice"), nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_script_result_length_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0)}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_holder_type_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), 12345}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + mockRedis := redisMocks.NewMockPersistentCmdable(ctrl) + c.setupMock(mockRedis) + locker := &redisLocker{c: mockRedis, holder: "holder"} + ok, val, err := locker.BackoffLockWithValue(context.Background(), "key1", "val1", c.expiresIn, c.maxWait) + if c.wantErr != (err != nil) { + t.Fatalf("err: got %v, wantErr %v", err, c.wantErr) + } + if ok != c.wantOk { + t.Errorf("ok: got %v, want %v", ok, c.wantOk) + } + if val != c.wantVal { + t.Errorf("val: got %q, want %q", val, c.wantVal) + } + }) + } +} + func TestRedisLocker_renewLock_ContextDoneUnlocksAndReturns(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/infra/lock/mocks/lock.go b/backend/infra/lock/mocks/lock.go index 810a9f7c7..0a081f4b5 100644 --- a/backend/infra/lock/mocks/lock.go +++ b/backend/infra/lock/mocks/lock.go @@ -42,6 +42,37 @@ func (m *MockILocker) EXPECT() *MockILockerMockRecorder { return m.recorder } +// BackoffLockWithValue mocks base method. +func (m *MockILocker) BackoffLockWithValue(ctx context.Context, key, val string, expiresIn, backoff time.Duration) (bool, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BackoffLockWithValue", ctx, key, val, expiresIn, backoff) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// BackoffLockWithValue indicates an expected call of BackoffLockWithValue. +func (mr *MockILockerMockRecorder) BackoffLockWithValue(ctx, key, val, expiresIn, backoff any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackoffLockWithValue", reflect.TypeOf((*MockILocker)(nil).BackoffLockWithValue), ctx, key, val, expiresIn, backoff) +} + +// Exists mocks base method. +func (m *MockILocker) Exists(ctx context.Context, key string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, key) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockILockerMockRecorder) Exists(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockILocker)(nil).Exists), ctx, key) +} + // ExpireLockIn mocks base method. func (m *MockILocker) ExpireLockIn(key string, expiresIn time.Duration) (bool, error) { m.ctrl.T.Helper() @@ -136,6 +167,36 @@ func (mr *MockILockerMockRecorder) Unlock(key any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unlock", reflect.TypeOf((*MockILocker)(nil).Unlock), key) } +// UnlockForce mocks base method. +func (m *MockILocker) UnlockForce(ctx context.Context, key string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockForce", ctx, key) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UnlockForce indicates an expected call of UnlockForce. +func (mr *MockILockerMockRecorder) UnlockForce(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockForce", reflect.TypeOf((*MockILocker)(nil).UnlockForce), ctx, key) +} + +// UnlockWithValue mocks base method. +func (m *MockILocker) UnlockWithValue(ctx context.Context, key, val string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockWithValue", ctx, key, val) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UnlockWithValue indicates an expected call of UnlockWithValue. +func (mr *MockILockerMockRecorder) UnlockWithValue(ctx, key, val any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockWithValue", reflect.TypeOf((*MockILocker)(nil).UnlockWithValue), ctx, key, val) +} + // WithHolder mocks base method. func (m *MockILocker) WithHolder(holder string) lock.ILocker { m.ctrl.T.Helper() diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go index 3bb93f7b7..db88c59e8 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go @@ -988,6 +988,7 @@ type Experiment struct { MaxAliveTime *int64 `thrift:"max_alive_time,41,optional" frugal:"41,optional,i64" form:"max_alive_time" json:"max_alive_time,omitempty" query:"max_alive_time"` SourceType *SourceType `thrift:"source_type,42,optional" frugal:"42,optional,SourceType" form:"source_type" json:"source_type,omitempty" query:"source_type"` SourceID *string `thrift:"source_id,43,optional" frugal:"43,optional,string" form:"source_id" json:"source_id,omitempty" query:"source_id"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` // 补充的评估器id+version关联评估器方式，和evaluator_version_ids共同使用，兼容老逻辑 EvaluatorIDVersionList []*evaluator.EvaluatorIDVersionItem `thrift:"evaluator_id_version_list,51,optional" frugal:"51,optional,list" form:"evaluator_id_version_list" json:"evaluator_id_version_list,omitempty" query:"evaluator_id_version_list"` ExptTemplateMeta *ExptTemplateMeta `thrift:"expt_template_meta,60,optional" frugal:"60,optional,ExptTemplateMeta" form:"expt_template_meta" json:"expt_template_meta,omitempty" query:"expt_template_meta"` @@ -1315,6 +1316,18 @@ func (p *Experiment) GetSourceID() (v string) { return *p.SourceID } +var Experiment_ItemRetryNum_DEFAULT int32 + +func (p *Experiment) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return Experiment_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var Experiment_EvaluatorIDVersionList_DEFAULT []*evaluator.EvaluatorIDVersionItem func (p *Experiment) GetEvaluatorIDVersionList() (v []*evaluator.EvaluatorIDVersionItem) { @@ -1440,6 +1453,9 @@ func (p *Experiment) SetSourceType(val *SourceType) { func (p *Experiment) SetSourceID(val *string) { p.SourceID = val } +func (p *Experiment) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *Experiment) SetEvaluatorIDVersionList(val []*evaluator.EvaluatorIDVersionItem) { p.EvaluatorIDVersionList = val } @@ -1480,6 +1496,7 @@ var fieldIDToName_Experiment = map[int16]string{ 41: "max_alive_time", 42: "source_type", 43: "source_id", + 45: "item_retry_num", 51: "evaluator_id_version_list", 60: "expt_template_meta", 61: "score_weight_config", @@ -1590,6 +1607,10 @@ func (p *Experiment) IsSetSourceID() bool { return p.SourceID != nil } +func (p *Experiment) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *Experiment) IsSetEvaluatorIDVersionList() bool { return p.EvaluatorIDVersionList != nil } @@ -1832,6 +1853,14 @@ func (p *Experiment) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 51: if fieldTypeId == thrift.LIST { if err = p.ReadField51(iprot); err != nil { @@ -2200,6 +2229,17 @@ func (p *Experiment) ReadField43(iprot thrift.TProtocol) error { p.SourceID = _field return nil } +func (p *Experiment) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *Experiment) ReadField51(iprot thrift.TProtocol) error { _, size, err := iprot.ReadListBegin() if err != nil { @@ -2361,6 +2401,10 @@ func (p *Experiment) Write(oprot thrift.TProtocol) (err error) { fieldId = 43 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField51(oprot); err != nil { fieldId = 51 goto WriteFieldError @@ -2887,6 +2931,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 43 end error: ", p), err) } +func (p *Experiment) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *Experiment) writeField51(oprot thrift.TProtocol) (err error) { if p.IsSetEvaluatorIDVersionList() { if err = oprot.WriteFieldBegin("evaluator_id_version_list", thrift.LIST, 51); err != nil { @@ -3060,6 +3122,9 @@ func (p *Experiment) DeepEqual(ano *Experiment) bool { if !p.Field43DeepEqual(ano.SourceID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field51DeepEqual(ano.EvaluatorIDVersionList) { return false } @@ -3360,6 +3425,18 @@ func (p *Experiment) Field43DeepEqual(src *string) bool { } return true } +func (p *Experiment) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *Experiment) Field51DeepEqual(src []*evaluator.EvaluatorIDVersionItem) bool { if len(p.EvaluatorIDVersionList) != len(src) { @@ -4645,6 +4722,7 @@ type ExptFieldMapping struct { EvaluatorFieldMapping []*EvaluatorFieldMapping `thrift:"evaluator_field_mapping,2,optional" frugal:"2,optional,list" form:"evaluator_field_mapping" json:"evaluator_field_mapping,omitempty" query:"evaluator_field_mapping"` TargetRuntimeParam *common.RuntimeParam `thrift:"target_runtime_param,3,optional" frugal:"3,optional,common.RuntimeParam" form:"target_runtime_param" json:"target_runtime_param,omitempty" query:"target_runtime_param"` ItemConcurNum *int32 `thrift:"item_concur_num,4,optional" frugal:"4,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty" query:"item_concur_num"` + ItemRetryNum *int32 `thrift:"item_retry_num,5,optional" frugal:"5,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` } func NewExptFieldMapping() *ExptFieldMapping { @@ -4701,6 +4779,18 @@ func (p *ExptFieldMapping) GetItemConcurNum() (v int32) { } return *p.ItemConcurNum } + +var ExptFieldMapping_ItemRetryNum_DEFAULT int32 + +func (p *ExptFieldMapping) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return ExptFieldMapping_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} func (p *ExptFieldMapping) SetTargetFieldMapping(val *TargetFieldMapping) { p.TargetFieldMapping = val } @@ -4713,12 +4803,16 @@ func (p *ExptFieldMapping) SetTargetRuntimeParam(val *common.RuntimeParam) { func (p *ExptFieldMapping) SetItemConcurNum(val *int32) { p.ItemConcurNum = val } +func (p *ExptFieldMapping) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} var fieldIDToName_ExptFieldMapping = map[int16]string{ 1: "target_field_mapping", 2: "evaluator_field_mapping", 3: "target_runtime_param", 4: "item_concur_num", + 5: "item_retry_num", } func (p *ExptFieldMapping) IsSetTargetFieldMapping() bool { @@ -4737,6 +4831,10 @@ func (p *ExptFieldMapping) IsSetItemConcurNum() bool { return p.ItemConcurNum != nil } +func (p *ExptFieldMapping) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *ExptFieldMapping) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -4787,6 +4885,14 @@ func (p *ExptFieldMapping) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 5: + if fieldTypeId == thrift.I32 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -4866,6 +4972,17 @@ func (p *ExptFieldMapping) ReadField4(iprot thrift.TProtocol) error { p.ItemConcurNum = _field return nil } +func (p *ExptFieldMapping) ReadField5(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *ExptFieldMapping) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -4889,6 +5006,10 @@ func (p *ExptFieldMapping) Write(oprot thrift.TProtocol) (err error) { fieldId = 4 goto WriteFieldError } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -4987,6 +5108,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) } +func (p *ExptFieldMapping) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} func (p *ExptFieldMapping) String() string { if p == nil { @@ -5014,6 +5153,9 @@ func (p *ExptFieldMapping) DeepEqual(ano *ExptFieldMapping) bool { if !p.Field4DeepEqual(ano.ItemConcurNum) { return false } + if !p.Field5DeepEqual(ano.ItemRetryNum) { + return false + } return true } @@ -5056,6 +5198,18 @@ func (p *ExptFieldMapping) Field4DeepEqual(src *int32) bool { } return true } +func (p *ExptFieldMapping) Field5DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} // 实验评估器得分加权配置 type ExptScoreWeight struct { diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go index 0803ff63e..b52dd6740 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go @@ -418,6 +418,20 @@ func (p *Experiment) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 51: if fieldTypeId == thrift.LIST { l, err = p.FastReadField51(buf[offset:]) @@ -882,6 +896,20 @@ func (p *Experiment) FastReadField43(buf []byte) (int, error) { return offset, nil } +func (p *Experiment) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *Experiment) FastReadField51(buf []byte) (int, error) { offset := 0 @@ -961,6 +989,7 @@ func (p *Experiment) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { offset += p.fastWriteField27(buf[offset:], w) offset += p.fastWriteField28(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField62(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) @@ -1016,6 +1045,7 @@ func (p *Experiment) BLength() int { l += p.field41Length() l += p.field42Length() l += p.field43Length() + l += p.field45Length() l += p.field51Length() l += p.field60Length() l += p.field61Length() @@ -1280,6 +1310,15 @@ func (p *Experiment) fastWriteField43(buf []byte, w thrift.NocopyWriter) int { return offset } +func (p *Experiment) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *Experiment) fastWriteField51(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetEvaluatorIDVersionList() { @@ -1567,6 +1606,15 @@ func (p *Experiment) field43Length() int { return l } +func (p *Experiment) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *Experiment) field51Length() int { l := 0 if p.IsSetEvaluatorIDVersionList() { @@ -1806,6 +1854,11 @@ func (p *Experiment) DeepCopy(s interface{}) error { p.SourceID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.EvaluatorIDVersionList != nil { p.EvaluatorIDVersionList = make([]*evaluator.EvaluatorIDVersionItem, 0, len(src.EvaluatorIDVersionList)) for _, elem := range src.EvaluatorIDVersionList { @@ -2813,6 +2866,20 @@ func (p *ExptFieldMapping) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 5: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -2894,6 +2961,20 @@ func (p *ExptFieldMapping) FastReadField4(buf []byte) (int, error) { return offset, nil } +func (p *ExptFieldMapping) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *ExptFieldMapping) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -2902,6 +2983,7 @@ func (p *ExptFieldMapping) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) in offset := 0 if p != nil { offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) @@ -2917,6 +2999,7 @@ func (p *ExptFieldMapping) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field4Length() + l += p.field5Length() } l += thrift.Binary.FieldStopLength() return l @@ -2965,6 +3048,15 @@ func (p *ExptFieldMapping) fastWriteField4(buf []byte, w thrift.NocopyWriter) in return offset } +func (p *ExptFieldMapping) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 5) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *ExptFieldMapping) field1Length() int { l := 0 if p.IsSetTargetFieldMapping() { @@ -3005,6 +3097,15 @@ func (p *ExptFieldMapping) field4Length() int { return l } +func (p *ExptFieldMapping) field5Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *ExptFieldMapping) DeepCopy(s interface{}) error { src, ok := s.(*ExptFieldMapping) if !ok { @@ -3049,6 +3150,11 @@ func (p *ExptFieldMapping) DeepCopy(s interface{}) error { p.ItemConcurNum = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go index 124c98d96..d2ab53786 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go @@ -51,6 +51,7 @@ type CreateExperimentRequest struct { EnableWeightedScore *bool `thrift:"enable_weighted_score,41,optional" frugal:"41,optional,bool" json:"enable_weighted_score" form:"enable_weighted_score" ` EvaluatorScoreWeights map[int64]float64 `thrift:"evaluator_score_weights,42,optional" frugal:"42,optional,map" json:"evaluator_score_weights" form:"evaluator_score_weights" ` ExptTemplateID *int64 `thrift:"expt_template_id,43,optional" frugal:"43,optional,i64" json:"expt_template_id" form:"expt_template_id" ` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -321,6 +322,18 @@ func (p *CreateExperimentRequest) GetExptTemplateID() (v int64) { return *p.ExptTemplateID } +var CreateExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *CreateExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return CreateExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var CreateExperimentRequest_Session_DEFAULT *common.Session func (p *CreateExperimentRequest) GetSession() (v *common.Session) { @@ -410,6 +423,9 @@ func (p *CreateExperimentRequest) SetEvaluatorScoreWeights(val map[int64]float64 func (p *CreateExperimentRequest) SetExptTemplateID(val *int64) { p.ExptTemplateID = val } +func (p *CreateExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *CreateExperimentRequest) SetSession(val *common.Session) { p.Session = val } @@ -440,6 +456,7 @@ var fieldIDToName_CreateExperimentRequest = map[int16]string{ 41: "enable_weighted_score", 42: "evaluator_score_weights", 43: "expt_template_id", + 45: "item_retry_num", 200: "session", 255: "Base", } @@ -528,6 +545,10 @@ func (p *CreateExperimentRequest) IsSetExptTemplateID() bool { return p.ExptTemplateID != nil } +func (p *CreateExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *CreateExperimentRequest) IsSetSession() bool { return p.Session != nil } @@ -732,6 +753,14 @@ func (p *CreateExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 200: if fieldTypeId == thrift.STRUCT { if err = p.ReadField200(iprot); err != nil { @@ -1072,6 +1101,17 @@ func (p *CreateExperimentRequest) ReadField43(iprot thrift.TProtocol) error { p.ExptTemplateID = _field return nil } +func (p *CreateExperimentRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *CreateExperimentRequest) ReadField200(iprot thrift.TProtocol) error { _field := common.NewSession() if err := _field.Read(iprot); err != nil { @@ -1183,6 +1223,10 @@ func (p *CreateExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 43 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField200(oprot); err != nil { fieldId = 200 goto WriteFieldError @@ -1638,6 +1682,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 43 end error: ", p), err) } +func (p *CreateExperimentRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *CreateExperimentRequest) writeField200(oprot thrift.TProtocol) (err error) { if p.IsSetSession() { if err = oprot.WriteFieldBegin("session", thrift.STRUCT, 200); err != nil { @@ -1755,6 +1817,9 @@ func (p *CreateExperimentRequest) DeepEqual(ano *CreateExperimentRequest) bool { if !p.Field43DeepEqual(ano.ExptTemplateID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field200DeepEqual(ano.Session) { return false } @@ -2012,6 +2077,18 @@ func (p *CreateExperimentRequest) Field43DeepEqual(src *int64) bool { } return true } +func (p *CreateExperimentRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *CreateExperimentRequest) Field200DeepEqual(src *common.Session) bool { if !p.Session.DeepEqual(src) { @@ -2291,6 +2368,7 @@ type SubmitExperimentRequest struct { // 是否启用评估器得分加权汇总，以及各评估器的权重配置（key 为 evaluator_version_id，value 为权重） EnableWeightedScore *bool `thrift:"enable_weighted_score,41,optional" frugal:"41,optional,bool" json:"enable_weighted_score" form:"enable_weighted_score" ` ExptTemplateID *int64 `thrift:"expt_template_id,42,optional" frugal:"42,optional,i64" json:"expt_template_id" form:"expt_template_id" ` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` @@ -2550,6 +2628,18 @@ func (p *SubmitExperimentRequest) GetExptTemplateID() (v int64) { return *p.ExptTemplateID } +var SubmitExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *SubmitExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return SubmitExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var SubmitExperimentRequest_Ext_DEFAULT map[string]string func (p *SubmitExperimentRequest) GetExt() (v map[string]string) { @@ -2648,6 +2738,9 @@ func (p *SubmitExperimentRequest) SetEnableWeightedScore(val *bool) { func (p *SubmitExperimentRequest) SetExptTemplateID(val *int64) { p.ExptTemplateID = val } +func (p *SubmitExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *SubmitExperimentRequest) SetExt(val map[string]string) { p.Ext = val } @@ -2680,6 +2773,7 @@ var fieldIDToName_SubmitExperimentRequest = map[int16]string{ 40: "evaluator_id_version_list", 41: "enable_weighted_score", 42: "expt_template_id", + 45: "item_retry_num", 100: "ext", 200: "session", 255: "Base", @@ -2765,6 +2859,10 @@ func (p *SubmitExperimentRequest) IsSetExptTemplateID() bool { return p.ExptTemplateID != nil } +func (p *SubmitExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *SubmitExperimentRequest) IsSetExt() bool { return p.Ext != nil } @@ -2965,6 +3063,14 @@ func (p *SubmitExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -3284,6 +3390,17 @@ func (p *SubmitExperimentRequest) ReadField42(iprot thrift.TProtocol) error { p.ExptTemplateID = _field return nil } +func (p *SubmitExperimentRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *SubmitExperimentRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -3420,6 +3537,10 @@ func (p *SubmitExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 42 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -3850,6 +3971,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 42 end error: ", p), err) } +func (p *SubmitExperimentRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *SubmitExperimentRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -3993,6 +4132,9 @@ func (p *SubmitExperimentRequest) DeepEqual(ano *SubmitExperimentRequest) bool { if !p.Field42DeepEqual(ano.ExptTemplateID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -4240,6 +4382,18 @@ func (p *SubmitExperimentRequest) Field42DeepEqual(src *int64) bool { } return true } +func (p *SubmitExperimentRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *SubmitExperimentRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { @@ -7792,13 +7946,14 @@ func (p *BatchDeleteExperimentsResponse) Field255DeepEqual(src *base.BaseResp) b } type RunExperimentRequest struct { - WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` - ExptID *int64 `thrift:"expt_id,2,optional" frugal:"2,optional,i64" json:"expt_id" form:"expt_id" ` - ItemIds []int64 `thrift:"item_ids,3,optional" frugal:"3,optional,list" json:"item_ids" form:"item_ids" ` - ExptType *expt.ExptType `thrift:"expt_type,10,optional" frugal:"10,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` - Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` - Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` - Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` + WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` + ExptID *int64 `thrift:"expt_id,2,optional" frugal:"2,optional,i64" json:"expt_id" form:"expt_id" ` + ItemIds []int64 `thrift:"item_ids,3,optional" frugal:"3,optional,list" json:"item_ids" form:"item_ids" ` + ExptType *expt.ExptType `thrift:"expt_type,10,optional" frugal:"10,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,11,optional" frugal:"11,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty"` + Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` + Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` + Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } func NewRunExperimentRequest() *RunExperimentRequest { @@ -7856,6 +8011,18 @@ func (p *RunExperimentRequest) GetExptType() (v expt.ExptType) { return *p.ExptType } +var RunExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *RunExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return RunExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var RunExperimentRequest_Ext_DEFAULT map[string]string func (p *RunExperimentRequest) GetExt() (v map[string]string) { @@ -7903,6 +8070,9 @@ func (p *RunExperimentRequest) SetItemIds(val []int64) { func (p *RunExperimentRequest) SetExptType(val *expt.ExptType) { p.ExptType = val } +func (p *RunExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *RunExperimentRequest) SetExt(val map[string]string) { p.Ext = val } @@ -7918,6 +8088,7 @@ var fieldIDToName_RunExperimentRequest = map[int16]string{ 2: "expt_id", 3: "item_ids", 10: "expt_type", + 11: "item_retry_num", 100: "ext", 200: "session", 255: "Base", @@ -7939,6 +8110,10 @@ func (p *RunExperimentRequest) IsSetExptType() bool { return p.ExptType != nil } +func (p *RunExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *RunExperimentRequest) IsSetExt() bool { return p.Ext != nil } @@ -8001,6 +8176,14 @@ func (p *RunExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 11: + if fieldTypeId == thrift.I32 { + if err = p.ReadField11(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -8111,6 +8294,17 @@ func (p *RunExperimentRequest) ReadField10(iprot thrift.TProtocol) error { p.ExptType = _field return nil } +func (p *RunExperimentRequest) ReadField11(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *RunExperimentRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -8179,6 +8373,10 @@ func (p *RunExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 10 goto WriteFieldError } + if err = p.writeField11(oprot); err != nil { + fieldId = 11 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -8289,6 +8487,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 10 end error: ", p), err) } +func (p *RunExperimentRequest) writeField11(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 11); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 11 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 11 end error: ", p), err) +} func (p *RunExperimentRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -8381,6 +8597,9 @@ func (p *RunExperimentRequest) DeepEqual(ano *RunExperimentRequest) bool { if !p.Field10DeepEqual(ano.ExptType) { return false } + if !p.Field11DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -8442,6 +8661,18 @@ func (p *RunExperimentRequest) Field10DeepEqual(src *expt.ExptType) bool { } return true } +func (p *RunExperimentRequest) Field11DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *RunExperimentRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { @@ -17024,6 +17255,7 @@ type CreateExperimentTemplateRequest struct { DefaultEvaluatorsConcurNum *int32 `thrift:"default_evaluators_concur_num,21,optional" frugal:"21,optional,i32" form:"default_evaluators_concur_num" json:"default_evaluators_concur_num,omitempty"` // 调度配置（不在 ExptTemplate 结构中，保留在顶层） ScheduleCron *string `thrift:"schedule_cron,22,optional" frugal:"22,optional,string" form:"schedule_cron" json:"schedule_cron,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -17114,6 +17346,18 @@ func (p *CreateExperimentTemplateRequest) GetScheduleCron() (v string) { return *p.ScheduleCron } +var CreateExperimentTemplateRequest_ItemRetryNum_DEFAULT int32 + +func (p *CreateExperimentTemplateRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return CreateExperimentTemplateRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var CreateExperimentTemplateRequest_Session_DEFAULT *common.Session func (p *CreateExperimentTemplateRequest) GetSession() (v *common.Session) { @@ -17158,6 +17402,9 @@ func (p *CreateExperimentTemplateRequest) SetDefaultEvaluatorsConcurNum(val *int func (p *CreateExperimentTemplateRequest) SetScheduleCron(val *string) { p.ScheduleCron = val } +func (p *CreateExperimentTemplateRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *CreateExperimentTemplateRequest) SetSession(val *common.Session) { p.Session = val } @@ -17173,6 +17420,7 @@ var fieldIDToName_CreateExperimentTemplateRequest = map[int16]string{ 20: "create_eval_target_param", 21: "default_evaluators_concur_num", 22: "schedule_cron", + 45: "item_retry_num", 200: "session", 255: "Base", } @@ -17201,6 +17449,10 @@ func (p *CreateExperimentTemplateRequest) IsSetScheduleCron() bool { return p.ScheduleCron != nil } +func (p *CreateExperimentTemplateRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *CreateExperimentTemplateRequest) IsSetSession() bool { return p.Session != nil } @@ -17285,6 +17537,14 @@ func (p *CreateExperimentTemplateRequest) Read(iprot thrift.TProtocol) (err erro } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 200: if fieldTypeId == thrift.STRUCT { if err = p.ReadField200(iprot); err != nil { @@ -17401,6 +17661,17 @@ func (p *CreateExperimentTemplateRequest) ReadField22(iprot thrift.TProtocol) er p.ScheduleCron = _field return nil } +func (p *CreateExperimentTemplateRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *CreateExperimentTemplateRequest) ReadField200(iprot thrift.TProtocol) error { _field := common.NewSession() if err := _field.Read(iprot); err != nil { @@ -17452,6 +17723,10 @@ func (p *CreateExperimentTemplateRequest) Write(oprot thrift.TProtocol) (err err fieldId = 22 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField200(oprot); err != nil { fieldId = 200 goto WriteFieldError @@ -17602,6 +17877,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 22 end error: ", p), err) } +func (p *CreateExperimentTemplateRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *CreateExperimentTemplateRequest) writeField200(oprot thrift.TProtocol) (err error) { if p.IsSetSession() { if err = oprot.WriteFieldBegin("session", thrift.STRUCT, 200); err != nil { @@ -17674,6 +17967,9 @@ func (p *CreateExperimentTemplateRequest) DeepEqual(ano *CreateExperimentTemplat if !p.Field22DeepEqual(ano.ScheduleCron) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field200DeepEqual(ano.Session) { return false } @@ -17742,6 +18038,18 @@ func (p *CreateExperimentTemplateRequest) Field22DeepEqual(src *string) bool { } return true } +func (p *CreateExperimentTemplateRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *CreateExperimentTemplateRequest) Field200DeepEqual(src *common.Session) bool { if !p.Session.DeepEqual(src) { diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go index b7d619af5..7094169e7 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go @@ -366,6 +366,20 @@ func (p *CreateExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 200: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField200(buf[offset:]) @@ -774,6 +788,20 @@ func (p *CreateExperimentRequest) FastReadField43(buf []byte) (int, error) { return offset, nil } +func (p *CreateExperimentRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *CreateExperimentRequest) FastReadField200(buf []byte) (int, error) { offset := 0 _field := common.NewSession() @@ -815,6 +843,7 @@ func (p *CreateExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWri offset += p.fastWriteField31(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) offset += p.fastWriteField43(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField6(buf[offset:], w) @@ -859,6 +888,7 @@ func (p *CreateExperimentRequest) BLength() int { l += p.field41Length() l += p.field42Length() l += p.field43Length() + l += p.field45Length() l += p.field200Length() l += p.field255Length() } @@ -1091,6 +1121,15 @@ func (p *CreateExperimentRequest) fastWriteField43(buf []byte, w thrift.NocopyWr return offset } +func (p *CreateExperimentRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *CreateExperimentRequest) fastWriteField200(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetSession() { @@ -1317,6 +1356,15 @@ func (p *CreateExperimentRequest) field43Length() int { return l } +func (p *CreateExperimentRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *CreateExperimentRequest) field200Length() int { l := 0 if p.IsSetSession() { @@ -1501,6 +1549,11 @@ func (p *CreateExperimentRequest) DeepCopy(s interface{}) error { p.ExptTemplateID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _session *common.Session if src.Session != nil { _session = &common.Session{} @@ -2005,6 +2058,20 @@ func (p *SubmitExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -2395,6 +2462,20 @@ func (p *SubmitExperimentRequest) FastReadField42(buf []byte) (int, error) { return offset, nil } +func (p *SubmitExperimentRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *SubmitExperimentRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -2468,6 +2549,7 @@ func (p *SubmitExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWri offset += p.fastWriteField31(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) offset += p.fastWriteField42(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField6(buf[offset:], w) @@ -2511,6 +2593,7 @@ func (p *SubmitExperimentRequest) BLength() int { l += p.field40Length() l += p.field41Length() l += p.field42Length() + l += p.field45Length() l += p.field100Length() l += p.field200Length() l += p.field255Length() @@ -2727,6 +2810,15 @@ func (p *SubmitExperimentRequest) fastWriteField42(buf []byte, w thrift.NocopyWr return offset } +func (p *SubmitExperimentRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *SubmitExperimentRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -2959,6 +3051,15 @@ func (p *SubmitExperimentRequest) field42Length() int { return l } +func (p *SubmitExperimentRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *SubmitExperimentRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -3145,6 +3246,11 @@ func (p *SubmitExperimentRequest) DeepCopy(s interface{}) error { p.ExptTemplateID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { @@ -5819,6 +5925,20 @@ func (p *RunExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 11: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField11(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -5947,6 +6067,20 @@ func (p *RunExperimentRequest) FastReadField10(buf []byte) (int, error) { return offset, nil } +func (p *RunExperimentRequest) FastReadField11(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *RunExperimentRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -6012,6 +6146,7 @@ func (p *RunExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWriter if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) + offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField10(buf[offset:], w) offset += p.fastWriteField100(buf[offset:], w) @@ -6029,6 +6164,7 @@ func (p *RunExperimentRequest) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field10Length() + l += p.field11Length() l += p.field100Length() l += p.field200Length() l += p.field255Length() @@ -6080,6 +6216,15 @@ func (p *RunExperimentRequest) fastWriteField10(buf []byte, w thrift.NocopyWrite return offset } +func (p *RunExperimentRequest) fastWriteField11(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 11) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *RunExperimentRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -6153,6 +6298,15 @@ func (p *RunExperimentRequest) field10Length() int { return l } +func (p *RunExperimentRequest) field11Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *RunExperimentRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -6216,6 +6370,11 @@ func (p *RunExperimentRequest) DeepCopy(s interface{}) error { p.ExptType = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { @@ -12634,6 +12793,20 @@ func (p *CreateExperimentTemplateRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 200: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField200(buf[offset:]) @@ -12776,6 +12949,20 @@ func (p *CreateExperimentTemplateRequest) FastReadField22(buf []byte) (int, erro return offset, nil } +func (p *CreateExperimentTemplateRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *CreateExperimentTemplateRequest) FastReadField200(buf []byte) (int, error) { offset := 0 _field := common.NewSession() @@ -12809,6 +12996,7 @@ func (p *CreateExperimentTemplateRequest) FastWriteNocopy(buf []byte, w thrift.N if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField21(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField10(buf[offset:], w) offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField12(buf[offset:], w) @@ -12831,6 +13019,7 @@ func (p *CreateExperimentTemplateRequest) BLength() int { l += p.field20Length() l += p.field21Length() l += p.field22Length() + l += p.field45Length() l += p.field200Length() l += p.field255Length() } @@ -12899,6 +13088,15 @@ func (p *CreateExperimentTemplateRequest) fastWriteField22(buf []byte, w thrift. return offset } +func (p *CreateExperimentTemplateRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *CreateExperimentTemplateRequest) fastWriteField200(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetSession() { @@ -12978,6 +13176,15 @@ func (p *CreateExperimentTemplateRequest) field22Length() int { return l } +func (p *CreateExperimentTemplateRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *CreateExperimentTemplateRequest) field200Length() int { l := 0 if p.IsSetSession() { @@ -13053,6 +13260,11 @@ func (p *CreateExperimentTemplateRequest) DeepCopy(s interface{}) error { p.ScheduleCron = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _session *common.Session if src.Session != nil { _session = &common.Session{} diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go index 059c4dff1..1f2e948ab 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go @@ -14824,6 +14824,7 @@ type SubmitExperimentOApiRequest struct { // 运行信息 ItemConcurNum *int32 `thrift:"item_concur_num,20,optional" frugal:"20,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty"` TargetRuntimeParam *common.RuntimeParam `thrift:"target_runtime_param,22,optional" frugal:"22,optional,common.RuntimeParam" form:"target_runtime_param" json:"target_runtime_param,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -14954,6 +14955,18 @@ func (p *SubmitExperimentOApiRequest) GetTargetRuntimeParam() (v *common.Runtime return p.TargetRuntimeParam } +var SubmitExperimentOApiRequest_ItemRetryNum_DEFAULT int32 + +func (p *SubmitExperimentOApiRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return SubmitExperimentOApiRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var SubmitExperimentOApiRequest_Base_DEFAULT *base.Base func (p *SubmitExperimentOApiRequest) GetBase() (v *base.Base) { @@ -14995,6 +15008,9 @@ func (p *SubmitExperimentOApiRequest) SetItemConcurNum(val *int32) { func (p *SubmitExperimentOApiRequest) SetTargetRuntimeParam(val *common.RuntimeParam) { p.TargetRuntimeParam = val } +func (p *SubmitExperimentOApiRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *SubmitExperimentOApiRequest) SetBase(val *base.Base) { p.Base = val } @@ -15010,6 +15026,7 @@ var fieldIDToName_SubmitExperimentOApiRequest = map[int16]string{ 8: "evaluator_field_mapping", 20: "item_concur_num", 22: "target_runtime_param", + 45: "item_retry_num", 255: "Base", } @@ -15053,6 +15070,10 @@ func (p *SubmitExperimentOApiRequest) IsSetTargetRuntimeParam() bool { return p.TargetRuntimeParam != nil } +func (p *SubmitExperimentOApiRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *SubmitExperimentOApiRequest) IsSetBase() bool { return p.Base != nil } @@ -15155,6 +15176,14 @@ func (p *SubmitExperimentOApiRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 255: if fieldTypeId == thrift.STRUCT { if err = p.ReadField255(iprot); err != nil { @@ -15314,6 +15343,17 @@ func (p *SubmitExperimentOApiRequest) ReadField22(iprot thrift.TProtocol) error p.TargetRuntimeParam = _field return nil } +func (p *SubmitExperimentOApiRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *SubmitExperimentOApiRequest) ReadField255(iprot thrift.TProtocol) error { _field := base.NewBase() if err := _field.Read(iprot); err != nil { @@ -15369,6 +15409,10 @@ func (p *SubmitExperimentOApiRequest) Write(oprot thrift.TProtocol) (err error) fieldId = 22 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField255(oprot); err != nil { fieldId = 255 goto WriteFieldError @@ -15587,6 +15631,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 22 end error: ", p), err) } +func (p *SubmitExperimentOApiRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *SubmitExperimentOApiRequest) writeField255(oprot thrift.TProtocol) (err error) { if p.IsSetBase() { if err = oprot.WriteFieldBegin("Base", thrift.STRUCT, 255); err != nil { @@ -15650,6 +15712,9 @@ func (p *SubmitExperimentOApiRequest) DeepEqual(ano *SubmitExperimentOApiRequest if !p.Field22DeepEqual(ano.TargetRuntimeParam) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field255DeepEqual(ano.Base) { return false } @@ -15758,6 +15823,18 @@ func (p *SubmitExperimentOApiRequest) Field22DeepEqual(src *common.RuntimeParam) } return true } +func (p *SubmitExperimentOApiRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *SubmitExperimentOApiRequest) Field255DeepEqual(src *base.Base) bool { if !p.Base.DeepEqual(src) { diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go index ea00cf97b..b51df057c 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go @@ -10652,6 +10652,20 @@ func (p *SubmitExperimentOApiRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 255: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField255(buf[offset:]) @@ -10838,6 +10852,20 @@ func (p *SubmitExperimentOApiRequest) FastReadField22(buf []byte) (int, error) { return offset, nil } +func (p *SubmitExperimentOApiRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *SubmitExperimentOApiRequest) FastReadField255(buf []byte) (int, error) { offset := 0 _field := base.NewBase() @@ -10859,6 +10887,7 @@ func (p *SubmitExperimentOApiRequest) FastWriteNocopy(buf []byte, w thrift.Nocop if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField20(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) @@ -10886,6 +10915,7 @@ func (p *SubmitExperimentOApiRequest) BLength() int { l += p.field8Length() l += p.field20Length() l += p.field22Length() + l += p.field45Length() l += p.field255Length() } l += thrift.Binary.FieldStopLength() @@ -10996,6 +11026,15 @@ func (p *SubmitExperimentOApiRequest) fastWriteField22(buf []byte, w thrift.Noco return offset } +func (p *SubmitExperimentOApiRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *SubmitExperimentOApiRequest) fastWriteField255(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetBase() { @@ -11103,6 +11142,15 @@ func (p *SubmitExperimentOApiRequest) field22Length() int { return l } +func (p *SubmitExperimentOApiRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *SubmitExperimentOApiRequest) field255Length() int { l := 0 if p.IsSetBase() { @@ -11210,6 +11258,11 @@ func (p *SubmitExperimentOApiRequest) DeepCopy(s interface{}) error { } p.TargetRuntimeParam = _targetRuntimeParam + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _base *base.Base if src.Base != nil { _base = &base.Base{} diff --git a/backend/modules/evaluation/application/convertor/experiment/expt.go b/backend/modules/evaluation/application/convertor/experiment/expt.go index 3ddc96126..566b6e1e5 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt.go @@ -55,6 +55,9 @@ func (e *EvalConfConvert) ConvertToEntity(cer *expt.CreateExperimentRequest, eva } ec.ConnectorConf.EvaluatorsConf = evalsConf } + if cer.GetItemRetryNum() > 0 { + ec.ItemRetryNum = gptr.Of(int(cer.GetItemRetryNum())) + } return ec, nil } @@ -361,8 +364,15 @@ func ToExptDTO(experiment *entity.Experiment) *domain_expt.Experiment { if experiment.EndAt != nil { res.EndTime = gptr.Of(experiment.EndAt.Unix()) } - if experiment.EvalConf != nil && experiment.EvalConf.ItemConcurNum != nil { - res.ItemConcurNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemConcurNum))) + if experiment.EvalConf != nil { + if experiment.EvalConf.ItemConcurNum != nil { + res.ItemConcurNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemConcurNum))) + } + if experiment.EvalConf.ItemRetryNum != nil { + res.ItemRetryNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemRetryNum))) + } else { + res.ItemRetryNum = gptr.Of(int32(0)) + } } // 填充权重配置（score_weight_config 和 enable_weighted_score） @@ -490,6 +500,18 @@ func ConvertCreateReq(cer *expt.CreateExperimentRequest, evaluatorVersionRunConf if cer.IsSetExptTemplateID() { param.ExptTemplateID = cer.GetExptTemplateID() } - return param, nil } + +func ConvRetryMode(m domain_expt.ExptRetryMode) entity.ExptRunMode { + switch m { + case domain_expt.ExptRetryMode_RetryFailure: + return entity.EvaluationModeFailRetry + case domain_expt.ExptRetryMode_RetryAll: + return entity.EvaluationModeRetryAll + case domain_expt.ExptRetryMode_RetryTargetItems: + return entity.EvaluationModeRetryItems + default: + return entity.EvaluationModeUnknown + } +} diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_template.go b/backend/modules/evaluation/application/convertor/experiment/expt_template.go index 62c8665b3..73020b073 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_template.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_template.go @@ -6,6 +6,7 @@ package experiment import ( "fmt" + "github.com/bytedance/gg/gcond" "github.com/bytedance/gg/gptr" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/common" @@ -232,6 +233,7 @@ func buildTemplateConfForCreate( templateConf := &entity.ExptTemplateConfiguration{ ItemConcurNum: ptr.ConvIntPtr[int32, int](itemConcurNum), EvaluatorsConcurNum: ptr.ConvIntPtr[int32, int](req.DefaultEvaluatorsConcurNum), + ItemRetryNum: gcond.If(req.GetFieldMappingConfig().GetItemRetryNum() > 0, gptr.Of(int(req.GetFieldMappingConfig().GetItemRetryNum())), nil), } if targetFieldMapping == nil && len(evaluatorConfs) == 0 { @@ -598,8 +600,15 @@ func buildTemplateFieldMappingDTO(template *entity.ExptTemplate) *domain_expt.Ex return nil } + var itemRetryNum *int32 + if template.TemplateConf != nil && gptr.Indirect(template.TemplateConf.ItemRetryNum) > 0 { + itemRetryNum = gptr.Of(int32(gptr.Indirect(template.TemplateConf.ItemRetryNum))) + } else { + itemRetryNum = gptr.Of(int32(0)) + } fieldMapping := &domain_expt.ExptFieldMapping{ ItemConcurNum: ptr.ConvIntPtr[int, int32](template.FieldMappingConfig.ItemConcurNum), + ItemRetryNum: itemRetryNum, } if template.FieldMappingConfig.TargetFieldMapping != nil { @@ -958,6 +967,7 @@ func ConvertUpdateExptTemplateReq(req *expt.UpdateExperimentTemplateRequest) (*e templateConf := &entity.ExptTemplateConfiguration{ ItemConcurNum: ptr.ConvIntPtr[int32, int](itemConcurNum), EvaluatorsConcurNum: ptr.ConvIntPtr[int32, int](req.DefaultEvaluatorsConcurNum), + ItemRetryNum: gcond.If(req.GetFieldMappingConfig().GetItemRetryNum() > 0, gptr.Of(int(req.GetFieldMappingConfig().GetItemRetryNum())), nil), } // 构建 ConnectorConf diff --git a/backend/modules/evaluation/application/experiment_app.go b/backend/modules/evaluation/application/experiment_app.go index ccdcf4807..9f833275e 100644 --- a/backend/modules/evaluation/application/experiment_app.go +++ b/backend/modules/evaluation/application/experiment_app.go @@ -459,6 +459,7 @@ func (e *experimentApplication) SubmitExperiment(ctx context.Context, req *expt. Session: req.Session, EnableWeightedScore: req.EnableWeightedScore, // EvaluatorScoreWeights 会在 CreateExperiment 的 resolveEvaluatorVersionIDsFromCreateReq 中解析 + ItemRetryNum: req.ItemRetryNum, } if req.IsSetExptTemplateID() { createReq.ExptTemplateID = gptr.Of(req.GetExptTemplateID()) @@ -469,11 +470,12 @@ func (e *experimentApplication) SubmitExperiment(ctx context.Context, req *expt. } rresp, err := e.RunExperiment(ctx, &expt.RunExperimentRequest{ - WorkspaceID: gptr.Of(req.GetWorkspaceID()), - ExptID: cresp.GetExperiment().ID, - ExptType: req.ExptType, - Session: req.Session, - Ext: req.Ext, + WorkspaceID: gptr.Of(req.GetWorkspaceID()), + ExptID: cresp.GetExperiment().ID, + ExptType: req.ExptType, + ItemRetryNum: req.ItemRetryNum, + Session: req.Session, + Ext: req.Ext, }) if err != nil { return nil, err @@ -967,13 +969,14 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run evalMode := experiment.ExptType2EvalMode(req.GetExptType()) - if err := e.manager.LogRun(ctx, req.GetExptID(), runID, evalMode, req.GetWorkspaceID(), session); err != nil { + if err := e.manager.LogRun(ctx, req.GetExptID(), runID, evalMode, req.GetWorkspaceID(), nil, session); err != nil { return nil, err } - if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), session, evalMode, req.GetExt()); err != nil { + if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), int(req.GetItemRetryNum()), session, evalMode, req.GetExt()); err != nil { return nil, err } + return &expt.RunExperimentResponse{ RunID: gptr.Of(runID), BaseResp: base.NewBaseResp(), @@ -981,7 +984,15 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run } func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.RetryExperimentRequest) (r *expt.RetryExperimentResponse, err error) { - session := entity.NewSession(ctx) + if req.GetRetryMode() == 0 { + req.RetryMode = domain_expt.ExptRetryModePtr(domain_expt.ExptRetryMode_RetryFailure) + } + + var ( + runID int64 + session = entity.NewSession(ctx) + runMode = experiment.ConvRetryMode(req.GetRetryMode()) + ) got, err := e.manager.Get(ctx, req.GetExptID(), req.GetWorkspaceID(), session) if err != nil { @@ -998,17 +1009,29 @@ func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.R return nil, err } - runID, err := e.idgen.GenID(ctx) - if err != nil { - return nil, err - } - - if err := e.manager.LogRun(ctx, req.GetExptID(), runID, entity.EvaluationModeFailRetry, req.GetWorkspaceID(), session); err != nil { - return nil, err - } + switch runMode { + case entity.EvaluationModeRetryItems: + rid, retried, err := e.manager.LogRetryItemsRun(ctx, req.GetExptID(), runMode, req.GetWorkspaceID(), req.GetItemIds(), session) + if err != nil { + return nil, err + } + runID = rid - if err := e.manager.RetryUnSuccess(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), session, req.GetExt()); err != nil { - return nil, err + if !retried { + if err := e.manager.RetryItems(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), gptr.Indirect(got.EvalConf.ItemRetryNum), req.GetItemIds(), session, req.GetExt()); err != nil { + return nil, err + } + } + default: + if runID, err = e.idgen.GenID(ctx); err != nil { + return nil, err + } + if err := e.manager.LogRun(ctx, req.GetExptID(), runID, runMode, req.GetWorkspaceID(), nil, session); err != nil { + return nil, err + } + if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), gptr.Indirect(got.EvalConf.ItemRetryNum), session, runMode, req.GetExt()); err != nil { + return nil, err + } } return &expt.RetryExperimentResponse{ diff --git a/backend/modules/evaluation/application/experiment_app_test.go b/backend/modules/evaluation/application/experiment_app_test.go index 86ba0f37f..4a1e359e9 100644 --- a/backend/modules/evaluation/application/experiment_app_test.go +++ b/backend/modules/evaluation/application/experiment_app_test.go @@ -694,6 +694,7 @@ func TestExperimentApplication_SubmitExperiment(t *testing.T) { validRunID, gomock.Any(), validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -703,6 +704,7 @@ func TestExperimentApplication_SubmitExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, gomock.Any(), gomock.Any(), @@ -825,8 +827,8 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). Return(nil) @@ -861,8 +863,8 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). Return(errors.New("update error")) @@ -897,8 +899,8 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) // 不应该调用 UpdateExptInfo mockTemplateManager.EXPECT().UpdateExptInfo(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) @@ -2341,6 +2343,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validRunID, entity.EvaluationModeSubmit, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -2351,6 +2354,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, entity.EvaluationModeSubmit, gomock.Any(), @@ -2386,6 +2390,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validRunID, entity.EvaluationModeSubmit, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -2396,6 +2401,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, entity.EvaluationModeSubmit, gomock.Any(), @@ -2462,11 +2468,12 @@ func TestExperimentApplication_RetryExperiment(t *testing.T) { ExptID: gptr.Of(validExptID), }, mockSetup: func() { - // 获取实验信息 + itemRetryNum := 0 mockManager.EXPECT().Get(gomock.Any(), validExptID, validWorkspaceID, gomock.Any()).Return(&entity.Experiment{ ID: validExptID, SpaceID: validWorkspaceID, CreatedBy: strconv.FormatInt(validUserID, 10), + EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: &itemRetryNum}, }, nil) // 权限验证 @@ -2482,10 +2489,10 @@ func TestExperimentApplication_RetryExperiment(t *testing.T) { mockIDGen.EXPECT().GenID(gomock.Any()).Return(validRunID, nil) // 记录运行日志 - mockManager.EXPECT().LogRun(gomock.Any(), validExptID, validRunID, entity.EvaluationModeFailRetry, validWorkspaceID, gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), validExptID, validRunID, entity.EvaluationModeFailRetry, validWorkspaceID, gomock.Any(), gomock.Any()).Return(nil) // 重试失败的实验 - mockManager.EXPECT().RetryUnSuccess(gomock.Any(), validExptID, validRunID, validWorkspaceID, gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), validExptID, validRunID, validWorkspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) }, wantResp: &exptpb.RetryExperimentResponse{ RunID: gptr.Of(validRunID), diff --git a/backend/modules/evaluation/application/wire_gen.go b/backend/modules/evaluation/application/wire_gen.go index 6deb32bd5..2fe9de54e 100644 --- a/backend/modules/evaluation/application/wire_gen.go +++ b/backend/modules/evaluation/application/wire_gen.go @@ -150,7 +150,7 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d iExptTemplateManager := service.NewExptTemplateManager(iExptTemplateRepo, idgen2, serviceEvaluatorService, iEvalTargetService, iEvaluationSetService, evaluationSetVersionService, iLatestWriteTracker) iNotifyRPCAdapter := notify.NewNotifyRPCAdapter() iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptTurnResultRepo, componentIConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, serviceEvaluatorService, benefitSvc, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider) - schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager) + schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo) exptSchedulerEvent := service.NewExptSchedulerSvc(iExptManager, iExperimentRepo, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExptRunLogRepo, idempotentService, componentIConfiger, quotaRepo, iLocker, exptEventPublisher, auditClient, exptMetric, exptResultService, idgen2, evaluationSetItemService, schedulerModeFactory) iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) @@ -376,7 +376,7 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL iExptTemplateManager := service.NewExptTemplateManager(iExptTemplateRepo, idgen2, evaluatorService, iEvalTargetService, iEvaluationSetService, evaluationSetVersionService, iLatestWriteTracker) iNotifyRPCAdapter := notify.NewNotifyRPCAdapter() iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptTurnResultRepo, iConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, evaluatorService, benefitService, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider) - schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, iConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager) + schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, iConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo) exptSchedulerEvent := service.NewExptSchedulerSvc(iExptManager, iExperimentRepo, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExptRunLogRepo, idempotentService, iConfiger, quotaRepo, iLocker, exptEventPublisher, auditClient, exptMetric, exptResultService, idgen2, evaluationSetItemService, schedulerModeFactory) exptItemEvalEvent := service.NewExptRecordEvalService(iExptManager, iConfiger, exptEventPublisher, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExperimentRepo, quotaRepo, iLocker, idempotentService, auditClient, exptMetric, exptResultService, iEvalTargetService, evaluationSetItemService, evaluatorRecordService, evaluatorService, idgen2, benefitService, iEvalAsyncRepo) iExptAnnotateService := service.NewExptAnnotateService(db2, iExptAnnotateRepo, iExptTurnResultRepo, exptEventPublisher, evaluationSetItemService, iExperimentRepo, exptResultService, iExptTurnResultFilterRepo, iExptAggrResultRepo) diff --git a/backend/modules/evaluation/domain/entity/event.go b/backend/modules/evaluation/domain/entity/event.go index 2a01f02a1..2dee9802c 100644 --- a/backend/modules/evaluation/domain/entity/event.go +++ b/backend/modules/evaluation/domain/entity/event.go @@ -14,7 +14,8 @@ type ExptScheduleEvent struct { Ext map[string]string Session *Session - RetryTimes int + ItemRetryTimes int + ExecEvalSetItemIDs []int64 } type ExptItemEvalEvent struct { @@ -26,10 +27,15 @@ type ExptItemEvalEvent struct { EvalSetItemID int64 AsyncReportTrigger bool - CreateAt int64 - RetryTimes int - Ext map[string]string - Session *Session + CreateAt int64 + RetryTimes int + MaxRetryTimes int + Ext map[string]string + Session *Session +} + +func (e *ExptItemEvalEvent) IgnoreExistedResult() bool { + return (e.ExptRunMode == EvaluationModeRetryItems || e.ExptRunMode == EvaluationModeRetryAll) && e.RetryTimes == 0 } func (e *ExptItemEvalEvent) GetExptID() int64 { diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index aacc4dcc5..ada136478 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -11,6 +11,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/mitchellh/mapstructure" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" ) @@ -51,14 +52,13 @@ const ( SourceType_Trace SourceType = 2 ) -// TODO type ExptRunLog struct { ID int64 SpaceID int64 CreatedBy string ExptID int64 ExptRunID int64 - ItemIds []byte + ItemIds []ExptRunLogItems Mode int32 Status int64 PendingCnt int32 @@ -73,6 +73,36 @@ type ExptRunLog struct { UpdatedAt time.Time } +func (e *ExptRunLog) GetItemIDs() []int64 { + var itemIDs []int64 + for _, items := range e.ItemIds { + itemIDs = append(itemIDs, items.ItemIDs...) + } + return itemIDs +} + +func (e *ExptRunLog) AppendItemIDs(itemIDs []int64) error { + if e == nil { + return errorx.New("ExptRunLog AppendItemIDs must init first") + } + exists := make(map[int64]bool) + for _, chunk := range e.ItemIds { + for _, itemID := range chunk.ItemIDs { + exists[itemID] = true + } + } + rlItems := ExptRunLogItems{CreateAt: gptr.Of(time.Now().Unix())} + for _, itemID := range itemIDs { + if exists[itemID] { + return errorx.NewByCode(errno.EvalItemAlreadyRetryingCode, errorx.WithExtraMsg(fmt.Sprintf("existed item_id: %v", itemID))) + } else { + rlItems.ItemIDs = append(rlItems.ItemIDs, itemID) + } + } + e.ItemIds = append(e.ItemIds, rlItems) + return nil +} + type Experiment struct { ID int64 SpaceID int64 @@ -160,6 +190,7 @@ func (e *ExptEvaluatorVersionRef) String() string { type EvaluationConfiguration struct { ConnectorConf Connector ItemConcurNum *int + ItemRetryNum *int } type Connector struct { @@ -340,3 +371,8 @@ type InvokeExptReq struct { Ext map[string]string } + +type ExptRunLogItems struct { + ItemIDs []int64 + CreateAt *int64 +} diff --git a/backend/modules/evaluation/domain/entity/expt_result.go b/backend/modules/evaluation/domain/entity/expt_result.go index 669f8ec59..cb3f161b9 100644 --- a/backend/modules/evaluation/domain/entity/expt_result.go +++ b/backend/modules/evaluation/domain/entity/expt_result.go @@ -9,9 +9,12 @@ import ( "strconv" "time" + "gorm.io/gorm/clause" + "github.com/coze-dev/coze-loop/backend/infra/middleware/session" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" + gslice "github.com/coze-dev/coze-loop/backend/pkg/lang/slices" ) type FieldType int64 @@ -244,6 +247,12 @@ type ExptItemEvalResult struct { TurnResultRunLogs map[int64]*ExptTurnResultRunLog } +type ExptEvalItems []*ExptEvalItem + +func (e ExptEvalItems) GetItemIDs() []int64 { + return gslice.Map(e, func(f *ExptEvalItem) int64 { return f.ItemID }) +} + type ExptEvalItem struct { ExptID int64 EvalSetVersionID int64 @@ -465,9 +474,15 @@ func (e *ExptTemplateFilterFields) IsValid() bool { type ExptItemRunLogFilter struct { Status []ItemRunState ResultState *ExptItemResultState + + RawFilter bool + RawCond clause.Expr } func (e *ExptItemRunLogFilter) GetResultState() ExptItemResultState { + if e.ResultState == nil { + return 0 + } return *e.ResultState } diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index 5873af3c5..e41a414a0 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -19,6 +19,8 @@ import ( type ExptRunMode int32 const ( + EvaluationModeUnknown ExptRunMode = 0 + // EvaluationModeSubmit 创建后提交 EvaluationModeSubmit ExptRunMode = 1 @@ -27,6 +29,9 @@ const ( // EvaluationModeAppend 追加模式 EvaluationModeAppend ExptRunMode = 3 + + EvaluationModeRetryAll ExptRunMode = 4 + EvaluationModeRetryItems ExptRunMode = 5 ) type ItemRunState int64 diff --git a/backend/modules/evaluation/domain/entity/expt_template.go b/backend/modules/evaluation/domain/entity/expt_template.go index 023e4c1c4..cfb4f14ae 100644 --- a/backend/modules/evaluation/domain/entity/expt_template.go +++ b/backend/modules/evaluation/domain/entity/expt_template.go @@ -131,6 +131,7 @@ type ExptTemplateConfiguration struct { // 默认评估器并发数 EvaluatorsConcurNum *int + ItemRetryNum *int } // ToEvaluatorRefDO 转换为评估器引用DO diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index eb05122af..e958d6872 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -218,26 +218,22 @@ type RunEvaluatorRequest struct { } type CreateExptParam struct { - WorkspaceID int64 `thrift:"workspace_id,1,required" frugal:"1,required,i64" json:"workspace_id" form:"workspace_id,required" ` - EvalSetVersionID int64 `thrift:"eval_set_version_id,2,optional" frugal:"2,optional,i64" json:"eval_set_version_id" form:"eval_set_version_id" ` - TargetVersionID int64 `thrift:"target_version_id,3,optional" frugal:"3,optional,i64" json:"target_version_id" form:"target_version_id" ` - EvaluatorVersionIds []int64 `thrift:"evaluator_version_ids,4,optional" frugal:"4,optional,list" json:"evaluator_version_ids" form:"evaluator_version_ids" ` - Name string `thrift:"name,5,optional" frugal:"5,optional,string" form:"name" json:"name,omitempty"` - Desc string `thrift:"desc,6,optional" frugal:"6,optional,string" form:"desc" json:"desc,omitempty"` - EvalSetID int64 `thrift:"eval_set_id,7,optional" frugal:"7,optional,i64" json:"eval_set_id" form:"eval_set_id" ` - TargetID *int64 `thrift:"target_id,8,optional" frugal:"8,optional,i64" json:"target_id" form:"target_id" ` - // TargetFieldMapping *TargetFieldMapping `thrift:"target_field_mapping,20,optional" frugal:"20,optional,TargetFieldMapping" form:"target_field_mapping" json:"target_field_mapping,omitempty"` - // EvaluatorFieldMapping []*EvaluatorFieldMapping `thrift:"evaluator_field_mapping,21,optional" frugal:"21,optional,list" form:"evaluator_field_mapping" json:"evaluator_field_mapping,omitempty"` - // ItemConcurNum int32 `thrift:"item_concur_num,22,optional" frugal:"22,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty"` - // EvaluatorsConcurNum int32 `thrift:"evaluators_concur_num,23,optional" frugal:"23,optional,i32" form:"evaluators_concur_num" json:"evaluators_concur_num,omitempty"` - CreateEvalTargetParam *CreateEvalTargetParam `thrift:"create_eval_target_param,24,optional" frugal:"24,optional,eval_target.CreateEvalTargetParam" form:"create_eval_target_param" json:"create_eval_target_param,omitempty"` - ExptType ExptType `thrift:"expt_type,30,optional" frugal:"30,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` - MaxAliveTime int64 `thrift:"max_alive_time,31,optional" frugal:"31,optional,i64" form:"max_alive_time" json:"max_alive_time,omitempty"` - SourceType SourceType `thrift:"source_type,32,optional" frugal:"32,optional,SourceType" form:"source_type" json:"source_type,omitempty"` - SourceID string `thrift:"source_id,33,optional" frugal:"33,optional,string" form:"source_id" json:"source_id,omitempty"` - ExptTemplateID int64 `thrift:"expt_template_id,34,optional" frugal:"34,optional,i64" form:"expt_template_id" json:"expt_template_id,omitempty"` - - ExptConf *EvaluationConfiguration + WorkspaceID int64 `json:"workspace_id"` + EvalSetVersionID int64 `json:"eval_set_version_id"` + TargetVersionID int64 `json:"target_version_id"` + EvaluatorVersionIds []int64 `json:"evaluator_version_ids"` + Name string `json:"name"` + Desc string `json:"desc"` + EvalSetID int64 `json:"eval_set_id"` + TargetID *int64 `json:"target_id,omitempty"` + CreateEvalTargetParam *CreateEvalTargetParam `json:"create_eval_target_param,omitempty"` + ExptType ExptType `json:"expt_type"` + MaxAliveTime int64 `json:"max_alive_time"` + SourceType SourceType `json:"source_type"` + SourceID string `json:"source_id"` + ExptTemplateID int64 `json:"expt_template_id"` + ExptConf *EvaluationConfiguration `json:"expt_conf"` + ItemRetryNum *int `json:"item_retry_num,omitempty"` } type ExptRunCheckOption struct { diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index 65e338314..20b354461 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -38,10 +38,12 @@ type IExptItemResultRepo interface { BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*entity.ExptItemResult) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64) (results []*entity.ExptItemResult, ncursor int64, err error) + MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) (results []*entity.ExptItemResult, err error) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*entity.ExptItemResult, int64, error) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error GetItemTurnResults(ctx context.Context, spaceID, exptID, itemID int64) ([]*entity.ExptTurnResult, error) + MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) UpdateItemsResult(ctx context.Context, spaceID, exptID int64, itemID []int64, ufields map[string]any) error GetMaxItemIdxByExptID(ctx context.Context, exptID, spaceID int64) (int32, error) diff --git a/backend/modules/evaluation/domain/repo/mocks/expt.go b/backend/modules/evaluation/domain/repo/mocks/expt.go index 66ae8648f..bd2eca539 100644 --- a/backend/modules/evaluation/domain/repo/mocks/expt.go +++ b/backend/modules/evaluation/domain/repo/mocks/expt.go @@ -457,6 +457,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) ListItemResultsByExptID(ctx, expt return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListItemResultsByExptID", reflect.TypeOf((*MockIExptItemResultRepo)(nil).ListItemResultsByExptID), ctx, exptID, spaceID, page, desc) } +// MGetItemResults mocks base method. +func (m *MockIExptItemResultRepo) MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) ([]*entity.ExptItemResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MGetItemResults", ctx, exptID, itemIDs, spaceID) + ret0, _ := ret[0].([]*entity.ExptItemResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemResults indicates an expected call of MGetItemResults. +func (mr *MockIExptItemResultRepoMockRecorder) MGetItemResults(ctx, exptID, itemIDs, spaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemResults), ctx, exptID, itemIDs, spaceID) +} + // MGetItemRunLog mocks base method. func (m *MockIExptItemResultRepo) MGetItemRunLog(ctx context.Context, exptID, exptRunID int64, itemIDs []int64, spaceID int64) ([]*entity.ExptItemResultRunLog, error) { m.ctrl.T.Helper() @@ -472,6 +487,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) MGetItemRunLog(ctx, exptID, exptR return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemRunLog", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemRunLog), ctx, exptID, exptRunID, itemIDs, spaceID) } +// MGetItemTurnResults mocks base method. +func (m *MockIExptItemResultRepo) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MGetItemTurnResults", ctx, spaceID, exptID, itemIDs) + ret0, _ := ret[0].([]*entity.ExptTurnResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemTurnResults indicates an expected call of MGetItemTurnResults. +func (mr *MockIExptItemResultRepoMockRecorder) MGetItemTurnResults(ctx, spaceID, exptID, itemIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemTurnResults), ctx, spaceID, exptID, itemIDs) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultRepo) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/expt_manage.go b/backend/modules/evaluation/domain/service/expt_manage.go index 315b1d1b9..daf052a33 100644 --- a/backend/modules/evaluation/domain/service/expt_manage.go +++ b/backend/modules/evaluation/domain/service/expt_manage.go @@ -39,8 +39,8 @@ type IExptConfigManager interface { // IExptExecutionManager 实验执行控制接口（负责实验的运行、监控和状态管理） type IExptExecutionManager interface { CheckRun(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.ExptRunCheckOptionFn) error - Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error - RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error + Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error + RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error Invoke(ctx context.Context, invokeExptReq *entity.InvokeExptReq) error Finish(ctx context.Context, exptID *entity.Experiment, exptRunID int64, session *entity.Session) error @@ -48,11 +48,14 @@ type IExptExecutionManager interface { PendRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error PendExpt(ctx context.Context, exptID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error + // IsCompletingRun returns true if the given run is currently in the completing phase. + IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) CompleteRun(ctx context.Context, exptID, exptRunID int64, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error CompleteExpt(ctx context.Context, exptID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error // SetExptTerminating Set experiment/run_log status to "terminating". SetExptTerminating(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error - LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error + LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error + LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (int64, bool, error) GetRunLog(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) (*entity.ExptRunLog, error) } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index d61f53382..ed9ab1880 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -269,7 +269,7 @@ func (e *ExptMangerImpl) CheckBenefit(ctx context.Context, expt *entity.Experime return nil } -func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { +func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { if err := NewQuotaService(e.quotaRepo, e.configer).AllowExptRun(ctx, exptID, spaceID, session); err != nil { return err } @@ -280,27 +280,29 @@ func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, } if err := e.publisher.PublishExptScheduleEvent(ctx, &entity.ExptScheduleEvent{ - SpaceID: spaceID, - ExptID: exptID, - ExptRunID: runID, - ExptRunMode: runMode, - ExptType: expt.ExptType, - CreatedAt: time.Now().Unix(), - Session: session, - Ext: ext, + SpaceID: spaceID, + ExptID: exptID, + ExptRunID: runID, + ExptRunMode: runMode, + ExptType: expt.ExptType, + CreatedAt: time.Now().Unix(), + ItemRetryTimes: itemRetryNum, + Session: session, + Ext: ext, }, gptr.Of(time.Second*3)); err != nil { return err } - err = e.sendExptNotify(ctx, expt) - if err != nil { - logs.CtxWarn(ctx, "[Run] sendExptNotify failed, expt_id: %v, error: %v", exptID, err) + switch runMode { + case entity.EvaluationModeSubmit: + if err = e.sendExptNotify(ctx, expt); err != nil { + logs.CtxWarn(ctx, "[Run] SendExptNotify failed, expt_id: %v, error: %v", exptID, err) + } } - return nil } -func (e *ExptMangerImpl) RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error { +func (e *ExptMangerImpl) RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error { if err := NewQuotaService(e.quotaRepo, e.configer).AllowExptRun(ctx, exptID, spaceID, session); err != nil { return err } @@ -311,14 +313,16 @@ func (e *ExptMangerImpl) RetryUnSuccess(ctx context.Context, exptID, runID, spac } if err := e.publisher.PublishExptScheduleEvent(ctx, &entity.ExptScheduleEvent{ - SpaceID: spaceID, - ExptID: exptID, - ExptRunID: runID, - ExptRunMode: entity.EvaluationModeFailRetry, - ExptType: expt.ExptType, - CreatedAt: time.Now().Unix(), - Session: session, - Ext: ext, + SpaceID: spaceID, + ExptID: exptID, + ExptRunID: runID, + ExptRunMode: entity.EvaluationModeRetryItems, + ExptType: expt.ExptType, + CreatedAt: time.Now().Unix(), + ItemRetryTimes: itemRetryNum, + ExecEvalSetItemIDs: itemIDs, + Session: session, + Ext: ext, }, gptr.Of(time.Second*3)); err != nil { return err } @@ -349,6 +353,11 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID int6 } } + if err := e.lockCompletingRun(ctx, exptID, exptRunID, spaceID, session); err != nil { + return err + } + defer func() { _ = e.unlockCompletingRun(ctx, exptID, exptRunID, spaceID, session) }() + runLog, err := e.runLogRepo.Get(ctx, exptID, exptRunID) if err != nil { return err @@ -358,7 +367,7 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID int6 return err } - if _, err := e.mutex.Unlock(e.makeExptMutexLockKey(exptID)); err != nil { + if _, err := e.mutex.UnlockForce(ctx, e.makeExptMutexLockKey(exptID)); err != nil { return err } @@ -908,7 +917,27 @@ func (e *ExptMangerImpl) PendExpt(ctx context.Context, exptID, spaceID int64, se return nil } -func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error { +func (e *ExptMangerImpl) IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { + return e.mutex.Exists(ctx, e.makeExptCompletingLockKey(exptID, exptRunID)) +} + +func (e *ExptMangerImpl) lockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + locked, err := e.mutex.Lock(ctx, e.makeExptCompletingLockKey(exptID, exptRunID), time.Minute*3) + if err != nil { + return err + } + if !locked { + return errorx.New("lockCompletingRun fail, expt_id: %v, expt_run_id: %v", exptID, exptRunID) + } + return nil +} + +func (e *ExptMangerImpl) unlockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + _, err := e.mutex.Unlock(e.makeExptCompletingLockKey(exptID, exptRunID)) + return err +} + +func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { duration := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second locked, err := e.mutex.LockBackoff(ctx, e.makeExptMutexLockKey(exptID), duration, time.Second) if err != nil { @@ -920,7 +949,7 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo defer e.mtr.EmitExptExecRun(spaceID, int64(mode)) - if err := e.runLogRepo.Create(ctx, &entity.ExptRunLog{ + rl := &entity.ExptRunLog{ ID: exptRunID, SpaceID: spaceID, CreatedBy: session.UserID, @@ -928,7 +957,12 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo ExptRunID: exptRunID, Mode: int32(mode), Status: int64(entity.ExptStatus_Pending), - }); err != nil { + } + if len(itemIDs) > 0 { + rl.ItemIds = []entity.ExptRunLogItems{{ItemIDs: itemIDs, CreateAt: gptr.Of(time.Now().Unix())}} + } + + if err := e.runLogRepo.Create(ctx, rl); err != nil { return err } @@ -942,6 +976,78 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo return nil } +func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (runID int64, retried bool, err error) { + expireAt := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second + retryTime := time.Second + runID, err = e.idgenerator.GenID(ctx) + if err != nil { + return 0, false, err + } + + locked, existedRunID, err := e.mutex.BackoffLockWithValue(ctx, e.makeExptMutexLockKey(exptID), strconv.FormatInt(runID, 10), expireAt, retryTime) + if err != nil { + return 0, false, err + } + + var rl *entity.ExptRunLog + retried = !locked + + if retried { + runID, err = strconv.ParseInt(existedRunID, 10, 64) + if err != nil { + logs.CtxError(ctx, "parsing expt run lock value to runid failed, raw: %v", existedRunID) + return 0, false, errorx.NewByCode(errno.ExperimentRunningExistedCode) + } + + completing, err := e.IsCompletingRun(ctx, exptID, runID, spaceID) + if err != nil { + return 0, false, err + } + if completing { + return 0, false, errorx.NewByCode(errno.ExperimentIsCompletingCode) + } + + rl, err = e.runLogRepo.Get(ctx, exptID, runID) + if err != nil { + return 0, false, err + } + + if rl == nil { + return 0, false, errorx.Wrapf(err, "target runlog %v not found, expt_id: %v", runID, exptID) + } + + if err := rl.AppendItemIDs(itemIDs); err != nil { + return 0, false, err + } + } else { + rl = &entity.ExptRunLog{ + ID: runID, + SpaceID: spaceID, + CreatedBy: session.UserID, + ExptID: exptID, + ExptRunID: runID, + Mode: int32(mode), + Status: int64(entity.ExptStatus_Pending), + } + if len(itemIDs) > 0 { + rl.ItemIds = []entity.ExptRunLogItems{{ItemIDs: itemIDs, CreateAt: gptr.Of(time.Now().Unix())}} + } + } + + if err := e.runLogRepo.Save(ctx, rl); err != nil { + return 0, false, err + } + + if !retried { + if err := e.exptRepo.Update(ctx, &entity.Experiment{ID: exptID, LatestRunID: runID}); err != nil { + return 0, false, err + } + e.mtr.EmitExptExecRun(spaceID, int64(mode)) + } + + return runID, !locked, nil +} + func (e *ExptMangerImpl) GetRunLog(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) (*entity.ExptRunLog, error) { return e.runLogRepo.Get(ctx, exptID, exptRunID) } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go deleted file mode 100755 index ff2bc0a94..000000000 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - "errors" - "testing" - - "github.com/bytedance/gg/gptr" - "go.uber.org/mock/gomock" - - "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" - "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - svcMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" -) - -// Phase 3: Runtime param validation integration tests -func TestExptMangerImpl_checkTargetConnector_WithRuntimeParam(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mgr := newTestExptManager(ctrl) - ctx := context.Background() - session := &entity.Session{UserID: "1"} - - tests := []struct { - name string - expt *entity.Experiment - setup func() - wantErr bool - }{ - { - name: "valid_runtime_param_success", - expt: &entity.Experiment{ - ID: 1, - TargetVersionID: 1, - TargetType: entity.EvalTargetTypeLoopPrompt, - Target: &entity.EvalTarget{ - EvalTargetType: entity.EvalTargetTypeLoopPrompt, - EvalTargetVersion: &entity.EvalTargetVersion{ - OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, - }, - }, - EvalSet: &entity.EvaluationSet{ - EvaluationSetVersion: &entity.EvaluationSetVersion{ - EvaluationSetSchema: &entity.EvaluationSetSchema{ - FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, - }, - }, - }, - EvalConf: &entity.EvaluationConfiguration{ - ConnectorConf: entity.Connector{ - TargetConf: &entity.TargetConf{ - TargetVersionID: 1, - IngressConf: &entity.TargetIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - CustomConf: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{ - { - FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, - Value: `{"model_config":{"model_id":"test_model"}}`, - }, - }, - }, - }, - }, - EvaluatorsConf: &entity.EvaluatorsConf{ - EvaluatorConf: []*entity.EvaluatorConf{ - { - EvaluatorVersionID: 1, - IngressConf: &entity.EvaluatorIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - TargetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, - }, - }, - }, - }, - }, - }, - }, - }, - setup: func() { - mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). - EXPECT(). - ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, `{"model_config":{"model_id":"test_model"}}`). - Return(nil) - }, - wantErr: false, - }, - { - name: "invalid_runtime_param_format_error", - expt: &entity.Experiment{ - ID: 1, - TargetVersionID: 1, - TargetType: entity.EvalTargetTypeLoopPrompt, - Target: &entity.EvalTarget{ - EvalTargetType: entity.EvalTargetTypeLoopPrompt, - EvalTargetVersion: &entity.EvalTargetVersion{ - OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, - }, - }, - EvalSet: &entity.EvaluationSet{ - EvaluationSetVersion: &entity.EvaluationSetVersion{ - EvaluationSetSchema: &entity.EvaluationSetSchema{ - FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, - }, - }, - }, - EvalConf: &entity.EvaluationConfiguration{ - ConnectorConf: entity.Connector{ - TargetConf: &entity.TargetConf{ - TargetVersionID: 1, - IngressConf: &entity.TargetIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - CustomConf: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{ - { - FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, - Value: `invalid_json`, - }, - }, - }, - }, - }, - EvaluatorsConf: &entity.EvaluatorsConf{ - EvaluatorConf: []*entity.EvaluatorConf{ - { - EvaluatorVersionID: 1, - IngressConf: &entity.EvaluatorIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - TargetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, - }, - }, - }, - }, - }, - }, - }, - }, - setup: func() { - mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). - EXPECT(). - ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, "invalid_json"). - Return(errors.New("invalid JSON format")) - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.setup() - err := mgr.checkTargetConnector(ctx, tt.expt, session) - if (err != nil) != tt.wantErr { - t.Errorf("checkTargetConnector() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index b536e5382..4d465bdc6 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -1,5 +1,6 @@ // Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 runMode: entity.EvaluationModeSubmit, +// SPDX-License-Identifier: Apache-2.0 + package service import ( @@ -18,6 +19,7 @@ import ( idgenMocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" lockMocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" lwtMocks "github.com/coze-dev/coze-loop/backend/infra/platestwrite/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" idemMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" metricsMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" componentMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" @@ -208,7 +210,7 @@ func TestExptMangerImpl_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setup() - err := mgr.Run(ctx, tt.exptID, tt.runID, tt.spaceID, session, tt.runMode, tt.ext) + err := mgr.Run(ctx, tt.exptID, tt.runID, tt.spaceID, 0, session, tt.runMode, tt.ext) if (err != nil) != tt.wantErr { t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) } @@ -245,6 +247,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { spaceID: 789, opts: []entity.CompleteExptOptionFn{}, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -267,13 +274,18 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Save(ctx, gomock.Any()). Return(nil) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: false, }, @@ -294,6 +306,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { Exist(ctx, "CompleteRun:test_cid"). Return(false, nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -312,7 +329,7 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). @@ -320,6 +337,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { Save(ctx, gomock.Any()). Return(nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) + mgr.idem.(*idemMocks.MockIdempotentService). EXPECT(). Set(ctx, "CompleteRun:test_cid", time.Second*60*3). @@ -354,6 +376,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { entity.WithCompleteInterval(time.Millisecond * 100), }, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -373,13 +400,18 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Save(ctx, gomock.Any()). Return(nil) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: false, }, @@ -391,10 +423,20 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { spaceID: 789, opts: []entity.CompleteExptOptionFn{}, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Get(ctx, int64(123), int64(456)). Return(nil, errors.New("run log not found")) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: true, }, @@ -522,136 +564,6 @@ func TestExptMangerImpl_Kill(t *testing.T) { } } -func TestExptMangerImpl_RetryUnSuccess(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mgr := newTestExptManager(ctrl) - ctx := context.Background() - session := &entity.Session{UserID: "test_user"} - - tests := []struct { - name string - exptID int64 - runID int64 - spaceID int64 - ext map[string]string - setup func() - wantErr bool - }{ - { - name: "successful_retry", - exptID: 123, - runID: 456, - spaceID: 789, - ext: map[string]string{"retry": "true"}, - setup: func() { - // Mock lwt.CheckWriteFlagByID - mgr.lwt.(*lwtMocks.MockILatestWriteTracker). - EXPECT(). - CheckWriteFlagByID(ctx, gomock.Any(), int64(123)). - Return(false).AnyTimes() - - // Mock MGetByID for experiment retrieval - mgr.exptRepo.(*repoMocks.MockIExperimentRepo). - EXPECT(). - MGetByID(ctx, []int64{123}, int64(789)). - Return([]*entity.Experiment{{ID: 123, SpaceID: 789}}, nil).AnyTimes() - - // Mock GetEvaluationSet - mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). - EXPECT(). - GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&entity.EvaluationSet{}, nil).AnyTimes() - - // Mock MGetStats - mgr.exptResultService.(*svcMocks.MockExptResultService). - EXPECT(). - MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptStats{}, nil).AnyTimes() - - // Mock BatchGetExptAggrResultByExperimentIDs - mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). - EXPECT(). - BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() - - mgr.quotaRepo.(*repoMocks.MockQuotaRepo). - EXPECT(). - CreateOrUpdate(ctx, int64(789), gomock.Any(), session). - Return(nil) - mgr.configer.(*componentMocks.MockIConfiger). - EXPECT(). - GetExptExecConf(ctx, int64(789)).AnyTimes(). - Return(&entity.ExptExecConf{ - SpaceExptConcurLimit: 10, - }) - mgr.publisher.(*eventsMocks.MockExptEventPublisher). - EXPECT(). - PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). - Do(func(ctx context.Context, event *entity.ExptScheduleEvent, timeout *time.Duration) { - assert.Equal(t, entity.EvaluationModeFailRetry, event.ExptRunMode) - }). - Return(nil) - }, - wantErr: false, - }, - { - name: "quota_check_failure_on_retry", - exptID: 123, - runID: 456, - spaceID: 789, - ext: map[string]string{}, - setup: func() { - // Mock lwt.CheckWriteFlagByID - mgr.lwt.(*lwtMocks.MockILatestWriteTracker). - EXPECT(). - CheckWriteFlagByID(ctx, gomock.Any(), int64(123)). - Return(false).AnyTimes() - - // Mock MGetByID for experiment retrieval - mgr.exptRepo.(*repoMocks.MockIExperimentRepo). - EXPECT(). - MGetByID(ctx, []int64{123}, int64(789)). - Return([]*entity.Experiment{{ID: 123, SpaceID: 789}}, nil).AnyTimes() - - // Mock GetEvaluationSet - mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). - EXPECT(). - GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&entity.EvaluationSet{}, nil).AnyTimes() - - // Mock MGetStats - mgr.exptResultService.(*svcMocks.MockExptResultService). - EXPECT(). - MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptStats{}, nil).AnyTimes() - - // Mock BatchGetExptAggrResultByExperimentIDs - mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). - EXPECT(). - BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() - - mgr.quotaRepo.(*repoMocks.MockQuotaRepo). - EXPECT(). - CreateOrUpdate(ctx, int64(789), gomock.Any(), session). - Return(errors.New("quota exceeded")) - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.setup() - err := mgr.RetryUnSuccess(ctx, tt.exptID, tt.runID, tt.spaceID, session, tt.ext) - if (err != nil) != tt.wantErr { - t.Errorf("RetryUnSuccess() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - func TestExptMangerImpl_LogRun(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -767,12 +679,57 @@ func TestExptMangerImpl_LogRun(t *testing.T) { }, wantErr: true, }, + { + name: "successful_log_run_with_item_ids", + exptID: 123, + exptRunID: 456, + mode: entity.EvaluationModeSubmit, + spaceID: 789, + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, int64(789)).AnyTimes(). + Return(&entity.ExptExecConf{ + ZombieIntervalSecond: 300, + }) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + LockBackoff(ctx, gomock.Any(), time.Duration(300)*time.Second, time.Second). + Return(true, nil) + + mgr.mtr.(*metricsMocks.MockExptMetric). + EXPECT(). + EmitExptExecRun(int64(789), int64(entity.EvaluationModeSubmit)) + + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT(). + Create(ctx, gomock.Any()). + Do(func(ctx context.Context, runLog *entity.ExptRunLog) { + assert.Equal(t, int64(456), runLog.ID) + assert.Equal(t, int64(123), runLog.ExptID) + assert.Len(t, runLog.ItemIds, 1) + assert.Equal(t, []int64{1, 2}, runLog.ItemIds[0].ItemIDs) + }). + Return(nil) + + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT(). + Update(ctx, gomock.Any()). + Return(nil) + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setup() - err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, session) + itemIDs := []int64(nil) + if tt.name == "successful_log_run_with_item_ids" { + itemIDs = []int64{1, 2} + } + err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, itemIDs, session) if (err != nil) != tt.wantErr { t.Errorf("LogRun() error = %v, wantErr %v", err, tt.wantErr) } @@ -780,6 +737,283 @@ func TestExptMangerImpl_LogRun(t *testing.T) { } } +func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "test_user"} + exptID := int64(123) + spaceID := int64(789) + mode := entity.EvaluationModeRetryItems + itemIDs := []int64{1, 2} + + tests := []struct { + name string + setup func() + wantRunID int64 + wantRetry bool + wantErr bool + }{ + { + name: "locked_success_new_run", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1001), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1001", 300*time.Second, time.Second). + Return(true, "1001", nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()). + Do(func(ctx context.Context, rl *entity.ExptRunLog) { + assert.Equal(t, int64(1001), rl.ExptRunID) + assert.Len(t, rl.ItemIds, 1) + assert.Equal(t, itemIDs, rl.ItemIds[0].ItemIDs) + }). + Return(nil) + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().Update(ctx, &entity.Experiment{ID: exptID, LatestRunID: 1001}).Return(nil) + mgr.mtr.(*metricsMocks.MockExptMetric). + EXPECT().EmitExptExecRun(spaceID, int64(mode)) + }, + wantRunID: 1001, + wantRetry: false, + wantErr: false, + }, + { + name: "retried_append_to_existing_run", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1002), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1002", 300*time.Second, time.Second). + Return(false, "1001", nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) + existingLog := &entity.ExptRunLog{ID: 1001, ExptID: exptID, ExptRunID: 1001} + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Get(ctx, exptID, int64(1001)). + Return(existingLog, nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()).Return(nil) + }, + wantRunID: 1001, + wantRetry: true, + wantErr: false, + }, + { + name: "idgen_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(0), errors.New("idgen failed")) + }, + wantErr: true, + }, + { + name: "lock_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1003), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1003", 300*time.Second, time.Second). + Return(false, "", errors.New("redis error")) + }, + wantErr: true, + }, + { + name: "retried_parse_run_id_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1004), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1004", 300*time.Second, time.Second). + Return(false, "not_a_number", nil) + }, + wantErr: true, + }, + { + name: "retried_get_run_log_returns_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1005), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1005", 300*time.Second, time.Second). + Return(false, "1001", nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Get(ctx, exptID, int64(1001)). + Return(nil, errors.New("get run log failed")) + }, + wantErr: true, + }, + { + name: "save_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1006), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1006", 300*time.Second, time.Second). + Return(true, "1006", nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()).Return(errors.New("save failed")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + runID, retried, err := mgr.LogRetryItemsRun(ctx, exptID, mode, spaceID, itemIDs, session) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantRunID, runID) + assert.Equal(t, tt.wantRetry, retried) + }) + } +} + +func TestExptMangerImpl_RetryItems(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "test_user"} + exptID := int64(123) + runID := int64(456) + spaceID := int64(789) + itemRetryNum := 1 + itemIDs := []int64{1, 2} + ext := map[string]string{"k": "v"} + + tests := []struct { + name string + setup func() + wantErr bool + }{ + { + name: "success_publish_event", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(nil) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID, ExptType: 1}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + mgr.publisher.(*eventsMocks.MockExptEventPublisher). + EXPECT(). + PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). + Return(nil) + }, + wantErr: false, + }, + { + name: "quota_check_failure", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(errors.New("quota exceeded")) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + }, + wantErr: true, + }, + { + name: "publish_event_failure", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(nil) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + mgr.publisher.(*eventsMocks.MockExptEventPublisher). + EXPECT(). + PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). + Return(errors.New("publish failed")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + err := mgr.RetryItems(ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) + if (err != nil) != tt.wantErr { + t.Errorf("RetryItems() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func TestExptMangerImpl_GetRunLog(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -2285,3 +2519,155 @@ func TestExptMangerImpl_Invoke_ExtField(t *testing.T) { }) } } + +func TestExptMangerImpl_checkTargetConnector_WithRuntimeParam(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "1"} + + tests := []struct { + name string + expt *entity.Experiment + setup func() + wantErr bool + }{ + { + name: "valid_runtime_param_success", + expt: &entity.Experiment{ + ID: 1, + TargetVersionID: 1, + TargetType: entity.EvalTargetTypeLoopPrompt, + Target: &entity.EvalTarget{ + EvalTargetType: entity.EvalTargetTypeLoopPrompt, + EvalTargetVersion: &entity.EvalTargetVersion{ + OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, + }, + }, + EvalSet: &entity.EvaluationSet{ + EvaluationSetVersion: &entity.EvaluationSetVersion{ + EvaluationSetSchema: &entity.EvaluationSetSchema{ + FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, + }, + }, + }, + EvalConf: &entity.EvaluationConfiguration{ + ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{ + TargetVersionID: 1, + IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + CustomConf: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{ + { + FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, + Value: `{"model_config":{"model_id":"test_model"}}`, + }, + }, + }, + }, + }, + EvaluatorsConf: &entity.EvaluatorsConf{ + EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + TargetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, + }, + }, + }, + }, + }, + }, + }, + }, + setup: func() { + mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). + EXPECT(). + ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, `{"model_config":{"model_id":"test_model"}}`). + Return(nil) + }, + wantErr: false, + }, + { + name: "invalid_runtime_param_format_error", + expt: &entity.Experiment{ + ID: 1, + TargetVersionID: 1, + TargetType: entity.EvalTargetTypeLoopPrompt, + Target: &entity.EvalTarget{ + EvalTargetType: entity.EvalTargetTypeLoopPrompt, + EvalTargetVersion: &entity.EvalTargetVersion{ + OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, + }, + }, + EvalSet: &entity.EvaluationSet{ + EvaluationSetVersion: &entity.EvaluationSetVersion{ + EvaluationSetSchema: &entity.EvaluationSetSchema{ + FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, + }, + }, + }, + EvalConf: &entity.EvaluationConfiguration{ + ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{ + TargetVersionID: 1, + IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + CustomConf: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{ + { + FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, + Value: `invalid_json`, + }, + }, + }, + }, + }, + EvaluatorsConf: &entity.EvaluatorsConf{ + EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + TargetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, + }, + }, + }, + }, + }, + }, + }, + }, + setup: func() { + mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). + EXPECT(). + ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, "invalid_json"). + Return(errors.New("invalid JSON format")) + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + err := mgr.checkTargetConnector(ctx, tt.expt, session) + if (err != nil) != tt.wantErr { + t.Errorf("checkTargetConnector() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_manage_impl.go b/backend/modules/evaluation/domain/service/expt_manage_impl.go index 9a9525fdf..042cddd5d 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_impl.go @@ -12,6 +12,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/bytedance/gg/gslice" + "github.com/coze-dev/coze-loop/backend/infra/external/audit" "github.com/coze-dev/coze-loop/backend/infra/external/benefit" "github.com/coze-dev/coze-loop/backend/infra/idgen" @@ -315,6 +316,10 @@ func (e *ExptMangerImpl) makeExptMutexLockKey(exptID int64) string { return fmt.Sprintf("expt_run_mutex_lock:%d", exptID) } +func (e *ExptMangerImpl) makeExptCompletingLockKey(exptID, exptRunID int64) string { + return fmt.Sprintf("expt_completing_mutex_lock:%d:%d", exptID, exptRunID) +} + func (e *ExptMangerImpl) getTupleByExpt(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.GetExptTupleOptionFn) (*entity.ExptTuple, error) { return e.getExptTupleByID(ctx, e.packTupleID(ctx, expt), spaceID, session, opts...) } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 70b10d7c1..c536f273f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -158,6 +158,9 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R retryConf := e.configer.GetErrRetryConf(ctx, event.SpaceID, nextErr) needRetry := event.RetryTimes < retryConf.GetRetryTimes() + if event.MaxRetryTimes > 0 { + needRetry = event.RetryTimes < event.MaxRetryTimes + } defer func() { code, stable, _ := errno.ParseStatusError(nextErr) @@ -204,7 +207,7 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R func (e *ExptItemEventEvalServiceImpl) HandleEventLock(next RecordEvalEndPoint) RecordEvalEndPoint { return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { lockKey := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, event.EvalSetItemID) - locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*20, time.Second*60*60) + locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*10, time.Second*60*60) if err != nil { return err } @@ -324,7 +327,7 @@ func (e *ExptItemEventEvalServiceImpl) GetExistExptRecordEvalResult(ctx context. turnRunResultMap := make(map[int64]*entity.ExptTurnResultRunLog, len(turnRunLogs)) for _, result := range turnRunLogs { - turnRunResultMap[result.ItemID] = result + turnRunResultMap[result.TurnID] = result } itemRunLog, err := e.exptItemResultRepo.GetItemRunLog(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) @@ -371,6 +374,11 @@ func NewRecordEvalMode( resultSvc: resultSvc, idgen: idgen, }, nil + case entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems: + return &ExptRecordEvalModeRetryIgnoreResult{ + exptTurnResultRepo: exptTurnResultRepo, + idgen: idgen, + }, nil default: return nil, fmt.Errorf("NewRecordEvalMode with unknown expt mode: %v", event.ExptRunMode) } @@ -384,14 +392,13 @@ type ExptRecordEvalModeSubmit struct { } func (e *ExptRecordEvalModeSubmit) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { + return nil + } + event := eiec.Event turns := eiec.EvalSetItem.Turns - // if err := e.exptItemResultRepo.UpdateItemRunLog(ctx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, map[string]any{"status": int32(entity.ItemRunState_Processing)}, - // event.SpaceID); err != nil { - // return err - // } - got, err := e.exptTurnResultRepo.GetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) if err != nil { return err @@ -503,3 +510,50 @@ func (e *ExptRecordEvalModeFailRetry) PreEval(ctx context.Context, eiec *entity. func (e *ExptRecordEvalModeFailRetry) PostEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { return nil } + +type ExptRecordEvalModeRetryIgnoreResult struct { + exptTurnResultRepo repo.IExptTurnResultRepo + idgen idgen.IIDGenerator +} + +func (e *ExptRecordEvalModeRetryIgnoreResult) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { + return nil + } + + event := eiec.Event + logID := logs.GetLogID(ctx) + + ids, err := e.idgen.GenMultiIDs(ctx, len(eiec.EvalSetItem.Turns)) + if err != nil { + return err + } + + turnRunLogs := make([]*entity.ExptTurnResultRunLog, 0, len(eiec.EvalSetItem.Turns)) + for idx, turn := range eiec.EvalSetItem.Turns { + turnRunLogs = append(turnRunLogs, &entity.ExptTurnResultRunLog{ + ID: ids[idx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: event.EvalSetItemID, + TurnID: turn.ID, + Status: entity.TurnRunState_Processing, + LogID: logID, + }) + } + + if err := e.exptTurnResultRepo.BatchCreateNXRunLog(ctx, turnRunLogs); err != nil { + return err + } + + eiec.ExistItemEvalResult.TurnResultRunLogs = gslice.ToMap(turnRunLogs, func(t *entity.ExptTurnResultRunLog) (int64, *entity.ExptTurnResultRunLog) { + return t.TurnID, t + }) + + return nil +} + +func (e *ExptRecordEvalModeRetryIgnoreResult) PostEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + return nil +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go index 4aca3e7b8..b3d297e9f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go @@ -562,6 +562,7 @@ func TestExptItemEventEvalServiceImpl_GetExistExptRecordEvalResult(t *testing.T) { ID: 1, ItemID: 1, + TurnID: 1, }, } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 94cdfb115..45009dca0 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -169,7 +169,14 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * } if evalErr != nil { - errMsg := e.Configer.GetErrCtrl(ctx).ConvertErrMsg(evalErr.Error()) + var rawErrMsg string + if se, ok := errorx.FromStatusError(evalErr); ok && (se.Code() == errno.CustomEvalTargetInvokeFailCode || se.Code() == errno.CustomRPCEvaluatorRunFailedCode) { + rawErrMsg = errorx.ErrorWithoutStack(evalErr) + } else { + rawErrMsg = evalErr.Error() + } + + errMsg := e.Configer.GetErrCtrl(ctx).ConvertErrMsg(rawErrMsg) logs.CtxWarn(ctx, "[ExptTurnEval] store turn run err, before: %v, after: %v", evalErr, errMsg) ei, ok := errno.ParseErrImpl(evalErr) @@ -270,7 +277,7 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error) error { if evalErr != nil { - if retry, _ := e.evalErrNeedRetry(ctx, event.SpaceID, event.RetryTimes, evalErr); retry { + if retry, _ := e.evalErrNeedRetry(ctx, event, evalErr); retry { return evalErr } } @@ -300,12 +307,18 @@ func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, event *en return nil } -func (e *ExptItemEvalCtxExecutor) evalErrNeedRetry(ctx context.Context, spaceID int64, retryTimes int, evalErr error) (bool, time.Duration) { +func (e *ExptItemEvalCtxExecutor) evalErrNeedRetry(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error) (bool, time.Duration) { if evalErr == nil { return false, 0 } + spaceID := event.SpaceID + retryTimes := event.RetryTimes conf := e.Configer.GetErrRetryConf(ctx, spaceID, evalErr) - return retryTimes < conf.GetRetryTimes(), conf.GetRetryInterval() + maxRetryTimes := conf.GetRetryTimes() + if event.MaxRetryTimes > 0 { + maxRetryTimes = event.MaxRetryTimes + } + return retryTimes < maxRetryTimes, conf.GetRetryInterval() } func (e *ExptItemEvalCtxExecutor) evalErrNeedTerminateExpt(ctx context.Context, spaceID int64, evalErr error) bool { diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 31a9613d7..5a7fae8ae 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -100,7 +100,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en return &entity.EvalTargetRecord{EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: make(map[string]*entity.Content)}}, nil } - if existRecord := e.existedTargetRecord(etec); existRecord != nil { + if existRecord := e.existedTargetRecord(etec); !etec.Event.IgnoreExistedResult() && existRecord != nil { logs.CtxInfo(ctx, "CallTarget return with existed target record, record_id: %v", existRecord.ID) return existRecord, nil } @@ -266,7 +266,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallEvaluators(ctx context.Context, etec for _, evaluatorVersion := range expt.Evaluators { existResult := etec.ExptTurnRunResult.GetEvaluatorRecord(evaluatorVersion.GetEvaluatorVersionID()) - if existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { + if !etec.Event.IgnoreExistedResult() && existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { evaluatorResults[existResult.ID] = existResult continue } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 8d5180fec..83216f2ff 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -1855,6 +1855,7 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators_EdgeCases(t *testing.T) { }, }, }, + Event: &entity.ExptItemEvalEvent{}, // Event required: CallEvaluators uses Event.IgnoreExistedResult() }, ExptTurnRunResult: &entity.ExptTurnRunResult{ EvaluatorResults: map[int64]*entity.EvaluatorRecord{ diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 0fecb3f7c..5f5878f7e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -163,7 +163,7 @@ func (e *ExptSchedulerImpl) makeExptRunExecLockKey(exptID, exptRunID int64) stri func (e *ExptSchedulerImpl) HandleEventLock(next SchedulerEndPoint) SchedulerEndPoint { return func(ctx context.Context, event *entity.ExptScheduleEvent) error { key := e.makeExptRunExecLockKey(event.ExptID, event.ExptRunID) - locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*20, time.Second*60*5) + locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*10, time.Second*60*5) if err != nil { return err } @@ -258,21 +258,24 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche } complete = append(complete, zombies...) + logs.CtxInfo(ctx, "expt scheduler scan item, to_submit: %v, incomplete: %v, complete: %v", + entity.ExptEvalItems(toSubmit).GetItemIDs(), entity.ExptEvalItems(incomplete).GetItemIDs(), entity.ExptEvalItems(complete).GetItemIDs()) + if err = e.recordEvalItemRunLogs(ctx, event, complete, mode); err != nil { return err } - err = mode.ScheduleEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) - if err != nil { + if err = e.handleToSubmits(ctx, event, toSubmit); err != nil { return err } - nextTick, err := mode.ExptEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) + err = mode.ScheduleEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) if err != nil { return err } - if err = e.handleToSubmits(ctx, event, toSubmit); err != nil { + nextTick, err := mode.ExptEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) + if err != nil { return err } @@ -359,6 +362,7 @@ func (e *ExptSchedulerImpl) handleToSubmits(ctx context.Context, event *entity.E ExptRunMode: event.ExptRunMode, EvalSetItemID: ts.ItemID, CreateAt: now, + MaxRetryTimes: event.ItemRetryTimes, Ext: event.Ext, Session: event.Session, }) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index be7a1d66d..6bba199f8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -9,6 +9,8 @@ import ( "time" "github.com/bytedance/gg/gptr" + "github.com/bytedance/gg/gslice" + "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/infra/idgen" "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" @@ -17,6 +19,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/pkg/json" "github.com/coze-dev/coze-loop/backend/pkg/lang/conv" "github.com/coze-dev/coze-loop/backend/pkg/lang/maps" "github.com/coze-dev/coze-loop/backend/pkg/logs" @@ -43,6 +46,7 @@ func NewSchedulerModeFactory( evaluatorRecordService EvaluatorRecordService, resultSvc ExptResultService, templateManager IExptTemplateManager, + exptRunLogRepo repo.IExptRunLogRepo, ) SchedulerModeFactory { return &DefaultSchedulerModeFactory{ manager: manager, @@ -58,6 +62,7 @@ func NewSchedulerModeFactory( evaluatorRecordService: evaluatorRecordService, resultSvc: resultSvc, templateManager: templateManager, + exptRunLogRepo: exptRunLogRepo, } } @@ -76,6 +81,7 @@ type DefaultSchedulerModeFactory struct { evaluatorRecordService EvaluatorRecordService resultSvc ExptResultService templateManager IExptTemplateManager + exptRunLogRepo repo.IExptRunLogRepo } func (f *DefaultSchedulerModeFactory) NewSchedulerMode( @@ -88,6 +94,10 @@ func (f *DefaultSchedulerModeFactory) NewSchedulerMode( return NewExptFailRetryMode(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil case entity.EvaluationModeAppend: return NewExptAppendMode(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil + case entity.EvaluationModeRetryAll: + return NewExptRetryAllExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil + case entity.EvaluationModeRetryItems: + return NewExptRetryItemsExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager, f.exptRunLogRepo), nil default: return nil, fmt.Errorf("NewSchedulerMode with unknown mode: %v", mode) } @@ -752,28 +762,48 @@ func newExptBaseExec( } func (e *exptBaseExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { - incomplete, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{ - Status: []entity.ItemRunState{entity.ItemRunState_Processing}, - }, 0) + incomplete, complete, err = e.scanIncompleteAndComplete(ctx, event, expt) if err != nil { return nil, nil, nil, err } if submitCnt := e.getItemConcurNum(ctx, expt) - len(incomplete); submitCnt > 0 { - toSubmit, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{Status: []entity.ItemRunState{entity.ItemRunState_Queueing}}, int64(submitCnt)) + toSubmit, err = e.scanToSubmit(ctx, event, expt, int64(submitCnt)) if err != nil { return nil, nil, nil, err } } - complete, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{ - ResultState: gptr.Of(entity.ExptItemResultStateLogged), - }, 0) + return toSubmit, incomplete, complete, nil +} + +func (e *exptBaseExec) scanIncompleteAndComplete(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (incomplete, complete []*entity.ExptEvalItem, err error) { + rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{ + RawFilter: true, + RawCond: clause.Expr{SQL: "status IN (?) OR result_state = ?", Vars: []interface{}{[]int32{int32(entity.ItemRunState_Processing)}, int32(entity.ExptItemResultStateLogged)}}, + }, 0, 0, event.SpaceID) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - - return toSubmit, incomplete, complete, nil + incomplete = make([]*entity.ExptEvalItem, 0) + complete = make([]*entity.ExptEvalItem, 0) + evalSetVersionID := expt.EvalSet.EvaluationSetVersion.ID + for _, log := range rls { + item := &entity.ExptEvalItem{ + ExptID: event.ExptID, + EvalSetVersionID: evalSetVersionID, + ItemID: log.ItemID, + State: entity.ItemRunState(log.Status), + UpdatedAt: log.UpdatedAt, + } + if log.Status == int32(entity.ItemRunState_Processing) { + incomplete = append(incomplete, item) + } + if log.ResultState == int32(entity.ExptItemResultStateLogged) { + complete = append(complete, item) + } + } + return incomplete, complete, nil } func (e *exptBaseExec) getItemConcurNum(ctx context.Context, expt *entity.Experiment) int { @@ -785,8 +815,8 @@ func (e *exptBaseExec) getItemConcurNum(ctx context.Context, expt *entity.Experi return concurNum } -func (e *exptBaseExec) ScanRunLogEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, filter *entity.ExptItemRunLogFilter, limit int64) (items []*entity.ExptEvalItem, err error) { - rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, filter, 0, limit, event.SpaceID) +func (e *exptBaseExec) scanToSubmit(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, limit int64) (items []*entity.ExptEvalItem, err error) { + rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{Status: []entity.ItemRunState{entity.ItemRunState_Queueing}}, 0, limit, event.SpaceID) if err != nil { return nil, err } @@ -888,3 +918,511 @@ func makeStartIdemKey(event *entity.ExptScheduleEvent) string { func makeEndIdemKey(event *entity.ExptScheduleEvent) string { return fmt.Sprintf("expt_end:%v%v", event.ExptID, event.ExptRunID) } + +func NewExptRetryAllExec( + manager IExptManager, + exptItemResultRepo repo.IExptItemResultRepo, + exptStatsRepo repo.IExptStatsRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + idgenerator idgen.IIDGenerator, + evaluationSetItemService EvaluationSetItemService, + exptRepo repo.IExperimentRepo, + idem idem.IdempotentService, + configer component.IConfiger, + publisher events.ExptEventPublisher, + evaluatorRecordService EvaluatorRecordService, + templateManager IExptTemplateManager, +) *ExptRetryAllExec { + return &ExptRetryAllExec{ + configer: configer, + evaluationSetItemService: evaluationSetItemService, + evaluatorRecordService: evaluatorRecordService, + exptItemResultRepo: exptItemResultRepo, + exptRepo: exptRepo, + exptStatsRepo: exptStatsRepo, + exptTurnResultRepo: exptTurnResultRepo, + idem: idem, + idgenerator: idgenerator, + manager: manager, + publisher: publisher, + templateManager: templateManager, + } +} + +type ExptRetryAllExec struct { + manager IExptManager + exptStatsRepo repo.IExptStatsRepo + exptItemResultRepo repo.IExptItemResultRepo + exptTurnResultRepo repo.IExptTurnResultRepo + idgenerator idgen.IIDGenerator + evaluationSetItemService EvaluationSetItemService + exptRepo repo.IExperimentRepo + idem idem.IdempotentService + configer component.IConfiger + publisher events.ExptEventPublisher + evaluatorRecordService EvaluatorRecordService + templateManager IExptTemplateManager +} + +func (e *ExptRetryAllExec) Mode() entity.ExptRunMode { + return entity.EvaluationModeRetryAll +} + +func (e *ExptRetryAllExec) ExptStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + idemKey := makeStartIdemKey(event) + exist, err := e.idem.Exist(ctx, idemKey) + if err != nil { + return err + } + if exist { + return nil + } + + var ( + evalSetID = expt.EvalSet.ID + evalSetVersionID = expt.EvalSet.EvaluationSetVersion.ID + + maxLoop = 10000 + page = int32(1) + pageSize = int32(100) + itemCnt = 0 + total = int64(0) + ) + + for i := 0; i < maxLoop; i++ { + logs.CtxInfo(ctx, "ExptRetryAllExec.ExptStart scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, page: %v, limit: %v, cur_cnt: %v, total: %v", + event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, page, pageSize, itemCnt, total) + + items, t, _, _, err := e.evaluationSetItemService.ListEvaluationSetItems(ctx, &entity.ListEvaluationSetItemsParam{ + SpaceID: event.SpaceID, + EvaluationSetID: evalSetID, + VersionID: &evalSetVersionID, + PageNumber: &page, + PageSize: &pageSize, + }) + if err != nil { + return err + } + + itemCnt += len(items) + page++ + total = gptr.Indirect(t) + + turnCnt := 0 + for _, item := range items { + turnCnt += len(item.Turns) + } + + ids, err := e.idgenerator.GenMultiIDs(ctx, len(items)+turnCnt) + if err != nil { + return err + } + + idIdx := 0 + itemIDs := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) + itemTurnIDs := make([]*entity.ItemTurnID, 0, len(items)) + for _, item := range items { + for _, turn := range item.Turns { + itemIDs[item.ItemID] = true + itemTurnIDs = append(itemTurnIDs, &entity.ItemTurnID{ + ItemID: item.ItemID, + TurnID: turn.ID, + }) + } + } + + itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDs)) + for itemID := range itemIDs { + itemRunLogs = append(itemRunLogs, &entity.ExptItemResultRunLog{ + ID: ids[idIdx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: itemID, + Status: int32(entity.ItemRunState_Queueing), + }) + idIdx++ + } + + if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ + "status": int32(entity.ItemRunState_Queueing), + "expt_run_id": event.ExptRunID, + }); err != nil { + return err + } + + if err := e.exptTurnResultRepo.UpdateTurnResults(ctx, event.ExptID, itemTurnIDs, event.SpaceID, map[string]any{ + "status": int32(entity.TurnRunState_Queueing), + }); err != nil { + return err + } + + if err := e.exptItemResultRepo.BatchCreateNXRunLogs(ctx, itemRunLogs); err != nil { + return err + } + + if itemCnt >= int(total) || len(items) == 0 { + break + } + + time.Sleep(time.Millisecond * 30) + } + + got, err := e.exptStatsRepo.Get(ctx, event.ExptID, event.SpaceID) + if err != nil { + return err + } + + pendingCnt := got.PendingItemCnt + got.FailItemCnt + got.TerminatedItemCnt + got.ProcessingItemCnt + got.SuccessItemCnt + got.PendingItemCnt = pendingCnt + got.FailItemCnt = 0 + got.TerminatedItemCnt = 0 + got.ProcessingItemCnt = 0 + got.SuccessItemCnt = 0 + + if err := e.exptStatsRepo.Save(ctx, got); err != nil { + return err + } + + logs.CtxInfo(ctx, "ExptRetryAllExec.ExptStart reset pending_cnt: %v, expt_id: %v", pendingCnt, event.ExptID) + + exptDo := &entity.Experiment{ + Status: entity.ExptStatus_Processing, + ID: event.ExptID, + SpaceID: event.SpaceID, + } + + if err := e.exptRepo.Update(ctx, exptDo); err != nil { + return err + } + + if e.templateManager != nil { + var templateID int64 + if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { + templateID = expt.ExptTemplateMeta.ID + } + if templateID > 0 { + if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { + logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryAllExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) + } + } + } + + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + return err + } + + time.Sleep(time.Second * 3) + + return nil +} + +func (e *ExptRetryAllExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).ScanEvalItems(ctx, event, expt) +} + +func (e *ExptRetryAllExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { + if toSubmit == 0 && incomplete == 0 { + logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + return false, newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt) + } + return true, nil +} + +func (e *ExptRetryAllExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + return nil +} + +func (e *ExptRetryAllExec) ScheduleEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) error { + return nil +} + +func (e *ExptRetryAllExec) NextTick(ctx context.Context, event *entity.ExptScheduleEvent, nextTick bool) error { + interval := e.configer.GetExptExecConf(ctx, event.SpaceID).GetDaemonInterval() + return e.publisher.PublishExptScheduleEvent(ctx, event, gptr.Of(interval)) +} + +func (e *ExptRetryAllExec) PublishResult(ctx context.Context, turnEvaluatorRefs []*entity.ExptTurnEvaluatorResultRef, event *entity.ExptScheduleEvent) error { + if event.ExptType == entity.ExptType_Offline { + return nil + } + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).publishResult(ctx, turnEvaluatorRefs, event) +} + +func NewExptRetryItemsExec( + manager IExptManager, + exptItemResultRepo repo.IExptItemResultRepo, + exptStatsRepo repo.IExptStatsRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + idgenerator idgen.IIDGenerator, + evaluationSetItemService EvaluationSetItemService, + exptRepo repo.IExperimentRepo, + idem idem.IdempotentService, + configer component.IConfiger, + publisher events.ExptEventPublisher, + evaluatorRecordService EvaluatorRecordService, + templateManager IExptTemplateManager, + exptRunLogRepo repo.IExptRunLogRepo, +) *ExptRetryItemsExec { + return &ExptRetryItemsExec{ + configer: configer, + evaluationSetItemService: evaluationSetItemService, + evaluatorRecordService: evaluatorRecordService, + exptItemResultRepo: exptItemResultRepo, + exptRepo: exptRepo, + exptStatsRepo: exptStatsRepo, + exptTurnResultRepo: exptTurnResultRepo, + idem: idem, + idgenerator: idgenerator, + manager: manager, + publisher: publisher, + templateManager: templateManager, + exptRunLogRepo: exptRunLogRepo, + } +} + +type ExptRetryItemsExec struct { + manager IExptManager + exptStatsRepo repo.IExptStatsRepo + exptItemResultRepo repo.IExptItemResultRepo + exptTurnResultRepo repo.IExptTurnResultRepo + idgenerator idgen.IIDGenerator + evaluationSetItemService EvaluationSetItemService + exptRepo repo.IExperimentRepo + idem idem.IdempotentService + configer component.IConfiger + publisher events.ExptEventPublisher + evaluatorRecordService EvaluatorRecordService + templateManager IExptTemplateManager + exptRunLogRepo repo.IExptRunLogRepo +} + +func (e *ExptRetryItemsExec) Mode() entity.ExptRunMode { + return entity.EvaluationModeRetryItems +} + +func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + idemKey := makeStartIdemKey(event) + exist, err := e.idem.Exist(ctx, idemKey) + if err != nil { + return err + } + if exist { + return nil + } + + if err := e.resetEvalItems(ctx, event, expt, event.ExecEvalSetItemIDs); err != nil { + return err + } + + if err := e.exptRepo.Update(ctx, &entity.Experiment{ + Status: entity.ExptStatus_Processing, + ID: event.ExptID, + SpaceID: event.SpaceID, + }); err != nil { + return err + } + + if e.templateManager != nil { + var templateID int64 + if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { + templateID = expt.ExptTemplateMeta.ID + } + if templateID > 0 { + if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { + logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryItemsExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) + } + } + } + + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + return err + } + + time.Sleep(time.Second * 3) + + return nil +} + +func (e *ExptRetryItemsExec) resetEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, itemIDs []int64) error { + got, err := e.exptStatsRepo.Get(ctx, event.ExptID, event.SpaceID) + if err != nil { + return err + } + + var ( + evalSetID = expt.EvalSet.ID + evalSetVersionID = expt.EvalSet.EvaluationSetVersion.ID + pageSize = int32(100) + ) + + for _, chunk := range gslice.Chunk(itemIDs, int(pageSize)) { + logs.CtxInfo(ctx, "ExptRetryItemsExec.resetEvalItems scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, item_ids: %v", + event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, chunk) + + items, err := e.evaluationSetItemService.BatchGetEvaluationSetItems(ctx, &entity.BatchGetEvaluationSetItemsParam{ + SpaceID: event.SpaceID, + EvaluationSetID: evalSetID, + VersionID: &evalSetVersionID, + ItemIDs: chunk, + }) + if err != nil { + return err + } + + turnCnt := 0 + for _, item := range items { + turnCnt += len(item.Turns) + } + + ids, err := e.idgenerator.GenMultiIDs(ctx, len(items)+turnCnt) + if err != nil { + return err + } + + idIdx := 0 + itemIDMap := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) + itemTurnIDs := make([]*entity.ItemTurnID, 0, len(items)) + for _, item := range items { + for _, turn := range item.Turns { + itemIDMap[item.ItemID] = true + itemTurnIDs = append(itemTurnIDs, &entity.ItemTurnID{ + ItemID: item.ItemID, + TurnID: turn.ID, + }) + } + } + + itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDMap)) + for itemID := range itemIDMap { + itemRunLogs = append(itemRunLogs, &entity.ExptItemResultRunLog{ + ID: ids[idIdx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: itemID, + Status: int32(entity.ItemRunState_Queueing), + }) + idIdx++ + } + + irs, err := e.exptItemResultRepo.MGetItemResults(ctx, event.ExptID, chunk, event.SpaceID) + if err != nil { + return err + } + + for _, ir := range irs { + switch ir.Status { + case entity.ItemRunState_Processing: + got.ProcessingItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Success: + got.SuccessItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Fail: + got.FailItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Terminal: + got.TerminatedItemCnt-- + got.PendingItemCnt++ + default: + } + } + + if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDMap, func(k int64, v bool) int64 { return k }), map[string]any{ + "status": int32(entity.ItemRunState_Queueing), + "expt_run_id": event.ExptRunID, + }); err != nil { + return err + } + + if err := e.exptTurnResultRepo.UpdateTurnResults(ctx, event.ExptID, itemTurnIDs, event.SpaceID, map[string]any{ + "status": int32(entity.TurnRunState_Queueing), + }); err != nil { + return err + } + + if err := e.exptItemResultRepo.BatchCreateNXRunLogs(ctx, itemRunLogs); err != nil { + return err + } + + time.Sleep(time.Millisecond * 30) + } + + if err := e.exptStatsRepo.Save(ctx, got); err != nil { + return err + } + + logs.CtxInfo(ctx, "ExptRetryItemsExec.resetEvalItems reset stat: %v, expt_id: %v", json.Jsonify(got), event.ExptID) + time.Sleep(time.Second * 3) + return nil +} + +func (e *ExptRetryItemsExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).ScanEvalItems(ctx, event, expt) +} + +func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { + if toSubmit > 0 || incomplete > 0 { + return true, nil + } + + logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + + got, err := e.exptRunLogRepo.Get(ctx, event.ExptID, event.ExptRunID) + if err != nil { + return false, err + } + + exist := gslice.ToMap(event.ExecEvalSetItemIDs, func(t int64) (int64, bool) { return t, true }) + for _, itemID := range got.GetItemIDs() { + if !exist[itemID] { + return true, nil + } + } + + if err := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt); err != nil { + return false, err + } + return false, nil +} + +func (e *ExptRetryItemsExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + rl, err := e.exptRunLogRepo.Get(ctx, event.ExptID, event.ExptRunID) + if err != nil { + return err + } + + var ( + absence []int64 + all = rl.GetItemIDs() + exist = gslice.ToMap(event.ExecEvalSetItemIDs, func(t int64) (int64, bool) { return t, true }) + ) + for _, itemID := range all { + if !exist[itemID] { + absence = append(absence, itemID) + } + } + event.ExecEvalSetItemIDs = all + logs.CtxInfo(ctx, "ExptRetryItemsExec.ScheduleStart found absent item_id: %v, expt_id: %v", absence, event.ExptID) + + return e.resetEvalItems(ctx, event, expt, absence) +} + +func (e *ExptRetryItemsExec) ScheduleEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) error { + return nil +} + +func (e *ExptRetryItemsExec) NextTick(ctx context.Context, event *entity.ExptScheduleEvent, nextTick bool) error { + interval := e.configer.GetExptExecConf(ctx, event.SpaceID).GetDaemonInterval() + return e.publisher.PublishExptScheduleEvent(ctx, event, gptr.Of(interval)) +} + +func (e *ExptRetryItemsExec) PublishResult(ctx context.Context, turnEvaluatorRefs []*entity.ExptTurnEvaluatorResultRef, event *entity.ExptScheduleEvent) error { + if event.ExptType == entity.ExptType_Offline { + return nil + } + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).publishResult(ctx, turnEvaluatorRefs, event) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go deleted file mode 100755 index bd4cd0bdd..000000000 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/bytedance/gg/gptr" - "go.uber.org/mock/gomock" - - idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" - idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" - configmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" - "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" - mock_repo "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" - svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" -) - -// Phase 3: Target skip functionality and scheduler updates tests -func TestExptAppendExec_ScheduleStart_WithTargetSkip(t *testing.T) { - testCases := []struct { - name string - event *entity.ExptScheduleEvent - expt *entity.Experiment - mockSetup func(mockRepo *mock_repo.MockIExperimentRepo) - wantErr bool - }{ - { - name: "experiment_with_target_normal_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 10, - TargetType: entity.EvalTargetTypeLoopPrompt, - MaxAliveTime: 3600000, // 1 hour - StartAt: gptr.Of(time.Now().Add(-30 * time.Minute)), // Started 30 minutes ago - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - // No update expected as experiment is within time limit - }, - wantErr: false, - }, - { - name: "experiment_without_target_skip_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 0, // No target - should be skipped - TargetType: 0, - MaxAliveTime: 3600000, - StartAt: gptr.Of(time.Now().Add(-30 * time.Minute)), - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - // No update expected as experiment without target continues normally - }, - wantErr: false, - }, - { - name: "experiment_max_alive_time_exceeded", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 10, - TargetType: entity.EvalTargetTypeLoopPrompt, - MaxAliveTime: 3600000, // 1 hour - StartAt: gptr.Of(time.Now().Add(-2 * time.Hour)), // Started 2 hours ago - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - mockRepo.EXPECT(). - Update(gomock.Any(), &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Draining, - }). - Return(nil) - }, - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockRepo := mock_repo.NewMockIExperimentRepo(ctrl) - tc.mockSetup(mockRepo) - - exec := &ExptAppendExec{ - exptRepo: mockRepo, - } - - err := exec.ScheduleStart(context.Background(), tc.event, tc.expt) - if (err != nil) != tc.wantErr { - t.Errorf("ScheduleStart() error = %v, wantErr %v", err, tc.wantErr) - } - }) - } -} - -func TestExptAppendExec_ExptEnd_WithTargetSkip(t *testing.T) { - type mockFields struct { - manager *svcmocks.MockIExptManager - idem *idemmocks.MockIdempotentService - configer *configmocks.MockIConfiger - itemRepo *mock_repo.MockIExptItemResultRepo - publisher *eventmocks.MockExptEventPublisher - } - - testCases := []struct { - name string - event *entity.ExptScheduleEvent - expt *entity.Experiment - toSubmit int - incomplete int - mockSetup func(f *mockFields) - wantTick bool - wantErr bool - }{ - { - name: "experiment_without_target_draining_complete", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - ExptRunID: 2, - SpaceID: 100, - ExptRunMode: entity.EvaluationModeAppend, - Session: &entity.Session{UserID: "user1"}, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Draining, - TargetID: 0, // No target - TargetType: 0, - }, - toSubmit: 0, - incomplete: 0, - mockSetup: func(f *mockFields) { - f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) - // CompleteRun: ctx, exptID, exptRunID, spaceID, session, WithCID, WithCompleteInterval (7个参数) - f.manager.EXPECT().CompleteRun(gomock.Any(), int64(1), int64(2), int64(100), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - // CompleteExpt: ctx, exptID, spaceID, session, WithCID, WithCompleteInterval (6个参数) - f.manager.EXPECT().CompleteExpt(gomock.Any(), int64(1), int64(100), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(100)).Return(&entity.ExptExecConf{ZombieIntervalSecond: 60}) - f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - }, - wantTick: false, - wantErr: false, - }, - { - name: "experiment_without_target_still_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - ExptRunID: 2, - SpaceID: 100, - ExptRunMode: entity.EvaluationModeAppend, - Session: &entity.Session{UserID: "user1"}, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 0, // No target - TargetType: 0, - }, - toSubmit: 0, - incomplete: 1, - mockSetup: func(f *mockFields) { - // No mock setup needed as function should return early - }, - wantTick: true, - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - f := &mockFields{ - manager: svcmocks.NewMockIExptManager(ctrl), - idem: idemmocks.NewMockIdempotentService(ctrl), - configer: configmocks.NewMockIConfiger(ctrl), - itemRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), - publisher: eventmocks.NewMockExptEventPublisher(ctrl), - } - - if tc.mockSetup != nil { - tc.mockSetup(f) - } - - exec := &ExptAppendExec{ - manager: f.manager, - idem: f.idem, - configer: f.configer, - exptItemResultRepo: f.itemRepo, - publisher: f.publisher, - } - - nextTick, err := exec.ExptEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) - - if (err != nil) != tc.wantErr { - t.Errorf("ExptEnd() error = %v, wantErr %v", err, tc.wantErr) - } - - if nextTick != tc.wantTick { - t.Errorf("ExptEnd() nextTick = %v, want %v", nextTick, tc.wantTick) - } - }) - } -} - -func TestDefaultSchedulerModeFactory_NewSchedulerMode_Integration(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Create all required mocks - mockManager := svcmocks.NewMockIExptManager(ctrl) - mockItemRepo := mock_repo.NewMockIExptItemResultRepo(ctrl) - mockStatsRepo := mock_repo.NewMockIExptStatsRepo(ctrl) - mockTurnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) - mockEvalSetItemService := svcmocks.NewMockEvaluationSetItemService(ctrl) - mockExptRepo := mock_repo.NewMockIExperimentRepo(ctrl) - mockIdem := idemmocks.NewMockIdempotentService(ctrl) - mockConfiger := configmocks.NewMockIConfiger(ctrl) - mockPublisher := eventmocks.NewMockExptEventPublisher(ctrl) - mockEvaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) - mockResultSvc := svcmocks.NewMockExptResultService(ctrl) - mockTemplateManager := svcmocks.NewMockIExptTemplateManager(ctrl) - - factory := NewSchedulerModeFactory( - mockManager, - mockItemRepo, - mockStatsRepo, - mockTurnRepo, - mockIDGen, - mockEvalSetItemService, - mockExptRepo, - mockIdem, - mockConfiger, - mockPublisher, - mockEvaluatorRecordService, - mockResultSvc, - mockTemplateManager, - ) - - tests := []struct { - name string - mode entity.ExptRunMode - expectedType string - wantErr bool - }{ - { - name: "submit_mode_success", - mode: entity.EvaluationModeSubmit, - expectedType: "*service.ExptSubmitExec", - wantErr: false, - }, - { - name: "fail_retry_mode_success", - mode: entity.EvaluationModeFailRetry, - expectedType: "*service.ExptFailRetryExec", - wantErr: false, - }, - { - name: "append_mode_success", - mode: entity.EvaluationModeAppend, - expectedType: "*service.ExptAppendExec", - wantErr: false, - }, - { - name: "unknown_mode_error", - mode: entity.ExptRunMode(999), - expectedType: "", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scheduler, err := factory.NewSchedulerMode(tt.mode) - - if (err != nil) != tt.wantErr { - t.Errorf("NewSchedulerMode() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if !tt.wantErr { - if scheduler == nil { - t.Errorf("NewSchedulerMode() returned nil scheduler") - return - } - - // Verify the returned scheduler mode matches the input - if scheduler.Mode() != tt.mode { - t.Errorf("NewSchedulerMode() returned scheduler with mode %v, want %v", scheduler.Mode(), tt.mode) - } - - // Verify the type is correct - actualType := fmt.Sprintf("%T", scheduler) - if actualType != tt.expectedType { - t.Errorf("NewSchedulerMode() returned type %v, want %v", actualType, tt.expectedType) - } - } - }) - } -} diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index e9eb99205..a09b2e60b 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -568,15 +568,13 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, @@ -614,6 +612,36 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { assert.Contains(t, err.Error(), "scan error") }, }, + { + name: "empty_incomplete_and_complete_then_to_submit_filled", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: 1, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Queueing)}, + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Queueing}, + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{}, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, } for _, tt := range tests { @@ -994,15 +1022,13 @@ func TestExptFailRetryExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *exptFailRetryExecFields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, @@ -1576,15 +1602,13 @@ func TestExptAppendExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, @@ -2063,6 +2087,7 @@ func TestNewSchedulerModeFactory(t *testing.T) { evaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) resultService := svcmocks.NewMockExptResultService(ctrl) templateManager := svcmocks.NewMockIExptTemplateManager(ctrl) + mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) factory := NewSchedulerModeFactory( manager, @@ -2078,6 +2103,7 @@ func TestNewSchedulerModeFactory(t *testing.T) { evaluatorRecordService, resultService, templateManager, + mockExptRunLogRepo, ) tests := []struct { @@ -2450,3 +2476,2056 @@ func TestExptBaseExec_publishResult(t *testing.T) { }) } } + +type exptRetryAllExecFields struct { + manager *svcmocks.MockIExptManager + exptItemResultRepo *mock_repo.MockIExptItemResultRepo + exptStatsRepo *mock_repo.MockIExptStatsRepo + exptTurnResultRepo *mock_repo.MockIExptTurnResultRepo + idgenerator *idgenmocks.MockIIDGenerator + evaluationSetItemService *svcmocks.MockEvaluationSetItemService + exptRepo *mock_repo.MockIExperimentRepo + idem *idemmocks.MockIdempotentService + configer *configmocks.MockIConfiger + publisher *eventmocks.MockExptEventPublisher + evaluatorRecordService *svcmocks.MockEvaluatorRecordService + templateManager *svcmocks.MockIExptTemplateManager +} + +type exptRetryItemsExecFields struct { + manager *svcmocks.MockIExptManager + exptItemResultRepo *mock_repo.MockIExptItemResultRepo + exptStatsRepo *mock_repo.MockIExptStatsRepo + exptTurnResultRepo *mock_repo.MockIExptTurnResultRepo + idgenerator *idgenmocks.MockIIDGenerator + evaluationSetItemService *svcmocks.MockEvaluationSetItemService + exptRepo *mock_repo.MockIExperimentRepo + idem *idemmocks.MockIdempotentService + configer *configmocks.MockIConfiger + publisher *eventmocks.MockExptEventPublisher + evaluatorRecordService *svcmocks.MockEvaluatorRecordService + templateManager *svcmocks.MockIExptTemplateManager + exptRunLogRepo *mock_repo.MockIExptRunLogRepo +} + +func buildRetryAllExecFields(ctrl *gomock.Controller) *exptRetryAllExecFields { + return &exptRetryAllExecFields{ + manager: svcmocks.NewMockIExptManager(ctrl), + exptItemResultRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), + exptStatsRepo: mock_repo.NewMockIExptStatsRepo(ctrl), + exptTurnResultRepo: mock_repo.NewMockIExptTurnResultRepo(ctrl), + idgenerator: idgenmocks.NewMockIIDGenerator(ctrl), + evaluationSetItemService: svcmocks.NewMockEvaluationSetItemService(ctrl), + exptRepo: mock_repo.NewMockIExperimentRepo(ctrl), + idem: idemmocks.NewMockIdempotentService(ctrl), + configer: configmocks.NewMockIConfiger(ctrl), + publisher: eventmocks.NewMockExptEventPublisher(ctrl), + evaluatorRecordService: svcmocks.NewMockEvaluatorRecordService(ctrl), + templateManager: svcmocks.NewMockIExptTemplateManager(ctrl), + } +} + +func buildRetryItemsExecFields(ctrl *gomock.Controller) *exptRetryItemsExecFields { + return &exptRetryItemsExecFields{ + manager: svcmocks.NewMockIExptManager(ctrl), + exptItemResultRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), + exptStatsRepo: mock_repo.NewMockIExptStatsRepo(ctrl), + exptTurnResultRepo: mock_repo.NewMockIExptTurnResultRepo(ctrl), + idgenerator: idgenmocks.NewMockIIDGenerator(ctrl), + evaluationSetItemService: svcmocks.NewMockEvaluationSetItemService(ctrl), + exptRepo: mock_repo.NewMockIExperimentRepo(ctrl), + idem: idemmocks.NewMockIdempotentService(ctrl), + configer: configmocks.NewMockIConfiger(ctrl), + publisher: eventmocks.NewMockExptEventPublisher(ctrl), + evaluatorRecordService: svcmocks.NewMockEvaluatorRecordService(ctrl), + templateManager: svcmocks.NewMockIExptTemplateManager(ctrl), + exptRunLogRepo: mock_repo.NewMockIExptRunLogRepo(ctrl), + } +} + +func buildMockExpt() *entity.Experiment { + return &entity.Experiment{ + ID: 1, + SpaceID: 3, + CreatedBy: "created_by", + Name: "created_by", + Description: "description", + EvalSetVersionID: 1, + EvalSetID: 1, + TargetType: 1, + TargetVersionID: 1, + TargetID: 1, + EvaluatorVersionRef: []*entity.ExptEvaluatorVersionRef{{EvaluatorID: 1, EvaluatorVersionID: 1}}, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{TargetVersionID: 1, IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{{FieldName: "field_name", FromField: "from_field"}}}, + }}, + EvaluatorsConf: &entity.EvaluatorsConf{EvaluatorConcurNum: ptr.Of(1), EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{EvalSetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{{FieldName: "field_name", FromField: "from_field"}}}}, + }, + }}, + }}, + Target: &entity.EvalTarget{ID: 1, SpaceID: 3, SourceTargetID: "source_target_id", EvalTargetType: 1, EvalTargetVersion: &entity.EvalTargetVersion{ID: 1, OutputSchema: []*entity.ArgsSchema{{Key: ptr.Of("key")}}}, BaseInfo: &entity.BaseInfo{}}, + EvalSet: &entity.EvaluationSet{ + ID: 1, SpaceID: 3, Name: "name", Description: "description", Status: 0, Spec: nil, Features: nil, ItemCount: 0, ChangeUncommitted: false, + EvaluationSetVersion: &entity.EvaluationSetVersion{ID: 1, AppID: 0, SpaceID: 3, EvaluationSetID: 1, Version: "version", VersionNum: 0, Description: "description", EvaluationSetSchema: nil, ItemCount: 0, BaseInfo: nil}, + LatestVersion: "", NextVersionNum: 0, BaseInfo: nil, BizCategory: strconv.Itoa(1), + }, + Evaluators: []*entity.Evaluator{{}}, + Status: 0, + StatusMessage: "", + LatestRunID: 0, + CreditCost: 0, + StartAt: nil, + EndAt: nil, + ExptType: 1, + MaxAliveTime: 0, + SourceType: 0, + SourceID: "", + Stats: nil, + AggregateResult: nil, + } +} + +func TestExptRetryAllExec_Mode(t *testing.T) { + exec := &ExptRetryAllExec{} + assert.Equal(t, entity.EvaluationModeRetryAll, exec.Mode()) +} + +func TestExptRetryAllExec_ScheduleStart(t *testing.T) { + tests := []struct { + name string + expt *entity.Experiment + event *entity.ExptScheduleEvent + wantErr bool + }{ + { + name: "normal_flow", + expt: &entity.Experiment{}, + event: &entity.ExptScheduleEvent{}, + wantErr: false, + }, + } + + exec := &ExptRetryAllExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleStart(context.Background(), tc.event, tc.expt) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ScheduleEnd(t *testing.T) { + tests := []struct { + name string + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + wantErr bool + }{ + { + name: "normal_flow", + event: &entity.ExptScheduleEvent{}, + expt: &entity.Experiment{}, + toSubmit: 0, + incomplete: 0, + wantErr: false, + }, + } + + exec := &ExptRetryAllExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ExptStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "list_eval_set_items_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, nil, nil, nil, errors.New("list error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "list error") + }, + }, + { + name: "gen_multi_ids_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return(nil, errors.New("gen id error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "gen id error") + }, + }, + { + name: "update_items_result_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update error") + }, + }, + { + name: "update_turn_results_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update turn error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update turn error") + }, + }, + { + name: "batch_create_run_logs_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(errors.New("batch create error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch create error") + }, + }, + { + name: "get_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("get stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "get stats error") + }, + }, + { + name: "save_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(errors.New("save stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "save stats error") + }, + }, + { + name: "update_expt_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errors.New("update expt error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update expt error") + }, + }, + { + name: "idem_set_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ZombieIntervalSecond: 10}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("idem set error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem set error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + err := e.ExptStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ScanEvalItems(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantToSubmit []*entity.ExptEvalItem + wantIncomplete []*entity.ExptEvalItem + wantComplete []*entity.ExptEvalItem + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "normal_flow", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, + }, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + }, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "scan_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "scan error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + toSubmit, incomplete, complete, err := e.ScanEvalItems(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + if !tt.wantErr { + assert.Equal(t, tt.wantToSubmit, toSubmit) + assert.Equal(t, tt.wantIncomplete, incomplete) + assert.Equal(t, tt.wantComplete, complete) + } + }) + } +} + +func TestExptRetryAllExec_ExptEnd(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantNextTick bool + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "all_completed", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ZombieIntervalSecond: 100}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "still_pending", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 1, + incomplete: 1, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) {}, + wantNextTick: true, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "complete_run_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(errors.New("complete run error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "complete run error") + }, + }, + { + name: "complete_expt_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(errors.New("complete expt error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "complete expt error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + nextTick, err := e.ExptEnd(tt.args.ctx, tt.args.event, tt.args.expt, tt.args.toSubmit, tt.args.incomplete) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + assert.Equal(t, tt.wantNextTick, nextTick) + }) + } +} + +func TestExptRetryAllExec_NextTick(t *testing.T) { + testUserID := "test_user_id_123" + + tests := []struct { + name string + nextTick bool + prepareMock func(f *exptRetryAllExecFields) + event *entity.ExptScheduleEvent + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "nextTick_true_publish_success", + nextTick: true, + prepareMock: func(f *exptRetryAllExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "nextTick_true_publish_error", + nextTick: true, + prepareMock: func(f *exptRetryAllExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish error")) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "publish error") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryAllExec{ + configer: f.configer, + publisher: f.publisher, + } + + err := exec.NextTick(context.Background(), tc.event, tc.nextTick) + if tc.assertErr != nil { + tc.assertErr(t, err) + } + }) + } +} + +func TestExptRetryAllExec_PublishResult(t *testing.T) { + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields) + event *entity.ExptScheduleEvent + refs []*entity.ExptTurnEvaluatorResultRef + wantErr bool + }{ + { + name: "offline_expt_skip_publish", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Offline, + }, + refs: nil, + wantErr: false, + }, + { + name: "online_expt_empty_refs", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Online, + }, + refs: []*entity.ExptTurnEvaluatorResultRef{}, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryAllExec{ + manager: f.manager, + idem: f.idem, + configer: f.configer, + exptItemResultRepo: f.exptItemResultRepo, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + } + + err := exec.PublishResult(context.Background(), tc.refs, tc.event) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_Mode(t *testing.T) { + exec := &ExptRetryItemsExec{} + assert.Equal(t, entity.EvaluationModeRetryItems, exec.Mode()) +} + +func TestExptRetryItemsExec_ScheduleEnd(t *testing.T) { + tests := []struct { + name string + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + wantErr bool + }{ + { + name: "normal_flow", + event: &entity.ExptScheduleEvent{}, + expt: &entity.Experiment{}, + toSubmit: 0, + incomplete: 0, + wantErr: false, + }, + } + + exec := &ExptRetryItemsExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_ExptStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "get_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "stats error") + }, + }, + { + name: "batch_get_eval_set_items_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, errors.New("batch get error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch get error") + }, + }, + { + name: "gen_multi_ids_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return(nil, errors.New("gen id error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "gen id error") + }, + }, + { + name: "mget_item_results_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("mget error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "mget error") + }, + }, + { + name: "update_items_result_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Processing}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update error") + }, + }, + { + name: "update_turn_results_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Success}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update turn error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update turn error") + }, + }, + { + name: "batch_create_run_logs_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Fail}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(errors.New("batch create error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch create error") + }, + }, + { + name: "save_expt_stats_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Terminal}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(errors.New("save stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "save stats error") + }, + }, + { + name: "update_expt_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errors.New("update expt error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update expt error") + }, + }, + { + name: "idem_set_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ZombieIntervalSecond: 10}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("idem set error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem set error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + err := e.ExptStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_ScanEvalItems(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantToSubmit []*entity.ExptEvalItem + wantIncomplete []*entity.ExptEvalItem + wantComplete []*entity.ExptEvalItem + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "normal_flow", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, + }, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + }, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "scan_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "scan error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + toSubmit, incomplete, complete, err := e.ScanEvalItems(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + if !tt.wantErr { + assert.Equal(t, tt.wantToSubmit, toSubmit) + assert.Equal(t, tt.wantIncomplete, incomplete) + assert.Equal(t, tt.wantComplete, complete) + } + }) + } +} + +func TestExptRetryItemsExec_ExptEnd(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantNextTick bool + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "all_completed", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1}}}, + }, nil).Times(1) + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ZombieIntervalSecond: 100}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "still_pending", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 1, + incomplete: 1, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) {}, + wantNextTick: true, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + nextTick, err := e.ExptEnd(tt.args.ctx, tt.args.event, tt.args.expt, tt.args.toSubmit, tt.args.incomplete) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + assert.Equal(t, tt.wantNextTick, nextTick) + }) + } +} + +func TestExptRetryItemsExec_ScheduleStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "get_run_log_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(nil, errors.New("get error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "get error") + }, + }, + { + name: "schedule_start_with_absent_items_reset_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1, 2, 3}}}, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "stats error") + }, + }, + { + name: "schedule_start_with_no_absent_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1, 2}}}, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + err := e.ScheduleStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_NextTick(t *testing.T) { + testUserID := "test_user_id_123" + + tests := []struct { + name string + nextTick bool + prepareMock func(f *exptRetryItemsExecFields) + event *entity.ExptScheduleEvent + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "nextTick_true_publish_success", + nextTick: true, + prepareMock: func(f *exptRetryItemsExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "nextTick_true_publish_error", + nextTick: true, + prepareMock: func(f *exptRetryItemsExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish error")) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "publish error") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryItemsExec{ + configer: f.configer, + publisher: f.publisher, + } + + err := exec.NextTick(context.Background(), tc.event, tc.nextTick) + if tc.assertErr != nil { + tc.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_PublishResult(t *testing.T) { + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields) + event *entity.ExptScheduleEvent + refs []*entity.ExptTurnEvaluatorResultRef + wantErr bool + }{ + { + name: "offline_expt_skip_publish", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Offline, + }, + refs: nil, + wantErr: false, + }, + { + name: "online_expt_empty_refs", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Online, + }, + refs: []*entity.ExptTurnEvaluatorResultRef{}, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryItemsExec{ + manager: f.manager, + idem: f.idem, + configer: f.configer, + exptItemResultRepo: f.exptItemResultRepo, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + } + + err := exec.PublishResult(context.Background(), tc.refs, tc.event) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestNewExptRetryAllExec(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + + exec := NewExptRetryAllExec( + f.manager, + f.exptItemResultRepo, + f.exptStatsRepo, + f.exptTurnResultRepo, + f.idgenerator, + f.evaluationSetItemService, + f.exptRepo, + f.idem, + f.configer, + f.publisher, + f.evaluatorRecordService, + f.templateManager, + ) + + assert.NotNil(t, exec) + assert.Equal(t, f.manager, exec.manager) + assert.Equal(t, f.exptItemResultRepo, exec.exptItemResultRepo) + assert.Equal(t, f.exptStatsRepo, exec.exptStatsRepo) + assert.Equal(t, f.exptTurnResultRepo, exec.exptTurnResultRepo) + assert.Equal(t, f.idgenerator, exec.idgenerator) + assert.Equal(t, f.evaluationSetItemService, exec.evaluationSetItemService) + assert.Equal(t, f.exptRepo, exec.exptRepo) + assert.Equal(t, f.idem, exec.idem) + assert.Equal(t, f.configer, exec.configer) + assert.Equal(t, f.publisher, exec.publisher) +} + +func TestNewExptRetryItemsExec(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + + exec := NewExptRetryItemsExec( + f.manager, + f.exptItemResultRepo, + f.exptStatsRepo, + f.exptTurnResultRepo, + f.idgenerator, + f.evaluationSetItemService, + f.exptRepo, + f.idem, + f.configer, + f.publisher, + f.evaluatorRecordService, + f.templateManager, + f.exptRunLogRepo, + ) + + assert.NotNil(t, exec) + assert.Equal(t, f.manager, exec.manager) + assert.Equal(t, f.exptItemResultRepo, exec.exptItemResultRepo) + assert.Equal(t, f.exptStatsRepo, exec.exptStatsRepo) + assert.Equal(t, f.exptTurnResultRepo, exec.exptTurnResultRepo) + assert.Equal(t, f.idgenerator, exec.idgenerator) + assert.Equal(t, f.evaluationSetItemService, exec.evaluationSetItemService) + assert.Equal(t, f.exptRepo, exec.exptRepo) + assert.Equal(t, f.idem, exec.idem) + assert.Equal(t, f.configer, exec.configer) + assert.Equal(t, f.publisher, exec.publisher) + assert.Equal(t, f.exptRunLogRepo, exec.exptRunLogRepo) +} + +func TestSchedulerModeFactory_NewSchedulerMode_RetryAll(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + manager := svcmocks.NewMockIExptManager(ctrl) + exptItemResultRepo := mock_repo.NewMockIExptItemResultRepo(ctrl) + exptStatsRepo := mock_repo.NewMockIExptStatsRepo(ctrl) + exptTurnResultRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + idgenerator := idgenmocks.NewMockIIDGenerator(ctrl) + evaluationSetItemService := svcmocks.NewMockEvaluationSetItemService(ctrl) + exptRepo := mock_repo.NewMockIExperimentRepo(ctrl) + idem := idemmocks.NewMockIdempotentService(ctrl) + configer := configmocks.NewMockIConfiger(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + evaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) + resultService := svcmocks.NewMockExptResultService(ctrl) + templateManager := svcmocks.NewMockIExptTemplateManager(ctrl) + mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) + + factory := NewSchedulerModeFactory( + manager, + exptItemResultRepo, + exptStatsRepo, + exptTurnResultRepo, + idgenerator, + evaluationSetItemService, + exptRepo, + idem, + configer, + publisher, + evaluatorRecordService, + resultService, + templateManager, + mockExptRunLogRepo, + ) + + tests := []struct { + name string + mode entity.ExptRunMode + wantType interface{} + wantError bool + }{ + { + name: "retryAll_mode", + mode: entity.EvaluationModeRetryAll, + wantType: &ExptRetryAllExec{}, + wantError: false, + }, + { + name: "retryItems_mode", + mode: entity.EvaluationModeRetryItems, + wantType: &ExptRetryItemsExec{}, + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, err := factory.NewSchedulerMode(tt.mode) + if tt.wantError { + assert.Error(t, err) + assert.Nil(t, mode) + } else { + assert.NoError(t, err) + assert.IsType(t, tt.wantType, mode) + } + }) + } +} diff --git a/backend/modules/evaluation/domain/service/file_name b/backend/modules/evaluation/domain/service/file_name deleted file mode 100644 index 46abf88e6..000000000 --- a/backend/modules/evaluation/domain/service/file_name +++ /dev/null @@ -1 +0,0 @@ -ID,status,test_field,test_evaluator,test_evaluator_reason,weightedScore,test_tag,logID,targetTraceID diff --git a/backend/modules/evaluation/domain/service/mocks/evaluation_set.go b/backend/modules/evaluation/domain/service/mocks/evaluation_set.go index 31e6f5d96..d6fb354d4 100644 --- a/backend/modules/evaluation/domain/service/mocks/evaluation_set.go +++ b/backend/modules/evaluation/domain/service/mocks/evaluation_set.go @@ -87,21 +87,6 @@ func (mr *MockIEvaluationSetServiceMockRecorder) CreateEvaluationSetWithImport(c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEvaluationSetWithImport", reflect.TypeOf((*MockIEvaluationSetService)(nil).CreateEvaluationSetWithImport), ctx, param) } -// ParseImportSourceFile mocks base method. -func (m *MockIEvaluationSetService) ParseImportSourceFile(ctx context.Context, param *entity.ParseImportSourceFileParam) (*entity.ParseImportSourceFileResult, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseImportSourceFile", ctx, param) - ret0, _ := ret[0].(*entity.ParseImportSourceFileResult) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseImportSourceFile indicates an expected call of ParseImportSourceFile. -func (mr *MockIEvaluationSetServiceMockRecorder) ParseImportSourceFile(ctx, param any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseImportSourceFile", reflect.TypeOf((*MockIEvaluationSetService)(nil).ParseImportSourceFile), ctx, param) -} - // DeleteEvaluationSet mocks base method. func (m *MockIEvaluationSetService) DeleteEvaluationSet(ctx context.Context, spaceID, evaluationSetID int64) error { m.ctrl.T.Helper() @@ -148,6 +133,21 @@ func (mr *MockIEvaluationSetServiceMockRecorder) ListEvaluationSets(ctx, param a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEvaluationSets", reflect.TypeOf((*MockIEvaluationSetService)(nil).ListEvaluationSets), ctx, param) } +// ParseImportSourceFile mocks base method. +func (m *MockIEvaluationSetService) ParseImportSourceFile(ctx context.Context, param *entity.ParseImportSourceFileParam) (*entity.ParseImportSourceFileResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ParseImportSourceFile", ctx, param) + ret0, _ := ret[0].(*entity.ParseImportSourceFileResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ParseImportSourceFile indicates an expected call of ParseImportSourceFile. +func (mr *MockIEvaluationSetServiceMockRecorder) ParseImportSourceFile(ctx, param any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseImportSourceFile", reflect.TypeOf((*MockIEvaluationSetService)(nil).ParseImportSourceFile), ctx, param) +} + // QueryItemSnapshotMappings mocks base method. func (m *MockIEvaluationSetService) QueryItemSnapshotMappings(ctx context.Context, spaceID, datasetID int64, versionID *int64) ([]*entity.ItemSnapshotFieldMapping, string, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej b/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej deleted file mode 100644 index 3d4f8e238..000000000 --- a/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej +++ /dev/null @@ -1,12 +0,0 @@ -diff a/backend/modules/evaluation/domain/service/mocks/expt_export.go b/backend/modules/evaluation/domain/service/mocks/expt_export.go (rejected hunks) -@@ -8,9 +8,8 @@ import ( - context "context" - reflect "reflect" - -- gomock "go.uber.org/mock/gomock" -- - entity "code.byted.org/flowdevops/cozeloop/backend/modules/evaluation/domain/entity" -+ "go.uber.org/mock/gomock" - ) - - // MockIExptResultExportService is a mock of IExptResultExportService interface. diff --git a/backend/modules/evaluation/domain/service/mocks/expt_manage.go b/backend/modules/evaluation/domain/service/mocks/expt_manage.go index 494544766..43a46519b 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_manage.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_manage.go @@ -235,6 +235,21 @@ func (mr *MockIExptManagerMockRecorder) Invoke(ctx, invokeExptReq any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Invoke", reflect.TypeOf((*MockIExptManager)(nil).Invoke), ctx, invokeExptReq) } +// IsRunCompleting mocks base method. +func (m *MockIExptManager) IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCompletingRun", ctx, exptID, exptRunID, spaceID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsRunCompleting indicates an expected call of IsRunCompleting. +func (mr *MockIExptManagerMockRecorder) IsRunCompleting(ctx, exptID, exptRunID, spaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).IsCompletingRun), ctx, exptID, exptRunID, spaceID) +} + // List mocks base method. func (m *MockIExptManager) List(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptListFilter, orders []*entity.OrderBy, session *entity.Session) ([]*entity.Experiment, int64, error) { m.ctrl.T.Helper() @@ -267,18 +282,34 @@ func (mr *MockIExptManagerMockRecorder) ListExptRaw(ctx, page, pageSize, spaceID return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListExptRaw", reflect.TypeOf((*MockIExptManager)(nil).ListExptRaw), ctx, page, pageSize, spaceID, filter) } +// LogRetryItemsRun mocks base method. +func (m *MockIExptManager) LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (int64, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LogRetryItemsRun", ctx, exptID, mode, spaceID, itemIDs, session) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// LogRetryItemsRun indicates an expected call of LogRetryItemsRun. +func (mr *MockIExptManagerMockRecorder) LogRetryItemsRun(ctx, exptID, mode, spaceID, itemIDs, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRetryItemsRun", reflect.TypeOf((*MockIExptManager)(nil).LogRetryItemsRun), ctx, exptID, mode, spaceID, itemIDs, session) +} + // LogRun mocks base method. -func (m *MockIExptManager) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error { +func (m *MockIExptManager) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LogRun", ctx, exptID, exptRunID, mode, spaceID, session) + ret := m.ctrl.Call(m, "LogRun", ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) ret0, _ := ret[0].(error) return ret0 } // LogRun indicates an expected call of LogRun. -func (mr *MockIExptManagerMockRecorder) LogRun(ctx, exptID, exptRunID, mode, spaceID, session any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) LogRun(ctx, exptID, exptRunID, mode, spaceID, itemIDs, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), ctx, exptID, exptRunID, mode, spaceID, session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) } // MDelete mocks base method. @@ -358,32 +389,32 @@ func (mr *MockIExptManagerMockRecorder) PendRun(ctx, exptID, exptRunID, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PendRun", reflect.TypeOf((*MockIExptManager)(nil).PendRun), ctx, exptID, exptRunID, spaceID, session) } -// RetryUnSuccess mocks base method. -func (m *MockIExptManager) RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error { +// RetryItems mocks base method. +func (m *MockIExptManager) RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RetryUnSuccess", ctx, exptID, runID, spaceID, session, ext) + ret := m.ctrl.Call(m, "RetryItems", ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) ret0, _ := ret[0].(error) return ret0 } -// RetryUnSuccess indicates an expected call of RetryUnSuccess. -func (mr *MockIExptManagerMockRecorder) RetryUnSuccess(ctx, exptID, runID, spaceID, session, ext any) *gomock.Call { +// RetryItems indicates an expected call of RetryItems. +func (mr *MockIExptManagerMockRecorder) RetryItems(ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryUnSuccess", reflect.TypeOf((*MockIExptManager)(nil).RetryUnSuccess), ctx, exptID, runID, spaceID, session, ext) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryItems", reflect.TypeOf((*MockIExptManager)(nil).RetryItems), ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) } // Run mocks base method. -func (m *MockIExptManager) Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { +func (m *MockIExptManager) Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", ctx, exptID, runID, spaceID, session, runMode, ext) + ret := m.ctrl.Call(m, "Run", ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) ret0, _ := ret[0].(error) return ret0 } // Run indicates an expected call of Run. -func (mr *MockIExptManagerMockRecorder) Run(ctx, exptID, runID, spaceID, session, runMode, ext any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Run(ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), ctx, exptID, runID, spaceID, session, runMode, ext) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) } // SetExptTerminating mocks base method. diff --git a/backend/modules/evaluation/domain/service/mocks/expt_result.go b/backend/modules/evaluation/domain/service/mocks/expt_result.go index 812fb85ee..d9447d5d6 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_result.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_result.go @@ -188,6 +188,20 @@ func (mr *MockExptResultServiceMockRecorder) ManualUpsertExptTurnResultFilter(ct return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManualUpsertExptTurnResultFilter", reflect.TypeOf((*MockExptResultService)(nil).ManualUpsertExptTurnResultFilter), ctx, spaceID, exptID, itemIDs) } +// RecalculateWeightedScore mocks base method. +func (m *MockExptResultService) RecalculateWeightedScore(ctx context.Context, spaceID, exptID, itemID, turnID int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecalculateWeightedScore", ctx, spaceID, exptID, itemID, turnID) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecalculateWeightedScore indicates an expected call of RecalculateWeightedScore. +func (mr *MockExptResultServiceMockRecorder) RecalculateWeightedScore(ctx, spaceID, exptID, itemID, turnID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecalculateWeightedScore", reflect.TypeOf((*MockExptResultService)(nil).RecalculateWeightedScore), ctx, spaceID, exptID, itemID, turnID) +} + // RecordItemRunLogs mocks base method. func (m *MockExptResultService) RecordItemRunLogs(ctx context.Context, exptID, exptRunID, itemID, spaceID int64) ([]*entity.ExptTurnEvaluatorResultRef, error) { m.ctrl.T.Helper() @@ -217,20 +231,6 @@ func (mr *MockExptResultServiceMockRecorder) UpsertExptTurnResultFilter(ctx, spa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertExptTurnResultFilter", reflect.TypeOf((*MockExptResultService)(nil).UpsertExptTurnResultFilter), ctx, spaceID, exptID, itemID) } -// RecalculateWeightedScore mocks base method. -func (m *MockExptResultService) RecalculateWeightedScore(ctx context.Context, spaceID, exptID, itemID, turnID int64) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecalculateWeightedScore", ctx, spaceID, exptID, itemID, turnID) - ret0, _ := ret[0].(error) - return ret0 -} - -// RecalculateWeightedScore indicates an expected call of RecalculateWeightedScore. -func (mr *MockExptResultServiceMockRecorder) RecalculateWeightedScore(ctx, spaceID, exptID, itemID, turnID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecalculateWeightedScore", reflect.TypeOf((*MockExptResultService)(nil).RecalculateWeightedScore), ctx, spaceID, exptID, itemID, turnID) -} - // MockExptAggrResultService is a mock of ExptAggrResultService interface. type MockExptAggrResultService struct { ctrl *gomock.Controller diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go index d4b4a00bb..cb1bbffe0 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go @@ -59,6 +59,18 @@ func (e ExptItemResultRepoImpl) GetItemTurnResults(ctx context.Context, spaceID, return results, nil } +func (e ExptItemResultRepoImpl) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) { + pos, err := e.exptItemResultDAO.MGetItemTurnResults(ctx, spaceID, exptID, itemIDs) + if err != nil { + return nil, err + } + results := make([]*entity.ExptTurnResult, 0) + for _, po := range pos { + results = append(results, convert.NewExptTurnResultConvertor().PO2DO(po, nil)) + } + return results, nil +} + func (e ExptItemResultRepoImpl) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error { pos := make([]*model.ExptItemResult, 0) for _, itemResult := range itemResults { @@ -112,6 +124,18 @@ func (e ExptItemResultRepoImpl) ScanItemResults(ctx context.Context, exptID, cur return results, ncursor, nil } +func (e ExptItemResultRepoImpl) MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) (results []*entity.ExptItemResult, err error) { + pos, err := e.exptItemResultDAO.MGetItemResults(ctx, spaceID, exptID, itemIDs) + if err != nil { + return nil, errorx.Wrapf(err, "MGetItemResults fail, exptID=%d, spaceID=%d", exptID, spaceID) + } + results = make([]*entity.ExptItemResult, 0, len(pos)) + for _, po := range pos { + results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + } + return results, nil +} + func (e ExptItemResultRepoImpl) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) { return e.exptItemResultDAO.GetItemIDListByExptID(ctx, exptID, spaceID) } diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go index 4b1a174c5..c90818cc7 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" @@ -492,6 +493,33 @@ func TestExptItemResultRepoImpl_ScanItemRunLogs(t *testing.T) { } } +func TestExptItemResultRepoImpl_ScanItemRunLogs_WithRawFilter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDAO := mocks.NewMockIExptItemResultDAO(ctrl) + repo := &ExptItemResultRepoImpl{exptItemResultDAO: mockDAO} + rawFilter := &entity.ExptItemRunLogFilter{ + RawFilter: true, + RawCond: clause.Expr{SQL: "status IN (?) OR result_state = ?", Vars: []interface{}{[]int32{1}, int32(2)}}, + } + + rs := int32(2) + mockDAO.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), rawFilter, int64(0), int64(0), int64(3)). + Return([]*model.ExptItemResultRunLog{ + {ItemID: 10, Status: 1}, + {ItemID: 20, Status: 2, ResultState: &rs}, + }, int64(0), nil) + + got, ncursor, err := repo.ScanItemRunLogs(context.Background(), 1, 2, rawFilter, 0, 0, 3) + assert.NoError(t, err) + assert.Equal(t, int64(0), ncursor) + assert.Len(t, got, 2) + assert.Equal(t, int64(10), got[0].ItemID) + assert.Equal(t, int64(20), got[1].ItemID) + assert.Equal(t, int32(2), got[1].ResultState) +} + func TestExptItemResultRepoImpl_BatchCreateNX(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go index 2f33a7542..b20d59e47 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go @@ -28,16 +28,17 @@ func (e *ExptRunLogImpl) Get(ctx context.Context, exptID, exptRunID int64) (*ent if err != nil { return nil, err } - do := convert.NewExptRunLogConvertor().PO2DO(po) - return do, nil + return convert.NewExptRunLogConvertor().PO2DO(po) } func (e *ExptRunLogImpl) Create(ctx context.Context, exptRunLog *entity.ExptRunLog) error { - po := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + po, err := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + if err != nil { + return err + } po.CreatedAt = time.Now() - err := e.exptRunLogDAO.Create(ctx, po) - if err != nil { + if err := e.exptRunLogDAO.Create(ctx, po); err != nil { return err } @@ -45,11 +46,13 @@ func (e *ExptRunLogImpl) Create(ctx context.Context, exptRunLog *entity.ExptRunL } func (e *ExptRunLogImpl) Save(ctx context.Context, exptRunLog *entity.ExptRunLog) error { - po := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + po, err := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + if err != nil { + return err + } po.UpdatedAt = time.Now() - err := e.exptRunLogDAO.Save(ctx, po) - if err != nil { + if err := e.exptRunLogDAO.Save(ctx, po); err != nil { return err } return nil diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go index 0648ba88d..a47535b90 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go @@ -73,9 +73,13 @@ func NewExptRunLogConvertor() ExptRunLogConvertor { return ExptRunLogConvertor{} } -func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { +func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) (*model.ExptRunLog, error) { if log == nil { - return nil + return nil, nil + } + itemIDsBytes, err := json.Marshal(log.ItemIds) + if err != nil { + return nil, errorx.Wrapf(err, "ExptRunLogItems list json marshal fail") } return &model.ExptRunLog{ ID: log.ID, @@ -83,7 +87,7 @@ func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { CreatedBy: log.CreatedBy, ExptID: log.ExptID, ExptRunID: log.ExptRunID, - ItemIds: gptr.Of(log.ItemIds), + ItemIds: gptr.Of(itemIDsBytes), Mode: gptr.Of(log.Mode), Status: gptr.Of(log.Status), PendingCnt: log.PendingCnt, @@ -96,20 +100,28 @@ func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { TerminatedCnt: log.TerminatedCnt, CreatedAt: log.CreatedAt, UpdatedAt: log.UpdatedAt, - } + }, nil } -func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) *entity.ExptRunLog { +func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) (*entity.ExptRunLog, error) { if log == nil { - return nil + return nil, nil } + + var itemIDs []entity.ExptRunLogItems + if log.ItemIds != nil && len(*log.ItemIds) > 0 { + if err := json.Unmarshal(*log.ItemIds, &itemIDs); err != nil { + return nil, errorx.Wrapf(err, "ExptRunLogItems list json unmarshal fail") + } + } + return &entity.ExptRunLog{ ID: log.ID, SpaceID: log.SpaceID, CreatedBy: log.CreatedBy, ExptID: log.ExptID, ExptRunID: log.ExptRunID, - ItemIds: gptr.Indirect(log.ItemIds), + ItemIds: itemIDs, Mode: gptr.Indirect(log.Mode), Status: gptr.Indirect(log.Status), PendingCnt: log.PendingCnt, @@ -122,5 +134,5 @@ func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) *entity.ExptRunLog { TerminatedCnt: log.TerminatedCnt, CreatedAt: log.CreatedAt, UpdatedAt: log.UpdatedAt, - } + }, nil } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 6fc2cd4c9..3b0c22b44 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -29,10 +29,12 @@ type IExptItemResultDAO interface { BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64, opts ...db.Option) (results []*model.ExptItemResult, ncursor int64, err error) + MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) (results []*model.ExptItemResult, err error) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*model.ExptItemResult, int64, error) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error GetItemTurnResults(ctx context.Context, spaceID, exptID, itemID int64, opts ...db.Option) ([]*model.ExptTurnResult, error) + MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) UpdateItemsResult(ctx context.Context, spaceID, exptID int64, itemID []int64, ufields map[string]any, opts ...db.Option) error GetMaxItemIdxByExptID(ctx context.Context, exptID, spaceID int64, opts ...db.Option) (int32, error) @@ -90,6 +92,19 @@ func (dao *exptItemResultDAOImpl) GetItemTurnResults(ctx context.Context, spaceI return finds, nil } +func (dao *exptItemResultDAOImpl) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) { + if len(itemIDs) == 0 { + return nil, nil + } + db := dao.provider.NewSession(ctx, opts...) + q := query.Use(db).ExptTurnResult + found, err := q.WithContext(ctx).Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.In(itemIDs...)).Find() + if err != nil { + return nil, err + } + return found, nil +} + func (dao *exptItemResultDAOImpl) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error { db := dao.provider.NewSession(ctx, opts...) q := query.Use(db).ExptItemResult @@ -191,6 +206,22 @@ func (dao *exptItemResultDAOImpl) ScanItemResults(ctx context.Context, exptID, c return res, res[len(res)-1].ID, nil } +func (dao *exptItemResultDAOImpl) MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) (results []*model.ExptItemResult, err error) { + if len(itemIDs) == 0 { + return nil, nil + } + db := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + db = db.Clauses(dbresolver.Write) + } + q := query.Use(db).ExptItemResult + res, err := q.WithContext(ctx).Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.In(itemIDs...)).Find() + if err != nil { + return nil, errorx.Wrapf(err, "MGetItemResults fail, exptID=%d, spaceID=%d, itemIDs=%v", exptID, spaceID, itemIDs) + } + return res, nil +} + func (dao *exptItemResultDAOImpl) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*model.ExptItemResult, int64, error) { db := dao.provider.NewSession(ctx) q := query.Use(db).ExptItemResult @@ -254,21 +285,46 @@ func (dao *exptItemResultDAOImpl) GetItemIDListByExptID(ctx context.Context, exp } func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, exptRunID int64, filter *entity.ExptItemRunLogFilter, cursor, limit, spaceID int64, opts ...db.Option) ([]*model.ExptItemResultRunLog, int64, error) { - db := dao.provider.NewSession(ctx, opts...) - q := query.Use(db).ExptItemResultRunLog + if filter == nil { + filter = &entity.ExptItemRunLogFilter{} + } + session := dao.provider.NewSession(ctx, opts...) + + // RawFilter: use raw gorm.DB Where(sql, vars...) to avoid gen clause conversion / unknown clause issues. + if filter.RawFilter && filter.RawCond.SQL != "" { + var res []*model.ExptItemResultRunLog + tx := session.WithContext(ctx).Model(&model.ExptItemResultRunLog{}). + Clauses(hints.ForceIndex("uk_expt_run_item_turn")). + Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", spaceID, exptID, exptRunID). + Where(filter.RawCond.SQL, filter.RawCond.Vars...) + if cursor > 0 { + tx = tx.Where("id > ?", cursor) + } + tx = tx.Order("id asc") + if limit > 0 { + tx = tx.Limit(int(limit)) + } + if err := tx.Find(&res).Error; err != nil { + return nil, 0, errorx.Wrapf(err, "ScanItemRunLogs fail, exptID=%d, exptRunID=%d, cursor=%d", exptID, exptRunID, cursor) + } + if len(res) == 0 { + return nil, 0, nil + } + return res, res[len(res)-1].ID, nil + } + + q := query.Use(session).ExptItemResultRunLog conds := []gen.Condition{ q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ExptRunID.Eq(exptRunID), } - if filter.ResultState != nil { conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) } if len(filter.Status) > 0 { conds = append(conds, q.Status.In(filter.GetStatus()...)) } - if cursor > 0 { conds = append(conds, q.ID.Gt(cursor)) } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go index 70685b05c..0d915174c 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go @@ -192,6 +192,26 @@ func (mr *MockIExptItemResultDAOMockRecorder) ListItemResultsByExptID(ctx, exptI return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListItemResultsByExptID", reflect.TypeOf((*MockIExptItemResultDAO)(nil).ListItemResultsByExptID), ctx, exptID, spaceID, page, desc) } +// MGetItemResults mocks base method. +func (m *MockIExptItemResultDAO) MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "MGetItemResults", varargs...) + ret0, _ := ret[0].([]*model.ExptItemResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemResults indicates an expected call of MGetItemResults. +func (mr *MockIExptItemResultDAOMockRecorder) MGetItemResults(ctx, spaceID, exptID, itemIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemResults), varargs...) +} + // MGetItemRunLog mocks base method. func (m *MockIExptItemResultDAO) MGetItemRunLog(ctx context.Context, exptID, exptRunID int64, itemIDs []int64, spaceID int64, opts ...db.Option) ([]*model.ExptItemResultRunLog, error) { m.ctrl.T.Helper() @@ -212,6 +232,26 @@ func (mr *MockIExptItemResultDAOMockRecorder) MGetItemRunLog(ctx, exptID, exptRu return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemRunLog", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemRunLog), varargs...) } +// MGetItemTurnResults mocks base method. +func (m *MockIExptItemResultDAO) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "MGetItemTurnResults", varargs...) + ret0, _ := ret[0].([]*model.ExptTurnResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemTurnResults indicates an expected call of MGetItemTurnResults. +func (mr *MockIExptItemResultDAOMockRecorder) MGetItemTurnResults(ctx, spaceID, exptID, itemIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemTurnResults), varargs...) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultDAO) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/pkg/errno/evaluation.go b/backend/modules/evaluation/pkg/errno/evaluation.go index 081e8174e..8d10456ce 100644 --- a/backend/modules/evaluation/pkg/errno/evaluation.go +++ b/backend/modules/evaluation/pkg/errno/evaluation.go @@ -124,6 +124,14 @@ const ( duplicateCalcExptAggrResultErrorMessage = "aggregated result calculation is already in progress" duplicateCalcExptAggrResultErrorNoAffectStability = true + EvalItemAlreadyRetryingCode = 601204015 // item already has been retrying + evalItemAlreadyRetryingMessage = "item already has been retrying" + evalItemAlreadyRetryingNoAffectStability = true + + ExperimentIsCompletingCode = 601204016 // experiment is completing, please try later + experimentIsCompletingMessage = "experiment is completing, please try later" + experimentIsCompletingNoAffectStability = true + ContentTypeNotSupportedCode = 601205000 // content type is not supported contentTypeNotSupportedMessage = "content type is not supported" contentTypeNotSupportedNoAffectStability = true @@ -537,6 +545,18 @@ func init() { code.WithAffectStability(!duplicateCalcExptAggrResultErrorNoAffectStability), ) + code.Register( + EvalItemAlreadyRetryingCode, + evalItemAlreadyRetryingMessage, + code.WithAffectStability(!evalItemAlreadyRetryingNoAffectStability), + ) + + code.Register( + ExperimentIsCompletingCode, + experimentIsCompletingMessage, + code.WithAffectStability(!experimentIsCompletingNoAffectStability), + ) + code.Register( ContentTypeNotSupportedCode, contentTypeNotSupportedMessage, diff --git a/backend/script/errorx/evaluation.yaml b/backend/script/errorx/evaluation.yaml index d849aac6a..fc1ea20a5 100644 --- a/backend/script/errorx/evaluation.yaml +++ b/backend/script/errorx/evaluation.yaml @@ -112,6 +112,18 @@ error_code: description: aggregated result calculation is already in progress no_affect_stability: true + - name: EvalItemAlreadyRetrying + code: 4015 + message: item already has been retrying + description: item already has been retrying + no_affect_stability: true + + - name: ExperimentIsCompleting + code: 4016 + message: experiment is completing, please try later + description: experiment is completing, please try later + no_affect_stability: true + - name: ContentTypeNotSupported code: 5000 message: content type is not supported diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift index 4f3f329c3..1835244bd 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift @@ -36,6 +36,7 @@ struct CreateExperimentRequest { 41: optional bool enable_weighted_score (api.body = 'enable_weighted_score', go.tag='json:"enable_weighted_score"') 42: optional map evaluator_score_weights (api.body = 'evaluator_score_weights', go.tag='json:"evaluator_score_weights"') 43: optional i64 expt_template_id (api.body='expt_template_id',api.js_conv='true', go.tag='json:"expt_template_id"') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') 200: optional common.Session session @@ -74,6 +75,7 @@ struct SubmitExperimentRequest { // 是否启用评估器得分加权汇总，以及各评估器的权重配置（key 为 evaluator_version_id，value 为权重） 41: optional bool enable_weighted_score (api.body = 'enable_weighted_score', go.tag='json:"enable_weighted_score"') 42: optional i64 expt_template_id (api.body='expt_template_id',api.js_conv='true', go.tag='json:"expt_template_id"') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') 100: optional map ext (api.body = 'ext') @@ -162,6 +164,7 @@ struct RunExperimentRequest { 2: optional i64 expt_id (api.body = 'expt_id', api.js_conv = 'true', go.tag = 'json:"expt_id"') 3: optional list item_ids (api.body = 'item_ids', api.js_conv = 'true', go.tag = 'json:"item_ids"') 10: optional expt.ExptType expt_type (api.body = 'expt_type') + 11: optional i32 item_retry_num (api.body = 'item_retry_num') 100: optional map ext (api.body = 'ext') diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift index c8b3883b4..6cf6da27c 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift @@ -346,6 +346,8 @@ struct SubmitExperimentOApiRequest { 20: optional i32 item_concur_num (api.body = 'item_concur_num') 22: optional common.RuntimeParam target_runtime_param (api.body = 'target_runtime_param') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') + 255: optional base.Base Base } diff --git a/idl/thrift/coze/loop/evaluation/domain/expt.thrift b/idl/thrift/coze/loop/evaluation/domain/expt.thrift index fb247339b..f7717be10 100644 --- a/idl/thrift/coze/loop/evaluation/domain/expt.thrift +++ b/idl/thrift/coze/loop/evaluation/domain/expt.thrift @@ -62,6 +62,7 @@ struct Experiment { 41: optional i64 max_alive_time 42: optional SourceType source_type 43: optional string source_id + 45: optional i32 item_retry_num 51: optional list evaluator_id_version_list // 补充的评估器id+version关联评估器方式，和evaluator_version_ids共同使用，兼容老逻辑 @@ -78,7 +79,6 @@ struct ExptTemplateMeta { 3: optional string name 4: optional string desc 5: optional ExptType expt_type // 模板对应的实验类型，当前主要为 Offline - } // 实验三元组配置 @@ -99,6 +99,7 @@ struct ExptFieldMapping { 2: optional list evaluator_field_mapping 3: optional common.RuntimeParam target_runtime_param 4: optional i32 item_concur_num + 5: optional i32 item_retry_num } // 实验评估器得分加权配置