From cc3666c4bb73eb4daf8c6561e07f8a0ae66739f2 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 6 Feb 2026 14:11:47 +0800 Subject: [PATCH 01/30] feat(evaluation): retry extension --- .../coze/loop/evaluation/coze.loop.evaluation.expt.thrift | 4 ++++ .../coze/loop/evaluation/coze.loop.evaluation.openapi.thrift | 2 ++ 2 files changed, 6 insertions(+) diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift index ce9b0be7d..e3f70a1f2 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift @@ -36,6 +36,7 @@ struct CreateExperimentRequest { 41: optional bool enable_weighted_score (api.body = 'enable_weighted_score', go.tag='json:"enable_weighted_score"') 42: optional map evaluator_score_weights (api.body = 'evaluator_score_weights', go.tag='json:"evaluator_score_weights"') 43: optional i64 expt_template_id (api.body='expt_template_id',api.js_conv='true', go.tag='json:"expt_template_id"') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') 200: optional common.Session session @@ -74,6 +75,7 @@ struct SubmitExperimentRequest { // 是否启用评估器得分加权汇总，以及各评估器的权重配置（key 为 evaluator_version_id，value 为权重） 41: optional bool enable_weighted_score (api.body = 'enable_weighted_score', go.tag='json:"enable_weighted_score"') 42: optional i64 expt_template_id (api.body='expt_template_id',api.js_conv='true', go.tag='json:"expt_template_id"') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') 100: optional map ext (api.body = 'ext') @@ -373,6 +375,8 @@ struct CreateExperimentTemplateRequest { // 调度配置（不在 ExptTemplate 结构中，保留在顶层） 22: optional string schedule_cron (api.body = 'schedule_cron') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') + 200: optional common.Session session 255: optional base.Base Base } diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift index f728a9f69..1ca1e29ec 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.openapi.thrift @@ -346,6 +346,8 @@ struct SubmitExperimentOApiRequest { 20: optional i32 item_concur_num (api.body = 'item_concur_num') 22: optional common.RuntimeParam target_runtime_param (api.body = 'target_runtime_param') + 45: optional i32 item_retry_num (api.boy = 'item_retry_num') + 255: optional base.Base Base } From e290ac10422100651aad8659d9989ffbd9c28aac Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 6 Feb 2026 14:41:57 +0800 Subject: [PATCH 02/30] feat(evaluation): retry gen --- .../api/handler/coze/loop/apis/wire_gen.go | 1 + .../expt/coze.loop.evaluation.expt.go | 231 ++++++++++++++++++ .../expt/k-coze.loop.evaluation.expt.go | 159 ++++++++++++ .../openapi/coze.loop.evaluation.openapi.go | 77 ++++++ .../openapi/k-coze.loop.evaluation.openapi.go | 53 ++++ 5 files changed, 521 insertions(+) diff --git a/backend/api/handler/coze/loop/apis/wire_gen.go b/backend/api/handler/coze/loop/apis/wire_gen.go index b0ce0120b..a7dcb28b3 100644 --- a/backend/api/handler/coze/loop/apis/wire_gen.go +++ b/backend/api/handler/coze/loop/apis/wire_gen.go @@ -8,6 +8,7 @@ package apis import ( "context" + "github.com/cloudwego/kitex/pkg/endpoint" "github.com/coze-dev/coze-loop/backend/infra/ck" "github.com/coze-dev/coze-loop/backend/infra/db" diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go index c9db81e5e..6d8c7f7dc 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go @@ -51,6 +51,7 @@ type CreateExperimentRequest struct { EnableWeightedScore *bool `thrift:"enable_weighted_score,41,optional" frugal:"41,optional,bool" json:"enable_weighted_score" form:"enable_weighted_score" ` EvaluatorScoreWeights map[int64]float64 `thrift:"evaluator_score_weights,42,optional" frugal:"42,optional,map" json:"evaluator_score_weights" form:"evaluator_score_weights" ` ExptTemplateID *int64 `thrift:"expt_template_id,43,optional" frugal:"43,optional,i64" json:"expt_template_id" form:"expt_template_id" ` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -321,6 +322,18 @@ func (p *CreateExperimentRequest) GetExptTemplateID() (v int64) { return *p.ExptTemplateID } +var CreateExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *CreateExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return CreateExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var CreateExperimentRequest_Session_DEFAULT *common.Session func (p *CreateExperimentRequest) GetSession() (v *common.Session) { @@ -410,6 +423,9 @@ func (p *CreateExperimentRequest) SetEvaluatorScoreWeights(val map[int64]float64 func (p *CreateExperimentRequest) SetExptTemplateID(val *int64) { p.ExptTemplateID = val } +func (p *CreateExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *CreateExperimentRequest) SetSession(val *common.Session) { p.Session = val } @@ -440,6 +456,7 @@ var fieldIDToName_CreateExperimentRequest = map[int16]string{ 41: "enable_weighted_score", 42: "evaluator_score_weights", 43: "expt_template_id", + 45: "item_retry_num", 200: "session", 255: "Base", } @@ -528,6 +545,10 @@ func (p *CreateExperimentRequest) IsSetExptTemplateID() bool { return p.ExptTemplateID != nil } +func (p *CreateExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *CreateExperimentRequest) IsSetSession() bool { return p.Session != nil } @@ -732,6 +753,14 @@ func (p *CreateExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 200: if fieldTypeId == thrift.STRUCT { if err = p.ReadField200(iprot); err != nil { @@ -1072,6 +1101,17 @@ func (p *CreateExperimentRequest) ReadField43(iprot thrift.TProtocol) error { p.ExptTemplateID = _field return nil } +func (p *CreateExperimentRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *CreateExperimentRequest) ReadField200(iprot thrift.TProtocol) error { _field := common.NewSession() if err := _field.Read(iprot); err != nil { @@ -1183,6 +1223,10 @@ func (p *CreateExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 43 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField200(oprot); err != nil { fieldId = 200 goto WriteFieldError @@ -1638,6 +1682,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 43 end error: ", p), err) } +func (p *CreateExperimentRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *CreateExperimentRequest) writeField200(oprot thrift.TProtocol) (err error) { if p.IsSetSession() { if err = oprot.WriteFieldBegin("session", thrift.STRUCT, 200); err != nil { @@ -1755,6 +1817,9 @@ func (p *CreateExperimentRequest) DeepEqual(ano *CreateExperimentRequest) bool { if !p.Field43DeepEqual(ano.ExptTemplateID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field200DeepEqual(ano.Session) { return false } @@ -2012,6 +2077,18 @@ func (p *CreateExperimentRequest) Field43DeepEqual(src *int64) bool { } return true } +func (p *CreateExperimentRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *CreateExperimentRequest) Field200DeepEqual(src *common.Session) bool { if !p.Session.DeepEqual(src) { @@ -2291,6 +2368,7 @@ type SubmitExperimentRequest struct { // 是否启用评估器得分加权汇总，以及各评估器的权重配置（key 为 evaluator_version_id，value 为权重） EnableWeightedScore *bool `thrift:"enable_weighted_score,41,optional" frugal:"41,optional,bool" json:"enable_weighted_score" form:"enable_weighted_score" ` ExptTemplateID *int64 `thrift:"expt_template_id,42,optional" frugal:"42,optional,i64" json:"expt_template_id" form:"expt_template_id" ` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` @@ -2550,6 +2628,18 @@ func (p *SubmitExperimentRequest) GetExptTemplateID() (v int64) { return *p.ExptTemplateID } +var SubmitExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *SubmitExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return SubmitExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var SubmitExperimentRequest_Ext_DEFAULT map[string]string func (p *SubmitExperimentRequest) GetExt() (v map[string]string) { @@ -2648,6 +2738,9 @@ func (p *SubmitExperimentRequest) SetEnableWeightedScore(val *bool) { func (p *SubmitExperimentRequest) SetExptTemplateID(val *int64) { p.ExptTemplateID = val } +func (p *SubmitExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *SubmitExperimentRequest) SetExt(val map[string]string) { p.Ext = val } @@ -2680,6 +2773,7 @@ var fieldIDToName_SubmitExperimentRequest = map[int16]string{ 40: "evaluator_id_version_list", 41: "enable_weighted_score", 42: "expt_template_id", + 45: "item_retry_num", 100: "ext", 200: "session", 255: "Base", @@ -2765,6 +2859,10 @@ func (p *SubmitExperimentRequest) IsSetExptTemplateID() bool { return p.ExptTemplateID != nil } +func (p *SubmitExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *SubmitExperimentRequest) IsSetExt() bool { return p.Ext != nil } @@ -2965,6 +3063,14 @@ func (p *SubmitExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -3284,6 +3390,17 @@ func (p *SubmitExperimentRequest) ReadField42(iprot thrift.TProtocol) error { p.ExptTemplateID = _field return nil } +func (p *SubmitExperimentRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *SubmitExperimentRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -3420,6 +3537,10 @@ func (p *SubmitExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 42 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -3850,6 +3971,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 42 end error: ", p), err) } +func (p *SubmitExperimentRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *SubmitExperimentRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -3993,6 +4132,9 @@ func (p *SubmitExperimentRequest) DeepEqual(ano *SubmitExperimentRequest) bool { if !p.Field42DeepEqual(ano.ExptTemplateID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -4240,6 +4382,18 @@ func (p *SubmitExperimentRequest) Field42DeepEqual(src *int64) bool { } return true } +func (p *SubmitExperimentRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *SubmitExperimentRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { @@ -17024,6 +17178,7 @@ type CreateExperimentTemplateRequest struct { DefaultEvaluatorsConcurNum *int32 `thrift:"default_evaluators_concur_num,21,optional" frugal:"21,optional,i32" form:"default_evaluators_concur_num" json:"default_evaluators_concur_num,omitempty"` // 调度配置（不在 ExptTemplate 结构中，保留在顶层） ScheduleCron *string `thrift:"schedule_cron,22,optional" frugal:"22,optional,string" form:"schedule_cron" json:"schedule_cron,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -17114,6 +17269,18 @@ func (p *CreateExperimentTemplateRequest) GetScheduleCron() (v string) { return *p.ScheduleCron } +var CreateExperimentTemplateRequest_ItemRetryNum_DEFAULT int32 + +func (p *CreateExperimentTemplateRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return CreateExperimentTemplateRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var CreateExperimentTemplateRequest_Session_DEFAULT *common.Session func (p *CreateExperimentTemplateRequest) GetSession() (v *common.Session) { @@ -17158,6 +17325,9 @@ func (p *CreateExperimentTemplateRequest) SetDefaultEvaluatorsConcurNum(val *int func (p *CreateExperimentTemplateRequest) SetScheduleCron(val *string) { p.ScheduleCron = val } +func (p *CreateExperimentTemplateRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *CreateExperimentTemplateRequest) SetSession(val *common.Session) { p.Session = val } @@ -17173,6 +17343,7 @@ var fieldIDToName_CreateExperimentTemplateRequest = map[int16]string{ 20: "create_eval_target_param", 21: "default_evaluators_concur_num", 22: "schedule_cron", + 45: "item_retry_num", 200: "session", 255: "Base", } @@ -17201,6 +17372,10 @@ func (p *CreateExperimentTemplateRequest) IsSetScheduleCron() bool { return p.ScheduleCron != nil } +func (p *CreateExperimentTemplateRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *CreateExperimentTemplateRequest) IsSetSession() bool { return p.Session != nil } @@ -17285,6 +17460,14 @@ func (p *CreateExperimentTemplateRequest) Read(iprot thrift.TProtocol) (err erro } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 200: if fieldTypeId == thrift.STRUCT { if err = p.ReadField200(iprot); err != nil { @@ -17401,6 +17584,17 @@ func (p *CreateExperimentTemplateRequest) ReadField22(iprot thrift.TProtocol) er p.ScheduleCron = _field return nil } +func (p *CreateExperimentTemplateRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *CreateExperimentTemplateRequest) ReadField200(iprot thrift.TProtocol) error { _field := common.NewSession() if err := _field.Read(iprot); err != nil { @@ -17452,6 +17646,10 @@ func (p *CreateExperimentTemplateRequest) Write(oprot thrift.TProtocol) (err err fieldId = 22 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField200(oprot); err != nil { fieldId = 200 goto WriteFieldError @@ -17602,6 +17800,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 22 end error: ", p), err) } +func (p *CreateExperimentTemplateRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *CreateExperimentTemplateRequest) writeField200(oprot thrift.TProtocol) (err error) { if p.IsSetSession() { if err = oprot.WriteFieldBegin("session", thrift.STRUCT, 200); err != nil { @@ -17674,6 +17890,9 @@ func (p *CreateExperimentTemplateRequest) DeepEqual(ano *CreateExperimentTemplat if !p.Field22DeepEqual(ano.ScheduleCron) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field200DeepEqual(ano.Session) { return false } @@ -17742,6 +17961,18 @@ func (p *CreateExperimentTemplateRequest) Field22DeepEqual(src *string) bool { } return true } +func (p *CreateExperimentTemplateRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *CreateExperimentTemplateRequest) Field200DeepEqual(src *common.Session) bool { if !p.Session.DeepEqual(src) { diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go index 5bf1546f8..671bf1fe4 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go @@ -366,6 +366,20 @@ func (p *CreateExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 200: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField200(buf[offset:]) @@ -774,6 +788,20 @@ func (p *CreateExperimentRequest) FastReadField43(buf []byte) (int, error) { return offset, nil } +func (p *CreateExperimentRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *CreateExperimentRequest) FastReadField200(buf []byte) (int, error) { offset := 0 _field := common.NewSession() @@ -815,6 +843,7 @@ func (p *CreateExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWri offset += p.fastWriteField31(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) offset += p.fastWriteField43(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField6(buf[offset:], w) @@ -859,6 +888,7 @@ func (p *CreateExperimentRequest) BLength() int { l += p.field41Length() l += p.field42Length() l += p.field43Length() + l += p.field45Length() l += p.field200Length() l += p.field255Length() } @@ -1091,6 +1121,15 @@ func (p *CreateExperimentRequest) fastWriteField43(buf []byte, w thrift.NocopyWr return offset } +func (p *CreateExperimentRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *CreateExperimentRequest) fastWriteField200(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetSession() { @@ -1317,6 +1356,15 @@ func (p *CreateExperimentRequest) field43Length() int { return l } +func (p *CreateExperimentRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *CreateExperimentRequest) field200Length() int { l := 0 if p.IsSetSession() { @@ -1501,6 +1549,11 @@ func (p *CreateExperimentRequest) DeepCopy(s interface{}) error { p.ExptTemplateID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _session *common.Session if src.Session != nil { _session = &common.Session{} @@ -2005,6 +2058,20 @@ func (p *SubmitExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -2395,6 +2462,20 @@ func (p *SubmitExperimentRequest) FastReadField42(buf []byte) (int, error) { return offset, nil } +func (p *SubmitExperimentRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *SubmitExperimentRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -2468,6 +2549,7 @@ func (p *SubmitExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWri offset += p.fastWriteField31(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) offset += p.fastWriteField42(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField6(buf[offset:], w) @@ -2511,6 +2593,7 @@ func (p *SubmitExperimentRequest) BLength() int { l += p.field40Length() l += p.field41Length() l += p.field42Length() + l += p.field45Length() l += p.field100Length() l += p.field200Length() l += p.field255Length() @@ -2727,6 +2810,15 @@ func (p *SubmitExperimentRequest) fastWriteField42(buf []byte, w thrift.NocopyWr return offset } +func (p *SubmitExperimentRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *SubmitExperimentRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -2959,6 +3051,15 @@ func (p *SubmitExperimentRequest) field42Length() int { return l } +func (p *SubmitExperimentRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *SubmitExperimentRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -3145,6 +3246,11 @@ func (p *SubmitExperimentRequest) DeepCopy(s interface{}) error { p.ExptTemplateID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { @@ -12634,6 +12740,20 @@ func (p *CreateExperimentTemplateRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 200: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField200(buf[offset:]) @@ -12776,6 +12896,20 @@ func (p *CreateExperimentTemplateRequest) FastReadField22(buf []byte) (int, erro return offset, nil } +func (p *CreateExperimentTemplateRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *CreateExperimentTemplateRequest) FastReadField200(buf []byte) (int, error) { offset := 0 _field := common.NewSession() @@ -12809,6 +12943,7 @@ func (p *CreateExperimentTemplateRequest) FastWriteNocopy(buf []byte, w thrift.N if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField21(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField10(buf[offset:], w) offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField12(buf[offset:], w) @@ -12831,6 +12966,7 @@ func (p *CreateExperimentTemplateRequest) BLength() int { l += p.field20Length() l += p.field21Length() l += p.field22Length() + l += p.field45Length() l += p.field200Length() l += p.field255Length() } @@ -12899,6 +13035,15 @@ func (p *CreateExperimentTemplateRequest) fastWriteField22(buf []byte, w thrift. return offset } +func (p *CreateExperimentTemplateRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *CreateExperimentTemplateRequest) fastWriteField200(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetSession() { @@ -12978,6 +13123,15 @@ func (p *CreateExperimentTemplateRequest) field22Length() int { return l } +func (p *CreateExperimentTemplateRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *CreateExperimentTemplateRequest) field200Length() int { l := 0 if p.IsSetSession() { @@ -13053,6 +13207,11 @@ func (p *CreateExperimentTemplateRequest) DeepCopy(s interface{}) error { p.ScheduleCron = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _session *common.Session if src.Session != nil { _session = &common.Session{} diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go index 059c4dff1..1f2e948ab 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/coze.loop.evaluation.openapi.go @@ -14824,6 +14824,7 @@ type SubmitExperimentOApiRequest struct { // 运行信息 ItemConcurNum *int32 `thrift:"item_concur_num,20,optional" frugal:"20,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty"` TargetRuntimeParam *common.RuntimeParam `thrift:"target_runtime_param,22,optional" frugal:"22,optional,common.RuntimeParam" form:"target_runtime_param" json:"target_runtime_param,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } @@ -14954,6 +14955,18 @@ func (p *SubmitExperimentOApiRequest) GetTargetRuntimeParam() (v *common.Runtime return p.TargetRuntimeParam } +var SubmitExperimentOApiRequest_ItemRetryNum_DEFAULT int32 + +func (p *SubmitExperimentOApiRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return SubmitExperimentOApiRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var SubmitExperimentOApiRequest_Base_DEFAULT *base.Base func (p *SubmitExperimentOApiRequest) GetBase() (v *base.Base) { @@ -14995,6 +15008,9 @@ func (p *SubmitExperimentOApiRequest) SetItemConcurNum(val *int32) { func (p *SubmitExperimentOApiRequest) SetTargetRuntimeParam(val *common.RuntimeParam) { p.TargetRuntimeParam = val } +func (p *SubmitExperimentOApiRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *SubmitExperimentOApiRequest) SetBase(val *base.Base) { p.Base = val } @@ -15010,6 +15026,7 @@ var fieldIDToName_SubmitExperimentOApiRequest = map[int16]string{ 8: "evaluator_field_mapping", 20: "item_concur_num", 22: "target_runtime_param", + 45: "item_retry_num", 255: "Base", } @@ -15053,6 +15070,10 @@ func (p *SubmitExperimentOApiRequest) IsSetTargetRuntimeParam() bool { return p.TargetRuntimeParam != nil } +func (p *SubmitExperimentOApiRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *SubmitExperimentOApiRequest) IsSetBase() bool { return p.Base != nil } @@ -15155,6 +15176,14 @@ func (p *SubmitExperimentOApiRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 255: if fieldTypeId == thrift.STRUCT { if err = p.ReadField255(iprot); err != nil { @@ -15314,6 +15343,17 @@ func (p *SubmitExperimentOApiRequest) ReadField22(iprot thrift.TProtocol) error p.TargetRuntimeParam = _field return nil } +func (p *SubmitExperimentOApiRequest) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *SubmitExperimentOApiRequest) ReadField255(iprot thrift.TProtocol) error { _field := base.NewBase() if err := _field.Read(iprot); err != nil { @@ -15369,6 +15409,10 @@ func (p *SubmitExperimentOApiRequest) Write(oprot thrift.TProtocol) (err error) fieldId = 22 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField255(oprot); err != nil { fieldId = 255 goto WriteFieldError @@ -15587,6 +15631,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 22 end error: ", p), err) } +func (p *SubmitExperimentOApiRequest) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *SubmitExperimentOApiRequest) writeField255(oprot thrift.TProtocol) (err error) { if p.IsSetBase() { if err = oprot.WriteFieldBegin("Base", thrift.STRUCT, 255); err != nil { @@ -15650,6 +15712,9 @@ func (p *SubmitExperimentOApiRequest) DeepEqual(ano *SubmitExperimentOApiRequest if !p.Field22DeepEqual(ano.TargetRuntimeParam) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field255DeepEqual(ano.Base) { return false } @@ -15758,6 +15823,18 @@ func (p *SubmitExperimentOApiRequest) Field22DeepEqual(src *common.RuntimeParam) } return true } +func (p *SubmitExperimentOApiRequest) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *SubmitExperimentOApiRequest) Field255DeepEqual(src *base.Base) bool { if !p.Base.DeepEqual(src) { diff --git a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go index ea00cf97b..b51df057c 100644 --- a/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go +++ b/backend/kitex_gen/coze/loop/evaluation/openapi/k-coze.loop.evaluation.openapi.go @@ -10652,6 +10652,20 @@ func (p *SubmitExperimentOApiRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 255: if fieldTypeId == thrift.STRUCT { l, err = p.FastReadField255(buf[offset:]) @@ -10838,6 +10852,20 @@ func (p *SubmitExperimentOApiRequest) FastReadField22(buf []byte) (int, error) { return offset, nil } +func (p *SubmitExperimentOApiRequest) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *SubmitExperimentOApiRequest) FastReadField255(buf []byte) (int, error) { offset := 0 _field := base.NewBase() @@ -10859,6 +10887,7 @@ func (p *SubmitExperimentOApiRequest) FastWriteNocopy(buf []byte, w thrift.Nocop if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField20(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) @@ -10886,6 +10915,7 @@ func (p *SubmitExperimentOApiRequest) BLength() int { l += p.field8Length() l += p.field20Length() l += p.field22Length() + l += p.field45Length() l += p.field255Length() } l += thrift.Binary.FieldStopLength() @@ -10996,6 +11026,15 @@ func (p *SubmitExperimentOApiRequest) fastWriteField22(buf []byte, w thrift.Noco return offset } +func (p *SubmitExperimentOApiRequest) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *SubmitExperimentOApiRequest) fastWriteField255(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetBase() { @@ -11103,6 +11142,15 @@ func (p *SubmitExperimentOApiRequest) field22Length() int { return l } +func (p *SubmitExperimentOApiRequest) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *SubmitExperimentOApiRequest) field255Length() int { l := 0 if p.IsSetBase() { @@ -11210,6 +11258,11 @@ func (p *SubmitExperimentOApiRequest) DeepCopy(s interface{}) error { } p.TargetRuntimeParam = _targetRuntimeParam + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + var _base *base.Base if src.Base != nil { _base = &base.Base{} From a28c726b07a00ed629d02d4587c73b2f9f32fb41 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Mon, 9 Feb 2026 13:10:00 +0800 Subject: [PATCH 03/30] feat(evaluation): retry extension --- .../coze/loop/evaluation/domain/expt/expt.go | 154 +++++++ .../loop/evaluation/domain/expt/k-expt.go | 106 +++++ .../expt/coze.loop.evaluation.expt.go | 91 +++- .../expt/k-coze.loop.evaluation.expt.go | 53 +++ .../application/convertor/experiment/expt.go | 18 +- .../convertor/experiment/expt_template.go | 3 + .../evaluation/application/experiment_app.go | 33 +- .../application/experiment_app_test.go | 11 +- .../modules/evaluation/domain/entity/event.go | 12 +- .../modules/evaluation/domain/entity/expt.go | 1 + .../evaluation/domain/entity/expt_run.go | 5 + .../evaluation/domain/entity/expt_template.go | 1 + .../modules/evaluation/domain/entity/param.go | 36 +- .../modules/evaluation/domain/repo/expt.go | 2 + .../evaluation/domain/repo/mocks/expt.go | 30 ++ .../evaluation/domain/service/expt_manage.go | 4 +- .../service/expt_manage_execution_impl.go | 48 +- .../expt_manage_execution_impl_test.go | 132 +----- .../domain/service/expt_manage_impl.go | 1 + .../service/expt_run_item_event_impl.go | 3 + .../service/expt_run_scheduler_event_impl.go | 1 + .../service/expt_run_scheduler_mode_impl.go | 430 ++++++++++++++++++ .../evaluation/domain/service/file_name | 1 - .../domain/service/mocks/expt_manage.go | 20 +- .../infra/repo/experiment/expt_item_result.go | 24 + .../repo/experiment/mysql/expt_item_result.go | 31 ++ .../mysql/mocks/expt_item_result.go | 40 ++ .../coze.loop.evaluation.expt.thrift | 1 + .../coze/loop/evaluation/domain/expt.thrift | 3 +- 29 files changed, 1082 insertions(+), 213 deletions(-) delete mode 100644 backend/modules/evaluation/domain/service/file_name diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go index b774ffbd3..fcc0ac537 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/expt.go @@ -988,6 +988,7 @@ type Experiment struct { MaxAliveTime *int64 `thrift:"max_alive_time,41,optional" frugal:"41,optional,i64" form:"max_alive_time" json:"max_alive_time,omitempty" query:"max_alive_time"` SourceType *SourceType `thrift:"source_type,42,optional" frugal:"42,optional,SourceType" form:"source_type" json:"source_type,omitempty" query:"source_type"` SourceID *string `thrift:"source_id,43,optional" frugal:"43,optional,string" form:"source_id" json:"source_id,omitempty" query:"source_id"` + ItemRetryNum *int32 `thrift:"item_retry_num,45,optional" frugal:"45,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` // 补充的评估器id+version关联评估器方式，和evaluator_version_ids共同使用，兼容老逻辑 EvaluatorIDVersionList []*evaluator.EvaluatorIDVersionItem `thrift:"evaluator_id_version_list,51,optional" frugal:"51,optional,list" form:"evaluator_id_version_list" json:"evaluator_id_version_list,omitempty" query:"evaluator_id_version_list"` ExptTemplateMeta *ExptTemplateMeta `thrift:"expt_template_meta,60,optional" frugal:"60,optional,ExptTemplateMeta" form:"expt_template_meta" json:"expt_template_meta,omitempty" query:"expt_template_meta"` @@ -1315,6 +1316,18 @@ func (p *Experiment) GetSourceID() (v string) { return *p.SourceID } +var Experiment_ItemRetryNum_DEFAULT int32 + +func (p *Experiment) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return Experiment_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var Experiment_EvaluatorIDVersionList_DEFAULT []*evaluator.EvaluatorIDVersionItem func (p *Experiment) GetEvaluatorIDVersionList() (v []*evaluator.EvaluatorIDVersionItem) { @@ -1440,6 +1453,9 @@ func (p *Experiment) SetSourceType(val *SourceType) { func (p *Experiment) SetSourceID(val *string) { p.SourceID = val } +func (p *Experiment) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *Experiment) SetEvaluatorIDVersionList(val []*evaluator.EvaluatorIDVersionItem) { p.EvaluatorIDVersionList = val } @@ -1480,6 +1496,7 @@ var fieldIDToName_Experiment = map[int16]string{ 41: "max_alive_time", 42: "source_type", 43: "source_id", + 45: "item_retry_num", 51: "evaluator_id_version_list", 60: "expt_template_meta", 61: "score_weight_config", @@ -1590,6 +1607,10 @@ func (p *Experiment) IsSetSourceID() bool { return p.SourceID != nil } +func (p *Experiment) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *Experiment) IsSetEvaluatorIDVersionList() bool { return p.EvaluatorIDVersionList != nil } @@ -1832,6 +1853,14 @@ func (p *Experiment) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 45: + if fieldTypeId == thrift.I32 { + if err = p.ReadField45(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 51: if fieldTypeId == thrift.LIST { if err = p.ReadField51(iprot); err != nil { @@ -2200,6 +2229,17 @@ func (p *Experiment) ReadField43(iprot thrift.TProtocol) error { p.SourceID = _field return nil } +func (p *Experiment) ReadField45(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *Experiment) ReadField51(iprot thrift.TProtocol) error { _, size, err := iprot.ReadListBegin() if err != nil { @@ -2361,6 +2401,10 @@ func (p *Experiment) Write(oprot thrift.TProtocol) (err error) { fieldId = 43 goto WriteFieldError } + if err = p.writeField45(oprot); err != nil { + fieldId = 45 + goto WriteFieldError + } if err = p.writeField51(oprot); err != nil { fieldId = 51 goto WriteFieldError @@ -2887,6 +2931,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 43 end error: ", p), err) } +func (p *Experiment) writeField45(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 45); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 45 end error: ", p), err) +} func (p *Experiment) writeField51(oprot thrift.TProtocol) (err error) { if p.IsSetEvaluatorIDVersionList() { if err = oprot.WriteFieldBegin("evaluator_id_version_list", thrift.LIST, 51); err != nil { @@ -3060,6 +3122,9 @@ func (p *Experiment) DeepEqual(ano *Experiment) bool { if !p.Field43DeepEqual(ano.SourceID) { return false } + if !p.Field45DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field51DeepEqual(ano.EvaluatorIDVersionList) { return false } @@ -3360,6 +3425,18 @@ func (p *Experiment) Field43DeepEqual(src *string) bool { } return true } +func (p *Experiment) Field45DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *Experiment) Field51DeepEqual(src []*evaluator.EvaluatorIDVersionItem) bool { if len(p.EvaluatorIDVersionList) != len(src) { @@ -4645,6 +4722,7 @@ type ExptFieldMapping struct { EvaluatorFieldMapping []*EvaluatorFieldMapping `thrift:"evaluator_field_mapping,2,optional" frugal:"2,optional,list" form:"evaluator_field_mapping" json:"evaluator_field_mapping,omitempty" query:"evaluator_field_mapping"` TargetRuntimeParam *common.RuntimeParam `thrift:"target_runtime_param,3,optional" frugal:"3,optional,common.RuntimeParam" form:"target_runtime_param" json:"target_runtime_param,omitempty" query:"target_runtime_param"` ItemConcurNum *int32 `thrift:"item_concur_num,4,optional" frugal:"4,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty" query:"item_concur_num"` + ItemRetryNum *int32 `thrift:"item_retry_num,5,optional" frugal:"5,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty" query:"item_retry_num"` } func NewExptFieldMapping() *ExptFieldMapping { @@ -4701,6 +4779,18 @@ func (p *ExptFieldMapping) GetItemConcurNum() (v int32) { } return *p.ItemConcurNum } + +var ExptFieldMapping_ItemRetryNum_DEFAULT int32 + +func (p *ExptFieldMapping) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return ExptFieldMapping_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} func (p *ExptFieldMapping) SetTargetFieldMapping(val *TargetFieldMapping) { p.TargetFieldMapping = val } @@ -4713,12 +4803,16 @@ func (p *ExptFieldMapping) SetTargetRuntimeParam(val *common.RuntimeParam) { func (p *ExptFieldMapping) SetItemConcurNum(val *int32) { p.ItemConcurNum = val } +func (p *ExptFieldMapping) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} var fieldIDToName_ExptFieldMapping = map[int16]string{ 1: "target_field_mapping", 2: "evaluator_field_mapping", 3: "target_runtime_param", 4: "item_concur_num", + 5: "item_retry_num", } func (p *ExptFieldMapping) IsSetTargetFieldMapping() bool { @@ -4737,6 +4831,10 @@ func (p *ExptFieldMapping) IsSetItemConcurNum() bool { return p.ItemConcurNum != nil } +func (p *ExptFieldMapping) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *ExptFieldMapping) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -4787,6 +4885,14 @@ func (p *ExptFieldMapping) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 5: + if fieldTypeId == thrift.I32 { + if err = p.ReadField5(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -4866,6 +4972,17 @@ func (p *ExptFieldMapping) ReadField4(iprot thrift.TProtocol) error { p.ItemConcurNum = _field return nil } +func (p *ExptFieldMapping) ReadField5(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *ExptFieldMapping) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -4889,6 +5006,10 @@ func (p *ExptFieldMapping) Write(oprot thrift.TProtocol) (err error) { fieldId = 4 goto WriteFieldError } + if err = p.writeField5(oprot); err != nil { + fieldId = 5 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -4987,6 +5108,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) } +func (p *ExptFieldMapping) writeField5(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 5); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) +} func (p *ExptFieldMapping) String() string { if p == nil { @@ -5014,6 +5153,9 @@ func (p *ExptFieldMapping) DeepEqual(ano *ExptFieldMapping) bool { if !p.Field4DeepEqual(ano.ItemConcurNum) { return false } + if !p.Field5DeepEqual(ano.ItemRetryNum) { + return false + } return true } @@ -5056,6 +5198,18 @@ func (p *ExptFieldMapping) Field4DeepEqual(src *int32) bool { } return true } +func (p *ExptFieldMapping) Field5DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} // 实验评估器得分加权配置 type ExptScoreWeight struct { diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go index 203907e7e..8123f407a 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/expt/k-expt.go @@ -418,6 +418,20 @@ func (p *Experiment) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 45: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField45(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 51: if fieldTypeId == thrift.LIST { l, err = p.FastReadField51(buf[offset:]) @@ -882,6 +896,20 @@ func (p *Experiment) FastReadField43(buf []byte) (int, error) { return offset, nil } +func (p *Experiment) FastReadField45(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *Experiment) FastReadField51(buf []byte) (int, error) { offset := 0 @@ -961,6 +989,7 @@ func (p *Experiment) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { offset += p.fastWriteField27(buf[offset:], w) offset += p.fastWriteField28(buf[offset:], w) offset += p.fastWriteField41(buf[offset:], w) + offset += p.fastWriteField45(buf[offset:], w) offset += p.fastWriteField62(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) @@ -1016,6 +1045,7 @@ func (p *Experiment) BLength() int { l += p.field41Length() l += p.field42Length() l += p.field43Length() + l += p.field45Length() l += p.field51Length() l += p.field60Length() l += p.field61Length() @@ -1280,6 +1310,15 @@ func (p *Experiment) fastWriteField43(buf []byte, w thrift.NocopyWriter) int { return offset } +func (p *Experiment) fastWriteField45(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 45) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *Experiment) fastWriteField51(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetEvaluatorIDVersionList() { @@ -1567,6 +1606,15 @@ func (p *Experiment) field43Length() int { return l } +func (p *Experiment) field45Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *Experiment) field51Length() int { l := 0 if p.IsSetEvaluatorIDVersionList() { @@ -1806,6 +1854,11 @@ func (p *Experiment) DeepCopy(s interface{}) error { p.SourceID = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.EvaluatorIDVersionList != nil { p.EvaluatorIDVersionList = make([]*evaluator.EvaluatorIDVersionItem, 0, len(src.EvaluatorIDVersionList)) for _, elem := range src.EvaluatorIDVersionList { @@ -2813,6 +2866,20 @@ func (p *ExptFieldMapping) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 5: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField5(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -2894,6 +2961,20 @@ func (p *ExptFieldMapping) FastReadField4(buf []byte) (int, error) { return offset, nil } +func (p *ExptFieldMapping) FastReadField5(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *ExptFieldMapping) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -2902,6 +2983,7 @@ func (p *ExptFieldMapping) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) in offset := 0 if p != nil { offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField5(buf[offset:], w) offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) @@ -2917,6 +2999,7 @@ func (p *ExptFieldMapping) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field4Length() + l += p.field5Length() } l += thrift.Binary.FieldStopLength() return l @@ -2965,6 +3048,15 @@ func (p *ExptFieldMapping) fastWriteField4(buf []byte, w thrift.NocopyWriter) in return offset } +func (p *ExptFieldMapping) fastWriteField5(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 5) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *ExptFieldMapping) field1Length() int { l := 0 if p.IsSetTargetFieldMapping() { @@ -3005,6 +3097,15 @@ func (p *ExptFieldMapping) field4Length() int { return l } +func (p *ExptFieldMapping) field5Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *ExptFieldMapping) DeepCopy(s interface{}) error { src, ok := s.(*ExptFieldMapping) if !ok { @@ -3049,6 +3150,11 @@ func (p *ExptFieldMapping) DeepCopy(s interface{}) error { p.ItemConcurNum = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go index 6d8c7f7dc..751a7c37c 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/coze.loop.evaluation.expt.go @@ -7946,13 +7946,14 @@ func (p *BatchDeleteExperimentsResponse) Field255DeepEqual(src *base.BaseResp) b } type RunExperimentRequest struct { - WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` - ExptID *int64 `thrift:"expt_id,2,optional" frugal:"2,optional,i64" json:"expt_id" form:"expt_id" ` - ItemIds []int64 `thrift:"item_ids,3,optional" frugal:"3,optional,list" json:"item_ids" form:"item_ids" ` - ExptType *expt.ExptType `thrift:"expt_type,10,optional" frugal:"10,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` - Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` - Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` - Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` + WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" json:"workspace_id" form:"workspace_id" ` + ExptID *int64 `thrift:"expt_id,2,optional" frugal:"2,optional,i64" json:"expt_id" form:"expt_id" ` + ItemIds []int64 `thrift:"item_ids,3,optional" frugal:"3,optional,list" json:"item_ids" form:"item_ids" ` + ExptType *expt.ExptType `thrift:"expt_type,10,optional" frugal:"10,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` + ItemRetryNum *int32 `thrift:"item_retry_num,11,optional" frugal:"11,optional,i32" form:"item_retry_num" json:"item_retry_num,omitempty"` + Ext map[string]string `thrift:"ext,100,optional" frugal:"100,optional,map" form:"ext" json:"ext,omitempty"` + Session *common.Session `thrift:"session,200,optional" frugal:"200,optional,common.Session" form:"session" json:"session,omitempty" query:"session"` + Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` } func NewRunExperimentRequest() *RunExperimentRequest { @@ -8010,6 +8011,18 @@ func (p *RunExperimentRequest) GetExptType() (v expt.ExptType) { return *p.ExptType } +var RunExperimentRequest_ItemRetryNum_DEFAULT int32 + +func (p *RunExperimentRequest) GetItemRetryNum() (v int32) { + if p == nil { + return + } + if !p.IsSetItemRetryNum() { + return RunExperimentRequest_ItemRetryNum_DEFAULT + } + return *p.ItemRetryNum +} + var RunExperimentRequest_Ext_DEFAULT map[string]string func (p *RunExperimentRequest) GetExt() (v map[string]string) { @@ -8057,6 +8070,9 @@ func (p *RunExperimentRequest) SetItemIds(val []int64) { func (p *RunExperimentRequest) SetExptType(val *expt.ExptType) { p.ExptType = val } +func (p *RunExperimentRequest) SetItemRetryNum(val *int32) { + p.ItemRetryNum = val +} func (p *RunExperimentRequest) SetExt(val map[string]string) { p.Ext = val } @@ -8072,6 +8088,7 @@ var fieldIDToName_RunExperimentRequest = map[int16]string{ 2: "expt_id", 3: "item_ids", 10: "expt_type", + 11: "item_retry_num", 100: "ext", 200: "session", 255: "Base", @@ -8093,6 +8110,10 @@ func (p *RunExperimentRequest) IsSetExptType() bool { return p.ExptType != nil } +func (p *RunExperimentRequest) IsSetItemRetryNum() bool { + return p.ItemRetryNum != nil +} + func (p *RunExperimentRequest) IsSetExt() bool { return p.Ext != nil } @@ -8155,6 +8176,14 @@ func (p *RunExperimentRequest) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 11: + if fieldTypeId == thrift.I32 { + if err = p.ReadField11(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 100: if fieldTypeId == thrift.MAP { if err = p.ReadField100(iprot); err != nil { @@ -8265,6 +8294,17 @@ func (p *RunExperimentRequest) ReadField10(iprot thrift.TProtocol) error { p.ExptType = _field return nil } +func (p *RunExperimentRequest) ReadField11(iprot thrift.TProtocol) error { + + var _field *int32 + if v, err := iprot.ReadI32(); err != nil { + return err + } else { + _field = &v + } + p.ItemRetryNum = _field + return nil +} func (p *RunExperimentRequest) ReadField100(iprot thrift.TProtocol) error { _, _, size, err := iprot.ReadMapBegin() if err != nil { @@ -8333,6 +8373,10 @@ func (p *RunExperimentRequest) Write(oprot thrift.TProtocol) (err error) { fieldId = 10 goto WriteFieldError } + if err = p.writeField11(oprot); err != nil { + fieldId = 11 + goto WriteFieldError + } if err = p.writeField100(oprot); err != nil { fieldId = 100 goto WriteFieldError @@ -8443,6 +8487,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 10 end error: ", p), err) } +func (p *RunExperimentRequest) writeField11(oprot thrift.TProtocol) (err error) { + if p.IsSetItemRetryNum() { + if err = oprot.WriteFieldBegin("item_retry_num", thrift.I32, 11); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI32(*p.ItemRetryNum); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 11 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 11 end error: ", p), err) +} func (p *RunExperimentRequest) writeField100(oprot thrift.TProtocol) (err error) { if p.IsSetExt() { if err = oprot.WriteFieldBegin("ext", thrift.MAP, 100); err != nil { @@ -8535,6 +8597,9 @@ func (p *RunExperimentRequest) DeepEqual(ano *RunExperimentRequest) bool { if !p.Field10DeepEqual(ano.ExptType) { return false } + if !p.Field11DeepEqual(ano.ItemRetryNum) { + return false + } if !p.Field100DeepEqual(ano.Ext) { return false } @@ -8596,6 +8661,18 @@ func (p *RunExperimentRequest) Field10DeepEqual(src *expt.ExptType) bool { } return true } +func (p *RunExperimentRequest) Field11DeepEqual(src *int32) bool { + + if p.ItemRetryNum == src { + return true + } else if p.ItemRetryNum == nil || src == nil { + return false + } + if *p.ItemRetryNum != *src { + return false + } + return true +} func (p *RunExperimentRequest) Field100DeepEqual(src map[string]string) bool { if len(p.Ext) != len(src) { diff --git a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go index 671bf1fe4..a6beed94e 100644 --- a/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go +++ b/backend/kitex_gen/coze/loop/evaluation/expt/k-coze.loop.evaluation.expt.go @@ -5925,6 +5925,20 @@ func (p *RunExperimentRequest) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 11: + if fieldTypeId == thrift.I32 { + l, err = p.FastReadField11(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 100: if fieldTypeId == thrift.MAP { l, err = p.FastReadField100(buf[offset:]) @@ -6053,6 +6067,20 @@ func (p *RunExperimentRequest) FastReadField10(buf []byte) (int, error) { return offset, nil } +func (p *RunExperimentRequest) FastReadField11(buf []byte) (int, error) { + offset := 0 + + var _field *int32 + if v, l, err := thrift.Binary.ReadI32(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.ItemRetryNum = _field + return offset, nil +} + func (p *RunExperimentRequest) FastReadField100(buf []byte) (int, error) { offset := 0 @@ -6118,6 +6146,7 @@ func (p *RunExperimentRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWriter if p != nil { offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) + offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField10(buf[offset:], w) offset += p.fastWriteField100(buf[offset:], w) @@ -6135,6 +6164,7 @@ func (p *RunExperimentRequest) BLength() int { l += p.field2Length() l += p.field3Length() l += p.field10Length() + l += p.field11Length() l += p.field100Length() l += p.field200Length() l += p.field255Length() @@ -6186,6 +6216,15 @@ func (p *RunExperimentRequest) fastWriteField10(buf []byte, w thrift.NocopyWrite return offset } +func (p *RunExperimentRequest) fastWriteField11(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetItemRetryNum() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I32, 11) + offset += thrift.Binary.WriteI32(buf[offset:], *p.ItemRetryNum) + } + return offset +} + func (p *RunExperimentRequest) fastWriteField100(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetExt() { @@ -6259,6 +6298,15 @@ func (p *RunExperimentRequest) field10Length() int { return l } +func (p *RunExperimentRequest) field11Length() int { + l := 0 + if p.IsSetItemRetryNum() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I32Length() + } + return l +} + func (p *RunExperimentRequest) field100Length() int { l := 0 if p.IsSetExt() { @@ -6322,6 +6370,11 @@ func (p *RunExperimentRequest) DeepCopy(s interface{}) error { p.ExptType = &tmp } + if src.ItemRetryNum != nil { + tmp := *src.ItemRetryNum + p.ItemRetryNum = &tmp + } + if src.Ext != nil { p.Ext = make(map[string]string, len(src.Ext)) for key, val := range src.Ext { diff --git a/backend/modules/evaluation/application/convertor/experiment/expt.go b/backend/modules/evaluation/application/convertor/experiment/expt.go index 3ddc96126..667ccde9d 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt.go @@ -55,6 +55,9 @@ func (e *EvalConfConvert) ConvertToEntity(cer *expt.CreateExperimentRequest, eva } ec.ConnectorConf.EvaluatorsConf = evalsConf } + if cer.GetItemRetryNum() > 0 { + ec.ItemRetryNum = gptr.Of(int(cer.GetItemRetryNum())) + } return ec, nil } @@ -363,6 +366,7 @@ func ToExptDTO(experiment *entity.Experiment) *domain_expt.Experiment { } if experiment.EvalConf != nil && experiment.EvalConf.ItemConcurNum != nil { res.ItemConcurNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemConcurNum))) + res.ItemRetryNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemRetryNum))) } // 填充权重配置（score_weight_config 和 enable_weighted_score） @@ -490,6 +494,18 @@ func ConvertCreateReq(cer *expt.CreateExperimentRequest, evaluatorVersionRunConf if cer.IsSetExptTemplateID() { param.ExptTemplateID = cer.GetExptTemplateID() } - return param, nil } + +func ConvRetryMode(m domain_expt.ExptRetryMode) entity.ExptRunMode { + switch m { + case domain_expt.ExptRetryMode_RetryFailure: + return entity.EvaluationModeFailRetry + case domain_expt.ExptRetryMode_RetryAll: + return entity.EvaluationModeRetryAll + case domain_expt.ExptRetryMode_RetryTargetItems: + return entity.EvaluationModeRetryItems + default: + return entity.EvaluationModeUnknown + } +} diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_template.go b/backend/modules/evaluation/application/convertor/experiment/expt_template.go index 62c8665b3..2e1d6159e 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_template.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_template.go @@ -6,6 +6,7 @@ package experiment import ( "fmt" + "github.com/bytedance/gg/gcond" "github.com/bytedance/gg/gptr" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/common" @@ -232,6 +233,7 @@ func buildTemplateConfForCreate( templateConf := &entity.ExptTemplateConfiguration{ ItemConcurNum: ptr.ConvIntPtr[int32, int](itemConcurNum), EvaluatorsConcurNum: ptr.ConvIntPtr[int32, int](req.DefaultEvaluatorsConcurNum), + ItemRetryNum: gcond.If(req.GetItemRetryNum() > 0, gptr.Of(int(req.GetItemRetryNum())), nil), } if targetFieldMapping == nil && len(evaluatorConfs) == 0 { @@ -600,6 +602,7 @@ func buildTemplateFieldMappingDTO(template *entity.ExptTemplate) *domain_expt.Ex fieldMapping := &domain_expt.ExptFieldMapping{ ItemConcurNum: ptr.ConvIntPtr[int, int32](template.FieldMappingConfig.ItemConcurNum), + ItemRetryNum: gcond.If(gptr.Indirect(template.TemplateConf.ItemRetryNum) > 0, gptr.Of(int32(gptr.Indirect(template.TemplateConf.ItemRetryNum))), nil), } if template.FieldMappingConfig.TargetFieldMapping != nil { diff --git a/backend/modules/evaluation/application/experiment_app.go b/backend/modules/evaluation/application/experiment_app.go index 83c62ae3c..751667ac7 100644 --- a/backend/modules/evaluation/application/experiment_app.go +++ b/backend/modules/evaluation/application/experiment_app.go @@ -467,6 +467,7 @@ func (e *experimentApplication) SubmitExperiment(ctx context.Context, req *expt. Session: req.Session, EnableWeightedScore: req.EnableWeightedScore, // EvaluatorScoreWeights 会在 CreateExperiment 的 resolveEvaluatorVersionIDsFromCreateReq 中解析 + ItemRetryNum: req.ItemRetryNum, } if req.IsSetExptTemplateID() { createReq.ExptTemplateID = gptr.Of(req.GetExptTemplateID()) @@ -477,11 +478,12 @@ func (e *experimentApplication) SubmitExperiment(ctx context.Context, req *expt. } rresp, err := e.RunExperiment(ctx, &expt.RunExperimentRequest{ - WorkspaceID: gptr.Of(req.GetWorkspaceID()), - ExptID: cresp.GetExperiment().ID, - ExptType: req.ExptType, - Session: req.Session, - Ext: req.Ext, + WorkspaceID: gptr.Of(req.GetWorkspaceID()), + ExptID: cresp.GetExperiment().ID, + ExptType: req.ExptType, + ItemRetryNum: req.ItemRetryNum, + Session: req.Session, + Ext: req.Ext, }) if err != nil { return nil, err @@ -979,7 +981,7 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run return nil, err } - if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), session, evalMode, req.GetExt()); err != nil { + if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), int(req.GetItemRetryNum()), session, evalMode, req.GetExt()); err != nil { return nil, err } return &expt.RunExperimentResponse{ @@ -991,6 +993,11 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.RetryExperimentRequest) (r *expt.RetryExperimentResponse, err error) { session := entity.NewSession(ctx) + if req.GetRetryMode() == 0 { + req.RetryMode = domain_expt.ExptRetryModePtr(domain_expt.ExptRetryMode_RetryFailure) + } + runMode := experiment.ConvRetryMode(req.GetRetryMode()) + got, err := e.manager.Get(ctx, req.GetExptID(), req.GetWorkspaceID(), session) if err != nil { return nil, err @@ -1011,12 +1018,20 @@ func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.R return nil, err } - if err := e.manager.LogRun(ctx, req.GetExptID(), runID, entity.EvaluationModeFailRetry, req.GetWorkspaceID(), session); err != nil { + if err := e.manager.LogRun(ctx, req.GetExptID(), runID, runMode, req.GetWorkspaceID(), session); err != nil { return nil, err } - if err := e.manager.RetryUnSuccess(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), session, req.GetExt()); err != nil { - return nil, err + itemRetryNum := gptr.Indirect(got.EvalConf.ItemRetryNum) + switch runMode { + case entity.EvaluationModeRetryItems: + if err := e.manager.RetryItems(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), itemRetryNum, req.GetItemIds(), session, req.GetExt()); err != nil { + return nil, err + } + default: + if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), itemRetryNum, session, runMode, req.GetExt()); err != nil { + return nil, err + } } return &expt.RetryExperimentResponse{ diff --git a/backend/modules/evaluation/application/experiment_app_test.go b/backend/modules/evaluation/application/experiment_app_test.go index 9147ee2bd..3cee1d40c 100644 --- a/backend/modules/evaluation/application/experiment_app_test.go +++ b/backend/modules/evaluation/application/experiment_app_test.go @@ -703,6 +703,7 @@ func TestExperimentApplication_SubmitExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, gomock.Any(), gomock.Any(), @@ -826,7 +827,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). Return(nil) @@ -862,7 +863,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). Return(errors.New("update error")) @@ -898,7 +899,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) - mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) // 不应该调用 UpdateExptInfo mockTemplateManager.EXPECT().UpdateExptInfo(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) @@ -2351,6 +2352,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, entity.EvaluationModeSubmit, gomock.Any(), @@ -2396,6 +2398,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validExptID, validRunID, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, entity.EvaluationModeSubmit, gomock.Any(), @@ -2485,7 +2488,7 @@ func TestExperimentApplication_RetryExperiment(t *testing.T) { mockManager.EXPECT().LogRun(gomock.Any(), validExptID, validRunID, entity.EvaluationModeFailRetry, validWorkspaceID, gomock.Any()).Return(nil) // 重试失败的实验 - mockManager.EXPECT().RetryUnSuccess(gomock.Any(), validExptID, validRunID, validWorkspaceID, gomock.Any(), gomock.Any()).Return(nil) + mockManager.EXPECT().Run(gomock.Any(), validExptID, validRunID, validWorkspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) }, wantResp: &exptpb.RetryExperimentResponse{ RunID: gptr.Of(validRunID), diff --git a/backend/modules/evaluation/domain/entity/event.go b/backend/modules/evaluation/domain/entity/event.go index 2a01f02a1..018cd0b8d 100644 --- a/backend/modules/evaluation/domain/entity/event.go +++ b/backend/modules/evaluation/domain/entity/event.go @@ -14,7 +14,8 @@ type ExptScheduleEvent struct { Ext map[string]string Session *Session - RetryTimes int + ItemRetryTimes int + ExecEvalSetItemIDs []int64 } type ExptItemEvalEvent struct { @@ -26,10 +27,11 @@ type ExptItemEvalEvent struct { EvalSetItemID int64 AsyncReportTrigger bool - CreateAt int64 - RetryTimes int - Ext map[string]string - Session *Session + CreateAt int64 + RetryTimes int + MaxRetryTimes int + Ext map[string]string + Session *Session } func (e *ExptItemEvalEvent) GetExptID() int64 { diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index aacc4dcc5..1a8f05de1 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -160,6 +160,7 @@ func (e *ExptEvaluatorVersionRef) String() string { type EvaluationConfiguration struct { ConnectorConf Connector ItemConcurNum *int + ItemRetryNum *int } type Connector struct { diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index 5873af3c5..e41a414a0 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -19,6 +19,8 @@ import ( type ExptRunMode int32 const ( + EvaluationModeUnknown ExptRunMode = 0 + // EvaluationModeSubmit 创建后提交 EvaluationModeSubmit ExptRunMode = 1 @@ -27,6 +29,9 @@ const ( // EvaluationModeAppend 追加模式 EvaluationModeAppend ExptRunMode = 3 + + EvaluationModeRetryAll ExptRunMode = 4 + EvaluationModeRetryItems ExptRunMode = 5 ) type ItemRunState int64 diff --git a/backend/modules/evaluation/domain/entity/expt_template.go b/backend/modules/evaluation/domain/entity/expt_template.go index 023e4c1c4..cfb4f14ae 100644 --- a/backend/modules/evaluation/domain/entity/expt_template.go +++ b/backend/modules/evaluation/domain/entity/expt_template.go @@ -131,6 +131,7 @@ type ExptTemplateConfiguration struct { // 默认评估器并发数 EvaluatorsConcurNum *int + ItemRetryNum *int } // ToEvaluatorRefDO 转换为评估器引用DO diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index 783859b6f..6d1e6c5b0 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -217,26 +217,22 @@ type RunEvaluatorRequest struct { } type CreateExptParam struct { - WorkspaceID int64 `thrift:"workspace_id,1,required" frugal:"1,required,i64" json:"workspace_id" form:"workspace_id,required" ` - EvalSetVersionID int64 `thrift:"eval_set_version_id,2,optional" frugal:"2,optional,i64" json:"eval_set_version_id" form:"eval_set_version_id" ` - TargetVersionID int64 `thrift:"target_version_id,3,optional" frugal:"3,optional,i64" json:"target_version_id" form:"target_version_id" ` - EvaluatorVersionIds []int64 `thrift:"evaluator_version_ids,4,optional" frugal:"4,optional,list" json:"evaluator_version_ids" form:"evaluator_version_ids" ` - Name string `thrift:"name,5,optional" frugal:"5,optional,string" form:"name" json:"name,omitempty"` - Desc string `thrift:"desc,6,optional" frugal:"6,optional,string" form:"desc" json:"desc,omitempty"` - EvalSetID int64 `thrift:"eval_set_id,7,optional" frugal:"7,optional,i64" json:"eval_set_id" form:"eval_set_id" ` - TargetID *int64 `thrift:"target_id,8,optional" frugal:"8,optional,i64" json:"target_id" form:"target_id" ` - // TargetFieldMapping *TargetFieldMapping `thrift:"target_field_mapping,20,optional" frugal:"20,optional,TargetFieldMapping" form:"target_field_mapping" json:"target_field_mapping,omitempty"` - // EvaluatorFieldMapping []*EvaluatorFieldMapping `thrift:"evaluator_field_mapping,21,optional" frugal:"21,optional,list" form:"evaluator_field_mapping" json:"evaluator_field_mapping,omitempty"` - // ItemConcurNum int32 `thrift:"item_concur_num,22,optional" frugal:"22,optional,i32" form:"item_concur_num" json:"item_concur_num,omitempty"` - // EvaluatorsConcurNum int32 `thrift:"evaluators_concur_num,23,optional" frugal:"23,optional,i32" form:"evaluators_concur_num" json:"evaluators_concur_num,omitempty"` - CreateEvalTargetParam *CreateEvalTargetParam `thrift:"create_eval_target_param,24,optional" frugal:"24,optional,eval_target.CreateEvalTargetParam" form:"create_eval_target_param" json:"create_eval_target_param,omitempty"` - ExptType ExptType `thrift:"expt_type,30,optional" frugal:"30,optional,ExptType" form:"expt_type" json:"expt_type,omitempty"` - MaxAliveTime int64 `thrift:"max_alive_time,31,optional" frugal:"31,optional,i64" form:"max_alive_time" json:"max_alive_time,omitempty"` - SourceType SourceType `thrift:"source_type,32,optional" frugal:"32,optional,SourceType" form:"source_type" json:"source_type,omitempty"` - SourceID string `thrift:"source_id,33,optional" frugal:"33,optional,string" form:"source_id" json:"source_id,omitempty"` - ExptTemplateID int64 `thrift:"expt_template_id,34,optional" frugal:"34,optional,i64" form:"expt_template_id" json:"expt_template_id,omitempty"` - - ExptConf *EvaluationConfiguration + WorkspaceID int64 `json:"workspace_id"` + EvalSetVersionID int64 `json:"eval_set_version_id"` + TargetVersionID int64 `json:"target_version_id"` + EvaluatorVersionIds []int64 `json:"evaluator_version_ids"` + Name string `json:"name"` + Desc string `json:"desc"` + EvalSetID int64 `json:"eval_set_id"` + TargetID *int64 `json:"target_id,omitempty"` + CreateEvalTargetParam *CreateEvalTargetParam `json:"create_eval_target_param,omitempty"` + ExptType ExptType `json:"expt_type"` + MaxAliveTime int64 `json:"max_alive_time"` + SourceType SourceType `json:"source_type"` + SourceID string `json:"source_id"` + ExptTemplateID int64 `json:"expt_template_id"` + ExptConf *EvaluationConfiguration `json:"expt_conf"` + ItemRetryNum *int `json:"item_retry_num,omitempty"` } type ExptRunCheckOption struct { diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index 65e338314..20b354461 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -38,10 +38,12 @@ type IExptItemResultRepo interface { BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*entity.ExptItemResult) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64) (results []*entity.ExptItemResult, ncursor int64, err error) + MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) (results []*entity.ExptItemResult, err error) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*entity.ExptItemResult, int64, error) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error GetItemTurnResults(ctx context.Context, spaceID, exptID, itemID int64) ([]*entity.ExptTurnResult, error) + MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) UpdateItemsResult(ctx context.Context, spaceID, exptID int64, itemID []int64, ufields map[string]any) error GetMaxItemIdxByExptID(ctx context.Context, exptID, spaceID int64) (int32, error) diff --git a/backend/modules/evaluation/domain/repo/mocks/expt.go b/backend/modules/evaluation/domain/repo/mocks/expt.go index 66ae8648f..bd2eca539 100644 --- a/backend/modules/evaluation/domain/repo/mocks/expt.go +++ b/backend/modules/evaluation/domain/repo/mocks/expt.go @@ -457,6 +457,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) ListItemResultsByExptID(ctx, expt return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListItemResultsByExptID", reflect.TypeOf((*MockIExptItemResultRepo)(nil).ListItemResultsByExptID), ctx, exptID, spaceID, page, desc) } +// MGetItemResults mocks base method. +func (m *MockIExptItemResultRepo) MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) ([]*entity.ExptItemResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MGetItemResults", ctx, exptID, itemIDs, spaceID) + ret0, _ := ret[0].([]*entity.ExptItemResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemResults indicates an expected call of MGetItemResults. +func (mr *MockIExptItemResultRepoMockRecorder) MGetItemResults(ctx, exptID, itemIDs, spaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemResults), ctx, exptID, itemIDs, spaceID) +} + // MGetItemRunLog mocks base method. func (m *MockIExptItemResultRepo) MGetItemRunLog(ctx context.Context, exptID, exptRunID int64, itemIDs []int64, spaceID int64) ([]*entity.ExptItemResultRunLog, error) { m.ctrl.T.Helper() @@ -472,6 +487,21 @@ func (mr *MockIExptItemResultRepoMockRecorder) MGetItemRunLog(ctx, exptID, exptR return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemRunLog", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemRunLog), ctx, exptID, exptRunID, itemIDs, spaceID) } +// MGetItemTurnResults mocks base method. +func (m *MockIExptItemResultRepo) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MGetItemTurnResults", ctx, spaceID, exptID, itemIDs) + ret0, _ := ret[0].([]*entity.ExptTurnResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemTurnResults indicates an expected call of MGetItemTurnResults. +func (mr *MockIExptItemResultRepoMockRecorder) MGetItemTurnResults(ctx, spaceID, exptID, itemIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultRepo)(nil).MGetItemTurnResults), ctx, spaceID, exptID, itemIDs) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultRepo) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/expt_manage.go b/backend/modules/evaluation/domain/service/expt_manage.go index 315b1d1b9..582817554 100644 --- a/backend/modules/evaluation/domain/service/expt_manage.go +++ b/backend/modules/evaluation/domain/service/expt_manage.go @@ -39,8 +39,8 @@ type IExptConfigManager interface { // IExptExecutionManager 实验执行控制接口（负责实验的运行、监控和状态管理） type IExptExecutionManager interface { CheckRun(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.ExptRunCheckOptionFn) error - Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error - RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error + Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error + RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error Invoke(ctx context.Context, invokeExptReq *entity.InvokeExptReq) error Finish(ctx context.Context, exptID *entity.Experiment, exptRunID int64, session *entity.Session) error diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index d61f53382..567cc8496 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -269,7 +269,7 @@ func (e *ExptMangerImpl) CheckBenefit(ctx context.Context, expt *entity.Experime return nil } -func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { +func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { if err := NewQuotaService(e.quotaRepo, e.configer).AllowExptRun(ctx, exptID, spaceID, session); err != nil { return err } @@ -280,27 +280,29 @@ func (e *ExptMangerImpl) Run(ctx context.Context, exptID, runID, spaceID int64, } if err := e.publisher.PublishExptScheduleEvent(ctx, &entity.ExptScheduleEvent{ - SpaceID: spaceID, - ExptID: exptID, - ExptRunID: runID, - ExptRunMode: runMode, - ExptType: expt.ExptType, - CreatedAt: time.Now().Unix(), - Session: session, - Ext: ext, + SpaceID: spaceID, + ExptID: exptID, + ExptRunID: runID, + ExptRunMode: runMode, + ExptType: expt.ExptType, + CreatedAt: time.Now().Unix(), + ItemRetryTimes: itemRetryNum, + Session: session, + Ext: ext, }, gptr.Of(time.Second*3)); err != nil { return err } - err = e.sendExptNotify(ctx, expt) - if err != nil { - logs.CtxWarn(ctx, "[Run] sendExptNotify failed, expt_id: %v, error: %v", exptID, err) + switch runMode { + case entity.EvaluationModeSubmit: + if err = e.sendExptNotify(ctx, expt); err != nil { + logs.CtxWarn(ctx, "[Run] SendExptNotify failed, expt_id: %v, error: %v", exptID, err) + } } - return nil } -func (e *ExptMangerImpl) RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error { +func (e *ExptMangerImpl) RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error { if err := NewQuotaService(e.quotaRepo, e.configer).AllowExptRun(ctx, exptID, spaceID, session); err != nil { return err } @@ -311,14 +313,16 @@ func (e *ExptMangerImpl) RetryUnSuccess(ctx context.Context, exptID, runID, spac } if err := e.publisher.PublishExptScheduleEvent(ctx, &entity.ExptScheduleEvent{ - SpaceID: spaceID, - ExptID: exptID, - ExptRunID: runID, - ExptRunMode: entity.EvaluationModeFailRetry, - ExptType: expt.ExptType, - CreatedAt: time.Now().Unix(), - Session: session, - Ext: ext, + SpaceID: spaceID, + ExptID: exptID, + ExptRunID: runID, + ExptRunMode: entity.EvaluationModeRetryItems, + ExptType: expt.ExptType, + CreatedAt: time.Now().Unix(), + ItemRetryTimes: itemRetryNum, + ExecEvalSetItemIDs: itemIDs, + Session: session, + Ext: ext, }, gptr.Of(time.Second*3)); err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index b536e5382..342f0e444 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -208,7 +208,7 @@ func TestExptMangerImpl_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setup() - err := mgr.Run(ctx, tt.exptID, tt.runID, tt.spaceID, session, tt.runMode, tt.ext) + err := mgr.Run(ctx, tt.exptID, tt.runID, tt.spaceID, 0, session, tt.runMode, tt.ext) if (err != nil) != tt.wantErr { t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) } @@ -522,136 +522,6 @@ func TestExptMangerImpl_Kill(t *testing.T) { } } -func TestExptMangerImpl_RetryUnSuccess(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mgr := newTestExptManager(ctrl) - ctx := context.Background() - session := &entity.Session{UserID: "test_user"} - - tests := []struct { - name string - exptID int64 - runID int64 - spaceID int64 - ext map[string]string - setup func() - wantErr bool - }{ - { - name: "successful_retry", - exptID: 123, - runID: 456, - spaceID: 789, - ext: map[string]string{"retry": "true"}, - setup: func() { - // Mock lwt.CheckWriteFlagByID - mgr.lwt.(*lwtMocks.MockILatestWriteTracker). - EXPECT(). - CheckWriteFlagByID(ctx, gomock.Any(), int64(123)). - Return(false).AnyTimes() - - // Mock MGetByID for experiment retrieval - mgr.exptRepo.(*repoMocks.MockIExperimentRepo). - EXPECT(). - MGetByID(ctx, []int64{123}, int64(789)). - Return([]*entity.Experiment{{ID: 123, SpaceID: 789}}, nil).AnyTimes() - - // Mock GetEvaluationSet - mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). - EXPECT(). - GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&entity.EvaluationSet{}, nil).AnyTimes() - - // Mock MGetStats - mgr.exptResultService.(*svcMocks.MockExptResultService). - EXPECT(). - MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptStats{}, nil).AnyTimes() - - // Mock BatchGetExptAggrResultByExperimentIDs - mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). - EXPECT(). - BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() - - mgr.quotaRepo.(*repoMocks.MockQuotaRepo). - EXPECT(). - CreateOrUpdate(ctx, int64(789), gomock.Any(), session). - Return(nil) - mgr.configer.(*componentMocks.MockIConfiger). - EXPECT(). - GetExptExecConf(ctx, int64(789)).AnyTimes(). - Return(&entity.ExptExecConf{ - SpaceExptConcurLimit: 10, - }) - mgr.publisher.(*eventsMocks.MockExptEventPublisher). - EXPECT(). - PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). - Do(func(ctx context.Context, event *entity.ExptScheduleEvent, timeout *time.Duration) { - assert.Equal(t, entity.EvaluationModeFailRetry, event.ExptRunMode) - }). - Return(nil) - }, - wantErr: false, - }, - { - name: "quota_check_failure_on_retry", - exptID: 123, - runID: 456, - spaceID: 789, - ext: map[string]string{}, - setup: func() { - // Mock lwt.CheckWriteFlagByID - mgr.lwt.(*lwtMocks.MockILatestWriteTracker). - EXPECT(). - CheckWriteFlagByID(ctx, gomock.Any(), int64(123)). - Return(false).AnyTimes() - - // Mock MGetByID for experiment retrieval - mgr.exptRepo.(*repoMocks.MockIExperimentRepo). - EXPECT(). - MGetByID(ctx, []int64{123}, int64(789)). - Return([]*entity.Experiment{{ID: 123, SpaceID: 789}}, nil).AnyTimes() - - // Mock GetEvaluationSet - mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). - EXPECT(). - GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&entity.EvaluationSet{}, nil).AnyTimes() - - // Mock MGetStats - mgr.exptResultService.(*svcMocks.MockExptResultService). - EXPECT(). - MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptStats{}, nil).AnyTimes() - - // Mock BatchGetExptAggrResultByExperimentIDs - mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). - EXPECT(). - BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()). - Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() - - mgr.quotaRepo.(*repoMocks.MockQuotaRepo). - EXPECT(). - CreateOrUpdate(ctx, int64(789), gomock.Any(), session). - Return(errors.New("quota exceeded")) - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.setup() - err := mgr.RetryUnSuccess(ctx, tt.exptID, tt.runID, tt.spaceID, session, tt.ext) - if (err != nil) != tt.wantErr { - t.Errorf("RetryUnSuccess() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - func TestExptMangerImpl_LogRun(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/modules/evaluation/domain/service/expt_manage_impl.go b/backend/modules/evaluation/domain/service/expt_manage_impl.go index 9a9525fdf..126609ad2 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_impl.go @@ -12,6 +12,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/bytedance/gg/gslice" + "github.com/coze-dev/coze-loop/backend/infra/external/audit" "github.com/coze-dev/coze-loop/backend/infra/external/benefit" "github.com/coze-dev/coze-loop/backend/infra/idgen" diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 70b10d7c1..2f5a112dd 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -158,6 +158,9 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R retryConf := e.configer.GetErrRetryConf(ctx, event.SpaceID, nextErr) needRetry := event.RetryTimes < retryConf.GetRetryTimes() + if event.MaxRetryTimes > 0 { + needRetry = event.RetryTimes < event.MaxRetryTimes + } defer func() { code, stable, _ := errno.ParseStatusError(nextErr) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 3536c240a..a3f26675a 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -355,6 +355,7 @@ func (e *ExptSchedulerImpl) handleToSubmits(ctx context.Context, event *entity.E ExptRunMode: event.ExptRunMode, EvalSetItemID: ts.ItemID, CreateAt: now, + MaxRetryTimes: event.ItemRetryTimes, Ext: event.Ext, Session: event.Session, }) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 7e4e639a7..f2b7d3cc5 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -9,6 +9,7 @@ import ( "time" "github.com/bytedance/gg/gptr" + "github.com/bytedance/gg/gslice" "github.com/coze-dev/coze-loop/backend/infra/idgen" "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" @@ -17,6 +18,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" + "github.com/coze-dev/coze-loop/backend/pkg/json" "github.com/coze-dev/coze-loop/backend/pkg/lang/conv" "github.com/coze-dev/coze-loop/backend/pkg/lang/maps" "github.com/coze-dev/coze-loop/backend/pkg/logs" @@ -888,3 +890,431 @@ func makeStartIdemKey(event *entity.ExptScheduleEvent) string { func makeEndIdemKey(event *entity.ExptScheduleEvent) string { return fmt.Sprintf("expt_end:%v%v", event.ExptID, event.ExptRunID) } + +func NewExptRetryAllExec( + configer component.IConfiger, + evaluationSetItemService EvaluationSetItemService, + evaluatorRecordService EvaluatorRecordService, + exptItemResultRepo repo.IExptItemResultRepo, + exptRepo repo.IExperimentRepo, + exptStatsRepo repo.IExptStatsRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + idem idem.IdempotentService, + idgenerator idgen.IIDGenerator, + manager IExptManager, + publisher events.ExptEventPublisher, + templateManager IExptTemplateManager, +) *ExptRetryAllExec { + return &ExptRetryAllExec{ + configer: configer, + evaluationSetItemService: evaluationSetItemService, + evaluatorRecordService: evaluatorRecordService, + exptItemResultRepo: exptItemResultRepo, + exptRepo: exptRepo, + exptStatsRepo: exptStatsRepo, + exptTurnResultRepo: exptTurnResultRepo, + idem: idem, + idgenerator: idgenerator, + manager: manager, + publisher: publisher, + templateManager: templateManager, + } +} + +type ExptRetryAllExec struct { + manager IExptManager + exptStatsRepo repo.IExptStatsRepo + exptItemResultRepo repo.IExptItemResultRepo + exptTurnResultRepo repo.IExptTurnResultRepo + idgenerator idgen.IIDGenerator + evaluationSetItemService EvaluationSetItemService + exptRepo repo.IExperimentRepo + idem idem.IdempotentService + configer component.IConfiger + publisher events.ExptEventPublisher + evaluatorRecordService EvaluatorRecordService + templateManager IExptTemplateManager +} + +func (e *ExptRetryAllExec) Mode() entity.ExptRunMode { + return entity.EvaluationModeRetryAll +} + +func (e *ExptRetryAllExec) ExptStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + idemKey := makeStartIdemKey(event) + exist, err := e.idem.Exist(ctx, idemKey) + if err != nil { + return err + } + if exist { + return nil + } + + var ( + evalSetID = expt.EvalSet.ID + evalSetVersionID = expt.EvalSet.EvaluationSetVersion.ID + + maxLoop = 10000 + page = int32(1) + pageSize = int32(100) + itemCnt = 0 + total = int64(0) + ) + + for i := 0; i < maxLoop; i++ { + logs.CtxInfo(ctx, "ExptRetryAllExec.ExptStart scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, page: %v, limit: %v, cur_cnt: %v, total: %v", + event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, page, pageSize, itemCnt, total) + + items, t, _, err := e.evaluationSetItemService.ListEvaluationSetItems(ctx, &entity.ListEvaluationSetItemsParam{ + SpaceID: event.SpaceID, + EvaluationSetID: evalSetID, + VersionID: &evalSetVersionID, + PageNumber: &page, + PageSize: &pageSize, + }) + if err != nil { + return err + } + + itemCnt += len(items) + page++ + total = gptr.Indirect(t) + + turnCnt := 0 + for _, item := range items { + turnCnt += len(item.Turns) + } + + ids, err := e.idgenerator.GenMultiIDs(ctx, len(items)+turnCnt) + if err != nil { + return err + } + + idIdx := 0 + itemIDs := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) + itemTurnIDs := make([]*entity.ItemTurnID, 0, len(items)) + for _, item := range items { + for _, turn := range item.Turns { + itemIDs[item.ItemID] = true + itemTurnIDs = append(itemTurnIDs, &entity.ItemTurnID{ + ItemID: item.ItemID, + TurnID: turn.ID, + }) + } + } + + itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDs)) + for itemID := range itemIDs { + itemRunLogs = append(itemRunLogs, &entity.ExptItemResultRunLog{ + ID: ids[idIdx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: itemID, + Status: int32(entity.ItemRunState_Queueing), + }) + idIdx++ + } + + if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ + "status": int32(entity.ItemRunState_Queueing), + "expt_run_id": event.ExptRunID, + }); err != nil { + return err + } + + if err := e.exptTurnResultRepo.UpdateTurnResults(ctx, event.ExptID, itemTurnIDs, event.SpaceID, map[string]any{ + "status": int32(entity.TurnRunState_Queueing), + }); err != nil { + return err + } + + if err := e.exptItemResultRepo.BatchCreateNXRunLogs(ctx, itemRunLogs); err != nil { + return err + } + + time.Sleep(time.Millisecond * 30) + } + + got, err := e.exptStatsRepo.Get(ctx, event.ExptID, event.SpaceID) + if err != nil { + return err + } + + pendingCnt := got.PendingItemCnt + got.FailItemCnt + got.TerminatedItemCnt + got.ProcessingItemCnt + got.SuccessItemCnt + got.PendingItemCnt = pendingCnt + got.FailItemCnt = 0 + got.TerminatedItemCnt = 0 + got.ProcessingItemCnt = 0 + got.SuccessItemCnt = 0 + + if err := e.exptStatsRepo.Save(ctx, got); err != nil { + return err + } + + logs.CtxInfo(ctx, "ExptRetryAllExec.ExptStart reset pending_cnt: %v, expt_id: %v", pendingCnt, event.ExptID) + + exptDo := &entity.Experiment{ + Status: entity.ExptStatus_Processing, + ID: event.ExptID, + SpaceID: event.SpaceID, + } + + if err := e.exptRepo.Update(ctx, exptDo); err != nil { + return err + } + + if e.templateManager != nil { + var templateID int64 + if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { + templateID = expt.ExptTemplateMeta.ID + } + if templateID > 0 { + if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { + logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryAllExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) + } + } + } + + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + return err + } + + time.Sleep(time.Second * 3) + + return nil +} + +func (e *ExptRetryAllExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).ScanEvalItems(ctx, event, expt) +} + +func (e *ExptRetryAllExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { + if toSubmit == 0 && incomplete == 0 { + logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + return false, newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt) + } + return true, nil +} + +func (e *ExptRetryAllExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + return nil +} + +func (e *ExptRetryAllExec) ScheduleEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) error { + return nil +} + +func (e *ExptRetryAllExec) NextTick(ctx context.Context, event *entity.ExptScheduleEvent, nextTick bool) error { + interval := e.configer.GetExptExecConf(ctx, event.SpaceID).GetDaemonInterval() + return e.publisher.PublishExptScheduleEvent(ctx, event, gptr.Of(interval)) +} + +func (e *ExptRetryAllExec) PublishResult(ctx context.Context, turnEvaluatorRefs []*entity.ExptTurnEvaluatorResultRef, event *entity.ExptScheduleEvent) error { + if event.ExptType == entity.ExptType_Offline { + return nil + } + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).publishResult(ctx, turnEvaluatorRefs, event) +} + +type ExptRetryItemsExec struct { + manager IExptManager + exptStatsRepo repo.IExptStatsRepo + exptItemResultRepo repo.IExptItemResultRepo + exptTurnResultRepo repo.IExptTurnResultRepo + idgenerator idgen.IIDGenerator + evaluationSetItemService EvaluationSetItemService + exptRepo repo.IExperimentRepo + idem idem.IdempotentService + configer component.IConfiger + publisher events.ExptEventPublisher + evaluatorRecordService EvaluatorRecordService + templateManager IExptTemplateManager +} + +func (e *ExptRetryItemsExec) Mode() entity.ExptRunMode { + return entity.EvaluationModeRetryItems +} + +func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + idemKey := makeStartIdemKey(event) + exist, err := e.idem.Exist(ctx, idemKey) + if err != nil { + return err + } + if exist { + return nil + } + + got, err := e.exptStatsRepo.Get(ctx, event.ExptID, event.SpaceID) + if err != nil { + return err + } + + var ( + evalSetID = expt.EvalSet.ID + evalSetVersionID = expt.EvalSet.EvaluationSetVersion.ID + pageSize = int32(100) + ) + + for _, chunk := range gslice.Chunk(event.ExecEvalSetItemIDs, int(pageSize)) { + logs.CtxInfo(ctx, "ExptRetryItemsExec.ExptStart scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, item_ids: %v", + event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, chunk) + + items, err := e.evaluationSetItemService.BatchGetEvaluationSetItems(ctx, &entity.BatchGetEvaluationSetItemsParam{ + SpaceID: event.SpaceID, + EvaluationSetID: evalSetID, + VersionID: &evalSetVersionID, + ItemIDs: chunk, + }) + if err != nil { + return err + } + + turnCnt := 0 + for _, item := range items { + turnCnt += len(item.Turns) + } + + ids, err := e.idgenerator.GenMultiIDs(ctx, len(items)+turnCnt) + if err != nil { + return err + } + + idIdx := 0 + itemIDs := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) + itemTurnIDs := make([]*entity.ItemTurnID, 0, len(items)) + for _, item := range items { + for _, turn := range item.Turns { + itemIDs[item.ItemID] = true + itemTurnIDs = append(itemTurnIDs, &entity.ItemTurnID{ + ItemID: item.ItemID, + TurnID: turn.ID, + }) + } + } + + itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDs)) + for itemID := range itemIDs { + itemRunLogs = append(itemRunLogs, &entity.ExptItemResultRunLog{ + ID: ids[idIdx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: itemID, + Status: int32(entity.ItemRunState_Queueing), + }) + idIdx++ + } + + irs, err := e.exptItemResultRepo.MGetItemResults(ctx, event.ExptID, chunk, event.SpaceID) + if err != nil { + return err + } + + for _, ir := range irs { + switch ir.Status { + case entity.ItemRunState_Processing: + got.ProcessingItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Success: + got.SuccessItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Fail: + got.FailItemCnt-- + got.PendingItemCnt++ + case entity.ItemRunState_Terminal: + got.TerminatedItemCnt-- + got.PendingItemCnt++ + default: + } + } + + if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ + "status": int32(entity.ItemRunState_Queueing), + "expt_run_id": event.ExptRunID, + }); err != nil { + return err + } + + if err := e.exptTurnResultRepo.UpdateTurnResults(ctx, event.ExptID, itemTurnIDs, event.SpaceID, map[string]any{ + "status": int32(entity.TurnRunState_Queueing), + }); err != nil { + return err + } + + if err := e.exptItemResultRepo.BatchCreateNXRunLogs(ctx, itemRunLogs); err != nil { + return err + } + + time.Sleep(time.Millisecond * 30) + } + + if err := e.exptStatsRepo.Save(ctx, got); err != nil { + return err + } + + logs.CtxInfo(ctx, "ExptRetryItemsExec.ExptStart reset stat: %v, expt_id: %v", json.Jsonify(got), event.ExptID) + + if err := e.exptRepo.Update(ctx, &entity.Experiment{ + Status: entity.ExptStatus_Processing, + ID: event.ExptID, + SpaceID: event.SpaceID, + }); err != nil { + return err + } + + if e.templateManager != nil { + var templateID int64 + if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { + templateID = expt.ExptTemplateMeta.ID + } + if templateID > 0 { + if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { + logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryItemsExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) + } + } + } + + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + return err + } + + time.Sleep(time.Second * 3) + + return nil +} + +func (e *ExptRetryItemsExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).ScanEvalItems(ctx, event, expt) +} + +func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { + if toSubmit == 0 && incomplete == 0 { + logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + return false, newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt) + } + return true, nil +} + +func (e *ExptRetryItemsExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { + return nil +} + +func (e *ExptRetryItemsExec) ScheduleEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) error { + return nil +} + +func (e *ExptRetryItemsExec) NextTick(ctx context.Context, event *entity.ExptScheduleEvent, nextTick bool) error { + interval := e.configer.GetExptExecConf(ctx, event.SpaceID).GetDaemonInterval() + return e.publisher.PublishExptScheduleEvent(ctx, event, gptr.Of(interval)) +} + +func (e *ExptRetryItemsExec) PublishResult(ctx context.Context, turnEvaluatorRefs []*entity.ExptTurnEvaluatorResultRef, event *entity.ExptScheduleEvent) error { + if event.ExptType == entity.ExptType_Offline { + return nil + } + return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).publishResult(ctx, turnEvaluatorRefs, event) +} diff --git a/backend/modules/evaluation/domain/service/file_name b/backend/modules/evaluation/domain/service/file_name deleted file mode 100644 index 46abf88e6..000000000 --- a/backend/modules/evaluation/domain/service/file_name +++ /dev/null @@ -1 +0,0 @@ -ID,status,test_field,test_evaluator,test_evaluator_reason,weightedScore,test_tag,logID,targetTraceID diff --git a/backend/modules/evaluation/domain/service/mocks/expt_manage.go b/backend/modules/evaluation/domain/service/mocks/expt_manage.go index 494544766..fabd6bf16 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_manage.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_manage.go @@ -358,32 +358,32 @@ func (mr *MockIExptManagerMockRecorder) PendRun(ctx, exptID, exptRunID, spaceID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PendRun", reflect.TypeOf((*MockIExptManager)(nil).PendRun), ctx, exptID, exptRunID, spaceID, session) } -// RetryUnSuccess mocks base method. -func (m *MockIExptManager) RetryUnSuccess(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, ext map[string]string) error { +// RetryItems mocks base method. +func (m *MockIExptManager) RetryItems(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, itemIDs []int64, session *entity.Session, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RetryUnSuccess", ctx, exptID, runID, spaceID, session, ext) + ret := m.ctrl.Call(m, "RetryItems", ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) ret0, _ := ret[0].(error) return ret0 } -// RetryUnSuccess indicates an expected call of RetryUnSuccess. -func (mr *MockIExptManagerMockRecorder) RetryUnSuccess(ctx, exptID, runID, spaceID, session, ext any) *gomock.Call { +// RetryItems indicates an expected call of RetryItems. +func (mr *MockIExptManagerMockRecorder) RetryItems(ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryUnSuccess", reflect.TypeOf((*MockIExptManager)(nil).RetryUnSuccess), ctx, exptID, runID, spaceID, session, ext) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryItems", reflect.TypeOf((*MockIExptManager)(nil).RetryItems), ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) } // Run mocks base method. -func (m *MockIExptManager) Run(ctx context.Context, exptID, runID, spaceID int64, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { +func (m *MockIExptManager) Run(ctx context.Context, exptID, runID, spaceID int64, itemRetryNum int, session *entity.Session, runMode entity.ExptRunMode, ext map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", ctx, exptID, runID, spaceID, session, runMode, ext) + ret := m.ctrl.Call(m, "Run", ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) ret0, _ := ret[0].(error) return ret0 } // Run indicates an expected call of Run. -func (mr *MockIExptManagerMockRecorder) Run(ctx, exptID, runID, spaceID, session, runMode, ext any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) Run(ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), ctx, exptID, runID, spaceID, session, runMode, ext) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIExptManager)(nil).Run), ctx, exptID, runID, spaceID, itemRetryNum, session, runMode, ext) } // SetExptTerminating mocks base method. diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go index d4b4a00bb..cb1bbffe0 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result.go @@ -59,6 +59,18 @@ func (e ExptItemResultRepoImpl) GetItemTurnResults(ctx context.Context, spaceID, return results, nil } +func (e ExptItemResultRepoImpl) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64) ([]*entity.ExptTurnResult, error) { + pos, err := e.exptItemResultDAO.MGetItemTurnResults(ctx, spaceID, exptID, itemIDs) + if err != nil { + return nil, err + } + results := make([]*entity.ExptTurnResult, 0) + for _, po := range pos { + results = append(results, convert.NewExptTurnResultConvertor().PO2DO(po, nil)) + } + return results, nil +} + func (e ExptItemResultRepoImpl) SaveItemResults(ctx context.Context, itemResults []*entity.ExptItemResult) error { pos := make([]*model.ExptItemResult, 0) for _, itemResult := range itemResults { @@ -112,6 +124,18 @@ func (e ExptItemResultRepoImpl) ScanItemResults(ctx context.Context, exptID, cur return results, ncursor, nil } +func (e ExptItemResultRepoImpl) MGetItemResults(ctx context.Context, exptID int64, itemIDs []int64, spaceID int64) (results []*entity.ExptItemResult, err error) { + pos, err := e.exptItemResultDAO.MGetItemResults(ctx, spaceID, exptID, itemIDs) + if err != nil { + return nil, errorx.Wrapf(err, "MGetItemResults fail, exptID=%d, spaceID=%d", exptID, spaceID) + } + results = make([]*entity.ExptItemResult, 0, len(pos)) + for _, po := range pos { + results = append(results, convert.NewExptItemResultConvertor().PO2DO(po)) + } + return results, nil +} + func (e ExptItemResultRepoImpl) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) { return e.exptItemResultDAO.GetItemIDListByExptID(ctx, exptID, spaceID) } diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 6fc2cd4c9..80a05a3ad 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -29,10 +29,12 @@ type IExptItemResultDAO interface { BatchGet(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) BatchCreateNX(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error ScanItemResults(ctx context.Context, exptID, cursor, limit int64, status []int32, spaceID int64, opts ...db.Option) (results []*model.ExptItemResult, ncursor int64, err error) + MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) (results []*model.ExptItemResult, err error) GetItemIDListByExptID(ctx context.Context, exptID, spaceID int64) (itemIDList []int64, err error) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*model.ExptItemResult, int64, error) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error GetItemTurnResults(ctx context.Context, spaceID, exptID, itemID int64, opts ...db.Option) ([]*model.ExptTurnResult, error) + MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) UpdateItemsResult(ctx context.Context, spaceID, exptID int64, itemID []int64, ufields map[string]any, opts ...db.Option) error GetMaxItemIdxByExptID(ctx context.Context, exptID, spaceID int64, opts ...db.Option) (int32, error) @@ -90,6 +92,19 @@ func (dao *exptItemResultDAOImpl) GetItemTurnResults(ctx context.Context, spaceI return finds, nil } +func (dao *exptItemResultDAOImpl) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) { + if len(itemIDs) == 0 { + return nil, nil + } + db := dao.provider.NewSession(ctx, opts...) + q := query.Use(db).ExptTurnResult + found, err := q.WithContext(ctx).Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.In(itemIDs...)).Find() + if err != nil { + return nil, err + } + return found, nil +} + func (dao *exptItemResultDAOImpl) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error { db := dao.provider.NewSession(ctx, opts...) q := query.Use(db).ExptItemResult @@ -191,6 +206,22 @@ func (dao *exptItemResultDAOImpl) ScanItemResults(ctx context.Context, exptID, c return res, res[len(res)-1].ID, nil } +func (dao *exptItemResultDAOImpl) MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) (results []*model.ExptItemResult, err error) { + if len(itemIDs) == 0 { + return nil, nil + } + db := dao.provider.NewSession(ctx, opts...) + if contexts.CtxWriteDB(ctx) { + db = db.Clauses(dbresolver.Write) + } + q := query.Use(db).ExptItemResult + res, err := q.WithContext(ctx).Where(q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ItemID.In(itemIDs...)).Find() + if err != nil { + return nil, errorx.Wrapf(err, "MGetItemResults fail, exptID=%d, spaceID=%d, itemIDs=%v", exptID, spaceID, itemIDs) + } + return res, nil +} + func (dao *exptItemResultDAOImpl) ListItemResultsByExptID(ctx context.Context, exptID, spaceID int64, page entity.Page, desc bool) ([]*model.ExptItemResult, int64, error) { db := dao.provider.NewSession(ctx) q := query.Use(db).ExptItemResult diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go index 70685b05c..0d915174c 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/mocks/expt_item_result.go @@ -192,6 +192,26 @@ func (mr *MockIExptItemResultDAOMockRecorder) ListItemResultsByExptID(ctx, exptI return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListItemResultsByExptID", reflect.TypeOf((*MockIExptItemResultDAO)(nil).ListItemResultsByExptID), ctx, exptID, spaceID, page, desc) } +// MGetItemResults mocks base method. +func (m *MockIExptItemResultDAO) MGetItemResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptItemResult, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "MGetItemResults", varargs...) + ret0, _ := ret[0].([]*model.ExptItemResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemResults indicates an expected call of MGetItemResults. +func (mr *MockIExptItemResultDAOMockRecorder) MGetItemResults(ctx, spaceID, exptID, itemIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemResults), varargs...) +} + // MGetItemRunLog mocks base method. func (m *MockIExptItemResultDAO) MGetItemRunLog(ctx context.Context, exptID, exptRunID int64, itemIDs []int64, spaceID int64, opts ...db.Option) ([]*model.ExptItemResultRunLog, error) { m.ctrl.T.Helper() @@ -212,6 +232,26 @@ func (mr *MockIExptItemResultDAOMockRecorder) MGetItemRunLog(ctx, exptID, exptRu return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemRunLog", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemRunLog), varargs...) } +// MGetItemTurnResults mocks base method. +func (m *MockIExptItemResultDAO) MGetItemTurnResults(ctx context.Context, spaceID, exptID int64, itemIDs []int64, opts ...db.Option) ([]*model.ExptTurnResult, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, spaceID, exptID, itemIDs} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "MGetItemTurnResults", varargs...) + ret0, _ := ret[0].([]*model.ExptTurnResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGetItemTurnResults indicates an expected call of MGetItemTurnResults. +func (mr *MockIExptItemResultDAOMockRecorder) MGetItemTurnResults(ctx, spaceID, exptID, itemIDs any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, spaceID, exptID, itemIDs}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGetItemTurnResults", reflect.TypeOf((*MockIExptItemResultDAO)(nil).MGetItemTurnResults), varargs...) +} + // SaveItemResults mocks base method. func (m *MockIExptItemResultDAO) SaveItemResults(ctx context.Context, itemResults []*model.ExptItemResult, opts ...db.Option) error { m.ctrl.T.Helper() diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift index e3f70a1f2..79d124b8d 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift @@ -164,6 +164,7 @@ struct RunExperimentRequest { 2: optional i64 expt_id (api.body = 'expt_id', api.js_conv = 'true', go.tag = 'json:"expt_id"') 3: optional list item_ids (api.body = 'item_ids', api.js_conv = 'true', go.tag = 'json:"item_ids"') 10: optional expt.ExptType expt_type (api.body = 'expt_type') + 11: optional i32 item_retry_num (api.body = 'item_retry_num') 100: optional map ext (api.body = 'ext') diff --git a/idl/thrift/coze/loop/evaluation/domain/expt.thrift b/idl/thrift/coze/loop/evaluation/domain/expt.thrift index 01f8d34b3..80e587b61 100644 --- a/idl/thrift/coze/loop/evaluation/domain/expt.thrift +++ b/idl/thrift/coze/loop/evaluation/domain/expt.thrift @@ -62,6 +62,7 @@ struct Experiment { 41: optional i64 max_alive_time 42: optional SourceType source_type 43: optional string source_id + 45: optional i32 item_retry_num 51: optional list evaluator_id_version_list // 补充的评估器id+version关联评估器方式，和evaluator_version_ids共同使用，兼容老逻辑 @@ -78,7 +79,6 @@ struct ExptTemplateMeta { 3: optional string name 4: optional string desc 5: optional ExptType expt_type // 模板对应的实验类型，当前主要为 Offline - } // 实验三元组配置 @@ -99,6 +99,7 @@ struct ExptFieldMapping { 2: optional list evaluator_field_mapping 3: optional common.RuntimeParam target_runtime_param 4: optional i32 item_concur_num + 5: optional i32 item_retry_num } // 实验评估器得分加权配置 From 7975d8b9334251a9c71617334eb02368d1734ef5 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Tue, 10 Feb 2026 22:29:56 +0800 Subject: [PATCH 04/30] fix(evaluation): event mode --- .../modules/evaluation/domain/entity/event.go | 4 ++ .../service/expt_run_item_event_impl.go | 61 +++++++++++++++++-- .../domain/service/expt_run_item_turn_impl.go | 4 +- .../service/expt_run_scheduler_mode_impl.go | 46 ++++++++++++-- 4 files changed, 102 insertions(+), 13 deletions(-) diff --git a/backend/modules/evaluation/domain/entity/event.go b/backend/modules/evaluation/domain/entity/event.go index 018cd0b8d..2dee9802c 100644 --- a/backend/modules/evaluation/domain/entity/event.go +++ b/backend/modules/evaluation/domain/entity/event.go @@ -34,6 +34,10 @@ type ExptItemEvalEvent struct { Session *Session } +func (e *ExptItemEvalEvent) IgnoreExistedResult() bool { + return (e.ExptRunMode == EvaluationModeRetryItems || e.ExptRunMode == EvaluationModeRetryAll) && e.RetryTimes == 0 +} + func (e *ExptItemEvalEvent) GetExptID() int64 { if e == nil { return 0 diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 2f5a112dd..ada6e5b3d 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -374,6 +374,11 @@ func NewRecordEvalMode( resultSvc: resultSvc, idgen: idgen, }, nil + case entity.EvaluationModeRetryAll, entity.EvaluationModeRetryItems: + return &ExptRecordEvalModeRetryIgnoreResult{ + exptTurnResultRepo: exptTurnResultRepo, + idgen: idgen, + }, nil default: return nil, fmt.Errorf("NewRecordEvalMode with unknown expt mode: %v", event.ExptRunMode) } @@ -387,14 +392,13 @@ type ExptRecordEvalModeSubmit struct { } func (e *ExptRecordEvalModeSubmit) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { + return nil + } + event := eiec.Event turns := eiec.EvalSetItem.Turns - // if err := e.exptItemResultRepo.UpdateItemRunLog(ctx, event.ExptID, event.ExptRunID, []int64{event.EvalSetItemID}, map[string]any{"status": int32(entity.ItemRunState_Processing)}, - // event.SpaceID); err != nil { - // return err - // } - got, err := e.exptTurnResultRepo.GetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) if err != nil { return err @@ -506,3 +510,50 @@ func (e *ExptRecordEvalModeFailRetry) PreEval(ctx context.Context, eiec *entity. func (e *ExptRecordEvalModeFailRetry) PostEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { return nil } + +type ExptRecordEvalModeRetryIgnoreResult struct { + exptTurnResultRepo repo.IExptTurnResultRepo + idgen idgen.IIDGenerator +} + +func (e *ExptRecordEvalModeRetryIgnoreResult) PreEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + if eiec.GetExistItemResultLog() != nil && len(eiec.GetExistTurnResultLogs()) > 0 { + return nil + } + + event := eiec.Event + logID := logs.GetLogID(ctx) + + ids, err := e.idgen.GenMultiIDs(ctx, len(eiec.EvalSetItem.Turns)) + if err != nil { + return err + } + + turnRunLogs := make([]*entity.ExptTurnResultRunLog, 0, len(eiec.EvalSetItem.Turns)) + for idx, turn := range eiec.EvalSetItem.Turns { + turnRunLogs = append(turnRunLogs, &entity.ExptTurnResultRunLog{ + ID: ids[idx], + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ItemID: event.EvalSetItemID, + TurnID: turn.ID, + Status: entity.TurnRunState_Processing, + LogID: logID, + }) + } + + if err := e.exptTurnResultRepo.BatchCreateNXRunLog(ctx, turnRunLogs); err != nil { + return err + } + + eiec.ExistItemEvalResult.TurnResultRunLogs = gslice.ToMap(turnRunLogs, func(t *entity.ExptTurnResultRunLog) (int64, *entity.ExptTurnResultRunLog) { + return t.TurnID, t + }) + + return nil +} + +func (e *ExptRecordEvalModeRetryIgnoreResult) PostEval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { + return nil +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 31a9613d7..f10d74360 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -100,7 +100,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en return &entity.EvalTargetRecord{EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: make(map[string]*entity.Content)}}, nil } - if existRecord := e.existedTargetRecord(etec); existRecord != nil { + if existRecord := e.existedTargetRecord(etec); etec.Event.IgnoreExistedResult() && existRecord != nil { logs.CtxInfo(ctx, "CallTarget return with existed target record, record_id: %v", existRecord.ID) return existRecord, nil } @@ -266,7 +266,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallEvaluators(ctx context.Context, etec for _, evaluatorVersion := range expt.Evaluators { existResult := etec.ExptTurnRunResult.GetEvaluatorRecord(evaluatorVersion.GetEvaluatorVersionID()) - if existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { + if etec.Event.IgnoreExistedResult() && existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { evaluatorResults[existResult.ID] = existResult continue } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index f2b7d3cc5..c10879f11 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -90,6 +90,10 @@ func (f *DefaultSchedulerModeFactory) NewSchedulerMode( return NewExptFailRetryMode(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil case entity.EvaluationModeAppend: return NewExptAppendMode(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil + case entity.EvaluationModeRetryAll: + return NewExptRetryAllExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil + case entity.EvaluationModeRetryItems: + return NewExptRetryItemsExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil default: return nil, fmt.Errorf("NewSchedulerMode with unknown mode: %v", mode) } @@ -892,17 +896,17 @@ func makeEndIdemKey(event *entity.ExptScheduleEvent) string { } func NewExptRetryAllExec( - configer component.IConfiger, - evaluationSetItemService EvaluationSetItemService, - evaluatorRecordService EvaluatorRecordService, + manager IExptManager, exptItemResultRepo repo.IExptItemResultRepo, - exptRepo repo.IExperimentRepo, exptStatsRepo repo.IExptStatsRepo, exptTurnResultRepo repo.IExptTurnResultRepo, - idem idem.IdempotentService, idgenerator idgen.IIDGenerator, - manager IExptManager, + evaluationSetItemService EvaluationSetItemService, + exptRepo repo.IExperimentRepo, + idem idem.IdempotentService, + configer component.IConfiger, publisher events.ExptEventPublisher, + evaluatorRecordService EvaluatorRecordService, templateManager IExptTemplateManager, ) *ExptRetryAllExec { return &ExptRetryAllExec{ @@ -1118,6 +1122,36 @@ func (e *ExptRetryAllExec) PublishResult(ctx context.Context, turnEvaluatorRefs return newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).publishResult(ctx, turnEvaluatorRefs, event) } +func NewExptRetryItemsExec( + manager IExptManager, + exptItemResultRepo repo.IExptItemResultRepo, + exptStatsRepo repo.IExptStatsRepo, + exptTurnResultRepo repo.IExptTurnResultRepo, + idgenerator idgen.IIDGenerator, + evaluationSetItemService EvaluationSetItemService, + exptRepo repo.IExperimentRepo, + idem idem.IdempotentService, + configer component.IConfiger, + publisher events.ExptEventPublisher, + evaluatorRecordService EvaluatorRecordService, + templateManager IExptTemplateManager, +) *ExptRetryItemsExec { + return &ExptRetryItemsExec{ + configer: configer, + evaluationSetItemService: evaluationSetItemService, + evaluatorRecordService: evaluatorRecordService, + exptItemResultRepo: exptItemResultRepo, + exptRepo: exptRepo, + exptStatsRepo: exptStatsRepo, + exptTurnResultRepo: exptTurnResultRepo, + idem: idem, + idgenerator: idgenerator, + manager: manager, + publisher: publisher, + templateManager: templateManager, + } +} + type ExptRetryItemsExec struct { manager IExptManager exptStatsRepo repo.IExptStatsRepo From 00e1b09de6f069d22a774173b4586adbd2588ed6 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 12 Feb 2026 15:09:15 +0800 Subject: [PATCH 05/30] fix(evaluation): retryall --- .../evaluation/domain/service/expt_run_scheduler_mode_impl.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index c10879f11..f4a46bf72 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -1037,6 +1037,10 @@ func (e *ExptRetryAllExec) ExptStart(ctx context.Context, event *entity.ExptSche return err } + if itemCnt >= int(total) || len(items) == 0 { + break + } + time.Sleep(time.Millisecond * 30) } From 08a4c535621b9eda85000be26d4f412ed36a4184 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 12 Feb 2026 17:11:30 +0800 Subject: [PATCH 06/30] fix(evaluation): retry --- .../evaluation/domain/service/expt_run_item_impl.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 94cdfb115..d9d74cfb2 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -270,7 +270,7 @@ func (e *ExptItemEvalCtxExecutor) buildExptTurnEvalCtx(ctx context.Context, turn func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error) error { if evalErr != nil { - if retry, _ := e.evalErrNeedRetry(ctx, event.SpaceID, event.RetryTimes, evalErr); retry { + if retry, _ := e.evalErrNeedRetry(ctx, event, evalErr); retry { return evalErr } } @@ -300,12 +300,18 @@ func (e *ExptItemEvalCtxExecutor) CompleteItemRun(ctx context.Context, event *en return nil } -func (e *ExptItemEvalCtxExecutor) evalErrNeedRetry(ctx context.Context, spaceID int64, retryTimes int, evalErr error) (bool, time.Duration) { +func (e *ExptItemEvalCtxExecutor) evalErrNeedRetry(ctx context.Context, event *entity.ExptItemEvalEvent, evalErr error) (bool, time.Duration) { if evalErr == nil { return false, 0 } + spaceID := event.SpaceID + retryTimes := event.RetryTimes conf := e.Configer.GetErrRetryConf(ctx, spaceID, evalErr) - return retryTimes < conf.GetRetryTimes(), conf.GetRetryInterval() + maxRetryTimes := conf.GetRetryTimes() + if event.MaxRetryTimes > 0 { + maxRetryTimes = event.MaxRetryTimes + } + return retryTimes < maxRetryTimes, conf.GetRetryInterval() } func (e *ExptItemEvalCtxExecutor) evalErrNeedTerminateExpt(ctx context.Context, spaceID int64, evalErr error) bool { From 01727521d91d51f6915377187ae6c232c90e0c43 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 12 Feb 2026 20:22:06 +0800 Subject: [PATCH 07/30] fix(evaluation): custom raw err --- .../evaluation/domain/service/expt_run_item_impl.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index d9d74cfb2..0163c247e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -169,7 +169,14 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * } if evalErr != nil { - errMsg := e.Configer.GetErrCtrl(ctx).ConvertErrMsg(evalErr.Error()) + var rawErrMsg string + if se, ok := errorx.FromStatusError(evalErr); ok && se.Code() == errno.CustomEvalTargetInvokeFailCode || se.Code() == errno.CustomRPCEvaluatorRunFailedCode { + rawErrMsg = errorx.ErrorWithoutStack(evalErr) + } else { + rawErrMsg = evalErr.Error() + } + + errMsg := e.Configer.GetErrCtrl(ctx).ConvertErrMsg(rawErrMsg) logs.CtxWarn(ctx, "[ExptTurnEval] store turn run err, before: %v, after: %v", evalErr, errMsg) ei, ok := errno.ParseErrImpl(evalErr) From aa2fe89d892c370b93e1d15ebafecfc7c53d1069 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 13 Feb 2026 10:39:11 +0800 Subject: [PATCH 08/30] fix(evaluation): errmsg --- backend/modules/evaluation/domain/service/expt_run_item_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 0163c247e..45009dca0 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -170,7 +170,7 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * if evalErr != nil { var rawErrMsg string - if se, ok := errorx.FromStatusError(evalErr); ok && se.Code() == errno.CustomEvalTargetInvokeFailCode || se.Code() == errno.CustomRPCEvaluatorRunFailedCode { + if se, ok := errorx.FromStatusError(evalErr); ok && (se.Code() == errno.CustomEvalTargetInvokeFailCode || se.Code() == errno.CustomRPCEvaluatorRunFailedCode) { rawErrMsg = errorx.ErrorWithoutStack(evalErr) } else { rawErrMsg = evalErr.Error() From 63b9578b8b843cd3a8c64ec0ebe19d836dac97a9 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 13 Feb 2026 17:26:41 +0800 Subject: [PATCH 09/30] fix(evaluation): existed turn result --- .../evaluation/domain/service/expt_run_item_event_impl.go | 2 +- .../evaluation/domain/service/expt_run_item_event_impl_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index ada6e5b3d..263384437 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -327,7 +327,7 @@ func (e *ExptItemEventEvalServiceImpl) GetExistExptRecordEvalResult(ctx context. turnRunResultMap := make(map[int64]*entity.ExptTurnResultRunLog, len(turnRunLogs)) for _, result := range turnRunLogs { - turnRunResultMap[result.ItemID] = result + turnRunResultMap[result.TurnID] = result } itemRunLog, err := e.exptItemResultRepo.GetItemRunLog(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go index 4aca3e7b8..b3d297e9f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl_test.go @@ -562,6 +562,7 @@ func TestExptItemEventEvalServiceImpl_GetExistExptRecordEvalResult(t *testing.T) { ID: 1, ItemID: 1, + TurnID: 1, }, } From 014bdb7e3c9c3e2228d570fe6e15b3e5ea690da8 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Sat, 14 Feb 2026 16:09:16 +0800 Subject: [PATCH 10/30] fix(evaluation): scanitems --- .../evaluation/domain/entity/expt_result.go | 15 ++++++ .../service/expt_run_scheduler_event_impl.go | 3 ++ .../service/expt_run_scheduler_mode_impl.go | 45 +++++++++++----- .../expt_run_scheduler_mode_impl_test.go | 54 +++++++++++++------ .../repo/experiment/expt_item_result_test.go | 28 ++++++++++ .../repo/experiment/mysql/expt_item_result.go | 14 +++-- 6 files changed, 127 insertions(+), 32 deletions(-) diff --git a/backend/modules/evaluation/domain/entity/expt_result.go b/backend/modules/evaluation/domain/entity/expt_result.go index 669f8ec59..cb3f161b9 100644 --- a/backend/modules/evaluation/domain/entity/expt_result.go +++ b/backend/modules/evaluation/domain/entity/expt_result.go @@ -9,9 +9,12 @@ import ( "strconv" "time" + "gorm.io/gorm/clause" + "github.com/coze-dev/coze-loop/backend/infra/middleware/session" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" + gslice "github.com/coze-dev/coze-loop/backend/pkg/lang/slices" ) type FieldType int64 @@ -244,6 +247,12 @@ type ExptItemEvalResult struct { TurnResultRunLogs map[int64]*ExptTurnResultRunLog } +type ExptEvalItems []*ExptEvalItem + +func (e ExptEvalItems) GetItemIDs() []int64 { + return gslice.Map(e, func(f *ExptEvalItem) int64 { return f.ItemID }) +} + type ExptEvalItem struct { ExptID int64 EvalSetVersionID int64 @@ -465,9 +474,15 @@ func (e *ExptTemplateFilterFields) IsValid() bool { type ExptItemRunLogFilter struct { Status []ItemRunState ResultState *ExptItemResultState + + RawFilter bool + RawCond clause.Expr } func (e *ExptItemRunLogFilter) GetResultState() ExptItemResultState { + if e.ResultState == nil { + return 0 + } return *e.ResultState } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index eb06dc7df..a3b9c0402 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -258,6 +258,9 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche } complete = append(complete, zombies...) + logs.CtxInfo(ctx, "expt scheduler scan item, to_submit: %v, incomplete: %v, complete: %v", + entity.ExptEvalItems(toSubmit).GetItemIDs(), entity.ExptEvalItems(incomplete).GetItemIDs(), entity.ExptEvalItems(complete).GetItemIDs()) + if err = e.recordEvalItemRunLogs(ctx, event, complete, mode); err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index f4a46bf72..55c9de1be 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -10,6 +10,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/bytedance/gg/gslice" + "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/infra/idgen" "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" @@ -758,28 +759,48 @@ func newExptBaseExec( } func (e *exptBaseExec) ScanEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (toSubmit, incomplete, complete []*entity.ExptEvalItem, err error) { - incomplete, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{ - Status: []entity.ItemRunState{entity.ItemRunState_Processing}, - }, 0) + incomplete, complete, err = e.scanIncompleteAndComplete(ctx, event, expt) if err != nil { return nil, nil, nil, err } if submitCnt := e.getItemConcurNum(ctx, expt) - len(incomplete); submitCnt > 0 { - toSubmit, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{Status: []entity.ItemRunState{entity.ItemRunState_Queueing}}, int64(submitCnt)) + toSubmit, err = e.scanToSubmit(ctx, event, expt, int64(submitCnt)) if err != nil { return nil, nil, nil, err } } - complete, err = e.ScanRunLogEvalItems(ctx, event, expt, &entity.ExptItemRunLogFilter{ - ResultState: gptr.Of(entity.ExptItemResultStateLogged), - }, 0) + return toSubmit, incomplete, complete, nil +} + +func (e *exptBaseExec) scanIncompleteAndComplete(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) (incomplete, complete []*entity.ExptEvalItem, err error) { + rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{ + RawFilter: true, + RawCond: clause.Expr{SQL: "status IN (?) OR result_state = ?", Vars: []interface{}{[]int32{int32(entity.ItemRunState_Processing)}, int32(entity.ExptItemResultStateLogged)}}, + }, 0, 0, event.SpaceID) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - - return toSubmit, incomplete, complete, nil + incomplete = make([]*entity.ExptEvalItem, 0) + complete = make([]*entity.ExptEvalItem, 0) + evalSetVersionID := expt.EvalSet.EvaluationSetVersion.ID + for _, log := range rls { + item := &entity.ExptEvalItem{ + ExptID: event.ExptID, + EvalSetVersionID: evalSetVersionID, + ItemID: log.ItemID, + State: entity.ItemRunState(log.Status), + UpdatedAt: log.UpdatedAt, + } + if log.Status == int32(entity.ItemRunState_Processing) { + incomplete = append(incomplete, item) + } + if log.ResultState == int32(entity.ExptItemResultStateLogged) { + complete = append(complete, item) + } + } + return incomplete, complete, nil } func (e *exptBaseExec) getItemConcurNum(ctx context.Context, expt *entity.Experiment) int { @@ -791,8 +812,8 @@ func (e *exptBaseExec) getItemConcurNum(ctx context.Context, expt *entity.Experi return concurNum } -func (e *exptBaseExec) ScanRunLogEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, filter *entity.ExptItemRunLogFilter, limit int64) (items []*entity.ExptEvalItem, err error) { - rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, filter, 0, limit, event.SpaceID) +func (e *exptBaseExec) scanToSubmit(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, limit int64) (items []*entity.ExptEvalItem, err error) { + rls, _, err := e.exptItemResultRepo.ScanItemRunLogs(ctx, event.ExptID, event.ExptRunID, &entity.ExptItemRunLogFilter{Status: []entity.ItemRunState{entity.ItemRunState_Queueing}}, 0, limit, event.SpaceID) if err != nil { return nil, err } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 67dc835ed..47dc11c63 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -568,15 +568,13 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, @@ -614,6 +612,36 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { assert.Contains(t, err.Error(), "scan error") }, }, + { + name: "empty_incomplete_and_complete_then_to_submit_filled", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: 1, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{}, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Queueing)}, + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Queueing}, + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{}, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, } for _, tt := range tests { @@ -994,15 +1022,13 @@ func TestExptFailRetryExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *exptFailRetryExecFields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, @@ -1576,15 +1602,13 @@ func TestExptAppendExec_ScanEvalItems(t *testing.T) { }, prepareMock: func(f *fields, ctrl *gomock.Controller, args args) { f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, - }, int64(1), nil).Times(1) + {ItemID: 3, Status: int32(entity.ItemRunState_Success), ResultState: int32(entity.ExptItemResultStateLogged)}, + }, int64(0), nil).Times(1) f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, }, int64(1), nil).Times(1) - f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ - {ItemID: 3, Status: int32(entity.ItemRunState_Success)}, - }, int64(1), nil).Times(1) }, wantToSubmit: []*entity.ExptEvalItem{ {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go index 4b1a174c5..c90818cc7 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_item_result_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "gorm.io/gorm/clause" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" model "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/mysql/gorm_gen/model" @@ -492,6 +493,33 @@ func TestExptItemResultRepoImpl_ScanItemRunLogs(t *testing.T) { } } +func TestExptItemResultRepoImpl_ScanItemRunLogs_WithRawFilter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDAO := mocks.NewMockIExptItemResultDAO(ctrl) + repo := &ExptItemResultRepoImpl{exptItemResultDAO: mockDAO} + rawFilter := &entity.ExptItemRunLogFilter{ + RawFilter: true, + RawCond: clause.Expr{SQL: "status IN (?) OR result_state = ?", Vars: []interface{}{[]int32{1}, int32(2)}}, + } + + rs := int32(2) + mockDAO.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), rawFilter, int64(0), int64(0), int64(3)). + Return([]*model.ExptItemResultRunLog{ + {ItemID: 10, Status: 1}, + {ItemID: 20, Status: 2, ResultState: &rs}, + }, int64(0), nil) + + got, ncursor, err := repo.ScanItemRunLogs(context.Background(), 1, 2, rawFilter, 0, 0, 3) + assert.NoError(t, err) + assert.Equal(t, int64(0), ncursor) + assert.Len(t, got, 2) + assert.Equal(t, int64(10), got[0].ItemID) + assert.Equal(t, int64(20), got[1].ItemID) + assert.Equal(t, int32(2), got[1].ResultState) +} + func TestExptItemResultRepoImpl_BatchCreateNX(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 80a05a3ad..d8af1ec34 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -293,11 +293,15 @@ func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, e q.ExptRunID.Eq(exptRunID), } - if filter.ResultState != nil { - conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) - } - if len(filter.Status) > 0 { - conds = append(conds, q.Status.In(filter.GetStatus()...)) + if filter.RawFilter { + conds = append(conds, gen.Cond(filter.RawCond)...) + } else { + if filter.ResultState != nil { + conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) + } + if len(filter.Status) > 0 { + conds = append(conds, q.Status.In(filter.GetStatus()...)) + } } if cursor > 0 { From cacd9969e31d04b5d5fa046fce4247f04af08f03 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Sat, 14 Feb 2026 17:13:04 +0800 Subject: [PATCH 11/30] fix(evaluation): scanitems --- .../infra/repo/experiment/mysql/expt_item_result.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index d8af1ec34..74666e224 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -293,9 +293,7 @@ func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, e q.ExptRunID.Eq(exptRunID), } - if filter.RawFilter { - conds = append(conds, gen.Cond(filter.RawCond)...) - } else { + if !filter.RawFilter { if filter.ResultState != nil { conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) } @@ -310,8 +308,11 @@ func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, e query := q.WithContext(ctx). Clauses(hints.ForceIndex("uk_expt_run_item_turn")). - Where(conds...). - Order(q.ID.Asc()) + Where(conds...) + if filter.RawFilter { + query = query.Clauses(filter.RawCond) + } + query = query.Order(q.ID.Asc()) if limit > 0 { query = query.Limit(int(limit)) } From 184263738e96ff5252ebeddd1589015c118cd13d Mon Sep 17 00:00:00 2001 From: liushengyang Date: Sat, 14 Feb 2026 17:29:14 +0800 Subject: [PATCH 12/30] fix(evaluation): scancond --- .../evaluation/infra/repo/experiment/mysql/expt_item_result.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 74666e224..25f6075bf 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -310,7 +310,7 @@ func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, e Clauses(hints.ForceIndex("uk_expt_run_item_turn")). Where(conds...) if filter.RawFilter { - query = query.Clauses(filter.RawCond) + query = query.Clauses(clause.Where{Exprs: []clause.Expression{filter.RawCond}}) } query = query.Order(q.ID.Asc()) if limit > 0 { From 1123075b27a050c4ed7df7f415e58722b6959b90 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Sat, 14 Feb 2026 17:41:12 +0800 Subject: [PATCH 13/30] fix(evaluation): gorm sql --- .../repo/experiment/mysql/expt_item_result.go | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go index 25f6075bf..3b0c22b44 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/expt_item_result.go @@ -285,34 +285,54 @@ func (dao *exptItemResultDAOImpl) GetItemIDListByExptID(ctx context.Context, exp } func (dao *exptItemResultDAOImpl) ScanItemRunLogs(ctx context.Context, exptID, exptRunID int64, filter *entity.ExptItemRunLogFilter, cursor, limit, spaceID int64, opts ...db.Option) ([]*model.ExptItemResultRunLog, int64, error) { - db := dao.provider.NewSession(ctx, opts...) - q := query.Use(db).ExptItemResultRunLog + if filter == nil { + filter = &entity.ExptItemRunLogFilter{} + } + session := dao.provider.NewSession(ctx, opts...) + + // RawFilter: use raw gorm.DB Where(sql, vars...) to avoid gen clause conversion / unknown clause issues. + if filter.RawFilter && filter.RawCond.SQL != "" { + var res []*model.ExptItemResultRunLog + tx := session.WithContext(ctx).Model(&model.ExptItemResultRunLog{}). + Clauses(hints.ForceIndex("uk_expt_run_item_turn")). + Where("space_id = ? AND expt_id = ? AND expt_run_id = ?", spaceID, exptID, exptRunID). + Where(filter.RawCond.SQL, filter.RawCond.Vars...) + if cursor > 0 { + tx = tx.Where("id > ?", cursor) + } + tx = tx.Order("id asc") + if limit > 0 { + tx = tx.Limit(int(limit)) + } + if err := tx.Find(&res).Error; err != nil { + return nil, 0, errorx.Wrapf(err, "ScanItemRunLogs fail, exptID=%d, exptRunID=%d, cursor=%d", exptID, exptRunID, cursor) + } + if len(res) == 0 { + return nil, 0, nil + } + return res, res[len(res)-1].ID, nil + } + + q := query.Use(session).ExptItemResultRunLog conds := []gen.Condition{ q.SpaceID.Eq(spaceID), q.ExptID.Eq(exptID), q.ExptRunID.Eq(exptRunID), } - - if !filter.RawFilter { - if filter.ResultState != nil { - conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) - } - if len(filter.Status) > 0 { - conds = append(conds, q.Status.In(filter.GetStatus()...)) - } + if filter.ResultState != nil { + conds = append(conds, q.ResultState.In(int32(filter.GetResultState()))) + } + if len(filter.Status) > 0 { + conds = append(conds, q.Status.In(filter.GetStatus()...)) } - if cursor > 0 { conds = append(conds, q.ID.Gt(cursor)) } query := q.WithContext(ctx). Clauses(hints.ForceIndex("uk_expt_run_item_turn")). - Where(conds...) - if filter.RawFilter { - query = query.Clauses(clause.Where{Exprs: []clause.Expression{filter.RawCond}}) - } - query = query.Order(q.ID.Asc()) + Where(conds...). + Order(q.ID.Asc()) if limit > 0 { query = query.Limit(int(limit)) } From f2fc240d02fa52642ca623d3d1b9be9e4e3e6f5e Mon Sep 17 00:00:00 2001 From: liushengyang Date: Wed, 25 Feb 2026 16:12:04 +0800 Subject: [PATCH 14/30] feat(evaluation): itemretry --- backend/go.sum | 1 + backend/infra/lock/lock.go | 58 ++++++++++ backend/infra/lock/mocks/lock.go | 16 +++ .../evaluation/application/experiment_app.go | 40 ++++--- .../application/experiment_app_test.go | 11 +- .../evaluation/application/wire_gen.go | 4 +- .../modules/evaluation/domain/entity/expt.go | 41 ++++++- .../evaluation/domain/service/expt_manage.go | 3 +- .../service/expt_manage_execution_impl.go | 75 ++++++++++++- .../expt_manage_execution_impl_test.go | 2 +- .../service/expt_run_scheduler_mode_impl.go | 106 ++++++++++++------ ...xpt_run_scheduler_mode_impl_phase3_test.go | 2 + .../expt_run_scheduler_mode_impl_test.go | 4 +- .../domain/service/mocks/expt_manage.go | 24 +++- .../infra/repo/experiment/expt_run_log.go | 19 ++-- .../experiment/mysql/convert/expt_run_log.go | 28 +++-- .../evaluation/pkg/errno/evaluation.go | 10 ++ backend/script/errorx/evaluation.yaml | 6 + 18 files changed, 363 insertions(+), 87 deletions(-) diff --git a/backend/go.sum b/backend/go.sum index a3801a02e..6285b5857 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -473,6 +473,7 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAx github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/backend/infra/lock/lock.go b/backend/infra/lock/lock.go index 56e764fa6..eddcd9a97 100644 --- a/backend/infra/lock/lock.go +++ b/backend/infra/lock/lock.go @@ -30,6 +30,8 @@ type ILocker interface { // 后退出续期。调用方做写操作前应检查 ctx.Done 以确认仍持有锁，发生错误时应调用 cancel 以主动释放锁。 LockBackoffWithRenew(parent context.Context, key string, ttl time.Duration, maxHold time.Duration) (locked bool, ctx context.Context, cancel func(), err error) LockWithRenew(parent context.Context, key string, ttl time.Duration, maxHold time.Duration) (locked bool, ctx context.Context, cancel func(), err error) + + BackoffLockWithValue(ctx context.Context, key, val string, expiresIn time.Duration, backoff time.Duration) (bool, string, error) } func NewRedisLocker(c redis.Cmdable) ILocker { @@ -203,3 +205,59 @@ func (r *redisLocker) ExpireLockIn(key string, expiresIn time.Duration) (bool, e } return rt == 1, nil } + +const setNXWithGetScript = ` +local ok = redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', ARGV[2]) +if ok then + return {1, ARGV[1]} +else + local cur = redis.call('GET', KEYS[1]) + return {0, cur or ''} +end +` + +func (r *redisLocker) BackoffLockWithValue(ctx context.Context, key, val string, expiresIn time.Duration, maxWait time.Duration) (bool, string, error) { + if expiresIn < time.Second { + return false, "", fmt.Errorf("lock ttl too short") + } + + var ok bool + var lastHolder string + bf := backoff.NewExponentialBackOff() + bf.InitialInterval = 50 * time.Millisecond + bf.MaxInterval = 300 * time.Millisecond + bf.MaxElapsedTime = maxWait + + errNotLocked := errors.New("lock hold by others") + err := backoff.Retry(func() error { + result, err := r.c.Eval(ctx, setNXWithGetScript, []string{key}, val, int64(expiresIn/time.Millisecond)).Result() + if err != nil { + return errors.WithMessage(err, fmt.Sprintf("redis setnx with get fail, key: %v", key)) + } + sl, okType := result.([]interface{}) + if !okType || len(sl) != 2 { + return errors.Errorf("unexpected script result type %T or length", result) + } + locked, _ := sl[0].(int64) + if locked == 1 { + ok = true + return nil + } + switch v := sl[1].(type) { + case string: + lastHolder = v + case []byte: + lastHolder = string(v) + default: + return errors.Errorf("unexpected lua script result type %T or length, key: %v", sl[1], key) + } + return errNotLocked + }, bf) + if err != nil { + if errors.Is(err, errNotLocked) { + return false, lastHolder, nil + } + return false, "", err + } + return ok, val, nil +} diff --git a/backend/infra/lock/mocks/lock.go b/backend/infra/lock/mocks/lock.go index 810a9f7c7..6ac00f057 100644 --- a/backend/infra/lock/mocks/lock.go +++ b/backend/infra/lock/mocks/lock.go @@ -42,6 +42,22 @@ func (m *MockILocker) EXPECT() *MockILockerMockRecorder { return m.recorder } +// BackoffLockWithValue mocks base method. +func (m *MockILocker) BackoffLockWithValue(ctx context.Context, key, val string, expiresIn, backoff time.Duration) (bool, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BackoffLockWithValue", ctx, key, val, expiresIn, backoff) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// BackoffLockWithValue indicates an expected call of BackoffLockWithValue. +func (mr *MockILockerMockRecorder) BackoffLockWithValue(ctx, key, val, expiresIn, backoff any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackoffLockWithValue", reflect.TypeOf((*MockILocker)(nil).BackoffLockWithValue), ctx, key, val, expiresIn, backoff) +} + // ExpireLockIn mocks base method. func (m *MockILocker) ExpireLockIn(key string, expiresIn time.Duration) (bool, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/application/experiment_app.go b/backend/modules/evaluation/application/experiment_app.go index b95048504..c7b984faa 100644 --- a/backend/modules/evaluation/application/experiment_app.go +++ b/backend/modules/evaluation/application/experiment_app.go @@ -969,13 +969,14 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run evalMode := experiment.ExptType2EvalMode(req.GetExptType()) - if err := e.manager.LogRun(ctx, req.GetExptID(), runID, evalMode, req.GetWorkspaceID(), session); err != nil { + if err := e.manager.LogRun(ctx, req.GetExptID(), runID, evalMode, req.GetWorkspaceID(), nil, session); err != nil { return nil, err } if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), int(req.GetItemRetryNum()), session, evalMode, req.GetExt()); err != nil { return nil, err } + return &expt.RunExperimentResponse{ RunID: gptr.Of(runID), BaseResp: base.NewBaseResp(), @@ -983,12 +984,15 @@ func (e *experimentApplication) RunExperiment(ctx context.Context, req *expt.Run } func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.RetryExperimentRequest) (r *expt.RetryExperimentResponse, err error) { - session := entity.NewSession(ctx) - if req.GetRetryMode() == 0 { req.RetryMode = domain_expt.ExptRetryModePtr(domain_expt.ExptRetryMode_RetryFailure) } - runMode := experiment.ConvRetryMode(req.GetRetryMode()) + + var ( + runID int64 + session = entity.NewSession(ctx) + runMode = experiment.ConvRetryMode(req.GetRetryMode()) + ) got, err := e.manager.Get(ctx, req.GetExptID(), req.GetWorkspaceID(), session) if err != nil { @@ -1005,23 +1009,27 @@ func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.R return nil, err } - runID, err := e.idgen.GenID(ctx) - if err != nil { - return nil, err - } - - if err := e.manager.LogRun(ctx, req.GetExptID(), runID, runMode, req.GetWorkspaceID(), session); err != nil { - return nil, err - } - - itemRetryNum := gptr.Indirect(got.EvalConf.ItemRetryNum) switch runMode { case entity.EvaluationModeRetryItems: - if err := e.manager.RetryItems(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), itemRetryNum, req.GetItemIds(), session, req.GetExt()); err != nil { + rid, retried, err := e.manager.LogRetryItemsRun(ctx, req.GetExptID(), runMode, req.GetWorkspaceID(), req.GetItemIds(), session) + if err != nil { return nil, err } + if !retried { + if err := e.manager.RetryItems(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), gptr.Indirect(got.EvalConf.ItemRetryNum), req.GetItemIds(), session, req.GetExt()); err != nil { + return nil, err + } + } + runID = rid default: - if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), itemRetryNum, session, runMode, req.GetExt()); err != nil { + runID, err = e.idgen.GenID(ctx) + if err != nil { + return nil, err + } + if err := e.manager.LogRun(ctx, req.GetExptID(), runID, runMode, req.GetWorkspaceID(), nil, session); err != nil { + return nil, err + } + if err := e.manager.Run(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), gptr.Indirect(got.EvalConf.ItemRetryNum), session, runMode, req.GetExt()); err != nil { return nil, err } } diff --git a/backend/modules/evaluation/application/experiment_app_test.go b/backend/modules/evaluation/application/experiment_app_test.go index b7e7c6b03..7802ef7d0 100644 --- a/backend/modules/evaluation/application/experiment_app_test.go +++ b/backend/modules/evaluation/application/experiment_app_test.go @@ -694,6 +694,7 @@ func TestExperimentApplication_SubmitExperiment(t *testing.T) { validRunID, gomock.Any(), validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -826,7 +827,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). @@ -862,7 +863,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockTemplateManager.EXPECT(). UpdateExptInfo(gomock.Any(), templateID, workspaceID, exptID, entity.ExptStatus_Pending, int64(1)). @@ -898,7 +899,7 @@ func TestExperimentApplication_SubmitExperiment_UpdateExptInfo(t *testing.T) { mockManager.EXPECT().CreateExpt(gomock.Any(), gomock.Any(), gomock.Any()). Return(&entity.Experiment{ID: exptID}, nil) mockIDGen.EXPECT().GenID(gomock.Any()).Return(runID, nil) - mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), exptID, runID, gomock.Any(), workspaceID, gomock.Any(), gomock.Any()).Return(nil) mockManager.EXPECT().Run(gomock.Any(), exptID, runID, workspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) // 不应该调用 UpdateExptInfo mockTemplateManager.EXPECT().UpdateExptInfo(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) @@ -2342,6 +2343,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validRunID, entity.EvaluationModeSubmit, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -2388,6 +2390,7 @@ func TestExperimentApplication_RunExperiment(t *testing.T) { validRunID, entity.EvaluationModeSubmit, validWorkspaceID, + gomock.Any(), &entity.Session{UserID: "789", AppID: 0}, ).Return(nil) @@ -2485,7 +2488,7 @@ func TestExperimentApplication_RetryExperiment(t *testing.T) { mockIDGen.EXPECT().GenID(gomock.Any()).Return(validRunID, nil) // 记录运行日志 - mockManager.EXPECT().LogRun(gomock.Any(), validExptID, validRunID, entity.EvaluationModeFailRetry, validWorkspaceID, gomock.Any()).Return(nil) + mockManager.EXPECT().LogRun(gomock.Any(), validExptID, validRunID, entity.EvaluationModeFailRetry, validWorkspaceID, gomock.Any(), gomock.Any()).Return(nil) // 重试失败的实验 mockManager.EXPECT().Run(gomock.Any(), validExptID, validRunID, validWorkspaceID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) diff --git a/backend/modules/evaluation/application/wire_gen.go b/backend/modules/evaluation/application/wire_gen.go index 6deb32bd5..2fe9de54e 100644 --- a/backend/modules/evaluation/application/wire_gen.go +++ b/backend/modules/evaluation/application/wire_gen.go @@ -150,7 +150,7 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d iExptTemplateManager := service.NewExptTemplateManager(iExptTemplateRepo, idgen2, serviceEvaluatorService, iEvalTargetService, iEvaluationSetService, evaluationSetVersionService, iLatestWriteTracker) iNotifyRPCAdapter := notify.NewNotifyRPCAdapter() iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptTurnResultRepo, componentIConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, serviceEvaluatorService, benefitSvc, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider) - schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager) + schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo) exptSchedulerEvent := service.NewExptSchedulerSvc(iExptManager, iExperimentRepo, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExptRunLogRepo, idempotentService, componentIConfiger, quotaRepo, iLocker, exptEventPublisher, auditClient, exptMetric, exptResultService, idgen2, evaluationSetItemService, schedulerModeFactory) iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) @@ -376,7 +376,7 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL iExptTemplateManager := service.NewExptTemplateManager(iExptTemplateRepo, idgen2, evaluatorService, iEvalTargetService, iEvaluationSetService, evaluationSetVersionService, iLatestWriteTracker) iNotifyRPCAdapter := notify.NewNotifyRPCAdapter() iExptManager := service.NewExptManager(exptResultService, iExperimentRepo, iExptRunLogRepo, iExptStatsRepo, iExptItemResultRepo, iExptTurnResultRepo, iConfiger, quotaRepo, iLocker, idempotentService, exptEventPublisher, auditClient, idgen2, exptMetric, iLatestWriteTracker, evaluationSetVersionService, iEvaluationSetService, iEvalTargetService, evaluatorService, benefitService, exptAggrResultService, iExptTemplateRepo, iExptTemplateManager, iNotifyRPCAdapter, iUserProvider) - schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, iConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager) + schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, idempotentService, iConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo) exptSchedulerEvent := service.NewExptSchedulerSvc(iExptManager, iExperimentRepo, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExptRunLogRepo, idempotentService, iConfiger, quotaRepo, iLocker, exptEventPublisher, auditClient, exptMetric, exptResultService, idgen2, evaluationSetItemService, schedulerModeFactory) exptItemEvalEvent := service.NewExptRecordEvalService(iExptManager, iConfiger, exptEventPublisher, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExperimentRepo, quotaRepo, iLocker, idempotentService, auditClient, exptMetric, exptResultService, iEvalTargetService, evaluationSetItemService, evaluatorRecordService, evaluatorService, idgen2, benefitService, iEvalAsyncRepo) iExptAnnotateService := service.NewExptAnnotateService(db2, iExptAnnotateRepo, iExptTurnResultRepo, exptEventPublisher, evaluationSetItemService, iExperimentRepo, exptResultService, iExptTurnResultFilterRepo, iExptAggrResultRepo) diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index 1a8f05de1..aa11985d8 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -11,6 +11,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/mitchellh/mapstructure" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" ) @@ -51,14 +52,13 @@ const ( SourceType_Trace SourceType = 2 ) -// TODO type ExptRunLog struct { ID int64 SpaceID int64 CreatedBy string ExptID int64 ExptRunID int64 - ItemIds []byte + ItemIds []ExptRunLogItems Mode int32 Status int64 PendingCnt int32 @@ -73,6 +73,38 @@ type ExptRunLog struct { UpdatedAt time.Time } +func (e *ExptRunLog) GetItemIDs() []int64 { + var itemIDs []int64 + for _, items := range e.ItemIds { + for _, itemID := range items.ItemIDs { + itemIDs = append(itemIDs, itemID) + } + } + return itemIDs +} + +func (e *ExptRunLog) AppendItemIDs(itemIDs []int64) error { + if e == nil { + return errorx.New("ExptRunLog AppendItemIDs must init first") + } + var exists map[int64]bool + for _, chunk := range e.ItemIds { + for _, itemID := range chunk.ItemIDs { + exists[itemID] = true + } + } + rlItems := ExptRunLogItems{CreateAt: gptr.Of(time.Now().Unix())} + for _, itemID := range itemIDs { + if exists[itemID] { + return errorx.NewByCode(errno.EvalItemAlreadyRetryingCode, errorx.WithExtraMsg(fmt.Sprintf("existed item_id: %v", itemID))) + } else { + rlItems.ItemIDs = append(rlItems.ItemIDs, itemID) + } + } + e.ItemIds = append(e.ItemIds, rlItems) + return nil +} + type Experiment struct { ID int64 SpaceID int64 @@ -341,3 +373,8 @@ type InvokeExptReq struct { Ext map[string]string } + +type ExptRunLogItems struct { + ItemIDs []int64 + CreateAt *int64 +} diff --git a/backend/modules/evaluation/domain/service/expt_manage.go b/backend/modules/evaluation/domain/service/expt_manage.go index 582817554..b60d65311 100644 --- a/backend/modules/evaluation/domain/service/expt_manage.go +++ b/backend/modules/evaluation/domain/service/expt_manage.go @@ -53,6 +53,7 @@ type IExptExecutionManager interface { // SetExptTerminating Set experiment/run_log status to "terminating". SetExptTerminating(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error - LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error + LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error + LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (int64, bool, error) GetRunLog(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) (*entity.ExptRunLog, error) } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index 567cc8496..f11cb67f8 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -912,7 +912,7 @@ func (e *ExptMangerImpl) PendExpt(ctx context.Context, exptID, spaceID int64, se return nil } -func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error { +func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { duration := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second locked, err := e.mutex.LockBackoff(ctx, e.makeExptMutexLockKey(exptID), duration, time.Second) if err != nil { @@ -924,7 +924,7 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo defer e.mtr.EmitExptExecRun(spaceID, int64(mode)) - if err := e.runLogRepo.Create(ctx, &entity.ExptRunLog{ + rl := &entity.ExptRunLog{ ID: exptRunID, SpaceID: spaceID, CreatedBy: session.UserID, @@ -932,7 +932,12 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo ExptRunID: exptRunID, Mode: int32(mode), Status: int64(entity.ExptStatus_Pending), - }); err != nil { + } + if len(itemIDs) > 0 { + rl.ItemIds = []entity.ExptRunLogItems{{ItemIDs: itemIDs, CreateAt: gptr.Of(time.Now().Unix())}} + } + + if err := e.runLogRepo.Create(ctx, rl); err != nil { return err } @@ -946,6 +951,70 @@ func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mo return nil } +func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (runID int64, retried bool, err error) { + expireAt := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second + retryTime := time.Second + runID, err = e.idgenerator.GenID(ctx) + if err != nil { + return 0, false, err + } + + locked, existedRunID, err := e.mutex.BackoffLockWithValue(ctx, e.makeExptMutexLockKey(exptID), strconv.FormatInt(runID, 10), expireAt, retryTime) + if err != nil { + return 0, false, err + } + + var rl *entity.ExptRunLog + retried = !locked + + if retried { + runID, err = strconv.ParseInt(existedRunID, 10, 64) + if err != nil { + logs.CtxError(ctx, "parsing expt run lock value to runid failed, raw: %v", existedRunID) + return 0, false, errorx.NewByCode(errno.ExperimentRunningExistedCode) + } + + rl, err = e.runLogRepo.Get(ctx, exptID, runID) + if err != nil { + return 0, false, err + } + + if rl == nil { + return 0, false, errorx.Wrapf(err, "target runlog %v not found, expt_id: %v", runID, exptID) + } + + if err := rl.AppendItemIDs(itemIDs); err != nil { + return 0, false, err + } + } else { + rl = &entity.ExptRunLog{ + ID: runID, + SpaceID: spaceID, + CreatedBy: session.UserID, + ExptID: exptID, + ExptRunID: runID, + Mode: int32(mode), + Status: int64(entity.ExptStatus_Pending), + } + if len(itemIDs) > 0 { + rl.ItemIds = []entity.ExptRunLogItems{{ItemIDs: itemIDs, CreateAt: gptr.Of(time.Now().Unix())}} + } + } + + if err := e.runLogRepo.Save(ctx, rl); err != nil { + return 0, false, err + } + + if !retried { + if err := e.exptRepo.Update(ctx, &entity.Experiment{ID: exptID, LatestRunID: runID}); err != nil { + return 0, false, err + } + e.mtr.EmitExptExecRun(spaceID, int64(mode)) + } + + return runID, !locked, nil +} + func (e *ExptMangerImpl) GetRunLog(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) (*entity.ExptRunLog, error) { return e.runLogRepo.Get(ctx, exptID, exptRunID) } diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index 342f0e444..c925e28e9 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -642,7 +642,7 @@ func TestExptMangerImpl_LogRun(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setup() - err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, session) + err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, nil, session) if (err != nil) != tt.wantErr { t.Errorf("LogRun() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 55c9de1be..61d653208 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -46,6 +46,7 @@ func NewSchedulerModeFactory( evaluatorRecordService EvaluatorRecordService, resultSvc ExptResultService, templateManager IExptTemplateManager, + exptRunLogRepo repo.IExptRunLogRepo, ) SchedulerModeFactory { return &DefaultSchedulerModeFactory{ manager: manager, @@ -61,6 +62,7 @@ func NewSchedulerModeFactory( evaluatorRecordService: evaluatorRecordService, resultSvc: resultSvc, templateManager: templateManager, + exptRunLogRepo: exptRunLogRepo, } } @@ -79,6 +81,7 @@ type DefaultSchedulerModeFactory struct { evaluatorRecordService EvaluatorRecordService resultSvc ExptResultService templateManager IExptTemplateManager + exptRunLogRepo repo.IExptRunLogRepo } func (f *DefaultSchedulerModeFactory) NewSchedulerMode( @@ -94,7 +97,7 @@ func (f *DefaultSchedulerModeFactory) NewSchedulerMode( case entity.EvaluationModeRetryAll: return NewExptRetryAllExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil case entity.EvaluationModeRetryItems: - return NewExptRetryItemsExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager), nil + return NewExptRetryItemsExec(f.manager, f.exptItemResultRepo, f.exptStatsRepo, f.exptTurnResultRepo, f.idgenerator, f.evaluationSetItemService, f.exptRepo, f.idem, f.configer, f.publisher, f.evaluatorRecordService, f.templateManager, f.exptRunLogRepo), nil default: return nil, fmt.Errorf("NewSchedulerMode with unknown mode: %v", mode) } @@ -1160,6 +1163,7 @@ func NewExptRetryItemsExec( publisher events.ExptEventPublisher, evaluatorRecordService EvaluatorRecordService, templateManager IExptTemplateManager, + exptRunLogRepo repo.IExptRunLogRepo, ) *ExptRetryItemsExec { return &ExptRetryItemsExec{ configer: configer, @@ -1174,6 +1178,7 @@ func NewExptRetryItemsExec( manager: manager, publisher: publisher, templateManager: templateManager, + exptRunLogRepo: exptRunLogRepo, } } @@ -1190,6 +1195,7 @@ type ExptRetryItemsExec struct { publisher events.ExptEventPublisher evaluatorRecordService EvaluatorRecordService templateManager IExptTemplateManager + exptRunLogRepo repo.IExptRunLogRepo } func (e *ExptRetryItemsExec) Mode() entity.ExptRunMode { @@ -1206,6 +1212,41 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc return nil } + if err := e.resetEvalItems(ctx, event, expt, event.ExecEvalSetItemIDs); err != nil { + return err + } + + if err := e.exptRepo.Update(ctx, &entity.Experiment{ + Status: entity.ExptStatus_Processing, + ID: event.ExptID, + SpaceID: event.SpaceID, + }); err != nil { + return err + } + + if e.templateManager != nil { + var templateID int64 + if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { + templateID = expt.ExptTemplateMeta.ID + } + if templateID > 0 { + if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { + logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryItemsExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) + } + } + } + + duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 + if err := e.idem.Set(ctx, idemKey, duration); err != nil { + return err + } + + time.Sleep(time.Second * 3) + + return nil +} + +func (e *ExptRetryItemsExec) resetEvalItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, itemIDs []int64) error { got, err := e.exptStatsRepo.Get(ctx, event.ExptID, event.SpaceID) if err != nil { return err @@ -1217,8 +1258,8 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc pageSize = int32(100) ) - for _, chunk := range gslice.Chunk(event.ExecEvalSetItemIDs, int(pageSize)) { - logs.CtxInfo(ctx, "ExptRetryItemsExec.ExptStart scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, item_ids: %v", + for _, chunk := range gslice.Chunk(itemIDs, int(pageSize)) { + logs.CtxInfo(ctx, "ExptRetryItemsExec.resetEvalItems scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, item_ids: %v", event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, chunk) items, err := e.evaluationSetItemService.BatchGetEvaluationSetItems(ctx, &entity.BatchGetEvaluationSetItemsParam{ @@ -1242,11 +1283,11 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc } idIdx := 0 - itemIDs := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) + itemIDMap := gslice.ToMap(items, func(t *entity.EvaluationSetItem) (int64, bool) { return t.ItemID, true }) itemTurnIDs := make([]*entity.ItemTurnID, 0, len(items)) for _, item := range items { for _, turn := range item.Turns { - itemIDs[item.ItemID] = true + itemIDMap[item.ItemID] = true itemTurnIDs = append(itemTurnIDs, &entity.ItemTurnID{ ItemID: item.ItemID, TurnID: turn.ID, @@ -1254,8 +1295,8 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc } } - itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDs)) - for itemID := range itemIDs { + itemRunLogs := make([]*entity.ExptItemResultRunLog, 0, len(itemIDMap)) + for itemID := range itemIDMap { itemRunLogs = append(itemRunLogs, &entity.ExptItemResultRunLog{ ID: ids[idIdx], SpaceID: event.SpaceID, @@ -1290,7 +1331,7 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc } } - if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDs, func(k int64, v bool) int64 { return k }), map[string]any{ + if err := e.exptItemResultRepo.UpdateItemsResult(ctx, event.SpaceID, event.ExptID, maps.ToSlice(itemIDMap, func(k int64, v bool) int64 { return k }), map[string]any{ "status": int32(entity.ItemRunState_Queueing), "expt_run_id": event.ExptRunID, }); err != nil { @@ -1314,35 +1355,8 @@ func (e *ExptRetryItemsExec) ExptStart(ctx context.Context, event *entity.ExptSc return err } - logs.CtxInfo(ctx, "ExptRetryItemsExec.ExptStart reset stat: %v, expt_id: %v", json.Jsonify(got), event.ExptID) - - if err := e.exptRepo.Update(ctx, &entity.Experiment{ - Status: entity.ExptStatus_Processing, - ID: event.ExptID, - SpaceID: event.SpaceID, - }); err != nil { - return err - } - - if e.templateManager != nil { - var templateID int64 - if expt.ExptTemplateMeta != nil && expt.ExptTemplateMeta.ID > 0 { - templateID = expt.ExptTemplateMeta.ID - } - if templateID > 0 { - if err := e.templateManager.UpdateExptInfo(ctx, templateID, event.SpaceID, event.ExptID, entity.ExptStatus_Processing, 0); err != nil { - logs.CtxError(ctx, "UpdateExptInfo failed in ExptRetryItemsExec.ExptStart, template_id: %v, expt_id: %v, err: %v", templateID, event.ExptID, err) - } - } - } - - duration := time.Duration(e.configer.GetExptExecConf(ctx, event.SpaceID).GetZombieIntervalSecond()) * time.Second * 2 - if err := e.idem.Set(ctx, idemKey, duration); err != nil { - return err - } - + logs.CtxInfo(ctx, "ExptRetryItemsExec.resetEvalItems reset stat: %v, expt_id: %v", json.Jsonify(got), event.ExptID) time.Sleep(time.Second * 3) - return nil } @@ -1359,7 +1373,25 @@ func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptSche } func (e *ExptRetryItemsExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { - return nil + rl, err := e.exptRunLogRepo.Get(ctx, event.ExptID, event.ExptRunID) + if err != nil { + return err + } + + var ( + absence []int64 + all = rl.GetItemIDs() + exist = gslice.ToMap(event.ExecEvalSetItemIDs, func(t int64) (int64, bool) { return t, true }) + ) + for _, itemID := range all { + if !exist[itemID] { + absence = append(absence, itemID) + } + } + event.ExecEvalSetItemIDs = all + logs.CtxInfo(ctx, "ExptRetryItemsExec.ScheduleStart found absent item_id: %v, expt_id: %v", absence, event.ExptID) + + return e.resetEvalItems(ctx, event, expt, absence) } func (e *ExptRetryItemsExec) ScheduleEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) error { diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go index bd4cd0bdd..a8a593892 100755 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go @@ -249,6 +249,7 @@ func TestDefaultSchedulerModeFactory_NewSchedulerMode_Integration(t *testing.T) mockEvaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) mockResultSvc := svcmocks.NewMockExptResultService(ctrl) mockTemplateManager := svcmocks.NewMockIExptTemplateManager(ctrl) + mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) factory := NewSchedulerModeFactory( mockManager, @@ -264,6 +265,7 @@ func TestDefaultSchedulerModeFactory_NewSchedulerMode_Integration(t *testing.T) mockEvaluatorRecordService, mockResultSvc, mockTemplateManager, + mockExptRunLogRepo, ) tests := []struct { diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 47dc11c63..54e0c938e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -640,7 +640,7 @@ func TestExptSubmitExec_ScanEvalItems(t *testing.T) { wantIncomplete: []*entity.ExptEvalItem{}, wantComplete: []*entity.ExptEvalItem{}, wantErr: false, - assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, }, } @@ -2087,6 +2087,7 @@ func TestNewSchedulerModeFactory(t *testing.T) { evaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) resultService := svcmocks.NewMockExptResultService(ctrl) templateManager := svcmocks.NewMockIExptTemplateManager(ctrl) + mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) factory := NewSchedulerModeFactory( manager, @@ -2102,6 +2103,7 @@ func TestNewSchedulerModeFactory(t *testing.T) { evaluatorRecordService, resultService, templateManager, + mockExptRunLogRepo, ) tests := []struct { diff --git a/backend/modules/evaluation/domain/service/mocks/expt_manage.go b/backend/modules/evaluation/domain/service/mocks/expt_manage.go index fabd6bf16..ecfb8c40c 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_manage.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_manage.go @@ -267,18 +267,34 @@ func (mr *MockIExptManagerMockRecorder) ListExptRaw(ctx, page, pageSize, spaceID return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListExptRaw", reflect.TypeOf((*MockIExptManager)(nil).ListExptRaw), ctx, page, pageSize, spaceID, filter) } +// LogRetryItemsRun mocks base method. +func (m *MockIExptManager) LogRetryItemsRun(ctx context.Context, exptID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) (int64, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LogRetryItemsRun", ctx, exptID, mode, spaceID, itemIDs, session) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// LogRetryItemsRun indicates an expected call of LogRetryItemsRun. +func (mr *MockIExptManagerMockRecorder) LogRetryItemsRun(ctx, exptID, mode, spaceID, itemIDs, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRetryItemsRun", reflect.TypeOf((*MockIExptManager)(nil).LogRetryItemsRun), ctx, exptID, mode, spaceID, itemIDs, session) +} + // LogRun mocks base method. -func (m *MockIExptManager) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, session *entity.Session) error { +func (m *MockIExptManager) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LogRun", ctx, exptID, exptRunID, mode, spaceID, session) + ret := m.ctrl.Call(m, "LogRun", ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) ret0, _ := ret[0].(error) return ret0 } // LogRun indicates an expected call of LogRun. -func (mr *MockIExptManagerMockRecorder) LogRun(ctx, exptID, exptRunID, mode, spaceID, session any) *gomock.Call { +func (mr *MockIExptManagerMockRecorder) LogRun(ctx, exptID, exptRunID, mode, spaceID, itemIDs, session any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), ctx, exptID, exptRunID, mode, spaceID, session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogRun", reflect.TypeOf((*MockIExptManager)(nil).LogRun), ctx, exptID, exptRunID, mode, spaceID, itemIDs, session) } // MDelete mocks base method. diff --git a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go index 2f33a7542..b20d59e47 100644 --- a/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/expt_run_log.go @@ -28,16 +28,17 @@ func (e *ExptRunLogImpl) Get(ctx context.Context, exptID, exptRunID int64) (*ent if err != nil { return nil, err } - do := convert.NewExptRunLogConvertor().PO2DO(po) - return do, nil + return convert.NewExptRunLogConvertor().PO2DO(po) } func (e *ExptRunLogImpl) Create(ctx context.Context, exptRunLog *entity.ExptRunLog) error { - po := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + po, err := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + if err != nil { + return err + } po.CreatedAt = time.Now() - err := e.exptRunLogDAO.Create(ctx, po) - if err != nil { + if err := e.exptRunLogDAO.Create(ctx, po); err != nil { return err } @@ -45,11 +46,13 @@ func (e *ExptRunLogImpl) Create(ctx context.Context, exptRunLog *entity.ExptRunL } func (e *ExptRunLogImpl) Save(ctx context.Context, exptRunLog *entity.ExptRunLog) error { - po := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + po, err := convert.NewExptRunLogConvertor().DO2PO(exptRunLog) + if err != nil { + return err + } po.UpdatedAt = time.Now() - err := e.exptRunLogDAO.Save(ctx, po) - if err != nil { + if err := e.exptRunLogDAO.Save(ctx, po); err != nil { return err } return nil diff --git a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go index 0648ba88d..a47535b90 100644 --- a/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go +++ b/backend/modules/evaluation/infra/repo/experiment/mysql/convert/expt_run_log.go @@ -73,9 +73,13 @@ func NewExptRunLogConvertor() ExptRunLogConvertor { return ExptRunLogConvertor{} } -func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { +func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) (*model.ExptRunLog, error) { if log == nil { - return nil + return nil, nil + } + itemIDsBytes, err := json.Marshal(log.ItemIds) + if err != nil { + return nil, errorx.Wrapf(err, "ExptRunLogItems list json marshal fail") } return &model.ExptRunLog{ ID: log.ID, @@ -83,7 +87,7 @@ func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { CreatedBy: log.CreatedBy, ExptID: log.ExptID, ExptRunID: log.ExptRunID, - ItemIds: gptr.Of(log.ItemIds), + ItemIds: gptr.Of(itemIDsBytes), Mode: gptr.Of(log.Mode), Status: gptr.Of(log.Status), PendingCnt: log.PendingCnt, @@ -96,20 +100,28 @@ func (ExptRunLogConvertor) DO2PO(log *entity.ExptRunLog) *model.ExptRunLog { TerminatedCnt: log.TerminatedCnt, CreatedAt: log.CreatedAt, UpdatedAt: log.UpdatedAt, - } + }, nil } -func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) *entity.ExptRunLog { +func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) (*entity.ExptRunLog, error) { if log == nil { - return nil + return nil, nil } + + var itemIDs []entity.ExptRunLogItems + if log.ItemIds != nil && len(*log.ItemIds) > 0 { + if err := json.Unmarshal(*log.ItemIds, &itemIDs); err != nil { + return nil, errorx.Wrapf(err, "ExptRunLogItems list json unmarshal fail") + } + } + return &entity.ExptRunLog{ ID: log.ID, SpaceID: log.SpaceID, CreatedBy: log.CreatedBy, ExptID: log.ExptID, ExptRunID: log.ExptRunID, - ItemIds: gptr.Indirect(log.ItemIds), + ItemIds: itemIDs, Mode: gptr.Indirect(log.Mode), Status: gptr.Indirect(log.Status), PendingCnt: log.PendingCnt, @@ -122,5 +134,5 @@ func (ExptRunLogConvertor) PO2DO(log *model.ExptRunLog) *entity.ExptRunLog { TerminatedCnt: log.TerminatedCnt, CreatedAt: log.CreatedAt, UpdatedAt: log.UpdatedAt, - } + }, nil } diff --git a/backend/modules/evaluation/pkg/errno/evaluation.go b/backend/modules/evaluation/pkg/errno/evaluation.go index 081e8174e..233f40438 100644 --- a/backend/modules/evaluation/pkg/errno/evaluation.go +++ b/backend/modules/evaluation/pkg/errno/evaluation.go @@ -124,6 +124,10 @@ const ( duplicateCalcExptAggrResultErrorMessage = "aggregated result calculation is already in progress" duplicateCalcExptAggrResultErrorNoAffectStability = true + EvalItemAlreadyRetryingCode = 601204015 // item already has been retrying + evalItemAlreadyRetryingMessage = "item already has been retrying" + evalItemAlreadyRetryingNoAffectStability = true + ContentTypeNotSupportedCode = 601205000 // content type is not supported contentTypeNotSupportedMessage = "content type is not supported" contentTypeNotSupportedNoAffectStability = true @@ -537,6 +541,12 @@ func init() { code.WithAffectStability(!duplicateCalcExptAggrResultErrorNoAffectStability), ) + code.Register( + EvalItemAlreadyRetryingCode, + evalItemAlreadyRetryingMessage, + code.WithAffectStability(!evalItemAlreadyRetryingNoAffectStability), + ) + code.Register( ContentTypeNotSupportedCode, contentTypeNotSupportedMessage, diff --git a/backend/script/errorx/evaluation.yaml b/backend/script/errorx/evaluation.yaml index d849aac6a..407c546b6 100644 --- a/backend/script/errorx/evaluation.yaml +++ b/backend/script/errorx/evaluation.yaml @@ -112,6 +112,12 @@ error_code: description: aggregated result calculation is already in progress no_affect_stability: true + - name: EvalItemAlreadyRetrying + code: 4015 + message: item already has been retrying + description: item already has been retrying + no_affect_stability: true + - name: ContentTypeNotSupported code: 5000 message: content type is not supported From dfc48ea45449f3b80ecbea3ac6424cec804271c1 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 13:08:45 +0800 Subject: [PATCH 15/30] fix(evaluation): ut --- backend/infra/lock/lock_test.go | 151 ++++++++ .../expt_manage_execution_impl_test.go | 323 +++++++++++++++++- 2 files changed, 472 insertions(+), 2 deletions(-) diff --git a/backend/infra/lock/lock_test.go b/backend/infra/lock/lock_test.go index f53cfc0b9..ae0d052e5 100644 --- a/backend/infra/lock/lock_test.go +++ b/backend/infra/lock/lock_test.go @@ -25,6 +25,157 @@ func newIntCmdResult(val int64, err error) *redis.Cmd { return cmd } +// helper to build a redis.Cmd for Eval script result (val is typically []interface{}{locked, holder}) +func newEvalCmdResult(val interface{}, err error) *redis.Cmd { + cmd := redis.NewCmd(context.Background()) + if err != nil { + cmd.SetErr(err) + return cmd + } + cmd.SetVal(val) + return cmd +} + +func TestRedisLocker_BackoffLockWithValue(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cases := []struct { + name string + expiresIn time.Duration + maxWait time.Duration + setupMock func(m *redisMocks.MockPersistentCmdable) + wantOk bool + wantVal string + wantErr bool + }{ + { + name: "ttl_too_short_returns_error", + expiresIn: 100 * time.Millisecond, + maxWait: time.Second, + setupMock: func(m *redisMocks.MockPersistentCmdable) {}, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "success_locked_on_first_try", + expiresIn: 2 * time.Second, + maxWait: time.Second, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, []string{"key1"}, "val1", int64(2000)). + Return(newEvalCmdResult([]interface{}{int64(1), "val1"}, nil)). + Times(1) + }, + wantOk: true, + wantVal: "val1", + wantErr: false, + }, + { + name: "locked_by_others_returns_holder_string", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), "other_holder"}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "other_holder", + wantErr: false, + }, + { + name: "locked_by_others_returns_holder_bytes", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), []byte("byte_holder")}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "byte_holder", + wantErr: false, + }, + { + name: "redis_error_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult(nil, context.DeadlineExceeded)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_script_result_type_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult(interface{}("not_a_slice"), nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_script_result_length_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0)}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + { + name: "unexpected_holder_type_returns_error", + expiresIn: 2 * time.Second, + maxWait: 100 * time.Millisecond, + setupMock: func(m *redisMocks.MockPersistentCmdable) { + m.EXPECT(). + Eval(gomock.Any(), setNXWithGetScript, gomock.Any(), gomock.Any(), gomock.Any()). + Return(newEvalCmdResult([]interface{}{int64(0), 12345}, nil)). + AnyTimes() + }, + wantOk: false, + wantVal: "", + wantErr: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + mockRedis := redisMocks.NewMockPersistentCmdable(ctrl) + c.setupMock(mockRedis) + locker := &redisLocker{c: mockRedis, holder: "holder"} + ok, val, err := locker.BackoffLockWithValue(context.Background(), "key1", "val1", c.expiresIn, c.maxWait) + if c.wantErr != (err != nil) { + t.Fatalf("err: got %v, wantErr %v", err, c.wantErr) + } + if ok != c.wantOk { + t.Errorf("ok: got %v, want %v", ok, c.wantOk) + } + if val != c.wantVal { + t.Errorf("val: got %q, want %q", val, c.wantVal) + } + }) + } +} + func TestRedisLocker_renewLock_ContextDoneUnlocksAndReturns(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index c925e28e9..48f0a3021 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -1,5 +1,6 @@ // Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 runMode: entity.EvaluationModeSubmit, +// SPDX-License-Identifier: Apache-2.0 + package service import ( @@ -637,12 +638,57 @@ func TestExptMangerImpl_LogRun(t *testing.T) { }, wantErr: true, }, + { + name: "successful_log_run_with_item_ids", + exptID: 123, + exptRunID: 456, + mode: entity.EvaluationModeSubmit, + spaceID: 789, + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, int64(789)).AnyTimes(). + Return(&entity.ExptExecConf{ + ZombieIntervalSecond: 300, + }) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + LockBackoff(ctx, gomock.Any(), time.Duration(300)*time.Second, time.Second). + Return(true, nil) + + mgr.mtr.(*metricsMocks.MockExptMetric). + EXPECT(). + EmitExptExecRun(int64(789), int64(entity.EvaluationModeSubmit)) + + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT(). + Create(ctx, gomock.Any()). + Do(func(ctx context.Context, runLog *entity.ExptRunLog) { + assert.Equal(t, int64(456), runLog.ID) + assert.Equal(t, int64(123), runLog.ExptID) + assert.Len(t, runLog.ItemIds, 1) + assert.Equal(t, []int64{1, 2}, runLog.ItemIds[0].ItemIDs) + }). + Return(nil) + + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT(). + Update(ctx, gomock.Any()). + Return(nil) + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setup() - err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, nil, session) + itemIDs := []int64(nil) + if tt.name == "successful_log_run_with_item_ids" { + itemIDs = []int64{1, 2} + } + err := mgr.LogRun(ctx, tt.exptID, tt.exptRunID, tt.mode, tt.spaceID, itemIDs, session) if (err != nil) != tt.wantErr { t.Errorf("LogRun() error = %v, wantErr %v", err, tt.wantErr) } @@ -650,6 +696,279 @@ func TestExptMangerImpl_LogRun(t *testing.T) { } } +func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "test_user"} + exptID := int64(123) + spaceID := int64(789) + mode := entity.EvaluationModeRetryItems + itemIDs := []int64{1, 2} + + tests := []struct { + name string + setup func() + wantRunID int64 + wantRetry bool + wantErr bool + }{ + { + name: "locked_success_new_run", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1001), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1001", 300*time.Second, time.Second). + Return(true, "1001", nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()). + Do(func(ctx context.Context, rl *entity.ExptRunLog) { + assert.Equal(t, int64(1001), rl.ExptRunID) + assert.Len(t, rl.ItemIds, 1) + assert.Equal(t, itemIDs, rl.ItemIds[0].ItemIDs) + }). + Return(nil) + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().Update(ctx, &entity.Experiment{ID: exptID, LatestRunID: 1001}).Return(nil) + mgr.mtr.(*metricsMocks.MockExptMetric). + EXPECT().EmitExptExecRun(spaceID, int64(mode)) + }, + wantRunID: 1001, + wantRetry: false, + wantErr: false, + }, + { + name: "retried_append_to_existing_run", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1002), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1002", 300*time.Second, time.Second). + Return(false, "1001", nil) + existingLog := &entity.ExptRunLog{ID: 1001, ExptID: exptID, ExptRunID: 1001} + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Get(ctx, exptID, int64(1001)). + Return(existingLog, nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()).Return(nil) + }, + wantRunID: 1001, + wantRetry: true, + wantErr: false, + }, + { + name: "idgen_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(0), errors.New("idgen failed")) + }, + wantErr: true, + }, + { + name: "lock_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1003), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1003", 300*time.Second, time.Second). + Return(false, "", errors.New("redis error")) + }, + wantErr: true, + }, + { + name: "retried_parse_run_id_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1004), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1004", 300*time.Second, time.Second). + Return(false, "not_a_number", nil) + }, + wantErr: true, + }, + { + name: "retried_get_run_log_returns_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1005), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1005", 300*time.Second, time.Second). + Return(false, "1001", nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Get(ctx, exptID, int64(1001)). + Return(nil, errors.New("get run log failed")) + }, + wantErr: true, + }, + { + name: "save_error", + setup: func() { + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT(). + GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{ZombieIntervalSecond: 300}) + mgr.idgenerator.(*idgenMocks.MockIIDGenerator). + EXPECT().GenID(ctx).Return(int64(1006), nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + BackoffLockWithValue(ctx, gomock.Any(), "1006", 300*time.Second, time.Second). + Return(true, "1006", nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). + EXPECT().Save(ctx, gomock.Any()).Return(errors.New("save failed")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + runID, retried, err := mgr.LogRetryItemsRun(ctx, exptID, mode, spaceID, itemIDs, session) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantRunID, runID) + assert.Equal(t, tt.wantRetry, retried) + }) + } +} + +func TestExptMangerImpl_RetryItems(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "test_user"} + exptID := int64(123) + runID := int64(456) + spaceID := int64(789) + itemRetryNum := 1 + itemIDs := []int64{1, 2} + ext := map[string]string{"k": "v"} + + tests := []struct { + name string + setup func() + wantErr bool + }{ + { + name: "success_publish_event", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(nil) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID, ExptType: 1}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + mgr.publisher.(*eventsMocks.MockExptEventPublisher). + EXPECT(). + PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). + Return(nil) + }, + wantErr: false, + }, + { + name: "quota_check_failure", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(errors.New("quota exceeded")) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + }, + wantErr: true, + }, + { + name: "publish_event_failure", + setup: func() { + mgr.quotaRepo.(*repoMocks.MockQuotaRepo). + EXPECT().CreateOrUpdate(ctx, spaceID, gomock.Any(), session).Return(nil) + mgr.configer.(*componentMocks.MockIConfiger). + EXPECT().GetExptExecConf(ctx, spaceID).AnyTimes(). + Return(&entity.ExptExecConf{SpaceExptConcurLimit: 10}) + mgr.lwt.(*lwtMocks.MockILatestWriteTracker). + EXPECT().CheckWriteFlagByID(ctx, gomock.Any(), exptID).Return(false).AnyTimes() + mgr.exptRepo.(*repoMocks.MockIExperimentRepo). + EXPECT().MGetByID(ctx, []int64{exptID}, spaceID). + Return([]*entity.Experiment{{ID: exptID, SpaceID: spaceID}}, nil).AnyTimes() + mgr.evaluationSetService.(*svcMocks.MockIEvaluationSetService). + EXPECT().GetEvaluationSet(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.EvaluationSet{}, nil).AnyTimes() + mgr.exptResultService.(*svcMocks.MockExptResultService). + EXPECT().MGetStats(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptStats{}, nil).AnyTimes() + mgr.exptAggrResultService.(*svcMocks.MockExptAggrResultService). + EXPECT().BatchGetExptAggrResultByExperimentIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptAggregateResult{}, nil).AnyTimes() + mgr.publisher.(*eventsMocks.MockExptEventPublisher). + EXPECT(). + PublishExptScheduleEvent(ctx, gomock.Any(), gptr.Of(time.Second*3)). + Return(errors.New("publish failed")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + err := mgr.RetryItems(ctx, exptID, runID, spaceID, itemRetryNum, itemIDs, session, ext) + if (err != nil) != tt.wantErr { + t.Errorf("RetryItems() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func TestExptMangerImpl_GetRunLog(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() From c12983a9741daddfb8e2c12cbcfcbd4148d9e3fc Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 14:12:07 +0800 Subject: [PATCH 16/30] fix(evaluation): ut --- .../expt_manage_execution_impl_phase3_test.go | 170 -- .../expt_manage_execution_impl_test.go | 153 ++ ...xpt_run_scheduler_mode_impl_phase3_test.go | 331 --- .../expt_run_scheduler_mode_impl_test.go | 2049 +++++++++++++++++ 4 files changed, 2202 insertions(+), 501 deletions(-) delete mode 100755 backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go delete mode 100755 backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go deleted file mode 100755 index ff2bc0a94..000000000 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_phase3_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - "errors" - "testing" - - "github.com/bytedance/gg/gptr" - "go.uber.org/mock/gomock" - - "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" - "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - svcMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" -) - -// Phase 3: Runtime param validation integration tests -func TestExptMangerImpl_checkTargetConnector_WithRuntimeParam(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mgr := newTestExptManager(ctrl) - ctx := context.Background() - session := &entity.Session{UserID: "1"} - - tests := []struct { - name string - expt *entity.Experiment - setup func() - wantErr bool - }{ - { - name: "valid_runtime_param_success", - expt: &entity.Experiment{ - ID: 1, - TargetVersionID: 1, - TargetType: entity.EvalTargetTypeLoopPrompt, - Target: &entity.EvalTarget{ - EvalTargetType: entity.EvalTargetTypeLoopPrompt, - EvalTargetVersion: &entity.EvalTargetVersion{ - OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, - }, - }, - EvalSet: &entity.EvaluationSet{ - EvaluationSetVersion: &entity.EvaluationSetVersion{ - EvaluationSetSchema: &entity.EvaluationSetSchema{ - FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, - }, - }, - }, - EvalConf: &entity.EvaluationConfiguration{ - ConnectorConf: entity.Connector{ - TargetConf: &entity.TargetConf{ - TargetVersionID: 1, - IngressConf: &entity.TargetIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - CustomConf: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{ - { - FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, - Value: `{"model_config":{"model_id":"test_model"}}`, - }, - }, - }, - }, - }, - EvaluatorsConf: &entity.EvaluatorsConf{ - EvaluatorConf: []*entity.EvaluatorConf{ - { - EvaluatorVersionID: 1, - IngressConf: &entity.EvaluatorIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - TargetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, - }, - }, - }, - }, - }, - }, - }, - }, - setup: func() { - mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). - EXPECT(). - ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, `{"model_config":{"model_id":"test_model"}}`). - Return(nil) - }, - wantErr: false, - }, - { - name: "invalid_runtime_param_format_error", - expt: &entity.Experiment{ - ID: 1, - TargetVersionID: 1, - TargetType: entity.EvalTargetTypeLoopPrompt, - Target: &entity.EvalTarget{ - EvalTargetType: entity.EvalTargetTypeLoopPrompt, - EvalTargetVersion: &entity.EvalTargetVersion{ - OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, - }, - }, - EvalSet: &entity.EvaluationSet{ - EvaluationSetVersion: &entity.EvaluationSetVersion{ - EvaluationSetSchema: &entity.EvaluationSetSchema{ - FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, - }, - }, - }, - EvalConf: &entity.EvaluationConfiguration{ - ConnectorConf: entity.Connector{ - TargetConf: &entity.TargetConf{ - TargetVersionID: 1, - IngressConf: &entity.TargetIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - CustomConf: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{ - { - FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, - Value: `invalid_json`, - }, - }, - }, - }, - }, - EvaluatorsConf: &entity.EvaluatorsConf{ - EvaluatorConf: []*entity.EvaluatorConf{ - { - EvaluatorVersionID: 1, - IngressConf: &entity.EvaluatorIngressConf{ - EvalSetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, - }, - TargetAdapter: &entity.FieldAdapter{ - FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, - }, - }, - }, - }, - }, - }, - }, - }, - setup: func() { - mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). - EXPECT(). - ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, "invalid_json"). - Return(errors.New("invalid JSON format")) - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.setup() - err := mgr.checkTargetConnector(ctx, tt.expt, session) - if (err != nil) != tt.wantErr { - t.Errorf("checkTargetConnector() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index 48f0a3021..051aa4ff8 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -19,6 +19,7 @@ import ( idgenMocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" lockMocks "github.com/coze-dev/coze-loop/backend/infra/lock/mocks" lwtMocks "github.com/coze-dev/coze-loop/backend/infra/platestwrite/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/consts" idemMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" metricsMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/metrics/mocks" componentMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" @@ -2474,3 +2475,155 @@ func TestExptMangerImpl_Invoke_ExtField(t *testing.T) { }) } } + +func TestExptMangerImpl_checkTargetConnector_WithRuntimeParam(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mgr := newTestExptManager(ctrl) + ctx := context.Background() + session := &entity.Session{UserID: "1"} + + tests := []struct { + name string + expt *entity.Experiment + setup func() + wantErr bool + }{ + { + name: "valid_runtime_param_success", + expt: &entity.Experiment{ + ID: 1, + TargetVersionID: 1, + TargetType: entity.EvalTargetTypeLoopPrompt, + Target: &entity.EvalTarget{ + EvalTargetType: entity.EvalTargetTypeLoopPrompt, + EvalTargetVersion: &entity.EvalTargetVersion{ + OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, + }, + }, + EvalSet: &entity.EvaluationSet{ + EvaluationSetVersion: &entity.EvaluationSetVersion{ + EvaluationSetSchema: &entity.EvaluationSetSchema{ + FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, + }, + }, + }, + EvalConf: &entity.EvaluationConfiguration{ + ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{ + TargetVersionID: 1, + IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + CustomConf: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{ + { + FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, + Value: `{"model_config":{"model_id":"test_model"}}`, + }, + }, + }, + }, + }, + EvaluatorsConf: &entity.EvaluatorsConf{ + EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + TargetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, + }, + }, + }, + }, + }, + }, + }, + }, + setup: func() { + mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). + EXPECT(). + ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, `{"model_config":{"model_id":"test_model"}}`). + Return(nil) + }, + wantErr: false, + }, + { + name: "invalid_runtime_param_format_error", + expt: &entity.Experiment{ + ID: 1, + TargetVersionID: 1, + TargetType: entity.EvalTargetTypeLoopPrompt, + Target: &entity.EvalTarget{ + EvalTargetType: entity.EvalTargetTypeLoopPrompt, + EvalTargetVersion: &entity.EvalTargetVersion{ + OutputSchema: []*entity.ArgsSchema{{Key: gptr.Of("output_field")}}, + }, + }, + EvalSet: &entity.EvaluationSet{ + EvaluationSetVersion: &entity.EvaluationSetVersion{ + EvaluationSetSchema: &entity.EvaluationSetSchema{ + FieldSchemas: []*entity.FieldSchema{{Name: "input_field"}}, + }, + }, + }, + EvalConf: &entity.EvaluationConfiguration{ + ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{ + TargetVersionID: 1, + IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + CustomConf: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{ + { + FieldName: consts.FieldAdapterBuiltinFieldNameRuntimeParam, + Value: `invalid_json`, + }, + }, + }, + }, + }, + EvaluatorsConf: &entity.EvaluatorsConf{ + EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "input_field"}}, + }, + TargetAdapter: &entity.FieldAdapter{ + FieldConfs: []*entity.FieldConf{{FromField: "output_field"}}, + }, + }, + }, + }, + }, + }, + }, + }, + setup: func() { + mgr.evalTargetService.(*svcMocks.MockIEvalTargetService). + EXPECT(). + ValidateRuntimeParam(ctx, entity.EvalTargetTypeLoopPrompt, "invalid_json"). + Return(errors.New("invalid JSON format")) + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + err := mgr.checkTargetConnector(ctx, tt.expt, session) + if (err != nil) != tt.wantErr { + t.Errorf("checkTargetConnector() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go deleted file mode 100755 index a8a593892..000000000 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_phase3_test.go +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/bytedance/gg/gptr" - "go.uber.org/mock/gomock" - - idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" - idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" - configmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" - "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" - mock_repo "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" - svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" -) - -// Phase 3: Target skip functionality and scheduler updates tests -func TestExptAppendExec_ScheduleStart_WithTargetSkip(t *testing.T) { - testCases := []struct { - name string - event *entity.ExptScheduleEvent - expt *entity.Experiment - mockSetup func(mockRepo *mock_repo.MockIExperimentRepo) - wantErr bool - }{ - { - name: "experiment_with_target_normal_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 10, - TargetType: entity.EvalTargetTypeLoopPrompt, - MaxAliveTime: 3600000, // 1 hour - StartAt: gptr.Of(time.Now().Add(-30 * time.Minute)), // Started 30 minutes ago - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - // No update expected as experiment is within time limit - }, - wantErr: false, - }, - { - name: "experiment_without_target_skip_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 0, // No target - should be skipped - TargetType: 0, - MaxAliveTime: 3600000, - StartAt: gptr.Of(time.Now().Add(-30 * time.Minute)), - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - // No update expected as experiment without target continues normally - }, - wantErr: false, - }, - { - name: "experiment_max_alive_time_exceeded", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - SpaceID: 100, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 10, - TargetType: entity.EvalTargetTypeLoopPrompt, - MaxAliveTime: 3600000, // 1 hour - StartAt: gptr.Of(time.Now().Add(-2 * time.Hour)), // Started 2 hours ago - }, - mockSetup: func(mockRepo *mock_repo.MockIExperimentRepo) { - mockRepo.EXPECT(). - Update(gomock.Any(), &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Draining, - }). - Return(nil) - }, - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockRepo := mock_repo.NewMockIExperimentRepo(ctrl) - tc.mockSetup(mockRepo) - - exec := &ExptAppendExec{ - exptRepo: mockRepo, - } - - err := exec.ScheduleStart(context.Background(), tc.event, tc.expt) - if (err != nil) != tc.wantErr { - t.Errorf("ScheduleStart() error = %v, wantErr %v", err, tc.wantErr) - } - }) - } -} - -func TestExptAppendExec_ExptEnd_WithTargetSkip(t *testing.T) { - type mockFields struct { - manager *svcmocks.MockIExptManager - idem *idemmocks.MockIdempotentService - configer *configmocks.MockIConfiger - itemRepo *mock_repo.MockIExptItemResultRepo - publisher *eventmocks.MockExptEventPublisher - } - - testCases := []struct { - name string - event *entity.ExptScheduleEvent - expt *entity.Experiment - toSubmit int - incomplete int - mockSetup func(f *mockFields) - wantTick bool - wantErr bool - }{ - { - name: "experiment_without_target_draining_complete", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - ExptRunID: 2, - SpaceID: 100, - ExptRunMode: entity.EvaluationModeAppend, - Session: &entity.Session{UserID: "user1"}, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Draining, - TargetID: 0, // No target - TargetType: 0, - }, - toSubmit: 0, - incomplete: 0, - mockSetup: func(f *mockFields) { - f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil) - // CompleteRun: ctx, exptID, exptRunID, spaceID, session, WithCID, WithCompleteInterval (7个参数) - f.manager.EXPECT().CompleteRun(gomock.Any(), int64(1), int64(2), int64(100), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - // CompleteExpt: ctx, exptID, spaceID, session, WithCID, WithCompleteInterval (6个参数) - f.manager.EXPECT().CompleteExpt(gomock.Any(), int64(1), int64(100), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(100)).Return(&entity.ExptExecConf{ZombieIntervalSecond: 60}) - f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - }, - wantTick: false, - wantErr: false, - }, - { - name: "experiment_without_target_still_processing", - event: &entity.ExptScheduleEvent{ - ExptID: 1, - ExptRunID: 2, - SpaceID: 100, - ExptRunMode: entity.EvaluationModeAppend, - Session: &entity.Session{UserID: "user1"}, - }, - expt: &entity.Experiment{ - ID: 1, - SpaceID: 100, - Status: entity.ExptStatus_Processing, - TargetID: 0, // No target - TargetType: 0, - }, - toSubmit: 0, - incomplete: 1, - mockSetup: func(f *mockFields) { - // No mock setup needed as function should return early - }, - wantTick: true, - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - f := &mockFields{ - manager: svcmocks.NewMockIExptManager(ctrl), - idem: idemmocks.NewMockIdempotentService(ctrl), - configer: configmocks.NewMockIConfiger(ctrl), - itemRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), - publisher: eventmocks.NewMockExptEventPublisher(ctrl), - } - - if tc.mockSetup != nil { - tc.mockSetup(f) - } - - exec := &ExptAppendExec{ - manager: f.manager, - idem: f.idem, - configer: f.configer, - exptItemResultRepo: f.itemRepo, - publisher: f.publisher, - } - - nextTick, err := exec.ExptEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) - - if (err != nil) != tc.wantErr { - t.Errorf("ExptEnd() error = %v, wantErr %v", err, tc.wantErr) - } - - if nextTick != tc.wantTick { - t.Errorf("ExptEnd() nextTick = %v, want %v", nextTick, tc.wantTick) - } - }) - } -} - -func TestDefaultSchedulerModeFactory_NewSchedulerMode_Integration(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Create all required mocks - mockManager := svcmocks.NewMockIExptManager(ctrl) - mockItemRepo := mock_repo.NewMockIExptItemResultRepo(ctrl) - mockStatsRepo := mock_repo.NewMockIExptStatsRepo(ctrl) - mockTurnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) - mockEvalSetItemService := svcmocks.NewMockEvaluationSetItemService(ctrl) - mockExptRepo := mock_repo.NewMockIExperimentRepo(ctrl) - mockIdem := idemmocks.NewMockIdempotentService(ctrl) - mockConfiger := configmocks.NewMockIConfiger(ctrl) - mockPublisher := eventmocks.NewMockExptEventPublisher(ctrl) - mockEvaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) - mockResultSvc := svcmocks.NewMockExptResultService(ctrl) - mockTemplateManager := svcmocks.NewMockIExptTemplateManager(ctrl) - mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) - - factory := NewSchedulerModeFactory( - mockManager, - mockItemRepo, - mockStatsRepo, - mockTurnRepo, - mockIDGen, - mockEvalSetItemService, - mockExptRepo, - mockIdem, - mockConfiger, - mockPublisher, - mockEvaluatorRecordService, - mockResultSvc, - mockTemplateManager, - mockExptRunLogRepo, - ) - - tests := []struct { - name string - mode entity.ExptRunMode - expectedType string - wantErr bool - }{ - { - name: "submit_mode_success", - mode: entity.EvaluationModeSubmit, - expectedType: "*service.ExptSubmitExec", - wantErr: false, - }, - { - name: "fail_retry_mode_success", - mode: entity.EvaluationModeFailRetry, - expectedType: "*service.ExptFailRetryExec", - wantErr: false, - }, - { - name: "append_mode_success", - mode: entity.EvaluationModeAppend, - expectedType: "*service.ExptAppendExec", - wantErr: false, - }, - { - name: "unknown_mode_error", - mode: entity.ExptRunMode(999), - expectedType: "", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scheduler, err := factory.NewSchedulerMode(tt.mode) - - if (err != nil) != tt.wantErr { - t.Errorf("NewSchedulerMode() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if !tt.wantErr { - if scheduler == nil { - t.Errorf("NewSchedulerMode() returned nil scheduler") - return - } - - // Verify the returned scheduler mode matches the input - if scheduler.Mode() != tt.mode { - t.Errorf("NewSchedulerMode() returned scheduler with mode %v, want %v", scheduler.Mode(), tt.mode) - } - - // Verify the type is correct - actualType := fmt.Sprintf("%T", scheduler) - if actualType != tt.expectedType { - t.Errorf("NewSchedulerMode() returned type %v, want %v", actualType, tt.expectedType) - } - } - }) - } -} diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 54e0c938e..762295f23 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -2476,3 +2476,2052 @@ func TestExptBaseExec_publishResult(t *testing.T) { }) } } + +type exptRetryAllExecFields struct { + manager *svcmocks.MockIExptManager + exptItemResultRepo *mock_repo.MockIExptItemResultRepo + exptStatsRepo *mock_repo.MockIExptStatsRepo + exptTurnResultRepo *mock_repo.MockIExptTurnResultRepo + idgenerator *idgenmocks.MockIIDGenerator + evaluationSetItemService *svcmocks.MockEvaluationSetItemService + exptRepo *mock_repo.MockIExperimentRepo + idem *idemmocks.MockIdempotentService + configer *configmocks.MockIConfiger + publisher *eventmocks.MockExptEventPublisher + evaluatorRecordService *svcmocks.MockEvaluatorRecordService + templateManager *svcmocks.MockIExptTemplateManager +} + +type exptRetryItemsExecFields struct { + manager *svcmocks.MockIExptManager + exptItemResultRepo *mock_repo.MockIExptItemResultRepo + exptStatsRepo *mock_repo.MockIExptStatsRepo + exptTurnResultRepo *mock_repo.MockIExptTurnResultRepo + idgenerator *idgenmocks.MockIIDGenerator + evaluationSetItemService *svcmocks.MockEvaluationSetItemService + exptRepo *mock_repo.MockIExperimentRepo + idem *idemmocks.MockIdempotentService + configer *configmocks.MockIConfiger + publisher *eventmocks.MockExptEventPublisher + evaluatorRecordService *svcmocks.MockEvaluatorRecordService + templateManager *svcmocks.MockIExptTemplateManager + exptRunLogRepo *mock_repo.MockIExptRunLogRepo +} + +func buildRetryAllExecFields(ctrl *gomock.Controller) *exptRetryAllExecFields { + return &exptRetryAllExecFields{ + manager: svcmocks.NewMockIExptManager(ctrl), + exptItemResultRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), + exptStatsRepo: mock_repo.NewMockIExptStatsRepo(ctrl), + exptTurnResultRepo: mock_repo.NewMockIExptTurnResultRepo(ctrl), + idgenerator: idgenmocks.NewMockIIDGenerator(ctrl), + evaluationSetItemService: svcmocks.NewMockEvaluationSetItemService(ctrl), + exptRepo: mock_repo.NewMockIExperimentRepo(ctrl), + idem: idemmocks.NewMockIdempotentService(ctrl), + configer: configmocks.NewMockIConfiger(ctrl), + publisher: eventmocks.NewMockExptEventPublisher(ctrl), + evaluatorRecordService: svcmocks.NewMockEvaluatorRecordService(ctrl), + templateManager: svcmocks.NewMockIExptTemplateManager(ctrl), + } +} + +func buildRetryItemsExecFields(ctrl *gomock.Controller) *exptRetryItemsExecFields { + return &exptRetryItemsExecFields{ + manager: svcmocks.NewMockIExptManager(ctrl), + exptItemResultRepo: mock_repo.NewMockIExptItemResultRepo(ctrl), + exptStatsRepo: mock_repo.NewMockIExptStatsRepo(ctrl), + exptTurnResultRepo: mock_repo.NewMockIExptTurnResultRepo(ctrl), + idgenerator: idgenmocks.NewMockIIDGenerator(ctrl), + evaluationSetItemService: svcmocks.NewMockEvaluationSetItemService(ctrl), + exptRepo: mock_repo.NewMockIExperimentRepo(ctrl), + idem: idemmocks.NewMockIdempotentService(ctrl), + configer: configmocks.NewMockIConfiger(ctrl), + publisher: eventmocks.NewMockExptEventPublisher(ctrl), + evaluatorRecordService: svcmocks.NewMockEvaluatorRecordService(ctrl), + templateManager: svcmocks.NewMockIExptTemplateManager(ctrl), + exptRunLogRepo: mock_repo.NewMockIExptRunLogRepo(ctrl), + } +} + +func buildMockExpt() *entity.Experiment { + return &entity.Experiment{ + ID: 1, + SpaceID: 3, + CreatedBy: "created_by", + Name: "created_by", + Description: "description", + EvalSetVersionID: 1, + EvalSetID: 1, + TargetType: 1, + TargetVersionID: 1, + TargetID: 1, + EvaluatorVersionRef: []*entity.ExptEvaluatorVersionRef{{EvaluatorID: 1, EvaluatorVersionID: 1}}, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{TargetVersionID: 1, IngressConf: &entity.TargetIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{{FieldName: "field_name", FromField: "from_field"}}}, + }}, + EvaluatorsConf: &entity.EvaluatorsConf{EvaluatorConcurNum: ptr.Of(1), EvaluatorConf: []*entity.EvaluatorConf{ + { + EvaluatorVersionID: 1, + IngressConf: &entity.EvaluatorIngressConf{EvalSetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{{FieldName: "field_name", FromField: "from_field"}}}}, + }, + }}, + }}, + Target: &entity.EvalTarget{ID: 1, SpaceID: 3, SourceTargetID: "source_target_id", EvalTargetType: 1, EvalTargetVersion: &entity.EvalTargetVersion{ID: 1, OutputSchema: []*entity.ArgsSchema{{Key: ptr.Of("key")}}}, BaseInfo: &entity.BaseInfo{}}, + EvalSet: &entity.EvaluationSet{ + ID: 1, SpaceID: 3, Name: "name", Description: "description", Status: 0, Spec: nil, Features: nil, ItemCount: 0, ChangeUncommitted: false, + EvaluationSetVersion: &entity.EvaluationSetVersion{ID: 1, AppID: 0, SpaceID: 3, EvaluationSetID: 1, Version: "version", VersionNum: 0, Description: "description", EvaluationSetSchema: nil, ItemCount: 0, BaseInfo: nil}, + LatestVersion: "", NextVersionNum: 0, BaseInfo: nil, BizCategory: strconv.Itoa(1), + }, + Evaluators: []*entity.Evaluator{{}}, + Status: 0, + StatusMessage: "", + LatestRunID: 0, + CreditCost: 0, + StartAt: nil, + EndAt: nil, + ExptType: 1, + MaxAliveTime: 0, + SourceType: 0, + SourceID: "", + Stats: nil, + AggregateResult: nil, + } +} + +func TestExptRetryAllExec_Mode(t *testing.T) { + exec := &ExptRetryAllExec{} + assert.Equal(t, entity.EvaluationModeRetryAll, exec.Mode()) +} + +func TestExptRetryAllExec_ScheduleStart(t *testing.T) { + tests := []struct { + name string + expt *entity.Experiment + event *entity.ExptScheduleEvent + wantErr bool + }{ + { + name: "normal_flow", + expt: &entity.Experiment{}, + event: &entity.ExptScheduleEvent{}, + wantErr: false, + }, + } + + exec := &ExptRetryAllExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleStart(context.Background(), tc.event, tc.expt) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ScheduleEnd(t *testing.T) { + tests := []struct { + name string + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + wantErr bool + }{ + { + name: "normal_flow", + event: &entity.ExptScheduleEvent{}, + expt: &entity.Experiment{}, + toSubmit: 0, + incomplete: 0, + wantErr: false, + }, + } + + exec := &ExptRetryAllExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ExptStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "list_eval_set_items_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, nil, nil, errors.New("list error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "list error") + }, + }, + { + name: "gen_multi_ids_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return(nil, errors.New("gen id error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "gen id error") + }, + }, + { + name: "update_items_result_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update error") + }, + }, + { + name: "update_turn_results_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update turn error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update turn error") + }, + }, + { + name: "batch_create_run_logs_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(errors.New("batch create error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch create error") + }, + }, + { + name: "get_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("get stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "get stats error") + }, + }, + { + name: "save_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(errors.New("save stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "save stats error") + }, + }, + { + name: "update_expt_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errors.New("update expt error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update expt error") + }, + }, + { + name: "idem_set_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + total := int64(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{ + PendingItemCnt: 5, + FailItemCnt: 3, + TerminatedItemCnt: 2, + ProcessingItemCnt: 1, + SuccessItemCnt: 4, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ZombieIntervalSecond: 10}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("idem set error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem set error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + err := e.ExptStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryAllExec_ScanEvalItems(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantToSubmit []*entity.ExptEvalItem + wantIncomplete []*entity.ExptEvalItem + wantComplete []*entity.ExptEvalItem + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "normal_flow", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, + }, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + }, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "scan_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "scan error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + toSubmit, incomplete, complete, err := e.ScanEvalItems(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + if !tt.wantErr { + assert.Equal(t, tt.wantToSubmit, toSubmit) + assert.Equal(t, tt.wantIncomplete, incomplete) + assert.Equal(t, tt.wantComplete, complete) + } + }) + } +} + +func TestExptRetryAllExec_ExptEnd(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + } + + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields, args args) + args args + wantNextTick bool + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "all_completed", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ZombieIntervalSecond: 100}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "still_pending", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 1, + incomplete: 1, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) {}, + wantNextTick: true, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "complete_run_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(errors.New("complete run error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "complete run error") + }, + }, + { + name: "complete_expt_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryAll, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryAllExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(errors.New("complete expt error")).Times(1) + }, + wantNextTick: false, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "complete expt error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryAllExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + } + + nextTick, err := e.ExptEnd(tt.args.ctx, tt.args.event, tt.args.expt, tt.args.toSubmit, tt.args.incomplete) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + assert.Equal(t, tt.wantNextTick, nextTick) + }) + } +} + +func TestExptRetryAllExec_NextTick(t *testing.T) { + testUserID := "test_user_id_123" + + tests := []struct { + name string + nextTick bool + prepareMock func(f *exptRetryAllExecFields) + event *entity.ExptScheduleEvent + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "nextTick_true_publish_success", + nextTick: true, + prepareMock: func(f *exptRetryAllExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "nextTick_true_publish_error", + nextTick: true, + prepareMock: func(f *exptRetryAllExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish error")) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "publish error") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryAllExec{ + configer: f.configer, + publisher: f.publisher, + } + + err := exec.NextTick(context.Background(), tc.event, tc.nextTick) + if tc.assertErr != nil { + tc.assertErr(t, err) + } + }) + } +} + +func TestExptRetryAllExec_PublishResult(t *testing.T) { + tests := []struct { + name string + prepareMock func(f *exptRetryAllExecFields) + event *entity.ExptScheduleEvent + refs []*entity.ExptTurnEvaluatorResultRef + wantErr bool + }{ + { + name: "offline_expt_skip_publish", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Offline, + }, + refs: nil, + wantErr: false, + }, + { + name: "online_expt_empty_refs", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Online, + }, + refs: []*entity.ExptTurnEvaluatorResultRef{}, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryAllExec{ + manager: f.manager, + idem: f.idem, + configer: f.configer, + exptItemResultRepo: f.exptItemResultRepo, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + } + + err := exec.PublishResult(context.Background(), tc.refs, tc.event) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_Mode(t *testing.T) { + exec := &ExptRetryItemsExec{} + assert.Equal(t, entity.EvaluationModeRetryItems, exec.Mode()) +} + +func TestExptRetryItemsExec_ScheduleEnd(t *testing.T) { + tests := []struct { + name string + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + wantErr bool + }{ + { + name: "normal_flow", + event: &entity.ExptScheduleEvent{}, + expt: &entity.Experiment{}, + toSubmit: 0, + incomplete: 0, + wantErr: false, + }, + } + + exec := &ExptRetryItemsExec{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := exec.ScheduleEnd(context.Background(), tc.event, tc.expt, tc.toSubmit, tc.incomplete) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_ExptStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "idem_already_exist", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "idem_check_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, errors.New("idem error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem error") + }, + }, + { + name: "get_expt_stats_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "stats error") + }, + }, + { + name: "batch_get_eval_set_items_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, errors.New("batch get error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch get error") + }, + }, + { + name: "gen_multi_ids_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return(nil, errors.New("gen id error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "gen id error") + }, + }, + { + name: "mget_item_results_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("mget error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "mget error") + }, + }, + { + name: "update_items_result_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Processing}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update error") + }, + }, + { + name: "update_turn_results_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Success}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update turn error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update turn error") + }, + }, + { + name: "batch_create_run_logs_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Fail}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(errors.New("batch create error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch create error") + }, + }, + { + name: "save_expt_stats_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Terminal}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(errors.New("save stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "save stats error") + }, + }, + { + name: "update_expt_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{ + {ItemID: 100, Status: entity.ItemRunState_Queueing}, + }, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errors.New("update expt error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "update expt error") + }, + }, + { + name: "idem_set_error_retry_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + mockItems := []*entity.EvaluationSetItem{ + {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, + } + f.evaluationSetItemService.EXPECT().BatchGetEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, nil).Times(1) + f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) + f.exptItemResultRepo.EXPECT().MGetItemResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResult{}, nil).Times(1) + f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptItemResultRepo.EXPECT().BatchCreateNXRunLogs(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.exptRepo.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ZombieIntervalSecond: 10}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("idem set error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "idem set error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + err := e.ExptStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_ScanEvalItems(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantToSubmit []*entity.ExptEvalItem + wantIncomplete []*entity.ExptEvalItem + wantComplete []*entity.ExptEvalItem + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "normal_flow", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), gomock.Any()).Return(&entity.ExptExecConf{ExptItemEvalConf: &entity.ExptItemEvalConf{ConcurNum: 3}}).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), int64(1), int64(2), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 1, Status: int32(entity.ItemRunState_Processing)}, + }, int64(0), nil).Times(1) + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return([]*entity.ExptItemResultRunLog{ + {ItemID: 2, Status: int32(entity.ItemRunState_Queueing)}, + }, int64(1), nil).Times(1) + }, + wantToSubmit: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 2, State: entity.ItemRunState_Queueing}, + }, + wantIncomplete: []*entity.ExptEvalItem{ + {ExptID: 1, EvalSetVersionID: 1, ItemID: 1, State: entity.ItemRunState_Processing}, + }, + wantComplete: []*entity.ExptEvalItem{}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "scan_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptItemResultRepo.EXPECT().ScanItemRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("scan error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "scan error") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + toSubmit, incomplete, complete, err := e.ScanEvalItems(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + if !tt.wantErr { + assert.Equal(t, tt.wantToSubmit, toSubmit) + assert.Equal(t, tt.wantIncomplete, incomplete) + assert.Equal(t, tt.wantComplete, complete) + } + }) + } +} + +func TestExptRetryItemsExec_ExptEnd(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + toSubmit int + incomplete int + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantNextTick bool + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "all_completed", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 0, + incomplete: 0, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) + f.configer.EXPECT().GetExptExecConf(gomock.Any(), args.event.SpaceID).Return(&entity.ExptExecConf{ZombieIntervalSecond: 100}).Times(1) + f.idem.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantNextTick: false, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "still_pending", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + }, + expt: mockExpt, + toSubmit: 1, + incomplete: 1, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) {}, + wantNextTick: true, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + nextTick, err := e.ExptEnd(tt.args.ctx, tt.args.event, tt.args.expt, tt.args.toSubmit, tt.args.incomplete) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + assert.Equal(t, tt.wantNextTick, nextTick) + }) + } +} + +func TestExptRetryItemsExec_ScheduleStart(t *testing.T) { + testUserID := "test_user_id_123" + mockExpt := buildMockExpt() + + type args struct { + ctx context.Context + event *entity.ExptScheduleEvent + expt *entity.Experiment + } + + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields, args args) + args args + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "get_run_log_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(nil, errors.New("get error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "get error") + }, + }, + { + name: "schedule_start_with_absent_items_reset_error", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1, 2, 3}}}, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("stats error")).Times(1) + }, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "stats error") + }, + }, + { + name: "schedule_start_with_no_absent_items", + args: args{ + ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), + event: &entity.ExptScheduleEvent{ + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1, 2}, + }, + expt: mockExpt, + }, + prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1, 2}}}, + }, nil).Times(1) + f.exptStatsRepo.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&entity.ExptStats{}, nil).Times(1) + f.exptStatsRepo.EXPECT().Save(gomock.Any(), gomock.Any()).Return(nil).Times(1) + }, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(f, tt.args) + } + + e := &ExptRetryItemsExec{ + manager: f.manager, + exptItemResultRepo: f.exptItemResultRepo, + exptStatsRepo: f.exptStatsRepo, + exptTurnResultRepo: f.exptTurnResultRepo, + idgenerator: f.idgenerator, + evaluationSetItemService: f.evaluationSetItemService, + exptRepo: f.exptRepo, + idem: f.idem, + configer: f.configer, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + templateManager: f.templateManager, + exptRunLogRepo: f.exptRunLogRepo, + } + + err := e.ScheduleStart(tt.args.ctx, tt.args.event, tt.args.expt) + if tt.assertErr != nil { + tt.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_NextTick(t *testing.T) { + testUserID := "test_user_id_123" + + tests := []struct { + name string + nextTick bool + prepareMock func(f *exptRetryItemsExecFields) + event *entity.ExptScheduleEvent + wantErr bool + assertErr func(t *testing.T, err error) + }{ + { + name: "nextTick_true_publish_success", + nextTick: true, + prepareMock: func(f *exptRetryItemsExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: false, + assertErr: func(t *testing.T, err error) { assert.NoError(t, err) }, + }, + { + name: "nextTick_true_publish_error", + nextTick: true, + prepareMock: func(f *exptRetryItemsExecFields) { + f.configer.EXPECT().GetExptExecConf(gomock.Any(), int64(1)).Return(&entity.ExptExecConf{DaemonIntervalSecond: 1}) + f.publisher.EXPECT().PublishExptScheduleEvent(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("publish error")) + }, + event: &entity.ExptScheduleEvent{SpaceID: 1, Session: &entity.Session{UserID: testUserID}}, + wantErr: true, + assertErr: func(t *testing.T, err error) { + assert.Error(t, err) + assert.Contains(t, err.Error(), "publish error") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryItemsExec{ + configer: f.configer, + publisher: f.publisher, + } + + err := exec.NextTick(context.Background(), tc.event, tc.nextTick) + if tc.assertErr != nil { + tc.assertErr(t, err) + } + }) + } +} + +func TestExptRetryItemsExec_PublishResult(t *testing.T) { + tests := []struct { + name string + prepareMock func(f *exptRetryItemsExecFields) + event *entity.ExptScheduleEvent + refs []*entity.ExptTurnEvaluatorResultRef + wantErr bool + }{ + { + name: "offline_expt_skip_publish", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Offline, + }, + refs: nil, + wantErr: false, + }, + { + name: "online_expt_empty_refs", + event: &entity.ExptScheduleEvent{ + ExptType: entity.ExptType_Online, + }, + refs: []*entity.ExptTurnEvaluatorResultRef{}, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + if tc.prepareMock != nil { + tc.prepareMock(f) + } + + exec := &ExptRetryItemsExec{ + manager: f.manager, + idem: f.idem, + configer: f.configer, + exptItemResultRepo: f.exptItemResultRepo, + publisher: f.publisher, + evaluatorRecordService: f.evaluatorRecordService, + } + + err := exec.PublishResult(context.Background(), tc.refs, tc.event) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestNewExptRetryAllExec(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryAllExecFields(ctrl) + + exec := NewExptRetryAllExec( + f.manager, + f.exptItemResultRepo, + f.exptStatsRepo, + f.exptTurnResultRepo, + f.idgenerator, + f.evaluationSetItemService, + f.exptRepo, + f.idem, + f.configer, + f.publisher, + f.evaluatorRecordService, + f.templateManager, + ) + + assert.NotNil(t, exec) + assert.Equal(t, f.manager, exec.manager) + assert.Equal(t, f.exptItemResultRepo, exec.exptItemResultRepo) + assert.Equal(t, f.exptStatsRepo, exec.exptStatsRepo) + assert.Equal(t, f.exptTurnResultRepo, exec.exptTurnResultRepo) + assert.Equal(t, f.idgenerator, exec.idgenerator) + assert.Equal(t, f.evaluationSetItemService, exec.evaluationSetItemService) + assert.Equal(t, f.exptRepo, exec.exptRepo) + assert.Equal(t, f.idem, exec.idem) + assert.Equal(t, f.configer, exec.configer) + assert.Equal(t, f.publisher, exec.publisher) +} + +func TestNewExptRetryItemsExec(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + f := buildRetryItemsExecFields(ctrl) + + exec := NewExptRetryItemsExec( + f.manager, + f.exptItemResultRepo, + f.exptStatsRepo, + f.exptTurnResultRepo, + f.idgenerator, + f.evaluationSetItemService, + f.exptRepo, + f.idem, + f.configer, + f.publisher, + f.evaluatorRecordService, + f.templateManager, + f.exptRunLogRepo, + ) + + assert.NotNil(t, exec) + assert.Equal(t, f.manager, exec.manager) + assert.Equal(t, f.exptItemResultRepo, exec.exptItemResultRepo) + assert.Equal(t, f.exptStatsRepo, exec.exptStatsRepo) + assert.Equal(t, f.exptTurnResultRepo, exec.exptTurnResultRepo) + assert.Equal(t, f.idgenerator, exec.idgenerator) + assert.Equal(t, f.evaluationSetItemService, exec.evaluationSetItemService) + assert.Equal(t, f.exptRepo, exec.exptRepo) + assert.Equal(t, f.idem, exec.idem) + assert.Equal(t, f.configer, exec.configer) + assert.Equal(t, f.publisher, exec.publisher) + assert.Equal(t, f.exptRunLogRepo, exec.exptRunLogRepo) +} + +func TestSchedulerModeFactory_NewSchedulerMode_RetryAll(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + manager := svcmocks.NewMockIExptManager(ctrl) + exptItemResultRepo := mock_repo.NewMockIExptItemResultRepo(ctrl) + exptStatsRepo := mock_repo.NewMockIExptStatsRepo(ctrl) + exptTurnResultRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + idgenerator := idgenmocks.NewMockIIDGenerator(ctrl) + evaluationSetItemService := svcmocks.NewMockEvaluationSetItemService(ctrl) + exptRepo := mock_repo.NewMockIExperimentRepo(ctrl) + idem := idemmocks.NewMockIdempotentService(ctrl) + configer := configmocks.NewMockIConfiger(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + evaluatorRecordService := svcmocks.NewMockEvaluatorRecordService(ctrl) + resultService := svcmocks.NewMockExptResultService(ctrl) + templateManager := svcmocks.NewMockIExptTemplateManager(ctrl) + mockExptRunLogRepo := mock_repo.NewMockIExptRunLogRepo(ctrl) + + factory := NewSchedulerModeFactory( + manager, + exptItemResultRepo, + exptStatsRepo, + exptTurnResultRepo, + idgenerator, + evaluationSetItemService, + exptRepo, + idem, + configer, + publisher, + evaluatorRecordService, + resultService, + templateManager, + mockExptRunLogRepo, + ) + + tests := []struct { + name string + mode entity.ExptRunMode + wantType interface{} + wantError bool + }{ + { + name: "retryAll_mode", + mode: entity.EvaluationModeRetryAll, + wantType: &ExptRetryAllExec{}, + wantError: false, + }, + { + name: "retryItems_mode", + mode: entity.EvaluationModeRetryItems, + wantType: &ExptRetryItemsExec{}, + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, err := factory.NewSchedulerMode(tt.mode) + if tt.wantError { + assert.Error(t, err) + assert.Nil(t, mode) + } else { + assert.NoError(t, err) + assert.IsType(t, tt.wantType, mode) + } + }) + } +} From 68b195ff6bc79449fc85fde9f6763f5a575fa99f Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 14:38:07 +0800 Subject: [PATCH 17/30] fix(evaluation): experiment template itemretrynum --- .../application/convertor/experiment/expt_template.go | 3 ++- .../coze/loop/evaluation/coze.loop.evaluation.expt.thrift | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_template.go b/backend/modules/evaluation/application/convertor/experiment/expt_template.go index 2e1d6159e..3c6ffc3d1 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_template.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_template.go @@ -233,7 +233,7 @@ func buildTemplateConfForCreate( templateConf := &entity.ExptTemplateConfiguration{ ItemConcurNum: ptr.ConvIntPtr[int32, int](itemConcurNum), EvaluatorsConcurNum: ptr.ConvIntPtr[int32, int](req.DefaultEvaluatorsConcurNum), - ItemRetryNum: gcond.If(req.GetItemRetryNum() > 0, gptr.Of(int(req.GetItemRetryNum())), nil), + ItemRetryNum: gcond.If(req.GetFieldMappingConfig().GetItemRetryNum() > 0, gptr.Of(int(req.GetFieldMappingConfig().GetItemRetryNum())), nil), } if targetFieldMapping == nil && len(evaluatorConfs) == 0 { @@ -961,6 +961,7 @@ func ConvertUpdateExptTemplateReq(req *expt.UpdateExperimentTemplateRequest) (*e templateConf := &entity.ExptTemplateConfiguration{ ItemConcurNum: ptr.ConvIntPtr[int32, int](itemConcurNum), EvaluatorsConcurNum: ptr.ConvIntPtr[int32, int](req.DefaultEvaluatorsConcurNum), + ItemRetryNum: gcond.If(req.GetFieldMappingConfig().GetItemRetryNum() > 0, gptr.Of(int(req.GetFieldMappingConfig().GetItemRetryNum())), nil), } // 构建 ConnectorConf diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift index e4314b5fa..1835244bd 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.expt.thrift @@ -376,8 +376,6 @@ struct CreateExperimentTemplateRequest { // 调度配置（不在 ExptTemplate 结构中，保留在顶层） 22: optional string schedule_cron (api.body = 'schedule_cron') - 45: optional i32 item_retry_num (api.boy = 'item_retry_num') - 200: optional common.Session session 255: optional base.Base Base } From 0e012df6e4a2998776a4ff12484653bdc3f45826 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 15:25:46 +0800 Subject: [PATCH 18/30] fix(evaluation): runid --- backend/modules/evaluation/application/experiment_app.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/modules/evaluation/application/experiment_app.go b/backend/modules/evaluation/application/experiment_app.go index c7b984faa..9f833275e 100644 --- a/backend/modules/evaluation/application/experiment_app.go +++ b/backend/modules/evaluation/application/experiment_app.go @@ -1015,15 +1015,15 @@ func (e *experimentApplication) RetryExperiment(ctx context.Context, req *expt.R if err != nil { return nil, err } + runID = rid + if !retried { if err := e.manager.RetryItems(ctx, req.GetExptID(), runID, req.GetWorkspaceID(), gptr.Indirect(got.EvalConf.ItemRetryNum), req.GetItemIds(), session, req.GetExt()); err != nil { return nil, err } } - runID = rid default: - runID, err = e.idgen.GenID(ctx) - if err != nil { + if runID, err = e.idgen.GenID(ctx); err != nil { return nil, err } if err := e.manager.LogRun(ctx, req.GetExptID(), runID, runMode, req.GetWorkspaceID(), nil, session); err != nil { From 0c61ec555d1f3af0518e178b02d72c8133288d1c Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 15:40:20 +0800 Subject: [PATCH 19/30] fix(evaluation): appenditemids panic --- backend/modules/evaluation/domain/entity/expt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index aa11985d8..f7d9601a1 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -87,7 +87,7 @@ func (e *ExptRunLog) AppendItemIDs(itemIDs []int64) error { if e == nil { return errorx.New("ExptRunLog AppendItemIDs must init first") } - var exists map[int64]bool + exists := make(map[int64]bool) for _, chunk := range e.ItemIds { for _, itemID := range chunk.ItemIDs { exists[itemID] = true From 5e07bad1f6c7f1fb582328600bf4dd740e8b4721 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 16:24:46 +0800 Subject: [PATCH 20/30] fix(evaluation): unlock --- backend/infra/lock/lock.go | 24 +++++++++++++++ backend/infra/lock/mocks/lock.go | 30 +++++++++++++++++++ .../service/expt_manage_execution_impl.go | 2 +- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/backend/infra/lock/lock.go b/backend/infra/lock/lock.go index eddcd9a97..5f51102c5 100644 --- a/backend/infra/lock/lock.go +++ b/backend/infra/lock/lock.go @@ -32,6 +32,9 @@ type ILocker interface { LockWithRenew(parent context.Context, key string, ttl time.Duration, maxHold time.Duration) (locked bool, ctx context.Context, cancel func(), err error) BackoffLockWithValue(ctx context.Context, key, val string, expiresIn time.Duration, backoff time.Duration) (bool, string, error) + UnlockWithValue(ctx context.Context, key, val string) (bool, error) + // UnlockForce deletes the key without comparing its value. + UnlockForce(ctx context.Context, key string) (bool, error) } func NewRedisLocker(c redis.Cmdable) ILocker { @@ -145,6 +148,27 @@ func (r *redisLocker) Unlock(key string) (bool, error) { return rt == 1, nil } +func (r *redisLocker) UnlockWithValue(ctx context.Context, key, val string) (bool, error) { + const unlockWithValueScript = `if redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]); return 1; end; return 0;` + result, err := r.c.Eval(ctx, unlockWithValueScript, []string{key}, val).Result() + if err != nil { + return false, errors.WithMessage(err, "unlock with lua script") + } + rt, ok := result.(int64) + if !ok { + return false, errors.Errorf("unknown result type %T", result) + } + return rt == 1, nil +} + +func (r *redisLocker) UnlockForce(ctx context.Context, key string) (bool, error) { + n, err := r.c.Del(ctx, key).Result() + if err != nil { + return false, errors.WithMessage(err, "unlock force del fail") + } + return n > 0, nil +} + func (r *redisLocker) renewLock(ctx context.Context, key string, ttl time.Duration, maxHold time.Duration) { t1 := time.After(maxHold) t2 := time.NewTicker(gvalue.Max(time.Second, ttl>>2)) diff --git a/backend/infra/lock/mocks/lock.go b/backend/infra/lock/mocks/lock.go index 6ac00f057..ead5e049b 100644 --- a/backend/infra/lock/mocks/lock.go +++ b/backend/infra/lock/mocks/lock.go @@ -152,6 +152,36 @@ func (mr *MockILockerMockRecorder) Unlock(key any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unlock", reflect.TypeOf((*MockILocker)(nil).Unlock), key) } +// UnlockForce mocks base method. +func (m *MockILocker) UnlockForce(ctx context.Context, key string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockForce", ctx, key) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UnlockForce indicates an expected call of UnlockForce. +func (mr *MockILockerMockRecorder) UnlockForce(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockForce", reflect.TypeOf((*MockILocker)(nil).UnlockForce), ctx, key) +} + +// UnlockWithValue mocks base method. +func (m *MockILocker) UnlockWithValue(ctx context.Context, key, val string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockWithValue", ctx, key, val) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UnlockWithValue indicates an expected call of UnlockWithValue. +func (mr *MockILockerMockRecorder) UnlockWithValue(ctx, key, val any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockWithValue", reflect.TypeOf((*MockILocker)(nil).UnlockWithValue), ctx, key, val) +} + // WithHolder mocks base method. func (m *MockILocker) WithHolder(holder string) lock.ILocker { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index f11cb67f8..9388b4a98 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -362,7 +362,7 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID int6 return err } - if _, err := e.mutex.Unlock(e.makeExptMutexLockKey(exptID)); err != nil { + if _, err := e.mutex.UnlockForce(ctx, e.makeExptMutexLockKey(exptID)); err != nil { return err } From c6c71eb1f5f71d757cea886a6c42867662a43d65 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 17:01:20 +0800 Subject: [PATCH 21/30] fix(evaluation): completing lock --- backend/infra/lock/lock.go | 12 +- backend/infra/lock/mocks/lock.go | 15 ++ .../evaluation/domain/service/expt_manage.go | 2 + .../service/expt_manage_execution_impl.go | 33 ++++ .../domain/service/expt_manage_impl.go | 4 + .../service/expt_run_scheduler_mode_impl.go | 13 +- .../service/mocks/evaluation_analysis.go | 88 +++++++++ .../domain/service/mocks/evaluation_set.go | 30 +-- .../domain/service/mocks/expt_manage.go | 15 ++ .../domain/service/mocks/expt_result.go | 28 +-- .../domain/service/mocks/expt_template.go | 176 ++++++++++++++++++ .../evaluation/pkg/errno/evaluation.go | 10 + backend/script/errorx/evaluation.yaml | 6 + 13 files changed, 398 insertions(+), 34 deletions(-) create mode 100644 backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go create mode 100644 backend/modules/evaluation/domain/service/mocks/expt_template.go diff --git a/backend/infra/lock/lock.go b/backend/infra/lock/lock.go index 5f51102c5..5084bc64f 100644 --- a/backend/infra/lock/lock.go +++ b/backend/infra/lock/lock.go @@ -35,6 +35,8 @@ type ILocker interface { UnlockWithValue(ctx context.Context, key, val string) (bool, error) // UnlockForce deletes the key without comparing its value. UnlockForce(ctx context.Context, key string) (bool, error) + // Exists returns true if the key exists. + Exists(ctx context.Context, key string) (bool, error) } func NewRedisLocker(c redis.Cmdable) ILocker { @@ -164,7 +166,15 @@ func (r *redisLocker) UnlockWithValue(ctx context.Context, key, val string) (boo func (r *redisLocker) UnlockForce(ctx context.Context, key string) (bool, error) { n, err := r.c.Del(ctx, key).Result() if err != nil { - return false, errors.WithMessage(err, "unlock force del fail") + return false, errors.WithMessage(err, "unlock force del") + } + return n > 0, nil +} + +func (r *redisLocker) Exists(ctx context.Context, key string) (bool, error) { + n, err := r.c.Exists(ctx, key).Result() + if err != nil { + return false, errors.WithMessage(err, "exists") } return n > 0, nil } diff --git a/backend/infra/lock/mocks/lock.go b/backend/infra/lock/mocks/lock.go index ead5e049b..0a081f4b5 100644 --- a/backend/infra/lock/mocks/lock.go +++ b/backend/infra/lock/mocks/lock.go @@ -58,6 +58,21 @@ func (mr *MockILockerMockRecorder) BackoffLockWithValue(ctx, key, val, expiresIn return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackoffLockWithValue", reflect.TypeOf((*MockILocker)(nil).BackoffLockWithValue), ctx, key, val, expiresIn, backoff) } +// Exists mocks base method. +func (m *MockILocker) Exists(ctx context.Context, key string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, key) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockILockerMockRecorder) Exists(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockILocker)(nil).Exists), ctx, key) +} + // ExpireLockIn mocks base method. func (m *MockILocker) ExpireLockIn(key string, expiresIn time.Duration) (bool, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/expt_manage.go b/backend/modules/evaluation/domain/service/expt_manage.go index b60d65311..daf052a33 100644 --- a/backend/modules/evaluation/domain/service/expt_manage.go +++ b/backend/modules/evaluation/domain/service/expt_manage.go @@ -48,6 +48,8 @@ type IExptExecutionManager interface { PendRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error PendExpt(ctx context.Context, exptID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error + // IsCompletingRun returns true if the given run is currently in the completing phase. + IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) CompleteRun(ctx context.Context, exptID, exptRunID int64, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error CompleteExpt(ctx context.Context, exptID, spaceID int64, session *entity.Session, opts ...entity.CompleteExptOptionFn) error // SetExptTerminating Set experiment/run_log status to "terminating". diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go index 9388b4a98..ed9ab1880 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl.go @@ -353,6 +353,11 @@ func (e *ExptMangerImpl) CompleteRun(ctx context.Context, exptID, exptRunID int6 } } + if err := e.lockCompletingRun(ctx, exptID, exptRunID, spaceID, session); err != nil { + return err + } + defer func() { _ = e.unlockCompletingRun(ctx, exptID, exptRunID, spaceID, session) }() + runLog, err := e.runLogRepo.Get(ctx, exptID, exptRunID) if err != nil { return err @@ -912,6 +917,26 @@ func (e *ExptMangerImpl) PendExpt(ctx context.Context, exptID, spaceID int64, se return nil } +func (e *ExptMangerImpl) IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { + return e.mutex.Exists(ctx, e.makeExptCompletingLockKey(exptID, exptRunID)) +} + +func (e *ExptMangerImpl) lockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + locked, err := e.mutex.Lock(ctx, e.makeExptCompletingLockKey(exptID, exptRunID), time.Minute*3) + if err != nil { + return err + } + if !locked { + return errorx.New("lockCompletingRun fail, expt_id: %v, expt_run_id: %v", exptID, exptRunID) + } + return nil +} + +func (e *ExptMangerImpl) unlockCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64, session *entity.Session) error { + _, err := e.mutex.Unlock(e.makeExptCompletingLockKey(exptID, exptRunID)) + return err +} + func (e *ExptMangerImpl) LogRun(ctx context.Context, exptID, exptRunID int64, mode entity.ExptRunMode, spaceID int64, itemIDs []int64, session *entity.Session) error { duration := time.Duration(e.configer.GetExptExecConf(ctx, spaceID).GetZombieIntervalSecond()) * time.Second locked, err := e.mutex.LockBackoff(ctx, e.makeExptMutexLockKey(exptID), duration, time.Second) @@ -974,6 +999,14 @@ func (e *ExptMangerImpl) LogRetryItemsRun(ctx context.Context, exptID int64, mod return 0, false, errorx.NewByCode(errno.ExperimentRunningExistedCode) } + completing, err := e.IsCompletingRun(ctx, exptID, runID, spaceID) + if err != nil { + return 0, false, err + } + if completing { + return 0, false, errorx.NewByCode(errno.ExperimentIsCompletingCode) + } + rl, err = e.runLogRepo.Get(ctx, exptID, runID) if err != nil { return 0, false, err diff --git a/backend/modules/evaluation/domain/service/expt_manage_impl.go b/backend/modules/evaluation/domain/service/expt_manage_impl.go index 126609ad2..042cddd5d 100644 --- a/backend/modules/evaluation/domain/service/expt_manage_impl.go +++ b/backend/modules/evaluation/domain/service/expt_manage_impl.go @@ -316,6 +316,10 @@ func (e *ExptMangerImpl) makeExptMutexLockKey(exptID int64) string { return fmt.Sprintf("expt_run_mutex_lock:%d", exptID) } +func (e *ExptMangerImpl) makeExptCompletingLockKey(exptID, exptRunID int64) string { + return fmt.Sprintf("expt_completing_mutex_lock:%d:%d", exptID, exptRunID) +} + func (e *ExptMangerImpl) getTupleByExpt(ctx context.Context, expt *entity.Experiment, spaceID int64, session *entity.Session, opts ...entity.GetExptTupleOptionFn) (*entity.ExptTuple, error) { return e.getExptTupleByID(ctx, e.packTupleID(ctx, expt), spaceID, session, opts...) } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 61d653208..7de2dfb56 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -1365,11 +1365,16 @@ func (e *ExptRetryItemsExec) ScanEvalItems(ctx context.Context, event *entity.Ex } func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, toSubmit, incomplete int) (nextTick bool, err error) { - if toSubmit == 0 && incomplete == 0 { - logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) - return false, newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt) + if toSubmit > 0 || incomplete > 0 { + return true, nil } - return true, nil + + logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + + if err := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt); err != nil { + return false, err + } + return false, nil } func (e *ExptRetryItemsExec) ScheduleStart(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment) error { diff --git a/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go b/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go new file mode 100644 index 000000000..9b85bc60f --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go @@ -0,0 +1,88 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IEvaluationAnalysisService) +// +// Generated by this command: +// +// mockgen -destination=mocks/evaluation_analysis.go -package=mocks . IEvaluationAnalysisService +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + service "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" + gomock "go.uber.org/mock/gomock" +) + +// MockIEvaluationAnalysisService is a mock of IEvaluationAnalysisService interface. +type MockIEvaluationAnalysisService struct { + ctrl *gomock.Controller + recorder *MockIEvaluationAnalysisServiceMockRecorder + isgomock struct{} +} + +// MockIEvaluationAnalysisServiceMockRecorder is the mock recorder for MockIEvaluationAnalysisService. +type MockIEvaluationAnalysisServiceMockRecorder struct { + mock *MockIEvaluationAnalysisService +} + +// NewMockIEvaluationAnalysisService creates a new mock instance. +func NewMockIEvaluationAnalysisService(ctrl *gomock.Controller) *MockIEvaluationAnalysisService { + mock := &MockIEvaluationAnalysisService{ctrl: ctrl} + mock.recorder = &MockIEvaluationAnalysisServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIEvaluationAnalysisService) EXPECT() *MockIEvaluationAnalysisServiceMockRecorder { + return m.recorder +} + +// BatchGetAnalysisRecordByUniqueKeys mocks base method. +func (m *MockIEvaluationAnalysisService) BatchGetAnalysisRecordByUniqueKeys(ctx context.Context, uniqueKey []string) (map[string]*entity.AnalysisRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchGetAnalysisRecordByUniqueKeys", ctx, uniqueKey) + ret0, _ := ret[0].(map[string]*entity.AnalysisRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BatchGetAnalysisRecordByUniqueKeys indicates an expected call of BatchGetAnalysisRecordByUniqueKeys. +func (mr *MockIEvaluationAnalysisServiceMockRecorder) BatchGetAnalysisRecordByUniqueKeys(ctx, uniqueKey any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchGetAnalysisRecordByUniqueKeys", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).BatchGetAnalysisRecordByUniqueKeys), ctx, uniqueKey) +} + +// GetAnalysisRecord mocks base method. +func (m *MockIEvaluationAnalysisService) GetAnalysisRecord(ctx context.Context, id, spaceID int64) (*entity.AnalysisRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAnalysisRecord", ctx, id, spaceID) + ret0, _ := ret[0].(*entity.AnalysisRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAnalysisRecord indicates an expected call of GetAnalysisRecord. +func (mr *MockIEvaluationAnalysisServiceMockRecorder) GetAnalysisRecord(ctx, id, spaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalysisRecord", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).GetAnalysisRecord), ctx, id, spaceID) +} + +// TrajectoryAnalysis mocks base method. +func (m *MockIEvaluationAnalysisService) TrajectoryAnalysis(ctx context.Context, param service.TrajectoryAnalysisParam) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TrajectoryAnalysis", ctx, param) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TrajectoryAnalysis indicates an expected call of TrajectoryAnalysis. +func (mr *MockIEvaluationAnalysisServiceMockRecorder) TrajectoryAnalysis(ctx, param any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TrajectoryAnalysis", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).TrajectoryAnalysis), ctx, param) +} diff --git a/backend/modules/evaluation/domain/service/mocks/evaluation_set.go b/backend/modules/evaluation/domain/service/mocks/evaluation_set.go index 31e6f5d96..d6fb354d4 100644 --- a/backend/modules/evaluation/domain/service/mocks/evaluation_set.go +++ b/backend/modules/evaluation/domain/service/mocks/evaluation_set.go @@ -87,21 +87,6 @@ func (mr *MockIEvaluationSetServiceMockRecorder) CreateEvaluationSetWithImport(c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEvaluationSetWithImport", reflect.TypeOf((*MockIEvaluationSetService)(nil).CreateEvaluationSetWithImport), ctx, param) } -// ParseImportSourceFile mocks base method. -func (m *MockIEvaluationSetService) ParseImportSourceFile(ctx context.Context, param *entity.ParseImportSourceFileParam) (*entity.ParseImportSourceFileResult, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseImportSourceFile", ctx, param) - ret0, _ := ret[0].(*entity.ParseImportSourceFileResult) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseImportSourceFile indicates an expected call of ParseImportSourceFile. -func (mr *MockIEvaluationSetServiceMockRecorder) ParseImportSourceFile(ctx, param any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseImportSourceFile", reflect.TypeOf((*MockIEvaluationSetService)(nil).ParseImportSourceFile), ctx, param) -} - // DeleteEvaluationSet mocks base method. func (m *MockIEvaluationSetService) DeleteEvaluationSet(ctx context.Context, spaceID, evaluationSetID int64) error { m.ctrl.T.Helper() @@ -148,6 +133,21 @@ func (mr *MockIEvaluationSetServiceMockRecorder) ListEvaluationSets(ctx, param a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEvaluationSets", reflect.TypeOf((*MockIEvaluationSetService)(nil).ListEvaluationSets), ctx, param) } +// ParseImportSourceFile mocks base method. +func (m *MockIEvaluationSetService) ParseImportSourceFile(ctx context.Context, param *entity.ParseImportSourceFileParam) (*entity.ParseImportSourceFileResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ParseImportSourceFile", ctx, param) + ret0, _ := ret[0].(*entity.ParseImportSourceFileResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ParseImportSourceFile indicates an expected call of ParseImportSourceFile. +func (mr *MockIEvaluationSetServiceMockRecorder) ParseImportSourceFile(ctx, param any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseImportSourceFile", reflect.TypeOf((*MockIEvaluationSetService)(nil).ParseImportSourceFile), ctx, param) +} + // QueryItemSnapshotMappings mocks base method. func (m *MockIEvaluationSetService) QueryItemSnapshotMappings(ctx context.Context, spaceID, datasetID int64, versionID *int64) ([]*entity.ItemSnapshotFieldMapping, string, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/mocks/expt_manage.go b/backend/modules/evaluation/domain/service/mocks/expt_manage.go index ecfb8c40c..43a46519b 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_manage.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_manage.go @@ -235,6 +235,21 @@ func (mr *MockIExptManagerMockRecorder) Invoke(ctx, invokeExptReq any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Invoke", reflect.TypeOf((*MockIExptManager)(nil).Invoke), ctx, invokeExptReq) } +// IsRunCompleting mocks base method. +func (m *MockIExptManager) IsCompletingRun(ctx context.Context, exptID, exptRunID, spaceID int64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCompletingRun", ctx, exptID, exptRunID, spaceID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsRunCompleting indicates an expected call of IsRunCompleting. +func (mr *MockIExptManagerMockRecorder) IsRunCompleting(ctx, exptID, exptRunID, spaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCompletingRun", reflect.TypeOf((*MockIExptManager)(nil).IsCompletingRun), ctx, exptID, exptRunID, spaceID) +} + // List mocks base method. func (m *MockIExptManager) List(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptListFilter, orders []*entity.OrderBy, session *entity.Session) ([]*entity.Experiment, int64, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/mocks/expt_result.go b/backend/modules/evaluation/domain/service/mocks/expt_result.go index 812fb85ee..d9447d5d6 100644 --- a/backend/modules/evaluation/domain/service/mocks/expt_result.go +++ b/backend/modules/evaluation/domain/service/mocks/expt_result.go @@ -188,6 +188,20 @@ func (mr *MockExptResultServiceMockRecorder) ManualUpsertExptTurnResultFilter(ct return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManualUpsertExptTurnResultFilter", reflect.TypeOf((*MockExptResultService)(nil).ManualUpsertExptTurnResultFilter), ctx, spaceID, exptID, itemIDs) } +// RecalculateWeightedScore mocks base method. +func (m *MockExptResultService) RecalculateWeightedScore(ctx context.Context, spaceID, exptID, itemID, turnID int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecalculateWeightedScore", ctx, spaceID, exptID, itemID, turnID) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecalculateWeightedScore indicates an expected call of RecalculateWeightedScore. +func (mr *MockExptResultServiceMockRecorder) RecalculateWeightedScore(ctx, spaceID, exptID, itemID, turnID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecalculateWeightedScore", reflect.TypeOf((*MockExptResultService)(nil).RecalculateWeightedScore), ctx, spaceID, exptID, itemID, turnID) +} + // RecordItemRunLogs mocks base method. func (m *MockExptResultService) RecordItemRunLogs(ctx context.Context, exptID, exptRunID, itemID, spaceID int64) ([]*entity.ExptTurnEvaluatorResultRef, error) { m.ctrl.T.Helper() @@ -217,20 +231,6 @@ func (mr *MockExptResultServiceMockRecorder) UpsertExptTurnResultFilter(ctx, spa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertExptTurnResultFilter", reflect.TypeOf((*MockExptResultService)(nil).UpsertExptTurnResultFilter), ctx, spaceID, exptID, itemID) } -// RecalculateWeightedScore mocks base method. -func (m *MockExptResultService) RecalculateWeightedScore(ctx context.Context, spaceID, exptID, itemID, turnID int64) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecalculateWeightedScore", ctx, spaceID, exptID, itemID, turnID) - ret0, _ := ret[0].(error) - return ret0 -} - -// RecalculateWeightedScore indicates an expected call of RecalculateWeightedScore. -func (mr *MockExptResultServiceMockRecorder) RecalculateWeightedScore(ctx, spaceID, exptID, itemID, turnID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecalculateWeightedScore", reflect.TypeOf((*MockExptResultService)(nil).RecalculateWeightedScore), ctx, spaceID, exptID, itemID, turnID) -} - // MockExptAggrResultService is a mock of ExptAggrResultService interface. type MockExptAggrResultService struct { ctrl *gomock.Controller diff --git a/backend/modules/evaluation/domain/service/mocks/expt_template.go b/backend/modules/evaluation/domain/service/mocks/expt_template.go new file mode 100644 index 000000000..9f5bc9578 --- /dev/null +++ b/backend/modules/evaluation/domain/service/mocks/expt_template.go @@ -0,0 +1,176 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IExptTemplateManager) +// +// Generated by this command: +// +// mockgen -destination ./mocks/expt_template.go --package mocks . IExptTemplateManager +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockIExptTemplateManager is a mock of IExptTemplateManager interface. +type MockIExptTemplateManager struct { + ctrl *gomock.Controller + recorder *MockIExptTemplateManagerMockRecorder + isgomock struct{} +} + +// MockIExptTemplateManagerMockRecorder is the mock recorder for MockIExptTemplateManager. +type MockIExptTemplateManagerMockRecorder struct { + mock *MockIExptTemplateManager +} + +// NewMockIExptTemplateManager creates a new mock instance. +func NewMockIExptTemplateManager(ctrl *gomock.Controller) *MockIExptTemplateManager { + mock := &MockIExptTemplateManager{ctrl: ctrl} + mock.recorder = &MockIExptTemplateManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIExptTemplateManager) EXPECT() *MockIExptTemplateManagerMockRecorder { + return m.recorder +} + +// CheckName mocks base method. +func (m *MockIExptTemplateManager) CheckName(ctx context.Context, name string, spaceID int64, session *entity.Session) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckName", ctx, name, spaceID, session) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckName indicates an expected call of CheckName. +func (mr *MockIExptTemplateManagerMockRecorder) CheckName(ctx, name, spaceID, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckName", reflect.TypeOf((*MockIExptTemplateManager)(nil).CheckName), ctx, name, spaceID, session) +} + +// Create mocks base method. +func (m *MockIExptTemplateManager) Create(ctx context.Context, param *entity.CreateExptTemplateParam, session *entity.Session) (*entity.ExptTemplate, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, param, session) + ret0, _ := ret[0].(*entity.ExptTemplate) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockIExptTemplateManagerMockRecorder) Create(ctx, param, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockIExptTemplateManager)(nil).Create), ctx, param, session) +} + +// Delete mocks base method. +func (m *MockIExptTemplateManager) Delete(ctx context.Context, templateID, spaceID int64, session *entity.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, templateID, spaceID, session) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockIExptTemplateManagerMockRecorder) Delete(ctx, templateID, spaceID, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptTemplateManager)(nil).Delete), ctx, templateID, spaceID, session) +} + +// Get mocks base method. +func (m *MockIExptTemplateManager) Get(ctx context.Context, templateID, spaceID int64, session *entity.Session) (*entity.ExptTemplate, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, templateID, spaceID, session) + ret0, _ := ret[0].(*entity.ExptTemplate) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockIExptTemplateManagerMockRecorder) Get(ctx, templateID, spaceID, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptTemplateManager)(nil).Get), ctx, templateID, spaceID, session) +} + +// List mocks base method. +func (m *MockIExptTemplateManager) List(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptTemplateListFilter, orderBys []*entity.OrderBy, session *entity.Session) ([]*entity.ExptTemplate, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, page, pageSize, spaceID, filter, orderBys, session) + ret0, _ := ret[0].([]*entity.ExptTemplate) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// List indicates an expected call of List. +func (mr *MockIExptTemplateManagerMockRecorder) List(ctx, page, pageSize, spaceID, filter, orderBys, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptTemplateManager)(nil).List), ctx, page, pageSize, spaceID, filter, orderBys, session) +} + +// MGet mocks base method. +func (m *MockIExptTemplateManager) MGet(ctx context.Context, templateIDs []int64, spaceID int64, session *entity.Session) ([]*entity.ExptTemplate, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MGet", ctx, templateIDs, spaceID, session) + ret0, _ := ret[0].([]*entity.ExptTemplate) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MGet indicates an expected call of MGet. +func (mr *MockIExptTemplateManagerMockRecorder) MGet(ctx, templateIDs, spaceID, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGet", reflect.TypeOf((*MockIExptTemplateManager)(nil).MGet), ctx, templateIDs, spaceID, session) +} + +// Update mocks base method. +func (m *MockIExptTemplateManager) Update(ctx context.Context, param *entity.UpdateExptTemplateParam, session *entity.Session) (*entity.ExptTemplate, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, param, session) + ret0, _ := ret[0].(*entity.ExptTemplate) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockIExptTemplateManagerMockRecorder) Update(ctx, param, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptTemplateManager)(nil).Update), ctx, param, session) +} + +// UpdateExptInfo mocks base method. +func (m *MockIExptTemplateManager) UpdateExptInfo(ctx context.Context, templateID, spaceID, exptID int64, exptStatus entity.ExptStatus, adjustCount int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateExptInfo", ctx, templateID, spaceID, exptID, exptStatus, adjustCount) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateExptInfo indicates an expected call of UpdateExptInfo. +func (mr *MockIExptTemplateManagerMockRecorder) UpdateExptInfo(ctx, templateID, spaceID, exptID, exptStatus, adjustCount any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExptInfo", reflect.TypeOf((*MockIExptTemplateManager)(nil).UpdateExptInfo), ctx, templateID, spaceID, exptID, exptStatus, adjustCount) +} + +// UpdateMeta mocks base method. +func (m *MockIExptTemplateManager) UpdateMeta(ctx context.Context, param *entity.UpdateExptTemplateMetaParam, session *entity.Session) (*entity.ExptTemplate, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateMeta", ctx, param, session) + ret0, _ := ret[0].(*entity.ExptTemplate) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateMeta indicates an expected call of UpdateMeta. +func (mr *MockIExptTemplateManagerMockRecorder) UpdateMeta(ctx, param, session any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMeta", reflect.TypeOf((*MockIExptTemplateManager)(nil).UpdateMeta), ctx, param, session) +} diff --git a/backend/modules/evaluation/pkg/errno/evaluation.go b/backend/modules/evaluation/pkg/errno/evaluation.go index 233f40438..8d10456ce 100644 --- a/backend/modules/evaluation/pkg/errno/evaluation.go +++ b/backend/modules/evaluation/pkg/errno/evaluation.go @@ -128,6 +128,10 @@ const ( evalItemAlreadyRetryingMessage = "item already has been retrying" evalItemAlreadyRetryingNoAffectStability = true + ExperimentIsCompletingCode = 601204016 // experiment is completing, please try later + experimentIsCompletingMessage = "experiment is completing, please try later" + experimentIsCompletingNoAffectStability = true + ContentTypeNotSupportedCode = 601205000 // content type is not supported contentTypeNotSupportedMessage = "content type is not supported" contentTypeNotSupportedNoAffectStability = true @@ -547,6 +551,12 @@ func init() { code.WithAffectStability(!evalItemAlreadyRetryingNoAffectStability), ) + code.Register( + ExperimentIsCompletingCode, + experimentIsCompletingMessage, + code.WithAffectStability(!experimentIsCompletingNoAffectStability), + ) + code.Register( ContentTypeNotSupportedCode, contentTypeNotSupportedMessage, diff --git a/backend/script/errorx/evaluation.yaml b/backend/script/errorx/evaluation.yaml index 407c546b6..fc1ea20a5 100644 --- a/backend/script/errorx/evaluation.yaml +++ b/backend/script/errorx/evaluation.yaml @@ -118,6 +118,12 @@ error_code: description: item already has been retrying no_affect_stability: true + - name: ExperimentIsCompleting + code: 4016 + message: experiment is completing, please try later + description: experiment is completing, please try later + no_affect_stability: true + - name: ContentTypeNotSupported code: 5000 message: content type is not supported From ffff714adf32f8ecdcbde5af1b3e3bce026fc121 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 18:27:07 +0800 Subject: [PATCH 22/30] fix(evaluation): retryitems epxtend --- .../domain/service/expt_run_scheduler_event_impl.go | 8 ++++---- .../domain/service/expt_run_scheduler_mode_impl.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index a3b9c0402..8d2e195c7 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -265,17 +265,17 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche return err } - err = mode.ScheduleEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) - if err != nil { + if err = e.handleToSubmits(ctx, event, toSubmit); err != nil { return err } - nextTick, err := mode.ExptEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) + err = mode.ScheduleEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) if err != nil { return err } - if err = e.handleToSubmits(ctx, event, toSubmit); err != nil { + nextTick, err := mode.ExptEnd(ctx, event, exptDetail, len(toSubmit), len(incomplete)) + if err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 7de2dfb56..ca52a295d 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -1371,6 +1371,18 @@ func (e *ExptRetryItemsExec) ExptEnd(ctx context.Context, event *entity.ExptSche logs.CtxInfo(ctx, "[ExptEval] expt daemon finished, expt_id: %v, expt_run_id: %v", event.ExptID, event.ExptRunID) + got, err := e.exptRunLogRepo.Get(ctx, event.ExptID, event.ExptRunID) + if err != nil { + return false, err + } + + exist := gslice.ToMap(event.ExecEvalSetItemIDs, func(t int64) (int64, bool) { return t, true }) + for _, itemID := range got.GetItemIDs() { + if !exist[itemID] { + return true, nil + } + } + if err := newExptBaseExec(e.manager, e.idem, e.configer, e.exptItemResultRepo, e.publisher, e.evaluatorRecordService).exptEnd(ctx, event, expt); err != nil { return false, err } From 0e73f1f249c90affea57f5a5aaad6ff1f363bd4f Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 18:39:51 +0800 Subject: [PATCH 23/30] fix(evaluation): evaluator reenter --- .../evaluation/domain/service/expt_run_item_turn_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index f10d74360..d2713347b 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -266,7 +266,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallEvaluators(ctx context.Context, etec for _, evaluatorVersion := range expt.Evaluators { existResult := etec.ExptTurnRunResult.GetEvaluatorRecord(evaluatorVersion.GetEvaluatorVersionID()) - if etec.Event.IgnoreExistedResult() && existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { + if !etec.Event.IgnoreExistedResult() && existResult != nil && existResult.Status == entity.EvaluatorRunStatusSuccess { evaluatorResults[existResult.ID] = existResult continue } From 6ded8f84df546da15996e5b9a205c359bf72b476 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 20:25:40 +0800 Subject: [PATCH 24/30] fix(evaluation): expt event expiration --- .../evaluation/domain/service/expt_run_item_event_impl.go | 2 +- .../evaluation/domain/service/expt_run_scheduler_event_impl.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go index 263384437..c536f273f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_event_impl.go @@ -207,7 +207,7 @@ func (e *ExptItemEventEvalServiceImpl) HandleEventErr(next RecordEvalEndPoint) R func (e *ExptItemEventEvalServiceImpl) HandleEventLock(next RecordEvalEndPoint) RecordEvalEndPoint { return func(ctx context.Context, event *entity.ExptItemEvalEvent) error { lockKey := fmt.Sprintf("expt_item_eval_run_lock:%d:%d", event.ExptID, event.EvalSetItemID) - locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*20, time.Second*60*60) + locked, ctx, cancel, err := e.mutex.LockWithRenew(ctx, lockKey, time.Second*10, time.Second*60*60) if err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 8d2e195c7..5f5878f7e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -163,7 +163,7 @@ func (e *ExptSchedulerImpl) makeExptRunExecLockKey(exptID, exptRunID int64) stri func (e *ExptSchedulerImpl) HandleEventLock(next SchedulerEndPoint) SchedulerEndPoint { return func(ctx context.Context, event *entity.ExptScheduleEvent) error { key := e.makeExptRunExecLockKey(event.ExptID, event.ExptRunID) - locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*20, time.Second*60*5) + locked, ctx, cancel, err := e.Mutex.LockWithRenew(ctx, key, time.Second*10, time.Second*60*5) if err != nil { return err } From 7f838b41d2611a55d48c43d5ecff911f8adc493f Mon Sep 17 00:00:00 2001 From: liushengyang Date: Thu, 26 Feb 2026 20:57:15 +0800 Subject: [PATCH 25/30] fix(evaluation): mockgen --- .../modules/evaluation/domain/entity/expt.go | 4 +- .../service/mocks/evaluation_analysis.go | 88 --------- .../domain/service/mocks/expt_export.go.rej | 12 -- .../domain/service/mocks/expt_template.go | 176 ------------------ 4 files changed, 1 insertion(+), 279 deletions(-) delete mode 100644 backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go delete mode 100644 backend/modules/evaluation/domain/service/mocks/expt_export.go.rej delete mode 100644 backend/modules/evaluation/domain/service/mocks/expt_template.go diff --git a/backend/modules/evaluation/domain/entity/expt.go b/backend/modules/evaluation/domain/entity/expt.go index f7d9601a1..ada136478 100644 --- a/backend/modules/evaluation/domain/entity/expt.go +++ b/backend/modules/evaluation/domain/entity/expt.go @@ -76,9 +76,7 @@ type ExptRunLog struct { func (e *ExptRunLog) GetItemIDs() []int64 { var itemIDs []int64 for _, items := range e.ItemIds { - for _, itemID := range items.ItemIDs { - itemIDs = append(itemIDs, itemID) - } + itemIDs = append(itemIDs, items.ItemIDs...) } return itemIDs } diff --git a/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go b/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go deleted file mode 100644 index 9b85bc60f..000000000 --- a/backend/modules/evaluation/domain/service/mocks/evaluation_analysis.go +++ /dev/null @@ -1,88 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IEvaluationAnalysisService) -// -// Generated by this command: -// -// mockgen -destination=mocks/evaluation_analysis.go -package=mocks . IEvaluationAnalysisService -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - service "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service" - gomock "go.uber.org/mock/gomock" -) - -// MockIEvaluationAnalysisService is a mock of IEvaluationAnalysisService interface. -type MockIEvaluationAnalysisService struct { - ctrl *gomock.Controller - recorder *MockIEvaluationAnalysisServiceMockRecorder - isgomock struct{} -} - -// MockIEvaluationAnalysisServiceMockRecorder is the mock recorder for MockIEvaluationAnalysisService. -type MockIEvaluationAnalysisServiceMockRecorder struct { - mock *MockIEvaluationAnalysisService -} - -// NewMockIEvaluationAnalysisService creates a new mock instance. -func NewMockIEvaluationAnalysisService(ctrl *gomock.Controller) *MockIEvaluationAnalysisService { - mock := &MockIEvaluationAnalysisService{ctrl: ctrl} - mock.recorder = &MockIEvaluationAnalysisServiceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockIEvaluationAnalysisService) EXPECT() *MockIEvaluationAnalysisServiceMockRecorder { - return m.recorder -} - -// BatchGetAnalysisRecordByUniqueKeys mocks base method. -func (m *MockIEvaluationAnalysisService) BatchGetAnalysisRecordByUniqueKeys(ctx context.Context, uniqueKey []string) (map[string]*entity.AnalysisRecord, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BatchGetAnalysisRecordByUniqueKeys", ctx, uniqueKey) - ret0, _ := ret[0].(map[string]*entity.AnalysisRecord) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// BatchGetAnalysisRecordByUniqueKeys indicates an expected call of BatchGetAnalysisRecordByUniqueKeys. -func (mr *MockIEvaluationAnalysisServiceMockRecorder) BatchGetAnalysisRecordByUniqueKeys(ctx, uniqueKey any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchGetAnalysisRecordByUniqueKeys", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).BatchGetAnalysisRecordByUniqueKeys), ctx, uniqueKey) -} - -// GetAnalysisRecord mocks base method. -func (m *MockIEvaluationAnalysisService) GetAnalysisRecord(ctx context.Context, id, spaceID int64) (*entity.AnalysisRecord, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAnalysisRecord", ctx, id, spaceID) - ret0, _ := ret[0].(*entity.AnalysisRecord) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAnalysisRecord indicates an expected call of GetAnalysisRecord. -func (mr *MockIEvaluationAnalysisServiceMockRecorder) GetAnalysisRecord(ctx, id, spaceID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalysisRecord", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).GetAnalysisRecord), ctx, id, spaceID) -} - -// TrajectoryAnalysis mocks base method. -func (m *MockIEvaluationAnalysisService) TrajectoryAnalysis(ctx context.Context, param service.TrajectoryAnalysisParam) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TrajectoryAnalysis", ctx, param) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// TrajectoryAnalysis indicates an expected call of TrajectoryAnalysis. -func (mr *MockIEvaluationAnalysisServiceMockRecorder) TrajectoryAnalysis(ctx, param any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TrajectoryAnalysis", reflect.TypeOf((*MockIEvaluationAnalysisService)(nil).TrajectoryAnalysis), ctx, param) -} diff --git a/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej b/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej deleted file mode 100644 index 3d4f8e238..000000000 --- a/backend/modules/evaluation/domain/service/mocks/expt_export.go.rej +++ /dev/null @@ -1,12 +0,0 @@ -diff a/backend/modules/evaluation/domain/service/mocks/expt_export.go b/backend/modules/evaluation/domain/service/mocks/expt_export.go (rejected hunks) -@@ -8,9 +8,8 @@ import ( - context "context" - reflect "reflect" - -- gomock "go.uber.org/mock/gomock" -- - entity "code.byted.org/flowdevops/cozeloop/backend/modules/evaluation/domain/entity" -+ "go.uber.org/mock/gomock" - ) - - // MockIExptResultExportService is a mock of IExptResultExportService interface. diff --git a/backend/modules/evaluation/domain/service/mocks/expt_template.go b/backend/modules/evaluation/domain/service/mocks/expt_template.go deleted file mode 100644 index 9f5bc9578..000000000 --- a/backend/modules/evaluation/domain/service/mocks/expt_template.go +++ /dev/null @@ -1,176 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service (interfaces: IExptTemplateManager) -// -// Generated by this command: -// -// mockgen -destination ./mocks/expt_template.go --package mocks . IExptTemplateManager -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - entity "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" - gomock "go.uber.org/mock/gomock" -) - -// MockIExptTemplateManager is a mock of IExptTemplateManager interface. -type MockIExptTemplateManager struct { - ctrl *gomock.Controller - recorder *MockIExptTemplateManagerMockRecorder - isgomock struct{} -} - -// MockIExptTemplateManagerMockRecorder is the mock recorder for MockIExptTemplateManager. -type MockIExptTemplateManagerMockRecorder struct { - mock *MockIExptTemplateManager -} - -// NewMockIExptTemplateManager creates a new mock instance. -func NewMockIExptTemplateManager(ctrl *gomock.Controller) *MockIExptTemplateManager { - mock := &MockIExptTemplateManager{ctrl: ctrl} - mock.recorder = &MockIExptTemplateManagerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockIExptTemplateManager) EXPECT() *MockIExptTemplateManagerMockRecorder { - return m.recorder -} - -// CheckName mocks base method. -func (m *MockIExptTemplateManager) CheckName(ctx context.Context, name string, spaceID int64, session *entity.Session) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckName", ctx, name, spaceID, session) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckName indicates an expected call of CheckName. -func (mr *MockIExptTemplateManagerMockRecorder) CheckName(ctx, name, spaceID, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckName", reflect.TypeOf((*MockIExptTemplateManager)(nil).CheckName), ctx, name, spaceID, session) -} - -// Create mocks base method. -func (m *MockIExptTemplateManager) Create(ctx context.Context, param *entity.CreateExptTemplateParam, session *entity.Session) (*entity.ExptTemplate, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, param, session) - ret0, _ := ret[0].(*entity.ExptTemplate) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create. -func (mr *MockIExptTemplateManagerMockRecorder) Create(ctx, param, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockIExptTemplateManager)(nil).Create), ctx, param, session) -} - -// Delete mocks base method. -func (m *MockIExptTemplateManager) Delete(ctx context.Context, templateID, spaceID int64, session *entity.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", ctx, templateID, spaceID, session) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockIExptTemplateManagerMockRecorder) Delete(ctx, templateID, spaceID, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockIExptTemplateManager)(nil).Delete), ctx, templateID, spaceID, session) -} - -// Get mocks base method. -func (m *MockIExptTemplateManager) Get(ctx context.Context, templateID, spaceID int64, session *entity.Session) (*entity.ExptTemplate, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ctx, templateID, spaceID, session) - ret0, _ := ret[0].(*entity.ExptTemplate) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockIExptTemplateManagerMockRecorder) Get(ctx, templateID, spaceID, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIExptTemplateManager)(nil).Get), ctx, templateID, spaceID, session) -} - -// List mocks base method. -func (m *MockIExptTemplateManager) List(ctx context.Context, page, pageSize int32, spaceID int64, filter *entity.ExptTemplateListFilter, orderBys []*entity.OrderBy, session *entity.Session) ([]*entity.ExptTemplate, int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", ctx, page, pageSize, spaceID, filter, orderBys, session) - ret0, _ := ret[0].([]*entity.ExptTemplate) - ret1, _ := ret[1].(int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// List indicates an expected call of List. -func (mr *MockIExptTemplateManagerMockRecorder) List(ctx, page, pageSize, spaceID, filter, orderBys, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIExptTemplateManager)(nil).List), ctx, page, pageSize, spaceID, filter, orderBys, session) -} - -// MGet mocks base method. -func (m *MockIExptTemplateManager) MGet(ctx context.Context, templateIDs []int64, spaceID int64, session *entity.Session) ([]*entity.ExptTemplate, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MGet", ctx, templateIDs, spaceID, session) - ret0, _ := ret[0].([]*entity.ExptTemplate) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MGet indicates an expected call of MGet. -func (mr *MockIExptTemplateManagerMockRecorder) MGet(ctx, templateIDs, spaceID, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MGet", reflect.TypeOf((*MockIExptTemplateManager)(nil).MGet), ctx, templateIDs, spaceID, session) -} - -// Update mocks base method. -func (m *MockIExptTemplateManager) Update(ctx context.Context, param *entity.UpdateExptTemplateParam, session *entity.Session) (*entity.ExptTemplate, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", ctx, param, session) - ret0, _ := ret[0].(*entity.ExptTemplate) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Update indicates an expected call of Update. -func (mr *MockIExptTemplateManagerMockRecorder) Update(ctx, param, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockIExptTemplateManager)(nil).Update), ctx, param, session) -} - -// UpdateExptInfo mocks base method. -func (m *MockIExptTemplateManager) UpdateExptInfo(ctx context.Context, templateID, spaceID, exptID int64, exptStatus entity.ExptStatus, adjustCount int64) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateExptInfo", ctx, templateID, spaceID, exptID, exptStatus, adjustCount) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateExptInfo indicates an expected call of UpdateExptInfo. -func (mr *MockIExptTemplateManagerMockRecorder) UpdateExptInfo(ctx, templateID, spaceID, exptID, exptStatus, adjustCount any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExptInfo", reflect.TypeOf((*MockIExptTemplateManager)(nil).UpdateExptInfo), ctx, templateID, spaceID, exptID, exptStatus, adjustCount) -} - -// UpdateMeta mocks base method. -func (m *MockIExptTemplateManager) UpdateMeta(ctx context.Context, param *entity.UpdateExptTemplateMetaParam, session *entity.Session) (*entity.ExptTemplate, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateMeta", ctx, param, session) - ret0, _ := ret[0].(*entity.ExptTemplate) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// UpdateMeta indicates an expected call of UpdateMeta. -func (mr *MockIExptTemplateManagerMockRecorder) UpdateMeta(ctx, param, session any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMeta", reflect.TypeOf((*MockIExptTemplateManager)(nil).UpdateMeta), ctx, param, session) -} From b1f28464ae3dbe328e662e9012f8d81a6a5fc4c7 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 27 Feb 2026 10:55:35 +0800 Subject: [PATCH 26/30] fix(evaluation): ut --- .../convertor/experiment/expt_template.go | 6 ++- .../application/experiment_app_test.go | 3 +- .../expt_manage_execution_impl_test.go | 50 +++++++++++++++++-- .../service/expt_run_item_turn_impl_test.go | 1 + .../expt_run_scheduler_mode_impl_test.go | 14 ++++-- 5 files changed, 64 insertions(+), 10 deletions(-) diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_template.go b/backend/modules/evaluation/application/convertor/experiment/expt_template.go index 3c6ffc3d1..9fc1e6399 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_template.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_template.go @@ -600,9 +600,13 @@ func buildTemplateFieldMappingDTO(template *entity.ExptTemplate) *domain_expt.Ex return nil } + var itemRetryNum *int32 + if template.TemplateConf != nil && gptr.Indirect(template.TemplateConf.ItemRetryNum) > 0 { + itemRetryNum = gptr.Of(int32(gptr.Indirect(template.TemplateConf.ItemRetryNum))) + } fieldMapping := &domain_expt.ExptFieldMapping{ ItemConcurNum: ptr.ConvIntPtr[int, int32](template.FieldMappingConfig.ItemConcurNum), - ItemRetryNum: gcond.If(gptr.Indirect(template.TemplateConf.ItemRetryNum) > 0, gptr.Of(int32(gptr.Indirect(template.TemplateConf.ItemRetryNum))), nil), + ItemRetryNum: itemRetryNum, } if template.FieldMappingConfig.TargetFieldMapping != nil { diff --git a/backend/modules/evaluation/application/experiment_app_test.go b/backend/modules/evaluation/application/experiment_app_test.go index 7802ef7d0..4a1e359e9 100644 --- a/backend/modules/evaluation/application/experiment_app_test.go +++ b/backend/modules/evaluation/application/experiment_app_test.go @@ -2468,11 +2468,12 @@ func TestExperimentApplication_RetryExperiment(t *testing.T) { ExptID: gptr.Of(validExptID), }, mockSetup: func() { - // 获取实验信息 + itemRetryNum := 0 mockManager.EXPECT().Get(gomock.Any(), validExptID, validWorkspaceID, gomock.Any()).Return(&entity.Experiment{ ID: validExptID, SpaceID: validWorkspaceID, CreatedBy: strconv.FormatInt(validUserID, 10), + EvalConf: &entity.EvaluationConfiguration{ItemRetryNum: &itemRetryNum}, }, nil) // 权限验证 diff --git a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go index 051aa4ff8..4d465bdc6 100755 --- a/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_manage_execution_impl_test.go @@ -247,6 +247,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { spaceID: 789, opts: []entity.CompleteExptOptionFn{}, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -269,13 +274,18 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Save(ctx, gomock.Any()). Return(nil) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: false, }, @@ -296,6 +306,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { Exist(ctx, "CompleteRun:test_cid"). Return(false, nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -314,7 +329,7 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). @@ -322,6 +337,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { Save(ctx, gomock.Any()). Return(nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) + mgr.idem.(*idemMocks.MockIdempotentService). EXPECT(). Set(ctx, "CompleteRun:test_cid", time.Second*60*3). @@ -356,6 +376,11 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { entity.WithCompleteInterval(time.Millisecond * 100), }, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + runLog := &entity.ExptRunLog{ ID: 456, ExptID: 123, @@ -375,13 +400,18 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { mgr.mutex.(*lockMocks.MockILocker). EXPECT(). - Unlock(gomock.Any()). + UnlockForce(ctx, "expt_run_mutex_lock:123"). Return(true, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Save(ctx, gomock.Any()). Return(nil) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: false, }, @@ -393,10 +423,20 @@ func TestExptMangerImpl_CompleteRun(t *testing.T) { spaceID: 789, opts: []entity.CompleteExptOptionFn{}, setup: func() { + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Lock(ctx, "expt_completing_mutex_lock:123:456", time.Minute*3). + Return(true, nil) + mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT(). Get(ctx, int64(123), int64(456)). Return(nil, errors.New("run log not found")) + + mgr.mutex.(*lockMocks.MockILocker). + EXPECT(). + Unlock(gomock.Any()). + Return(true, nil) }, wantErr: true, }, @@ -758,6 +798,8 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { EXPECT(). BackoffLockWithValue(ctx, gomock.Any(), "1002", 300*time.Second, time.Second). Return(false, "1001", nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) existingLog := &entity.ExptRunLog{ID: 1001, ExptID: exptID, ExptRunID: 1001} mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT().Get(ctx, exptID, int64(1001)). @@ -826,6 +868,8 @@ func TestExptMangerImpl_LogRetryItemsRun(t *testing.T) { EXPECT(). BackoffLockWithValue(ctx, gomock.Any(), "1005", 300*time.Second, time.Second). Return(false, "1001", nil) + mgr.mutex.(*lockMocks.MockILocker). + EXPECT().Exists(ctx, "expt_completing_mutex_lock:123:1001").Return(false, nil) mgr.runLogRepo.(*repoMocks.MockIExptRunLogRepo). EXPECT().Get(ctx, exptID, int64(1001)). Return(nil, errors.New("get run log failed")) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 8d5180fec..83216f2ff 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -1855,6 +1855,7 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators_EdgeCases(t *testing.T) { }, }, }, + Event: &entity.ExptItemEvalEvent{}, // Event required: CallEvaluators uses Event.IgnoreExistedResult() }, ExptTurnRunResult: &entity.ExptTurnRunResult{ EvaluatorResults: map[int64]*entity.EvaluatorRecord{ diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 762295f23..6a4e14711 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -4066,17 +4066,21 @@ func TestExptRetryItemsExec_ExptEnd(t *testing.T) { args: args{ ctx: session.WithCtxUser(context.Background(), &session.User{ID: testUserID}), event: &entity.ExptScheduleEvent{ - ExptID: 1, - ExptRunID: 2, - SpaceID: 3, - ExptRunMode: entity.EvaluationModeRetryItems, - Session: &entity.Session{UserID: testUserID}, + ExptID: 1, + ExptRunID: 2, + SpaceID: 3, + ExptRunMode: entity.EvaluationModeRetryItems, + Session: &entity.Session{UserID: testUserID}, + ExecEvalSetItemIDs: []int64{1}, }, expt: mockExpt, toSubmit: 0, incomplete: 0, }, prepareMock: func(f *exptRetryItemsExecFields, args args) { + f.exptRunLogRepo.EXPECT().Get(gomock.Any(), args.event.ExptID, args.event.ExptRunID).Return(&entity.ExptRunLog{ + ItemIds: []entity.ExptRunLogItems{{ItemIDs: []int64{1}}}, + }, nil).Times(1) f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) f.manager.EXPECT().CompleteRun(gomock.Any(), args.event.ExptID, args.event.ExptRunID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) f.manager.EXPECT().CompleteExpt(gomock.Any(), args.event.ExptID, args.event.SpaceID, args.event.Session, gomock.Any(), gomock.Any()).Return(nil).Times(1) From 0cfafd5517960f235f52e182c2626a5561af26b2 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 27 Feb 2026 11:30:32 +0800 Subject: [PATCH 27/30] fix(evaluation): IgnoreExistedResult --- .../evaluation/domain/service/expt_run_item_turn_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index d2713347b..5a7fae8ae 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -100,7 +100,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en return &entity.EvalTargetRecord{EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: make(map[string]*entity.Content)}}, nil } - if existRecord := e.existedTargetRecord(etec); etec.Event.IgnoreExistedResult() && existRecord != nil { + if existRecord := e.existedTargetRecord(etec); !etec.Event.IgnoreExistedResult() && existRecord != nil { logs.CtxInfo(ctx, "CallTarget return with existed target record, record_id: %v", existRecord.ID) return existRecord, nil } From 11fd0291127d63443108a202cafa60acc6c8aaa2 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 27 Feb 2026 14:50:04 +0800 Subject: [PATCH 28/30] fix(evaluation): itemretrynum --- .../application/convertor/experiment/expt.go | 12 +++++++++--- .../convertor/experiment/expt_template.go | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/modules/evaluation/application/convertor/experiment/expt.go b/backend/modules/evaluation/application/convertor/experiment/expt.go index 667ccde9d..566b6e1e5 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt.go @@ -364,9 +364,15 @@ func ToExptDTO(experiment *entity.Experiment) *domain_expt.Experiment { if experiment.EndAt != nil { res.EndTime = gptr.Of(experiment.EndAt.Unix()) } - if experiment.EvalConf != nil && experiment.EvalConf.ItemConcurNum != nil { - res.ItemConcurNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemConcurNum))) - res.ItemRetryNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemRetryNum))) + if experiment.EvalConf != nil { + if experiment.EvalConf.ItemConcurNum != nil { + res.ItemConcurNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemConcurNum))) + } + if experiment.EvalConf.ItemRetryNum != nil { + res.ItemRetryNum = gptr.Of(int32(gptr.Indirect(experiment.EvalConf.ItemRetryNum))) + } else { + res.ItemRetryNum = gptr.Of(int32(0)) + } } // 填充权重配置（score_weight_config 和 enable_weighted_score） diff --git a/backend/modules/evaluation/application/convertor/experiment/expt_template.go b/backend/modules/evaluation/application/convertor/experiment/expt_template.go index 9fc1e6399..73020b073 100644 --- a/backend/modules/evaluation/application/convertor/experiment/expt_template.go +++ b/backend/modules/evaluation/application/convertor/experiment/expt_template.go @@ -603,6 +603,8 @@ func buildTemplateFieldMappingDTO(template *entity.ExptTemplate) *domain_expt.Ex var itemRetryNum *int32 if template.TemplateConf != nil && gptr.Indirect(template.TemplateConf.ItemRetryNum) > 0 { itemRetryNum = gptr.Of(int32(gptr.Indirect(template.TemplateConf.ItemRetryNum))) + } else { + itemRetryNum = gptr.Of(int32(0)) } fieldMapping := &domain_expt.ExptFieldMapping{ ItemConcurNum: ptr.ConvIntPtr[int, int32](template.FieldMappingConfig.ItemConcurNum), From 474e7e14e0f3f4a9bcd4c856cdf738e41991f461 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 27 Feb 2026 15:05:52 +0800 Subject: [PATCH 29/30] fix(evaluation): ListEvaluationSetItems return --- .../evaluation/domain/service/expt_run_scheduler_mode_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go index 6d813f665..6bba199f8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl.go @@ -993,7 +993,7 @@ func (e *ExptRetryAllExec) ExptStart(ctx context.Context, event *entity.ExptSche logs.CtxInfo(ctx, "ExptRetryAllExec.ExptStart scan item, expt_id: %v, expt_run_id: %v, eval_set_id: %v, eval_set_ver_id: %v, page: %v, limit: %v, cur_cnt: %v, total: %v", event.ExptID, event.ExptRunID, evalSetID, evalSetVersionID, page, pageSize, itemCnt, total) - items, t, _, err := e.evaluationSetItemService.ListEvaluationSetItems(ctx, &entity.ListEvaluationSetItemsParam{ + items, t, _, _, err := e.evaluationSetItemService.ListEvaluationSetItems(ctx, &entity.ListEvaluationSetItemsParam{ SpaceID: event.SpaceID, EvaluationSetID: evalSetID, VersionID: &evalSetVersionID, From 3b56e5968c29d9bd5bb5a97c2a510c7f8e602694 Mon Sep 17 00:00:00 2001 From: liushengyang Date: Fri, 27 Feb 2026 15:31:19 +0800 Subject: [PATCH 30/30] fix(evaluation): ut --- .../expt_run_scheduler_mode_impl_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go index 104334036..a09b2e60b 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_mode_impl_test.go @@ -2727,7 +2727,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { }, prepareMock: func(f *exptRetryAllExecFields, args args) { f.idem.EXPECT().Exist(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, nil, nil, errors.New("list error")).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(nil, nil, nil, nil, errors.New("list error")).Times(1) }, wantErr: true, assertErr: func(t *testing.T, err error) { @@ -2754,7 +2754,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return(nil, errors.New("gen id error")).Times(1) }, wantErr: true, @@ -2782,7 +2782,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update error")).Times(1) }, @@ -2811,7 +2811,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("update turn error")).Times(1) @@ -2841,7 +2841,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -2872,7 +2872,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -2904,7 +2904,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -2943,7 +2943,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -2983,7 +2983,7 @@ func TestExptRetryAllExec_ExptStart(t *testing.T) { mockItems := []*entity.EvaluationSetItem{ {ItemID: 100, Turns: []*entity.Turn{{ID: 1000}}}, } - f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil).Times(1) + f.evaluationSetItemService.EXPECT().ListEvaluationSetItems(gomock.Any(), gomock.Any()).Return(mockItems, &total, nil, nil, nil).Times(1) f.idgenerator.EXPECT().GenMultiIDs(gomock.Any(), gomock.Any()).Return([]int64{1, 2}, nil).Times(1) f.exptItemResultRepo.EXPECT().UpdateItemsResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) f.exptTurnResultRepo.EXPECT().UpdateTurnResults(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)