Skip to content

Commit 15d77ce

Browse files
HynoRHynoR
authored
fix: change sync logic to aggressive process (#11795)
* f * f * f
1 parent af8fa67 commit 15d77ce

5 files changed

Lines changed: 145 additions & 27 deletions

File tree

‎agent/app/repo/app.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ type IAppRepo interface {
2020
WithResource(resource string) DBOption
2121
WithNotLocal() DBOption
2222
WithByLikeName(name string) DBOption
23+
WithKeyIn(keys []string) DBOption
2324
WithArch(arch string) DBOption
2425
WithPanelVersion(panelVersion string) DBOption
2526

@@ -55,6 +56,12 @@ func (a AppRepo) WithKey(key string) DBOption {
5556
}
5657
}
5758

59+
func (a AppRepo) WithKeyIn(keys []string) DBOption {
60+
return func(db *gorm.DB) *gorm.DB {
61+
return db.Where("`key` in (?)", keys)
62+
}
63+
}
64+
5865
func (a AppRepo) WithType(typeStr string) DBOption {
5966
return func(g *gorm.DB) *gorm.DB {
6067
return g.Where("type = ?", typeStr)

‎agent/app/service/app.go‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"reflect"
1313
"strconv"
1414
"strings"
15+
"sync"
1516

1617
"github.com/gin-gonic/gin"
1718

@@ -34,6 +35,11 @@ import (
3435
"gopkg.in/yaml.v3"
3536
)
3637

38+
var (
39+
appStoreSyncMu sync.Mutex
40+
appStoreSyncing bool
41+
)
42+
3743
type AppService struct {
3844
}
3945

@@ -840,6 +846,13 @@ func (a AppService) GetAppUpdate() (*response.AppUpdateRes, error) {
840846
res.CanUpdate = true
841847
return res, err
842848
}
849+
if appicon.IsIconFile(app.Icon) {
850+
fileName, _ := appicon.ParseIconField(app.Icon)
851+
if fileName == "" || !appicon.IconFileExists(fileName) {
852+
res.CanUpdate = true
853+
return res, err
854+
}
855+
}
843856
}
844857

845858
list, err := getAppList()
@@ -897,33 +910,61 @@ var InitTypes = map[string]struct{}{
897910
}
898911

899912
func deleteCustomApp() {
913+
installs, err := appInstallRepo.ListBy(context.Background())
914+
if err != nil {
915+
global.LOG.Errorf("[AppStore] deleteCustomApp: failed to list installs, skipping: %v", err)
916+
return
917+
}
900918
var appIDS []uint
901-
installs, _ := appInstallRepo.ListBy(context.Background())
902919
for _, install := range installs {
903920
appIDS = append(appIDS, install.AppId)
904921
}
905922
var ops []repo.DBOption
906-
ops = append(ops, repo.WithByIDNotIn(appIDS))
907923
if len(appIDS) > 0 {
908924
ops = append(ops, repo.WithByIDNotIn(appIDS))
909925
}
910-
apps, _ := appRepo.GetBy(ops...)
926+
apps, err := appRepo.GetBy(ops...)
927+
if err != nil {
928+
global.LOG.Errorf("[AppStore] deleteCustomApp: failed to get apps, skipping: %v", err)
929+
return
930+
}
911931
var deleteIDS []uint
912932
for _, app := range apps {
913933
if app.Resource == constant.AppResourceCustom {
914934
deleteIDS = append(deleteIDS, app.ID)
915935
}
916936
}
917-
_ = appRepo.DeleteByIDs(context.Background(), deleteIDS)
918-
_ = appDetailRepo.DeleteByAppIds(context.Background(), deleteIDS)
937+
if len(deleteIDS) == 0 {
938+
return
939+
}
940+
if err = appRepo.DeleteByIDs(context.Background(), deleteIDS); err != nil {
941+
global.LOG.Errorf("[AppStore] deleteCustomApp: failed to delete apps: %v", err)
942+
}
943+
if err = appDetailRepo.DeleteByAppIds(context.Background(), deleteIDS); err != nil {
944+
global.LOG.Errorf("[AppStore] deleteCustomApp: failed to delete app details: %v", err)
945+
}
919946
}
920947

921948
func (a AppService) SyncAppListFromRemote(taskID string) (err error) {
922949
if xpack.IsUseCustomApp() {
923950
return nil
924951
}
952+
953+
appStoreSyncMu.Lock()
954+
global.LOG.Info("[AppStore] sync app from remote task create start")
955+
if appStoreSyncing {
956+
appStoreSyncMu.Unlock()
957+
global.LOG.Info("[AppStore] sync already in progress, skipping")
958+
return nil
959+
}
960+
appStoreSyncing = true
961+
appStoreSyncMu.Unlock()
962+
925963
syncTask, err := task.NewTaskWithOps(i18n.GetMsgByKey("App"), task.TaskSync, task.TaskScopeAppStore, taskID, 0)
926964
if err != nil {
965+
appStoreSyncMu.Lock()
966+
appStoreSyncing = false
967+
appStoreSyncMu.Unlock()
927968
return err
928969
}
929970

@@ -933,13 +974,29 @@ func (a AppService) SyncAppListFromRemote(taskID string) (err error) {
933974
syncTask.AddSubTask(i18n.GetMsgByKey("SyncAppDetail"), a.createSyncAppStoreMetaTask(&sharedCtx), nil)
934975

935976
go func() {
977+
defer func() {
978+
if r := recover(); r != nil {
979+
global.LOG.Errorf("[AppStore] sync goroutine recovered from panic: %v", r)
980+
if updateErr := NewISettingService().Update("AppStoreSyncStatus", constant.StatusError); updateErr != nil {
981+
global.LOG.Warnf("[AppStore] failed to update sync status after panic: %v", updateErr)
982+
}
983+
}
984+
appStoreSyncMu.Lock()
985+
appStoreSyncing = false
986+
appStoreSyncMu.Unlock()
987+
}()
936988
if err := syncTask.Execute(); err != nil {
937-
_ = NewISettingService().Update("AppStoreLastModified", "0")
938-
_ = NewISettingService().Update("AppStoreSyncStatus", constant.StatusError)
989+
if updateErr := NewISettingService().Update("AppStoreLastModified", "0"); updateErr != nil {
990+
global.LOG.Warnf("[AppStore] failed to reset last modified: %v", updateErr)
991+
}
992+
if updateErr := NewISettingService().Update("AppStoreSyncStatus", constant.StatusError); updateErr != nil {
993+
global.LOG.Warnf("[AppStore] failed to update sync status to error: %v", updateErr)
994+
}
939995
return
940996
}
941997
}()
942998

999+
global.LOG.Info("[AppStore] sync app from remote task create ok")
9431000
return nil
9441001
}
9451002

‎agent/app/service/app_sync_task.go‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t
6868
}
6969

7070
settingService := NewISettingService()
71-
_ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncing)
71+
if err := settingService.Update("AppStoreSyncStatus", constant.StatusSyncing); err != nil {
72+
global.LOG.Warnf("[AppStore] failed to update sync status to syncing: %v", err)
73+
}
7274

7375
setting, err := settingService.GetSettingInfo()
7476
if err != nil {
@@ -111,8 +113,12 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t
111113
return err
112114
}
113115

114-
_ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncSuccess)
115-
_ = settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified))
116+
if err := settingService.Update("AppStoreSyncStatus", constant.StatusSyncSuccess); err != nil {
117+
global.LOG.Warnf("[AppStore] failed to update sync status to success: %v", err)
118+
}
119+
if err := settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified)); err != nil {
120+
global.LOG.Warnf("[AppStore] failed to update last modified: %v", err)
121+
}
116122
global.LOG.Infof("[AppStore] Appstore sync completed")
117123

118124
*sharedCtx = ctx
@@ -342,6 +348,31 @@ func (c *appSyncContext) classifyAndPersistAppsWithStats(addCount, updateCount,
342348
}
343349
}
344350

351+
if len(addAppArray) > 0 {
352+
addKeys := make([]string, 0, len(addAppArray))
353+
for _, app := range addAppArray {
354+
addKeys = append(addKeys, app.Key)
355+
}
356+
existingApps, _ := appRepo.GetBy(appRepo.WithKeyIn(addKeys))
357+
if len(existingApps) > 0 {
358+
existingMap := make(map[string]model.App, len(existingApps))
359+
for _, e := range existingApps {
360+
existingMap[e.Key] = e
361+
}
362+
filteredAdd := make([]model.App, 0, len(addAppArray))
363+
for _, app := range addAppArray {
364+
if existing, ok := existingMap[app.Key]; ok {
365+
app.ID = existing.ID
366+
app.Details = existing.Details
367+
updateAppArray = append(updateAppArray, app)
368+
} else {
369+
filteredAdd = append(filteredAdd, app)
370+
}
371+
}
372+
addAppArray = filteredAdd
373+
}
374+
}
375+
345376
*addCount = len(addAppArray)
346377
*updateCount = len(updateAppArray)
347378
*deleteCount = len(deleteAppArray)
@@ -394,13 +425,10 @@ func (c *appSyncContext) classifyAndPersistAppsWithStats(addCount, updateCount,
394425
for _, tag := range app.TagsKey {
395426
tagId, ok := tagMap[tag]
396427
if ok {
397-
exist, _ := appTagRepo.GetFirst(ctx, appTagRepo.WithByTagID(tagId), appTagRepo.WithByAppID(app.ID))
398-
if exist == nil {
399-
c.appTags = append(c.appTags, &model.AppTag{
400-
AppId: app.ID,
401-
TagId: tagId,
402-
})
403-
}
428+
c.appTags = append(c.appTags, &model.AppTag{
429+
AppId: app.ID,
430+
TagId: tagId,
431+
})
404432
}
405433
}
406434

@@ -449,8 +477,19 @@ func (c *appSyncContext) classifyAndPersistAppsWithStats(addCount, updateCount,
449477
}
450478
}
451479

452-
if len(c.oldAppIds) > 0 {
453-
if err = appTagRepo.DeleteByAppIds(ctx, deleteIds); err != nil {
480+
syncedAppIds := make([]uint, 0, len(addAppArray)+len(updateAppArray)+len(deleteIds))
481+
for _, app := range addAppArray {
482+
if app.ID > 0 {
483+
syncedAppIds = append(syncedAppIds, app.ID)
484+
}
485+
}
486+
for _, app := range updateAppArray {
487+
syncedAppIds = append(syncedAppIds, app.ID)
488+
}
489+
syncedAppIds = append(syncedAppIds, deleteIds...)
490+
491+
if len(syncedAppIds) > 0 {
492+
if err = appTagRepo.DeleteByAppIds(ctx, syncedAppIds); err != nil {
454493
return
455494
}
456495
}
@@ -461,6 +500,8 @@ func (c *appSyncContext) classifyAndPersistAppsWithStats(addCount, updateCount,
461500
}
462501
}
463502

464-
tx.Commit()
503+
if err = tx.Commit().Error; err != nil {
504+
return
505+
}
465506
return nil
466507
}

‎agent/app/service/app_utils.go‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,23 +2150,31 @@ func handleSSLConfig(appInstall *model.AppInstall, hasDefaultWebsite bool, sslRe
21502150
return nil
21512151
}
21522152

2153-
func SyncTags(remoteProperties dto.ExtraProperties) error {
2153+
func SyncTags(remoteProperties dto.ExtraProperties) (err error) {
21542154
tx, ctx := getTxAndContext()
2155-
defer tx.Rollback()
2156-
localTags, _ := tagRepo.All()
2155+
defer func() {
2156+
if err != nil {
2157+
tx.Rollback()
2158+
}
2159+
}()
2160+
localTags, err := tagRepo.All()
2161+
if err != nil {
2162+
return err
2163+
}
21572164
localTagsMap := make(map[string]*model.Tag)
21582165
for i := range localTags {
21592166
localTagsMap[localTags[i].Key] = &localTags[i]
21602167
}
2161-
var err error
21622168
remoteTagsMap := make(map[string]*dto.Tag)
21632169
for i := range remoteProperties.Tags {
21642170
remoteTagsMap[remoteProperties.Tags[i].Key] = &remoteProperties.Tags[i]
21652171
}
21662172

21672173
for key, localTag := range localTagsMap {
21682174
if _, exists := remoteTagsMap[key]; !exists {
2169-
_ = tagRepo.DeleteByID(ctx, localTag.ID)
2175+
if err = tagRepo.DeleteByID(ctx, localTag.ID); err != nil {
2176+
return err
2177+
}
21702178
}
21712179
}
21722180

@@ -2196,7 +2204,9 @@ func SyncTags(remoteProperties dto.ExtraProperties) error {
21962204
}
21972205
}
21982206

2199-
tx.Commit()
2207+
if err = tx.Commit().Error; err != nil {
2208+
return err
2209+
}
22002210
return nil
22012211
}
22022212

‎agent/init/business/business.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@ func syncApp() {
2323
if global.CONF.Base.IsOffLine {
2424
return
2525
}
26-
_ = service.NewISettingService().Update("AppStoreSyncStatus", constant.StatusSyncSuccess)
26+
setting, err := service.NewISettingService().GetSettingInfo()
27+
if err == nil && setting.AppStoreSyncStatus == constant.StatusSyncing {
28+
_ = service.NewISettingService().Update("AppStoreSyncStatus", constant.StatusSyncSuccess)
29+
}
2730
if err := service.NewIAppService().SyncAppListFromRemote(""); err != nil {
2831
global.LOG.Errorf("App Store synchronization failed")
2932
return

0 commit comments

Comments
 (0)