Skip to content

Commit ac72205

Browse files
ssongliussongliu
authored
fix: Fix the problem that the website directory is not backed up duri… (#8280)
1 parent c7b6e15 commit ac72205

26 files changed

Lines changed: 2944 additions & 2329 deletions

File tree

‎agent/app/service/app_install.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,13 @@ func (a *AppInstallService) SyncAll(systemInit bool) error {
469469
}
470470
continue
471471
}
472+
if i.Status == constant.StatusWaitingRestart {
473+
dockerComposePath := i.GetComposePath()
474+
_, _ = compose.Up(dockerComposePath)
475+
i.Status = constant.StatusRunning
476+
_ = appInstallRepo.Save(context.Background(), &i)
477+
continue
478+
}
472479
if !systemInit {
473480
if err = syncAppInstallStatus(&i, false); err != nil {
474481
global.LOG.Errorf("sync install app[%s] error,mgs: %s", i.Name, err.Error())

‎agent/app/service/snapshot.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error {
135135

136136
type SnapshotJson struct {
137137
BaseDir string `json:"baseDir"`
138+
OperestyDir string `json:"operestyDir"`
138139
BackupDataDir string `json:"backupDataDir"`
139140
Size uint64 `json:"size"`
140141
}
@@ -322,12 +323,12 @@ func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) {
322323
Path: path.Join(global.Dir.DataDir, fileItem.Name()),
323324
}
324325
switch itemData.Label {
325-
case "agent", "conf", "runtime", "docker", "secret", "task":
326+
case "agent", "runtime", "docker", "task", "geo", "secret", "uploads":
326327
itemData.IsDisable = true
327-
case "clamav":
328+
case "clamav", "download", "resource":
328329
panelPath := path.Join(global.Dir.DataDir, itemData.Label)
329330
itemData.Children, _ = loadFile(panelPath, 5, fileOp)
330-
default:
331+
case "apps", "backup", "log", "db", "tmp":
331332
continue
332333
}
333334
if fileItem.IsDir() {
@@ -351,6 +352,19 @@ func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) {
351352
data = append(data, itemData)
352353
}
353354

355+
openrestySite, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
356+
if len(openrestySite) != 0 && !strings.Contains(openrestySite, global.Dir.DataDir) {
357+
sizeItem, _ := fileOp.GetDirSize(openrestySite)
358+
data = append(data, dto.DataTree{
359+
ID: uuid.NewString(),
360+
Label: "www",
361+
IsCheck: true,
362+
IsDisable: true,
363+
Path: openrestySite,
364+
Size: uint64(sizeItem),
365+
})
366+
}
367+
354368
return data, nil
355369
}
356370

‎agent/app/service/snapshot_create.go‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
116116

117117
func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) error {
118118
rootDir := path.Join(global.Dir.TmpDir, "system", req.Name)
119-
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()}
119+
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
120+
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background(), OpenrestyDir: openrestyDir}
120121
baseDir := path.Join(rootDir, "base")
121122
_ = os.MkdirAll(baseDir, os.ModePerm)
122123

@@ -168,7 +169,7 @@ func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) err
168169
req.InterruptStep = ""
169170
}
170171

171-
taskItem.AddSubTask(
172+
taskItem.AddSubTaskWithAlias(
172173
"SnapCloseDBConn",
173174
func(t *task.Task) error {
174175
taskItem.Log("---------------------- 6 / 8 ----------------------")
@@ -214,6 +215,8 @@ type snapHelper struct {
214215
FileOp files.FileOp
215216
Wg *sync.WaitGroup
216217
Task task.Task
218+
219+
OpenrestyDir string
217220
}
218221

219222
func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error {
@@ -247,6 +250,12 @@ func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) erro
247250
return err
248251
}
249252
}
253+
if !req.WithTaskLog {
254+
err = os.Remove(path.Join(targetDir, "db/task.db"))
255+
if err != nil {
256+
return err
257+
}
258+
}
250259
if !req.WithOperationLog {
251260
err = snap.snapCoreDB.Exec("DELETE FROM operation_logs").Error
252261
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteOperationLog"), err)
@@ -312,6 +321,7 @@ func snapBaseData(snap snapHelper, targetDir string) error {
312321

313322
remarkInfo, _ := json.MarshalIndent(SnapshotJson{
314323
BaseDir: global.Dir.BaseDir,
324+
OperestyDir: snap.OpenrestyDir,
315325
BackupDataDir: global.Dir.LocalBackupDir,
316326
}, "", "\t")
317327
err = os.WriteFile(path.Join(targetDir, "snapshot.json"), remarkInfo, 0640)
@@ -327,13 +337,22 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) err
327337
snap.Task.Log("---------------------- 3 / 8 ----------------------")
328338
snap.Task.LogStart(i18n.GetMsgByKey("SnapInstallApp"))
329339

340+
var appInstalls []model.AppInstall
341+
_ = snap.snapAgentDB.Where("1 = 1").Find(&appInstalls).Error
342+
for _, item := range appInstalls {
343+
_ = snap.snapAgentDB.Where("id = ?", item.ID).Updates(map[string]interface{}{"status": constant.StatusWaitingRestart}).Error
344+
}
345+
330346
var imageList []string
331347
existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG")
332348
existImages := strings.Split(existStr, "\n")
333349
for _, app := range req.AppData {
334350
for _, item := range app.Children {
335351
if item.Label == "appImage" && item.IsCheck {
336352
for _, existImage := range existImages {
353+
if len(existImage) == 0 {
354+
continue
355+
}
337356
if existImage == item.Name {
338357
imageList = append(imageList, item.Name)
339358
}
@@ -342,10 +361,10 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) err
342361
}
343362
}
344363

364+
snap.Task.Log(strings.Join(imageList, " "))
345365
if len(imageList) != 0 {
346366
snap.Task.Logf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz"))
347367
std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz"))
348-
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std))
349368
if err != nil {
350369
snap.Task.LogFailedWithErr(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std))
351370
return errors.New(std)
@@ -421,25 +440,32 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) er
421440
}
422441
}
423442
}
424-
excludes = append(excludes, "./tmp")
425443
excludes = append(excludes, "./cache")
426-
excludes = append(excludes, "./uploads")
427444
excludes = append(excludes, "./db")
428-
excludes = append(excludes, "./resource")
445+
excludes = append(excludes, "./tmp")
429446
if !req.WithSystemLog {
430447
excludes = append(excludes, "./log/1Panel*")
431448
}
432449
if !req.WithTaskLog {
433-
excludes = append(excludes, "./log/App")
434-
excludes = append(excludes, "./log/Snapshot")
450+
excludes = append(excludes, "./log/AI")
435451
excludes = append(excludes, "./log/AppStore")
452+
excludes = append(excludes, "./log/Cronjob")
453+
excludes = append(excludes, "./log/Image")
454+
excludes = append(excludes, "./log/Compose")
455+
excludes = append(excludes, "./log/Database")
456+
excludes = append(excludes, "./log/RuntimeExtension")
436457
excludes = append(excludes, "./log/Website")
458+
excludes = append(excludes, "./log/App")
459+
excludes = append(excludes, "./log/Snapshot")
437460
}
438461

439462
rootDir := global.Dir.DataDir
440463
if strings.Contains(global.Dir.LocalBackupDir, rootDir) {
441464
excludes = append(excludes, "."+strings.ReplaceAll(global.Dir.LocalBackupDir, rootDir, ""))
442465
}
466+
if len(snap.OpenrestyDir) != 0 && strings.Contains(snap.OpenrestyDir, rootDir) {
467+
excludes = append(excludes, "."+strings.ReplaceAll(snap.OpenrestyDir, rootDir, ""))
468+
}
443469
ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore"))
444470
rules := strings.Split(ignoreVal.Value, ",")
445471
for _, ignore := range rules {
@@ -450,6 +476,16 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) er
450476
}
451477
err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";"))
452478
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressPanel"), err)
479+
if err != nil {
480+
return err
481+
}
482+
if len(snap.OpenrestyDir) != 0 {
483+
err := snap.FileOp.TarGzCompressPro(false, snap.OpenrestyDir, path.Join(targetDir, "website.tar.gz"), "", "")
484+
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapWebsite"), err)
485+
if err != nil {
486+
return err
487+
}
488+
}
453489

454490
return err
455491
}

‎agent/app/service/snapshot_recover.go‎

Lines changed: 37 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
package service
22

33
import (
4-
"context"
54
"encoding/json"
65
"fmt"
76
"os"
87
"path"
98
"strings"
10-
"sync"
119

1210
"github.com/1Panel-dev/1Panel/agent/app/repo"
1311

@@ -19,7 +17,6 @@ import (
1917
"github.com/1Panel-dev/1Panel/agent/i18n"
2018
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
2119
"github.com/1Panel-dev/1Panel/agent/utils/common"
22-
"github.com/1Panel-dev/1Panel/agent/utils/compose"
2320
"github.com/1Panel-dev/1Panel/agent/utils/files"
2421
"github.com/pkg/errors"
2522
)
@@ -160,6 +157,17 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
160157
itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name))
161158
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "")
162159
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
160+
if err != nil {
161+
return err
162+
}
163+
164+
if len(snapJson.OperestyDir) != 0 {
165+
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/website.tar.gz"), snapJson.OperestyDir, "")
166+
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverWebsite"), err)
167+
if err != nil {
168+
return err
169+
}
170+
}
163171
return err
164172
},
165173
nil,
@@ -219,11 +227,24 @@ func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
219227
if err != nil {
220228
return err
221229
}
222-
err = itemHelper.FileOp.CopyDirWithExclude(global.Dir.LocalBackupDir, rootDir, []string{"system_snapshot"})
223-
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.Dir.LocalBackupDir), err)
224-
if err != nil {
225-
return err
230+
231+
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
232+
if len(openrestyDir) != 0 && !strings.Contains(openrestyDir, global.Dir.DataDir) {
233+
err := itemHelper.FileOp.CopyDirWithExclude(openrestyDir, rootDir, nil)
234+
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", openrestyDir), err)
235+
if err != nil {
236+
return err
237+
}
238+
}
239+
240+
if len(global.Dir.LocalBackupDir) != 0 && !strings.Contains(global.Dir.LocalBackupDir, global.Dir.DataDir) {
241+
err = itemHelper.FileOp.CopyDirWithExclude(global.Dir.LocalBackupDir, rootDir, []string{"system_snapshot"})
242+
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.Dir.LocalBackupDir), err)
243+
if err != nil {
244+
return err
245+
}
226246
}
247+
227248
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir)
228249
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
229250
if err != nil {
@@ -235,19 +256,17 @@ func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
235256
if err != nil {
236257
return err
237258
}
238-
}
239-
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel-agent", baseDir)
240-
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
241-
if err != nil {
242-
return err
243-
}
244-
if global.IsMaster {
245259
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel-core.service", baseDir)
246260
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-core.service"), err)
247261
if err != nil {
248262
return err
249263
}
250264
}
265+
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel-agent", baseDir)
266+
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
267+
if err != nil {
268+
return err
269+
}
251270
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel-agent.service", baseDir)
252271
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-agent.service"), err)
253272
if err != nil {
@@ -294,35 +313,13 @@ func recoverAppData(src string, itemHelper *snapRecoverHelper) error {
294313
if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil {
295314
itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty"))
296315
return nil
297-
} else {
298-
std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
299-
if err != nil {
300-
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
301-
return fmt.Errorf("docker load images failed, err: %v", err)
302-
}
303-
itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
304316
}
305-
306-
appInstalls, err := appInstallRepo.ListBy(context.Background())
307-
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err)
317+
std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
308318
if err != nil {
309-
return err
310-
}
311-
312-
var wg sync.WaitGroup
313-
for i := 0; i < len(appInstalls); i++ {
314-
wg.Add(1)
315-
appInstalls[i].Status = constant.StatusRebuilding
316-
_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
317-
go func(app model.AppInstall) {
318-
defer wg.Done()
319-
dockerComposePath := app.GetComposePath()
320-
_, _ = compose.Up(dockerComposePath)
321-
app.Status = constant.StatusRunning
322-
_ = appInstallRepo.Save(context.Background(), &app)
323-
}(appInstalls[i])
319+
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
320+
return fmt.Errorf("docker load images failed, err: %v", err)
324321
}
325-
wg.Wait()
322+
itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
326323
return nil
327324
}
328325

‎agent/app/service/snapshot_rollback.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error {
8989
},
9090
nil,
9191
)
92+
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
93+
if len(openrestyDir) != 0 {
94+
taskItem.AddSubTask(
95+
i18n.GetWithName("SnapCopy", openrestyDir),
96+
func(t *task.Task) error {
97+
return FileOp.CopyDir(path.Join(rootDir, "www"), openrestyDir)
98+
},
99+
nil,
100+
)
101+
}
92102
taskItem.AddSubTask(
93103
i18n.GetWithName("SnapCopy", global.Dir.BaseDir),
94104
func(t *task.Task) error {

0 commit comments

Comments
 (0)