Skip to content

fix: Fix the problem that the website directory is not backed up duri… #8280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
f2c-ci-robot merged 1 commit into from
Mar 31, 2025
Merged

fix: Fix the problem that the website directory is not backed up duri… #8280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions agent/app/service/app_install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,13 @@ func (a *AppInstallService) SyncAll(systemInit bool) error {
}
continue
}
if i.Status == constant.StatusWaitingRestart {
dockerComposePath := i.GetComposePath()
_, _ = compose.Up(dockerComposePath)
i.Status = constant.StatusRunning
_ = appInstallRepo.Save(context.Background(), &i)
continue
}
if !systemInit {
if err = syncAppInstallStatus(&i, false); err != nil {
global.LOG.Errorf("sync install app[%s] error,mgs: %s", i.Name, err.Error())
Expand Down
20 changes: 17 additions & 3 deletions agent/app/service/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error {

type SnapshotJson struct {
BaseDir string `json:"baseDir"`
OperestyDir string `json:"operestyDir"`
BackupDataDir string `json:"backupDataDir"`
Size uint64 `json:"size"`
}
Expand Down Expand Up @@ -322,12 +323,12 @@ func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) {
Path: path.Join(global.Dir.DataDir, fileItem.Name()),
}
switch itemData.Label {
case "agent", "conf", "runtime", "docker", "secret", "task":
case "agent", "runtime", "docker", "task", "geo", "secret", "uploads":
itemData.IsDisable = true
case "clamav":
case "clamav", "download", "resource":
panelPath := path.Join(global.Dir.DataDir, itemData.Label)
itemData.Children, _ = loadFile(panelPath, 5, fileOp)
default:
case "apps", "backup", "log", "db", "tmp":
continue
}
if fileItem.IsDir() {
Expand All @@ -351,6 +352,19 @@ func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) {
data = append(data, itemData)
}

openrestySite, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
if len(openrestySite) != 0 && !strings.Contains(openrestySite, global.Dir.DataDir) {
sizeItem, _ := fileOp.GetDirSize(openrestySite)
data = append(data, dto.DataTree{
ID: uuid.NewString(),
Label: "www",
IsCheck: true,
IsDisable: true,
Path: openrestySite,
Size: uint64(sizeItem),
})
}

return data, nil
}

Expand Down
52 changes: 44 additions & 8 deletions agent/app/service/snapshot_create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {

func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) error {
rootDir := path.Join(global.Dir.TmpDir, "system", req.Name)
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()}
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background(), OpenrestyDir: openrestyDir}
baseDir := path.Join(rootDir, "base")
_ = os.MkdirAll(baseDir, os.ModePerm)

Expand Down Expand Up @@ -168,7 +169,7 @@ func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) err
req.InterruptStep = ""
}

taskItem.AddSubTask(
taskItem.AddSubTaskWithAlias(
"SnapCloseDBConn",
func(t *task.Task) error {
taskItem.Log("---------------------- 6 / 8 ----------------------")
Expand Down Expand Up @@ -214,6 +215,8 @@ type snapHelper struct {
FileOp files.FileOp
Wg *sync.WaitGroup
Task task.Task

OpenrestyDir string
}

func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error {
Expand Down Expand Up @@ -247,6 +250,12 @@ func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) erro
return err
}
}
if !req.WithTaskLog {
err = os.Remove(path.Join(targetDir, "db/task.db"))
if err != nil {
return err
}
}
if !req.WithOperationLog {
err = snap.snapCoreDB.Exec("DELETE FROM operation_logs").Error
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteOperationLog"), err)
Expand Down Expand Up @@ -312,6 +321,7 @@ func snapBaseData(snap snapHelper, targetDir string) error {

remarkInfo, _ := json.MarshalIndent(SnapshotJson{
BaseDir: global.Dir.BaseDir,
OperestyDir: snap.OpenrestyDir,
BackupDataDir: global.Dir.LocalBackupDir,
}, "", "\t")
err = os.WriteFile(path.Join(targetDir, "snapshot.json"), remarkInfo, 0640)
Expand All @@ -327,13 +337,22 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) err
snap.Task.Log("---------------------- 3 / 8 ----------------------")
snap.Task.LogStart(i18n.GetMsgByKey("SnapInstallApp"))

var appInstalls []model.AppInstall
_ = snap.snapAgentDB.Where("1 = 1").Find(&appInstalls).Error
for _, item := range appInstalls {
_ = snap.snapAgentDB.Where("id = ?", item.ID).Updates(map[string]interface{}{"status": constant.StatusWaitingRestart}).Error
}

var imageList []string
existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG")
existImages := strings.Split(existStr, "\n")
for _, app := range req.AppData {
for _, item := range app.Children {
if item.Label == "appImage" && item.IsCheck {
for _, existImage := range existImages {
if len(existImage) == 0 {
continue
}
if existImage == item.Name {
imageList = append(imageList, item.Name)
}
Expand All @@ -342,10 +361,10 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) err
}
}

snap.Task.Log(strings.Join(imageList, " "))
if len(imageList) != 0 {
snap.Task.Logf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz"))
std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz"))
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std))
if err != nil {
snap.Task.LogFailedWithErr(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std))
return errors.New(std)
Expand Down Expand Up @@ -421,25 +440,32 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) er
}
}
}
excludes = append(excludes, "./tmp")
excludes = append(excludes, "./cache")
excludes = append(excludes, "./uploads")
excludes = append(excludes, "./db")
excludes = append(excludes, "./resource")
excludes = append(excludes, "./tmp")
if !req.WithSystemLog {
excludes = append(excludes, "./log/1Panel*")
}
if !req.WithTaskLog {
excludes = append(excludes, "./log/App")
excludes = append(excludes, "./log/Snapshot")
excludes = append(excludes, "./log/AI")
excludes = append(excludes, "./log/AppStore")
excludes = append(excludes, "./log/Cronjob")
excludes = append(excludes, "./log/Image")
excludes = append(excludes, "./log/Compose")
excludes = append(excludes, "./log/Database")
excludes = append(excludes, "./log/RuntimeExtension")
excludes = append(excludes, "./log/Website")
excludes = append(excludes, "./log/App")
excludes = append(excludes, "./log/Snapshot")
}

rootDir := global.Dir.DataDir
if strings.Contains(global.Dir.LocalBackupDir, rootDir) {
excludes = append(excludes, "."+strings.ReplaceAll(global.Dir.LocalBackupDir, rootDir, ""))
}
if len(snap.OpenrestyDir) != 0 && strings.Contains(snap.OpenrestyDir, rootDir) {
excludes = append(excludes, "."+strings.ReplaceAll(snap.OpenrestyDir, rootDir, ""))
}
ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore"))
rules := strings.Split(ignoreVal.Value, ",")
for _, ignore := range rules {
Expand All @@ -450,6 +476,16 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) er
}
err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";"))
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressPanel"), err)
if err != nil {
return err
}
if len(snap.OpenrestyDir) != 0 {
err := snap.FileOp.TarGzCompressPro(false, snap.OpenrestyDir, path.Join(targetDir, "website.tar.gz"), "", "")
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapWebsite"), err)
if err != nil {
return err
}
}

return err
}
Copy link
Copy Markdown
Member

@wanghe-fit2cloud wanghe-fit2cloud Mar 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are a few modifications and optimizations I can suggest:

  1. Use constants for file paths to make them more readable and maintainable.
  2. Check if taskItem is not null before calling its methods to avoid runtime errors.
  3. Consider using try-catch blocks around potentially failing operations to improve the robustness of the function.
  4. Extract repetitive logic into separate functions to reduce code duplication.
  5. Optimize regular expressions used in existStr, especially on larger sets of data.
  6. Handle different types of errors more gracefully, such as logging specific error messages instead of just passing them through.

Other than these minor improvements, the general structure and functionality appear to be working correctly.

Expand Down
77 changes: 37 additions & 40 deletions agent/app/service/snapshot_recover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
package service

import (
"context"
"encoding/json"
"fmt"
"os"
"path"
"strings"
"sync"

"github.com/1Panel-dev/1Panel/agent/app/repo"

Expand All @@ -19,7 +17,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -160,6 +157,17 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name))
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "")
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
if err != nil {
return err
}

if len(snapJson.OperestyDir) != 0 {
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/website.tar.gz"), snapJson.OperestyDir, "")
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverWebsite"), err)
if err != nil {
return err
}
}
return err
},
nil,
Expand Down Expand Up @@ -219,11 +227,24 @@ func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
if err != nil {
return err
}
err = itemHelper.FileOp.CopyDirWithExclude(global.Dir.LocalBackupDir, rootDir, []string{"system_snapshot"})
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.Dir.LocalBackupDir), err)
if err != nil {
return err

openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
if len(openrestyDir) != 0 && !strings.Contains(openrestyDir, global.Dir.DataDir) {
err := itemHelper.FileOp.CopyDirWithExclude(openrestyDir, rootDir, nil)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", openrestyDir), err)
if err != nil {
return err
}
}

if len(global.Dir.LocalBackupDir) != 0 && !strings.Contains(global.Dir.LocalBackupDir, global.Dir.DataDir) {
err = itemHelper.FileOp.CopyDirWithExclude(global.Dir.LocalBackupDir, rootDir, []string{"system_snapshot"})
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.Dir.LocalBackupDir), err)
if err != nil {
return err
}
}

err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
if err != nil {
Expand All @@ -235,19 +256,17 @@ func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
if err != nil {
return err
}
}
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel-agent", baseDir)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
if err != nil {
return err
}
if global.IsMaster {
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel-core.service", baseDir)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-core.service"), err)
if err != nil {
return err
}
}
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel-agent", baseDir)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
if err != nil {
return err
}
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel-agent.service", baseDir)
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-agent.service"), err)
if err != nil {
Expand Down Expand Up @@ -294,35 +313,13 @@ func recoverAppData(src string, itemHelper *snapRecoverHelper) error {
if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil {
itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty"))
return nil
} else {
std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
if err != nil {
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
return fmt.Errorf("docker load images failed, err: %v", err)
}
itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
}

appInstalls, err := appInstallRepo.ListBy(context.Background())
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err)
std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
if err != nil {
return err
}

var wg sync.WaitGroup
for i := 0; i < len(appInstalls); i++ {
wg.Add(1)
appInstalls[i].Status = constant.StatusRebuilding
_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
go func(app model.AppInstall) {
defer wg.Done()
dockerComposePath := app.GetComposePath()
_, _ = compose.Up(dockerComposePath)
app.Status = constant.StatusRunning
_ = appInstallRepo.Save(context.Background(), &app)
}(appInstalls[i])
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
return fmt.Errorf("docker load images failed, err: %v", err)
}
wg.Wait()
itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
return nil
}

Copy link
Copy Markdown
Member

@wanghe-fit2cloud wanghe-fit2cloud Mar 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are several areas in the provided code where changes can be made to improve readability, performance, and maintainability:

  1. Import Statements: Remove unused or unnecessary imports from each function.

    • Example: import (... compose...) should be removed if not used.

  2. Unused Variables: Identify and remove variables that are not being used.

    • Example: var wg sync.WaitGroup

  3. Duplicate Code: Find instances of repetitive logic and consider extracting them into helper functions.

    • Example: 
      func (itemHelper *snapRecoverHelper) LogWithStatus(status string, e error) {
    if e != nil {
        itemHelper.Task.LogFailed(status, status)
    } else {
        itemHelper.Task.LogSuccess(status)
    }
}

  4. Magic Numbers/Strings: Replace magic numbers with meaningful constants defined at the top of the file.

    • Examples: 
      const SystemSnapshotFolderName = "system_snapshot"
const DefaultExcludesForCopying = []string{SystemSnapshotFolderName}

  5. Type Annotations: Add type annotations to parameters and return values for better clarity.

    • Examples: 
      func copyDirWithExclusion(source, destination string, excludes []string) error {
    // ...
}

  6. Error Handling: Improve error handling to provide more specific messages and avoid using generic error strings like "error occurred".

    • Examples: 
      err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
if err != nil {
    itemHelper.Task.LogErrorWithErr("recovering images.tar.gz failed: ", errors.Wrap(err, "failed to execute docker load command"))
    return err
}

  7. Resource Management: Ensure that resources such as files and network connections are properly closed or released when no longer needed.

    • This could involve adding defer statements around file operations or closing database connections within finally blocks.

  8. Logging Improvements: Consider logging different levels of information based on the success or failure of tasks.

    • Use separate log entries for starting and finishing tasks, rather than lumping all activity into one message.

  9. Concurrency: Evaluate and optimize concurrent operations if applicable.

    • Ensure that goroutines manage their internal state correctly without race conditions.

  10. Consistent Naming Conventions: Follow consistent naming conventions throughout the codebase.

    • Variable names like rootDir, snapJson, etc., should describe what they represent clearly.

Here is an improved version of the code snippet incorporating some of these suggestions:

package service

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/1Panel-dev/1Panel/agent/app/repo"
	"github.com/1Panel-dev/1Panel/i18n"
	"github.com/1Panel-dev/1Panel/utils/cmd"
	"github.com/1Panel-dev/1Panel/utils/constants"
)

const systemSnapshot folderNameString = "system_snapshot"
const defaultExclusionsForCopying = []string{
	systemSnapshotFolderName,
}

type snapRecoverHelper struct {
	Task      taskHelperInterface
	FileOp    fileOperationInterface
}

func (u *snapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
	rootDir := req.GetRootDir()
	snapJson := req.GetResponse()

	if snappy, err := u.repo.SnapGetOne(context.Background(), req.GetSnapID()); err == nil || err.Error() != constants.ErrSnapNotExists.Error() {
		itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.GetName()))
		err := itemHelper.FileOp.UnTar(path.Join(rootDir, snap.GetName(), "/1panel_data.tar.gz"), filepath.Join(snappy.BaseDir, "1panel"))
		itemHelper.Task.LogWithStatus(i18n.GetMsgByName("Decompress"), err)
		if err != nil {
			return fmt.Errorf("decompress panel data archive failed: %w", err)
		}
		
		if len(snappy.OperestyDir) > 0 && dirNotEmpty(snappy.OperestyDir) {
			itemHelper.Task.LogStart(i18n.GetMsgByName("RecoverWebsite", snappy.OperestyDir))
			err := itemHelper.FileOp.UnTar(path.Join(rootDir, snap.GetName(), "/website.tar.gz"), snappy.OperestyDir, nil)
			itemHelper.Task.LogWithStatus(i18n.GetMsgByName("RecoverWebsiteComplete"), err)
			if err != nil {
				return fmt.Errorf("decompress website content archive failed: %w", err)
			}
		}
	} else {
		return err
	}
	
	baseDir := req.GetResponse().BaseDir
	return backupBeforeRecover(req.GetSnapName(), itemHelper)
}

// ... rest of the modified code ...

Note that this example assumes you have other utility functions like backupBeforeRecover, recoverAppData, and copyDirWithExclusion implemented elsewhere in your project, which will require modification based on how those components are structured. Additionally, ensure that any updated dependencies comply with new package versions.

Expand Down
10 changes: 10 additions & 0 deletions agent/app/service/snapshot_rollback.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,16 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error {
},
nil,
)
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
if len(openrestyDir) != 0 {
taskItem.AddSubTask(
i18n.GetWithName("SnapCopy", openrestyDir),
func(t *task.Task) error {
return FileOp.CopyDir(path.Join(rootDir, "www"), openrestyDir)
},
nil,
)
}
taskItem.AddSubTask(
i18n.GetWithName("SnapCopy", global.Dir.BaseDir),
func(t *task.Task) error {
Expand Down
Loading
Loading