Refactor backup configuration and logging; added LastCopies feature for backup retention. Updated Docker configuration and improved README for clarity.

Author: jsfraz

.gitignore

@@ -1,5 +1,3 @@
 .vscode/
 __debug_bin*
-
-# testing settings with sensitive info
 backuper_test.json

README.md

@@ -6,7 +6,7 @@ Container for backing up other container's volumes and database dumps to Mega.nz
 
 DISCLAIMER: If Mega.nz API returns error 402, login in browser from the same IP address before running the container. (<https://github.com/rclone/rclone/issues/8270#issuecomment-2562047717>)
 
-This example config backups PostgreSQL database from `postgres-example` container every day at 12:00.
+This example config backups PostgreSQL database from `postgres-example` container every day at 12:00. It keeps last 10 copies in the output directory, older copies are moved to the rubbish bin.
 
 ### Example `backuper.json`
 
@@ -18,6 +18,7 @@ This example config backups PostgreSQL database from `postgres-example` containe
         {
             "name": "postgres",
             "megaDir": "postgres/",
+            "lastCopies": 10,
             "cron": "0 12 * * *",
             "type": "postgres",
             "pgUser": "postgres",
@@ -67,19 +68,24 @@ volumes:
 |----------|---------------------|----------------------------|----------|
 | email    | string              | Your Mega.nz e-mail        | true     |
 | password | string              | Your Mega.nz password      | true     |
-| backups  | backup object array | Individual backup settings | false    |
+| backups  | backup object array | Individual backup settings | true     |
 
 ### Backup object properties
 
-| Property | Type   | Description                           | Required |
-|----------|--------|---------------------------------------|----------|
-| name     | string | Backup name                           | true     |
-| megaDir  | string | Remote Mega.nz destination directory  | true     |
-| cron     | string | Cron expression for scheduling backup | true     |
-| type     | string | Backup type (postgres)                | true     |
+| Property         | Type   | Description                                           | Required |
+|------------------|--------|-------------------------------------------------------|----------|
+| name             | string | Backup name                                           | true     |
+| megaDir          | string | Remote Mega.nz destination directory                  | true     |
+| lastCopies       | int    | Number of last copies to keep                         | false    |
+<!-- FIXME https://github.com/t3rm1n4l/go-mega/pull/46 -->
+<!-- | destroyOldCopies | bool   | Destroy old copies instead moving them to rubbish bin | false    | -->
+| cron             | string | Cron expression for scheduling backup                 | true     |
+| type             | string | Backup type (postgres)                                | true     |
 
 #### PostgreSQL backup properties
 
+**DISCLAIMER: This project uses [`go-pgdump`](https://github.com/JCoupalK/go-pgdump) to dump PostgreSQL database. It doesn't feature all of `pg_dump` features and only supports dumping table contents, not triggers, views, functions, etc.**
+
 | Property   | Type   | Description                                                                              | Required |
 |------------|--------|------------------------------------------------------------------------------------------|----------|
 | pgUser     | string | PostgreSQL username                                                                      | true     |

docker-compose.yaml

@@ -1,3 +1,4 @@
+# debug config
 name: mega-backuper-example
 
 services:
@@ -11,15 +12,14 @@ services:
       - POSTGRES_DB=postgres
     volumes:
       - postgres:/var/lib/postgresql/data
-    restart: always
+    restart: unless-stopped
 
   mega-backuper:
-    image: ghcr.io/jsfraz/mega-backuper:latest
+    build:
+      context: .
     container_name: mega-backuper
-    restart: always
+    restart: unless-stopped
     volumes:
-      # TODO actually replace with actual config file
-      # - ./backuper.json:/app/backuper.json   # backuper settings
       - ./backuper_test.json:/app/backuper.json
 
 volumes:

go.mod

@@ -5,6 +5,7 @@ go 1.24.1
 require (
 	github.com/go-co-op/gocron v1.37.0
 	github.com/go-playground/validator/v10 v10.25.0
+	github.com/lnquy/cron v1.1.1
 	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5
 )
 

go.sum

@@ -30,6 +30,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lnquy/cron v1.1.1 h1:iaDX1ublgQ9LBhA8l9BVU+FrTE1PPSPAuvAdhgdnXgA=
+github.com/lnquy/cron v1.1.1/go.mod h1:hu2Y7H68/8oKk6T4+K4qdbopbnaP4rGltK3ylWiiDss=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

main.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/go-co-op/gocron"
+	"github.com/lnquy/cron"
 	"github.com/t3rm1n4l/go-mega"
 )
 
@@ -32,6 +33,7 @@ func main() {
 	// login or exit
 	utils.MegaLogin()
 
+	// TODO ping Postgres
 	// Check volumes
 	// TODO volumes
 	// utils.CheckVolumes()
@@ -72,7 +74,9 @@ func main() {
 			}
 		*/
 		scheduler.Cron(backup.Cron).Do(backupFunc)
-		log.Printf("Added [%s] backup job '%s'", backup.Type, backup.Name)
+		exprDesc, _ := cron.NewDescriptor()
+		desc, _ := exprDesc.ToDescription(backup.Cron, cron.Locale_en)
+		log.Printf("Added [%s] backup job '%s': %s", backup.Type, backup.Name, desc)
 	}
 	// check if job list is empty or not
 	if len(scheduler.Jobs()) != 0 {
@@ -90,11 +94,11 @@ func main() {
 //	@param backup
 //	@param backupFunc
 func handleBackup(backup models.Backup, backupFunc func(backup models.Backup) error) {
-	log.Printf("Backing up [%s] backup job '%s'...", backup.Type, backup.Name)
+	log.Printf("Running [%s] backup job '%s'...", backup.Type, backup.Name)
 	err := backupFunc(backup)
 	if err != nil {
-		log.Printf("Failed to backup [%s] backup job '%s': %s", backup.Type, backup.Name, err)
+		log.Printf("Failed to run [%s] backup job '%s': %s", backup.Type, backup.Name, err)
 	} else {
-		log.Printf("Successfully backed up [%s] backup job '%s'", backup.Type, backup.Name)
+		log.Printf("Successfully ran up [%s] backup job '%s'", backup.Type, backup.Name)
 	}
 }

models/backup.go

@@ -1,10 +1,11 @@
 package models
 
 type Backup struct {
-	Name    string `json:"name" validate:"required"`
-	MegaDir string `json:"megaDir" validate:"dirpath,required"`
-	// LastCopies       *int       `json:"lastCopies"`
-	// DestroyOldCopies *bool      `json:"destroyOldCopies"`
+	Name       string `json:"name" validate:"required"`
+	MegaDir    string `json:"megaDir" validate:"dirpath,required"`
+	LastCopies *int   `json:"lastCopies" validate:"omitempty,gt=0"`
+	// FIXME https://github.com/t3rm1n4l/go-mega/pull/46
+	// DestroyOldCopies bool       `json:"destroyOldCopies" validate:"required_with=LastCopies"`
 	Cron string     `json:"cron" validate:"cron,required"`
 	Type BackupType `json:"type" validate:"oneof=volume postgres mysql,required"`
 
@@ -15,12 +16,11 @@ type Backup struct {
 	PgHost     string `json:"pgHost" validate:"required_if=Type postgres,omitempty,required"`
 	PgPort     int    `json:"pgPort" validate:"required_if=Type postgres,omitempty,required"`
 
-	// Mysql
-	MysqlUser     string `json:"mysqlUser" validate:"required_if=Type mysql,omitempty,required"`
-	MysqlPassword string `json:"mysqlPassword" validate:"required_if=Type mysql,omitempty,required"`
-	MysqlDb       string `json:"mysqlDb" validate:"required_if=Type mysql,omitempty,required"`
+	// TODO volume
 
-	// Volume
-	// TODO Support backuping only selected subdirs of volume
-	// Subdirs []string `json:"subdirs"`
+	// TODO mysql
+	// Mysql
+	// MysqlUser     string `json:"mysqlUser" validate:"required_if=Type mysql,omitempty,required"`
+	// MysqlPassword string `json:"mysqlPassword" validate:"required_if=Type mysql,omitempty,required"`
+	// MysqlDb       string `json:"mysqlDb" validate:"required_if=Type mysql,omitempty,required"`
 }

models/backup_type.go

@@ -3,8 +3,7 @@ package models
 type BackupType string
 
 const (
-	// TODO volumes and mysql
-	// Volume   BackupType = "volume"
 	Postgres BackupType = "postgres"
+	// Volume   BackupType = "volume"
 	// Mysql    BackupType = "mysql"
 )

utils/backup.go

@@ -24,8 +24,8 @@ import (
 func createTarball(backup models.Backup, now time.Time) (string, string, error) {
 	// create tarball file
 	folderPath := fmt.Sprintf("/tmp/%s", backup.Name)
-	// file name: NAME_RFC3339.tar.gz
-	tarballFileName := fmt.Sprintf("%s_%s.tar.gz", backup.Name, now.Format(time.RFC3339))
+	// File name: NAME_UNIX_TIMESTAMP.tar.gz
+	tarballFileName := fmt.Sprintf("%s_%d.tar.gz", backup.Name, now.Unix())
 	tarballPath := fmt.Sprintf("/tmp/%s", tarballFileName)
 	tarballFile, err := os.Create(tarballPath)
 	if err != nil {
@@ -112,45 +112,16 @@ func uploadToMegaAndDelete(localFilePath string, fileName string, megaDir string
 	return uploadNode, nil
 }
 
-// Backup volume to Mega.
-//
-//	@param backup
-//	@return error
-func BackupVolume(backup models.Backup) error {
-	now := time.Now()
-	// make tarball
-	tarballPath, tarballFileName, err := createTarball(backup, now)
-	if err != nil {
-		return err
-	}
-	// upload to mega
-	_, err = uploadToMegaAndDelete(tarballPath, tarballFileName, backup.MegaDir)
-	if err != nil {
-		return err
-	}
-	/*
-		// delete oldest file(s)
-		if backup.LastCopies != nil {
-			// FIXME every time after first backup, it throws error 'Object (typically, node or user) not found'
-			// every cron iteration the error count increases
-			err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
-			if err != nil {
-				return err
-			}
-		}
-	*/
-	return nil
-}
-
 // Backup Postgres.
 //
 //	@param backup
 //	@return error
 func BackupPostgres(backup models.Backup) error {
+	currentTime := time.Now()
 	// Init dumper
 	dumper := pgdump.NewDumper(fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
 		backup.PgHost, backup.PgPort, backup.PgUser, backup.PgPassword, backup.PgDb), 50)
-	currentTime := time.Now()
+	// File name: DB_NAME_UNIX_TIMESTAMP.sql
 	dumpFilename := fmt.Sprintf("/tmp/%s-%d.sql", backup.PgDb, currentTime.Unix())
 
 	// Dump database
@@ -164,21 +135,39 @@ func BackupPostgres(backup models.Backup) error {
 	}
 
 	// Upload to mega
-	_, err = uploadToMegaAndDelete(dumpFilename, strings.Split(dumpFilename, "/tmp/")[1], backup.MegaDir)
+	uploadNode, err := uploadToMegaAndDelete(dumpFilename, strings.Split(dumpFilename, "/tmp/")[1], backup.MegaDir)
+	if err != nil {
+		return err
+	}
+	// Delete oldest file(s)
+	err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Backup volume to Mega.
+//
+//	@param backup
+//	@return error
+func BackupVolume(backup models.Backup) error {
+	currentTime := time.Now()
+	// make tarball
+	tarballPath, tarballFileName, err := createTarball(backup, currentTime)
+	if err != nil {
+		return err
+	}
+	// upload to mega
+	uploadNode, err := uploadToMegaAndDelete(tarballPath, tarballFileName, backup.MegaDir)
+	if err != nil {
+		return err
+	}
+	// delete oldest file(s)
+	err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
 	if err != nil {
 		return err
 	}
-	/*
-		// Delete oldest file(s)
-		if backup.LastCopies != nil {
-			// FIXME every time after first backup, it throws error 'Object (typically, node or user) not found'
-			// Every cron iteration the error count increases
-			err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
-			if err != nil {
-				return err
-			}
-		}
-	*/
 	return nil
 }
 

utils/mega.go

@@ -1,7 +1,9 @@
 package utils
 
 import (
+	"jsfraz/mega-backuper/models"
 	"log"
+	"sort"
 	"strings"
 
 	"github.com/t3rm1n4l/go-mega"
@@ -88,50 +90,33 @@ func MegaUpload(localFilePath string, node *mega.Node, fileName string) error {
 	return err
 }
 
-/*
-// Uploads file to Mega and keeps last n versions. Others are deleted.
+// Keeps last n versions. Others are deleted.
 //
 //	@param backup
 //	@param node Node to upload/delete files to/from.
 //	@return error
 func MegaDeleteFilesByLastCopyCount(backup models.Backup, node *mega.Node) error {
-	// get node children
-	m := GetSingleton().Mega
-	children, err := m.FS.GetChildren(node)
-	if err != nil {
-		return err
-	}
-	// filter all files matching name
-	var fileNodes []*mega.Node
-	// NAME_RFC3339.tar.gz
-	pattern := `^(` + backup.Name + `_)((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)(.tar.gz)$`
-	for _, child := range children {
-		match, _ := regexp.MatchString(pattern, child.GetName())
-		// check if child is file (type == 0) and if name matches pattern
-		if child.GetType() == 0 && match {
-			fileNodes = append(fileNodes, child)
+	if backup.LastCopies != nil {
+		// get node children
+		m := GetSingleton().Mega
+		fileNodes, err := m.FS.GetChildren(node)
+		if err != nil {
+			return err
 		}
-	}
-	var deleteErrs []error
-	// delete oldest file(s)
-	if len(fileNodes) > *backup.LastCopies {
-		// sort by newest
-		sort.Slice(fileNodes, func(i, j int) bool {
-			return fileNodes[i].GetTimeStamp().After(fileNodes[j].GetTimeStamp())
-		})
-		// delete
-		for _, file := range fileNodes[*backup.LastCopies:] {
-			deleteErr := m.Delete(file, *backup.DestroyOldCopies)
-			if deleteErr != nil {
-				deleteErrs = append(deleteErrs, deleteErr)
+		// delete oldest file(s)
+		if len(fileNodes) > *backup.LastCopies {
+			// sort by newest
+			sort.Slice(fileNodes, func(i, j int) bool {
+				return fileNodes[i].GetTimeStamp().After(fileNodes[j].GetTimeStamp())
+			})
+			// delete
+			for _, file := range fileNodes[*backup.LastCopies:] {
+				// FIXME https://github.com/t3rm1n4l/go-mega/pull/46
+				// m.Delete(file, backup.DestroyOldCopies)
+				m.Delete(file, false)
 			}
 		}
 	}
-	// return error
-	if len(deleteErrs) != 0 {
-		return errors.Join(deleteErrs...)
-	} else {
-		return nil
-	}
+	// Don't return delete errors, the deleted files are still fetched from FS even when they don't exist
+	return nil
 }
-*/