Skip to content

Commit 84f9412

Browse files
jsfrazjsfraz
authored
Merge pull request #1 from jsfraz/development
Development
2 parents 44e3ec4 + 258cd46 commit 84f9412

10 files changed

Lines changed: 106 additions & 122 deletions

File tree

‎.gitignore‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
11
.vscode/
22
__debug_bin*
3-
4-
# testing settings with sensitive info
53
backuper_test.json

‎README.md‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ Container for backing up other container's volumes and database dumps to Mega.nz
44

55
## Example usage
66

7-
This example config backups PostgreSQL database from `postgres-mega-backuper` container every day at 12:00.
7+
DISCLAIMER: If Mega.nz API returns error 402, login in browser from the same IP address before running the container. (<https://github.com/rclone/rclone/issues/8270#issuecomment-2562047717>)
88

9+
This example config backups PostgreSQL database from `postgres-example` container every day at 12:00. It keeps last 10 copies in the output directory, older copies are moved to the rubbish bin.
910

1011
### Example `backuper.json`
1112

@@ -17,6 +18,7 @@ This example config backups PostgreSQL database from `postgres-mega-backuper` co
1718
{
1819
"name": "postgres",
1920
"megaDir": "postgres/",
21+
"lastCopies": 10,
2022
"cron": "0 12 * * *",
2123
"type": "postgres",
2224
"pgUser": "postgres",
@@ -58,7 +60,6 @@ volumes:
5860
postgres:
5961
```
6062
61-
6263
## Config file properties
6364
6465
### General properties
@@ -67,23 +68,28 @@ volumes:
6768
|----------|---------------------|----------------------------|----------|
6869
| email | string | Your Mega.nz e-mail | true |
6970
| password | string | Your Mega.nz password | true |
70-
| backups | backup object array | Individual backup settings | false |
71+
| backups | backup object array | Individual backup settings | true |
7172
7273
### Backup object properties
7374
74-
| Property | Type | Description | Required |
75-
|----------|--------|---------------------------------------|----------|
76-
| name | string | Backup name | true |
77-
| megaDir | string | Remote Mega.nz destination directory | true |
78-
| cron | string | Cron expression for scheduling backup | true |
79-
| type | string | Backup type (postgres) | true |
75+
| Property | Type | Description | Required |
76+
|------------------|--------|-------------------------------------------------------|----------|
77+
| name | string | Backup name | true |
78+
| megaDir | string | Remote Mega.nz destination directory | true |
79+
| lastCopies | int | Number of last copies to keep | false |
80+
<!-- FIXME https://github.com/t3rm1n4l/go-mega/pull/46 -->
81+
<!-- | destroyOldCopies | bool | Destroy old copies instead moving them to rubbish bin | false | -->
82+
| cron | string | Cron expression for scheduling backup | true |
83+
| type | string | Backup type (postgres) | true |
8084
8185
#### PostgreSQL backup properties
8286
83-
| Property | Type | Description | Required |
84-
|------------|--------|---------------------------------------|----------|
85-
| pgUser | string | PostgreSQL username | true |
86-
| pgPassword | string | PostgreSQL password | true |
87-
| pgDb | string | PostgreSQL database name | true |
88-
| pgHost | string | PostgreSQL host | true |
89-
| pgPort | int | PostgreSQL port | true |
87+
**DISCLAIMER: This project uses [`go-pgdump`](https://github.com/JCoupalK/go-pgdump) to dump PostgreSQL database. It doesn't feature all of `pg_dump` features and only supports dumping table contents, not triggers, views, functions, etc.**
88+
89+
| Property | Type | Description | Required |
90+
|------------|--------|------------------------------------------------------------------------------------------|----------|
91+
| pgUser | string | PostgreSQL username | true |
92+
| pgPassword | string | PostgreSQL password | true |
93+
| pgDb | string | PostgreSQL database name | true |
94+
| pgHost | string | PostgreSQL host (or container name if running in the same network) | true |
95+
| pgPort | int | PostgreSQL port | true |

‎docker-compose.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# debug config
12
name: mega-backuper-example
23

34
services:
@@ -11,15 +12,14 @@ services:
1112
- POSTGRES_DB=postgres
1213
volumes:
1314
- postgres:/var/lib/postgresql/data
14-
restart: always
15+
restart: unless-stopped
1516

1617
mega-backuper:
17-
image: ghcr.io/jsfraz/mega-backuper:latest
18+
build:
19+
context: .
1820
container_name: mega-backuper
19-
restart: always
21+
restart: unless-stopped
2022
volumes:
21-
# TODO actually replace with actual config file
22-
# - ./backuper.json:/app/backuper.json # backuper settings
2323
- ./backuper_test.json:/app/backuper.json
2424

2525
volumes:

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.24.1
55
require (
66
github.com/go-co-op/gocron v1.37.0
77
github.com/go-playground/validator/v10 v10.25.0
8+
github.com/lnquy/cron v1.1.1
89
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5
910
)
1011

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
3030
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
3131
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
3232
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
33+
github.com/lnquy/cron v1.1.1 h1:iaDX1ublgQ9LBhA8l9BVU+FrTE1PPSPAuvAdhgdnXgA=
34+
github.com/lnquy/cron v1.1.1/go.mod h1:hu2Y7H68/8oKk6T4+K4qdbopbnaP4rGltK3ylWiiDss=
3335
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
3436
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
3537
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

‎main.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"time"
88

99
"github.com/go-co-op/gocron"
10+
"github.com/lnquy/cron"
1011
"github.com/t3rm1n4l/go-mega"
1112
)
1213

@@ -32,6 +33,7 @@ func main() {
3233
// login or exit
3334
utils.MegaLogin()
3435

36+
// TODO ping Postgres
3537
// Check volumes
3638
// TODO volumes
3739
// utils.CheckVolumes()
@@ -72,7 +74,9 @@ func main() {
7274
}
7375
*/
7476
scheduler.Cron(backup.Cron).Do(backupFunc)
75-
log.Printf("Added [%s] backup job '%s'", backup.Type, backup.Name)
77+
exprDesc, _ := cron.NewDescriptor()
78+
desc, _ := exprDesc.ToDescription(backup.Cron, cron.Locale_en)
79+
log.Printf("Added [%s] backup job '%s': %s", backup.Type, backup.Name, desc)
7680
}
7781
// check if job list is empty or not
7882
if len(scheduler.Jobs()) != 0 {
@@ -90,11 +94,11 @@ func main() {
9094
// @param backup
9195
// @param backupFunc
9296
func handleBackup(backup models.Backup, backupFunc func(backup models.Backup) error) {
93-
log.Printf("Backing up [%s] backup job '%s'...", backup.Type, backup.Name)
97+
log.Printf("Running [%s] backup job '%s'...", backup.Type, backup.Name)
9498
err := backupFunc(backup)
9599
if err != nil {
96-
log.Printf("Failed to backup [%s] backup job '%s': %s", backup.Type, backup.Name, err)
100+
log.Printf("Failed to run [%s] backup job '%s': %s", backup.Type, backup.Name, err)
97101
} else {
98-
log.Printf("Successfully backed up [%s] backup job '%s'", backup.Type, backup.Name)
102+
log.Printf("Successfully ran up [%s] backup job '%s'", backup.Type, backup.Name)
99103
}
100104
}

‎models/backup.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
package models
22

33
type Backup struct {
4-
Name string `json:"name" validate:"alphanum,required"`
5-
MegaDir string `json:"megaDir" validate:"dirpath,required"`
6-
// LastCopies *int `json:"lastCopies"`
7-
// DestroyOldCopies *bool `json:"destroyOldCopies"`
4+
Name string `json:"name" validate:"required"`
5+
MegaDir string `json:"megaDir" validate:"dirpath,required"`
6+
LastCopies *int `json:"lastCopies" validate:"omitempty,gt=0"`
7+
// FIXME https://github.com/t3rm1n4l/go-mega/pull/46
8+
// DestroyOldCopies bool `json:"destroyOldCopies" validate:"required_with=LastCopies"`
89
Cron string `json:"cron" validate:"cron,required"`
910
Type BackupType `json:"type" validate:"oneof=volume postgres mysql,required"`
1011

@@ -15,12 +16,11 @@ type Backup struct {
1516
PgHost string `json:"pgHost" validate:"required_if=Type postgres,omitempty,required"`
1617
PgPort int `json:"pgPort" validate:"required_if=Type postgres,omitempty,required"`
1718

18-
// Mysql
19-
MysqlUser string `json:"mysqlUser" validate:"required_if=Type mysql,omitempty,required"`
20-
MysqlPassword string `json:"mysqlPassword" validate:"required_if=Type mysql,omitempty,required"`
21-
MysqlDb string `json:"mysqlDb" validate:"required_if=Type mysql,omitempty,required"`
19+
// TODO volume
2220

23-
// Volume
24-
// TODO Support backuping only selected subdirs of volume
25-
// Subdirs []string `json:"subdirs"`
21+
// TODO mysql
22+
// Mysql
23+
// MysqlUser string `json:"mysqlUser" validate:"required_if=Type mysql,omitempty,required"`
24+
// MysqlPassword string `json:"mysqlPassword" validate:"required_if=Type mysql,omitempty,required"`
25+
// MysqlDb string `json:"mysqlDb" validate:"required_if=Type mysql,omitempty,required"`
2626
}

‎models/backup_type.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ package models
33
type BackupType string
44

55
const (
6-
// TODO volumes and mysql
7-
// Volume BackupType = "volume"
86
Postgres BackupType = "postgres"
7+
// Volume BackupType = "volume"
98
// Mysql BackupType = "mysql"
109
)

‎utils/backup.go‎

Lines changed: 34 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ import (
2424
func createTarball(backup models.Backup, now time.Time) (string, string, error) {
2525
// create tarball file
2626
folderPath := fmt.Sprintf("/tmp/%s", backup.Name)
27-
// file name: NAME_RFC3339.tar.gz
28-
tarballFileName := fmt.Sprintf("%s_%s.tar.gz", backup.Name, now.Format(time.RFC3339))
27+
// File name: NAME_UNIX_TIMESTAMP.tar.gz
28+
tarballFileName := fmt.Sprintf("%s_%d.tar.gz", backup.Name, now.Unix())
2929
tarballPath := fmt.Sprintf("/tmp/%s", tarballFileName)
3030
tarballFile, err := os.Create(tarballPath)
3131
if err != nil {
@@ -112,45 +112,16 @@ func uploadToMegaAndDelete(localFilePath string, fileName string, megaDir string
112112
return uploadNode, nil
113113
}
114114

115-
// Backup volume to Mega.
116-
//
117-
// @param backup
118-
// @return error
119-
func BackupVolume(backup models.Backup) error {
120-
now := time.Now()
121-
// make tarball
122-
tarballPath, tarballFileName, err := createTarball(backup, now)
123-
if err != nil {
124-
return err
125-
}
126-
// upload to mega
127-
_, err = uploadToMegaAndDelete(tarballPath, tarballFileName, backup.MegaDir)
128-
if err != nil {
129-
return err
130-
}
131-
/*
132-
// delete oldest file(s)
133-
if backup.LastCopies != nil {
134-
// FIXME every time after first backup, it throws error 'Object (typically, node or user) not found'
135-
// every cron iteration the error count increases
136-
err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
137-
if err != nil {
138-
return err
139-
}
140-
}
141-
*/
142-
return nil
143-
}
144-
145115
// Backup Postgres.
146116
//
147117
// @param backup
148118
// @return error
149119
func BackupPostgres(backup models.Backup) error {
120+
currentTime := time.Now()
150121
// Init dumper
151122
dumper := pgdump.NewDumper(fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
152123
backup.PgHost, backup.PgPort, backup.PgUser, backup.PgPassword, backup.PgDb), 50)
153-
currentTime := time.Now()
124+
// File name: DB_NAME_UNIX_TIMESTAMP.sql
154125
dumpFilename := fmt.Sprintf("/tmp/%s-%d.sql", backup.PgDb, currentTime.Unix())
155126

156127
// Dump database
@@ -164,21 +135,39 @@ func BackupPostgres(backup models.Backup) error {
164135
}
165136

166137
// Upload to mega
167-
_, err = uploadToMegaAndDelete(dumpFilename, strings.Split(dumpFilename, "/tmp/")[1], backup.MegaDir)
138+
uploadNode, err := uploadToMegaAndDelete(dumpFilename, strings.Split(dumpFilename, "/tmp/")[1], backup.MegaDir)
139+
if err != nil {
140+
return err
141+
}
142+
// Delete oldest file(s)
143+
err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
144+
if err != nil {
145+
return err
146+
}
147+
return nil
148+
}
149+
150+
// Backup volume to Mega.
151+
//
152+
// @param backup
153+
// @return error
154+
func BackupVolume(backup models.Backup) error {
155+
currentTime := time.Now()
156+
// make tarball
157+
tarballPath, tarballFileName, err := createTarball(backup, currentTime)
158+
if err != nil {
159+
return err
160+
}
161+
// upload to mega
162+
uploadNode, err := uploadToMegaAndDelete(tarballPath, tarballFileName, backup.MegaDir)
163+
if err != nil {
164+
return err
165+
}
166+
// delete oldest file(s)
167+
err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
168168
if err != nil {
169169
return err
170170
}
171-
/*
172-
// Delete oldest file(s)
173-
if backup.LastCopies != nil {
174-
// FIXME every time after first backup, it throws error 'Object (typically, node or user) not found'
175-
// Every cron iteration the error count increases
176-
err = MegaDeleteFilesByLastCopyCount(backup, uploadNode)
177-
if err != nil {
178-
return err
179-
}
180-
}
181-
*/
182171
return nil
183172
}
184173

‎utils/mega.go‎

Lines changed: 22 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package utils
22

33
import (
4+
"jsfraz/mega-backuper/models"
45
"log"
6+
"sort"
57
"strings"
68

79
"github.com/t3rm1n4l/go-mega"
@@ -88,50 +90,33 @@ func MegaUpload(localFilePath string, node *mega.Node, fileName string) error {
8890
return err
8991
}
9092

91-
/*
92-
// Uploads file to Mega and keeps last n versions. Others are deleted.
93+
// Keeps last n versions. Others are deleted.
9394
//
9495
// @param backup
9596
// @param node Node to upload/delete files to/from.
9697
// @return error
9798
func MegaDeleteFilesByLastCopyCount(backup models.Backup, node *mega.Node) error {
98-
// get node children
99-
m := GetSingleton().Mega
100-
children, err := m.FS.GetChildren(node)
101-
if err != nil {
102-
return err
103-
}
104-
// filter all files matching name
105-
var fileNodes []*mega.Node
106-
// NAME_RFC3339.tar.gz
107-
pattern := `^(` + backup.Name + `_)((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)(.tar.gz)$`
108-
for _, child := range children {
109-
match, _ := regexp.MatchString(pattern, child.GetName())
110-
// check if child is file (type == 0) and if name matches pattern
111-
if child.GetType() == 0 && match {
112-
fileNodes = append(fileNodes, child)
99+
if backup.LastCopies != nil {
100+
// get node children
101+
m := GetSingleton().Mega
102+
fileNodes, err := m.FS.GetChildren(node)
103+
if err != nil {
104+
return err
113105
}
114-
}
115-
var deleteErrs []error
116-
// delete oldest file(s)
117-
if len(fileNodes) > *backup.LastCopies {
118-
// sort by newest
119-
sort.Slice(fileNodes, func(i, j int) bool {
120-
return fileNodes[i].GetTimeStamp().After(fileNodes[j].GetTimeStamp())
121-
})
122-
// delete
123-
for _, file := range fileNodes[*backup.LastCopies:] {
124-
deleteErr := m.Delete(file, *backup.DestroyOldCopies)
125-
if deleteErr != nil {
126-
deleteErrs = append(deleteErrs, deleteErr)
106+
// delete oldest file(s)
107+
if len(fileNodes) > *backup.LastCopies {
108+
// sort by newest
109+
sort.Slice(fileNodes, func(i, j int) bool {
110+
return fileNodes[i].GetTimeStamp().After(fileNodes[j].GetTimeStamp())
111+
})
112+
// delete
113+
for _, file := range fileNodes[*backup.LastCopies:] {
114+
// FIXME https://github.com/t3rm1n4l/go-mega/pull/46
115+
// m.Delete(file, backup.DestroyOldCopies)
116+
m.Delete(file, false)
127117
}
128118
}
129119
}
130-
// return error
131-
if len(deleteErrs) != 0 {
132-
return errors.Join(deleteErrs...)
133-
} else {
134-
return nil
135-
}
120+
// Don't return delete errors, the deleted files are still fetched from FS even when they don't exist
121+
return nil
136122
}
137-
*/

0 commit comments

Comments
 (0)