Skip to content

Commit 01c24fb

Browse files
MinituffMinituff
authored
Add Retention policy (#670)
* Add backup retention * Cleanup logs * feat: enhance backup retention policy with minimum backups and per-container pruning * Update rentention * Add more tests * Update docs --------- Co-authored-by: James Tufarelli <minituff@users.noreply.github.com>
1 parent 828c62a commit 01c24fb

11 files changed

Lines changed: 1172 additions & 32 deletions

‎.claude/settings.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"hooks": {
3+
"Stop": [
4+
{
5+
"hooks": [
6+
{
7+
"type": "command",
8+
"command": "nb pytest",
9+
"timeout": 60,
10+
"statusMessage": "Running tests..."
11+
}
12+
]
13+
}
14+
]
15+
}
16+
}

‎AGENTS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ Key docs pages:
2626
- `docs/labels.md` — all Docker label options (per-container config)
2727
- `docs/arguments.md` — all environment variable options (global config)
2828
- `docs/introduction.md` — high-level overview
29+
- `app/defaults.env` — default environment variable values
2930

3031
## Tests — Do Not Change Without Explicit Justification
3132

‎app/backup.py‎

Lines changed: 171 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import copy
44
import os
5+
import shutil
56
import subprocess
67
import sys
78
import time
@@ -83,13 +84,13 @@ def _record_error(self, message: str) -> None:
8384

8485
def _record_container_skipped(self, c: Container, reason: str, message: str, log=True) -> None:
8586
self.containers_skipped.add(c.name)
86-
self.container_skip_reasons.setdefault(c.name, reason)
87+
self.container_skip_reasons.setdefault(c.name if c.name else str(c.id), reason)
8788
if log:
8889
self.log_this(message, "WARN")
8990

9091
def _record_container_failed(self, c: Container, reason: str, message: str, log=True) -> None:
9192
self.containers_failed.add(c.name)
92-
self.container_failure_reasons.setdefault(c.name, reason)
93+
self.container_failure_reasons.setdefault(c.name if c.name else str(c.id), reason)
9394
self._record_error(message)
9495
if log:
9596
self.log_this(message, "ERROR")
@@ -781,6 +782,168 @@ def _get_rsync_args(self, c: Optional[Container], log=False) -> str:
781782

782783
return f"{default_rsync_args} {custom_rsync_args}"
783784

785+
def _apply_retention_policy(self, base_dest_dir: Path) -> None:
786+
"""Delete old date-stamped backup folders, keeping the N most recent backups.
787+
788+
Only runs when NUMBER_OF_BACKUPS_TO_KEEP > 0 and USE_DEST_DATE_FOLDER is true.
789+
Folders whose names cannot be parsed with DEST_DATE_FORMAT are left untouched.
790+
When RETENTION_DRY_RUN is true, candidates are logged but nothing is deleted.
791+
792+
Path formats are pruned according to their folder shape:
793+
- container/date: destination/<container>/<date>/ — date folders pruned per container dir
794+
- date/container: destination/<date>/<container>/ — date folders pruned as atomic backup sets
795+
"""
796+
backups_to_keep: int = self.env.NUMBER_OF_BACKUPS_TO_KEEP
797+
if backups_to_keep <= 0 or str(self.env.USE_DEST_DATE_FOLDER).lower() != "true":
798+
return
799+
800+
if base_dest_dir.is_symlink() or not base_dest_dir.is_dir():
801+
self.log_this(f"Retention policy: destination '{base_dest_dir}' is not a directory; skipping", "DEBUG")
802+
return
803+
804+
min_backups_to_keep: int = self.env.MIN_BACKUPS_TO_KEEP
805+
if min_backups_to_keep > 0 and min_backups_to_keep > backups_to_keep:
806+
self.log_this(
807+
f"Retention policy: MIN_BACKUPS_TO_KEEP ({min_backups_to_keep}) overrides "
808+
f"NUMBER_OF_BACKUPS_TO_KEEP ({backups_to_keep})",
809+
"DEBUG",
810+
)
811+
backups_to_keep = min_backups_to_keep
812+
813+
is_dry_run: bool = self.env.RETENTION_DRY_RUN
814+
date_format: str = self.env.DEST_DATE_FORMAT
815+
log_tag: str = " (DRY RUN)" if is_dry_run else ""
816+
817+
def _parse_date(folder_name: str) -> Optional[datetime]:
818+
"""Return a datetime if folder_name matches date_format, otherwise None."""
819+
try:
820+
return datetime.strptime(folder_name, date_format)
821+
except ValueError:
822+
return None
823+
824+
def _record_retention_error(message: str, error: OSError) -> None:
825+
"""Record retention failures without aborting the completed backup."""
826+
log_message = f"Retention policy: {message}: {error}"
827+
self._record_error(log_message)
828+
self.log_this(log_message, "ERROR")
829+
830+
def _iter_child_dirs(parent: Path) -> List[Path]:
831+
"""Return real child directories, skipping symlinks to avoid pruning outside the destination."""
832+
try:
833+
return [child for child in parent.iterdir() if not child.is_symlink() and child.is_dir()]
834+
except OSError as error:
835+
_record_retention_error(f"unable to inspect '{parent}'", error)
836+
return []
837+
838+
def _log_and_delete(target_folder: Path) -> None:
839+
"""Log and conditionally delete a single backup folder."""
840+
if is_dry_run:
841+
self.log_this(f"Retention policy (DRY RUN): would remove '{target_folder}'", "INFO")
842+
else:
843+
self.log_this(f"Retention policy: removing '{target_folder}'", "INFO")
844+
try:
845+
shutil.rmtree(target_folder)
846+
except OSError as error:
847+
_record_retention_error(f"failed to remove '{target_folder}'", error)
848+
849+
def _log_retention_summary(
850+
container_name: str,
851+
date_label_newest: str,
852+
date_label_oldest_kept: str,
853+
num_kept: int,
854+
num_to_delete: int,
855+
kept_date_labels: str,
856+
) -> None:
857+
"""Emit the per-container retention summary at DEBUG/TRACE."""
858+
self.log_this(
859+
f"Retention policy{log_tag}: '{container_name}' keeping {num_kept} backup(s) "
860+
f"({date_label_oldest_kept} to {date_label_newest}), {num_to_delete} older backup(s) will be removed",
861+
"DEBUG",
862+
)
863+
self.log_this(f"Retention policy{log_tag}: keeping '{container_name}' in: {kept_date_labels}", "TRACE")
864+
865+
def _prune_container_date(container_dir: Path) -> None:
866+
"""Prune date folders inside a single container directory (container/date layout)."""
867+
if container_dir.is_symlink() or not container_dir.is_dir():
868+
return
869+
870+
# Collect all date-named subfolders for this container
871+
dated_backups: List[Tuple[datetime, Path]] = []
872+
for date_folder in _iter_child_dirs(container_dir):
873+
parsed_date = _parse_date(date_folder.name)
874+
if parsed_date is not None:
875+
dated_backups.append((parsed_date, date_folder))
876+
877+
dated_backups.sort(key=lambda entry: entry[0], reverse=True) # newest first
878+
backups_kept = dated_backups[:backups_to_keep]
879+
backups_to_remove = dated_backups[backups_to_keep:]
880+
881+
if not backups_kept and not backups_to_remove:
882+
return
883+
884+
if backups_kept:
885+
date_label_newest = backups_kept[0][1].name
886+
date_label_oldest_kept = backups_kept[-1][1].name
887+
kept_date_labels = ", ".join(folder.name for _, folder in backups_kept)
888+
_log_retention_summary(
889+
container_dir.name,
890+
date_label_newest,
891+
date_label_oldest_kept,
892+
len(backups_kept),
893+
len(backups_to_remove),
894+
kept_date_labels,
895+
)
896+
897+
for _, backup_folder in backups_to_remove:
898+
_log_and_delete(backup_folder)
899+
900+
def _prune_date_container(destination_dir: Path) -> None:
901+
"""Prune dated backup folders (date/container layout).
902+
903+
Treats each date folder as the atomic unit of retention. The N most
904+
recent date folders are kept; all older ones are removed entirely.
905+
"""
906+
if destination_dir.is_symlink() or not destination_dir.is_dir():
907+
return
908+
909+
dated_folders: List[Tuple[datetime, Path]] = []
910+
for entry in _iter_child_dirs(destination_dir):
911+
parsed_date = _parse_date(entry.name)
912+
if parsed_date is not None:
913+
dated_folders.append((parsed_date, entry))
914+
915+
if not dated_folders:
916+
return
917+
918+
dated_folders.sort(key=lambda e: e[0], reverse=True) # newest first
919+
folders_kept = dated_folders[:backups_to_keep]
920+
folders_to_remove = dated_folders[backups_to_keep:]
921+
922+
if folders_kept:
923+
date_label_newest = folders_kept[0][1].name
924+
date_label_oldest_kept = folders_kept[-1][1].name
925+
kept_labels = ", ".join(f.name for _, f in folders_kept)
926+
self.log_this(
927+
f"Retention policy{log_tag}: keeping {len(folders_kept)} date folder(s) "
928+
f"({date_label_oldest_kept} to {date_label_newest}), "
929+
f"{len(folders_to_remove)} older folder(s) will be removed",
930+
"DEBUG",
931+
)
932+
self.log_this(f"Retention policy{log_tag}: keeping: {kept_labels}", "TRACE")
933+
934+
for _, folder in folders_to_remove:
935+
_log_and_delete(folder)
936+
937+
if self.env.DEST_DATE_PATH_FORMAT == "container/date":
938+
for container_dir in _iter_child_dirs(base_dest_dir):
939+
_prune_container_date(container_dir)
940+
elif self.env.DEST_DATE_PATH_FORMAT == "date/container":
941+
_prune_date_container(base_dest_dir)
942+
else:
943+
self.log_this(
944+
f"Unknown DEST_DATE_PATH_FORMAT '{self.env.DEST_DATE_PATH_FORMAT}' for retention policy", "ERROR"
945+
)
946+
784947
def reset_db(self) -> None:
785948
"""Reset the database values to their defaults"""
786949
self.db.put("containers_completed", 0)
@@ -910,6 +1073,12 @@ def backup(self):
9101073

9111074
for dir in dest_dirs:
9121075
self._backup_additional_folders_standalone(BeforeOrAfter.AFTER, dir)
1076+
1077+
self._apply_retention_policy(Path(self.env.DEST_LOCATION))
1078+
if self.env.RETENTION_SECONDARY_DESTINATIONS:
1079+
for dir in self.env.SECONDARY_DEST_DIRS:
1080+
self._apply_retention_policy(dir)
1081+
9131082
self.end_time = datetime.now()
9141083
exeuction_time = self.end_time - self.start_time
9151084
duration = datetime.fromtimestamp(exeuction_time.total_seconds())

‎app/defaults.env‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@ DEST_DATE_FORMAT=%Y-%m-%d
2727
# Otherwise, use the time Nautical started the backup (not when the container was backed up)
2828
USE_CONTAINER_BACKUP_DATE=true
2929

30+
# Keep only the N most recent dated backup folders (0 = disabled)
31+
NUMBER_OF_BACKUPS_TO_KEEP=0
32+
33+
# Never reduce a container's backup count below this floor (0 = disabled)
34+
MIN_BACKUPS_TO_KEEP=0
35+
36+
# Log which folders would be deleted without actually removing them
37+
RETENTION_DRY_RUN=false
38+
39+
# Apply the retention policy to secondary destinations as well as the primary
40+
RETENTION_SECONDARY_DESTINATIONS=true
41+
3042
# Use the default rsync args "-ahq" (archive, human-readable, quiet)
3143
USE_DEFAULT_RSYNC_ARGS=true
3244

‎app/nautical_env.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,20 @@ def __init__(self) -> None:
7070

7171
self.STOP_TIMEOUT = int(os.environ.get("STOP_TIMEOUT", 10))
7272

73+
_keep = os.environ.get("NUMBER_OF_BACKUPS_TO_KEEP", "0")
74+
self.NUMBER_OF_BACKUPS_TO_KEEP = int(_keep) if _keep.isdigit() else 0
75+
76+
_min_keep = os.environ.get("MIN_BACKUPS_TO_KEEP", "0")
77+
self.MIN_BACKUPS_TO_KEEP = int(_min_keep) if _min_keep.isdigit() else 0
78+
79+
self.RETENTION_DRY_RUN = False
80+
if os.environ.get("RETENTION_DRY_RUN", "false").lower() == "true":
81+
self.RETENTION_DRY_RUN = True
82+
83+
self.RETENTION_SECONDARY_DESTINATIONS = True
84+
if os.environ.get("RETENTION_SECONDARY_DESTINATIONS", "true").lower() == "false":
85+
self.RETENTION_SECONDARY_DESTINATIONS = False
86+
7387
@staticmethod
7488
def _populate_override_dirs(env_name: str) -> Dict[str, str]:
7589
"""Translate the Enviornment variable from single string to Python Dict.

‎dev/.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
COMPOSE_BAKE=false
2+
NAUTICAL_STOP_GRACE_PERIOD=1s
3+
NAUTICAL_S6_SERVICES_GRACETIME=250
4+
NAUTICAL_S6_KILL_GRACETIME=250

‎dev/create_test_backups_container_date.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
# Creates backdated test backup folders in container/date format:
3+
# destination/<container>/<date>/
4+
#
5+
# Usage: ./create_test_backups_container_date.sh [N_DAYS] [CONTAINER]
6+
# Defaults: N_DAYS=30, CONTAINER=watchtower
7+
8+
set -euo pipefail
9+
10+
N_DAYS="${1:-30}"
11+
CONTAINER="${2:-watchtower}"
12+
DEST="$(dirname "$0")/destination"
13+
14+
echo "Creating $N_DAYS days of test backups in container/date format"
15+
echo " Destination : $DEST"
16+
echo " Container : $CONTAINER"
17+
echo ""
18+
19+
for i in $(seq 0 "$((N_DAYS - 1))"); do
20+
folder=$(date -d "$i days ago" +%Y-%m-%d)
21+
target="$DEST/$CONTAINER/$folder"
22+
mkdir -p "$target"
23+
echo " Created: $CONTAINER/$folder"
24+
done
25+
26+
echo ""
27+
echo "Done. $N_DAYS folder(s) created."

‎dev/create_test_backups_date_container.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
# Creates backdated test backup folders in date/container format:
3+
# destination/<date>/<container>/
4+
#
5+
# Usage: ./create_test_backups_date_container.sh [N_DAYS] [CONTAINER]
6+
# Defaults: N_DAYS=30, CONTAINER=watchtower
7+
8+
set -euo pipefail
9+
10+
N_DAYS="${1:-30}"
11+
CONTAINER="${2:-watchtower}"
12+
CONTAINER2="${3:-test-container-2}"
13+
DEST="$(dirname "$0")/destination"
14+
15+
echo "Creating $N_DAYS days of test backups in date/container format"
16+
echo " Destination : $DEST"
17+
echo " Container : $CONTAINER"
18+
echo ""
19+
20+
for i in $(seq 0 "$((N_DAYS - 1))"); do
21+
folder=$(date -d "$i days ago" +%Y-%m-%d)
22+
target="$DEST/$folder/$CONTAINER"
23+
target2="$DEST/$folder/$CONTAINER2"
24+
mkdir -p "$target"
25+
mkdir -p "$target2"
26+
echo " Created: $folder/$CONTAINER and $folder/$CONTAINER2"
27+
echo "$DEST/$folder" >> "$target/test-file.txt"
28+
echo "$DEST/$folder" >> "$target2/test-file.txt"
29+
done
30+
31+
echo ""
32+
echo "Done. $N_DAYS folder(s) created."

0 commit comments

Comments
 (0)