Skip to content

Commit 4bc494e

Browse files
SiumauricioSiumauricio
authored
Merge pull request #3637 from Dokploy/3392-conflict-between-daily-docker-cleanup-and-volume-backup-turn-off-container-during-backup
3392 conflict between daily docker cleanup and volume backup turn off container during backup
2 parents 2532934 + 110bdce commit 4bc494e

File tree

6 files changed

+61
-13
lines changed

6 files changed

+61
-13
lines changed

‎apps/dokploy/server/api/routers/settings.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {
2+
CLEANUP_CRON_JOB,
23
canAccessToTraefikFiles,
34
checkGPUStatus,
45
checkPortInUse,
@@ -298,12 +299,12 @@ export const settingsRouter = createTRPCRouter({
298299
}
299300
if (IS_CLOUD) {
300301
await schedule({
301-
cronSchedule: "0 0 * * *",
302+
cronSchedule: CLEANUP_CRON_JOB,
302303
serverId: input.serverId,
303304
type: "server",
304305
});
305306
} else {
306-
scheduleJob(server.serverId, "0 0 * * *", async () => {
307+
scheduleJob(server.serverId, CLEANUP_CRON_JOB, async () => {
307308
console.log(
308309
`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
309310
);
@@ -316,7 +317,7 @@ export const settingsRouter = createTRPCRouter({
316317
} else {
317318
if (IS_CLOUD) {
318319
await removeJob({
319-
cronSchedule: "0 0 * * *",
320+
cronSchedule: CLEANUP_CRON_JOB,
320321
serverId: input.serverId,
321322
type: "server",
322323
});
@@ -331,7 +332,7 @@ export const settingsRouter = createTRPCRouter({
331332
});
332333

333334
if (settingsUpdated?.enableDockerCleanup) {
334-
scheduleJob("docker-cleanup", "0 0 * * *", async () => {
335+
scheduleJob("docker-cleanup", CLEANUP_CRON_JOB, async () => {
335336
console.log(
336337
`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
337338
);

‎apps/dokploy/server/api/routers/volume-backups.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
} from "@dokploy/server/utils/process/execAsync";
2222
import { TRPCError } from "@trpc/server";
2323
import { observable } from "@trpc/server/observable";
24-
import { eq } from "drizzle-orm";
24+
import { desc, eq } from "drizzle-orm";
2525
import { z } from "zod";
2626
import { removeJob, schedule, updateJob } from "@/server/utils/backup";
2727
import { createTRPCRouter, protectedProcedure } from "../trpc";
@@ -54,6 +54,7 @@ export const volumeBackupsRouter = createTRPCRouter({
5454
redis: true,
5555
compose: true,
5656
},
57+
orderBy: [desc(volumeBackups.createdAt)],
5758
});
5859
}),
5960
create: protectedProcedure

‎apps/schedules/src/utils.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {
2+
CLEANUP_CRON_JOB,
23
cleanupAll,
34
findBackupById,
45
findScheduleById,
@@ -125,7 +126,7 @@ export const initializeJobs = async () => {
125126
scheduleJob({
126127
serverId,
127128
type: "server",
128-
cronSchedule: "0 0 * * *",
129+
cronSchedule: CLEANUP_CRON_JOB,
129130
});
130131
}
131132

‎packages/server/src/constants/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import path from "node:path";
22
import Docker from "dockerode";
33

44
export const IS_CLOUD = process.env.IS_CLOUD === "true";
5+
export const CLEANUP_CRON_JOB = "50 23 * * *";
56
export const docker = new Docker();
67

78
export const BETTER_AUTH_SECRET =
@@ -29,5 +30,6 @@ export const paths = (isServer = false) => {
2930
REGISTRY_PATH: `${BASE_PATH}/registry`,
3031
SCHEDULES_PATH: `${BASE_PATH}/schedules`,
3132
VOLUME_BACKUPS_PATH: `${BASE_PATH}/volume-backups`,
33+
VOLUME_BACKUP_LOCK_PATH: `${BASE_PATH}/volume-backup-lock`,
3234
};
3335
};

‎packages/server/src/utils/backups/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import path from "node:path";
2+
import { CLEANUP_CRON_JOB } from "@dokploy/server/constants";
23
import { member } from "@dokploy/server/db/schema";
34
import type { BackupSchedule } from "@dokploy/server/services/backup";
45
import { getAllServers } from "@dokploy/server/services/server";
@@ -29,7 +30,7 @@ export const initCronJobs = async () => {
2930
const webServerSettings = await getWebServerSettings();
3031

3132
if (webServerSettings?.enableDockerCleanup) {
32-
scheduleJob("docker-cleanup", "0 0 * * *", async () => {
33+
scheduleJob("docker-cleanup", CLEANUP_CRON_JOB, async () => {
3334
console.log(
3435
`Docker Cleanup ${new Date().toLocaleString()}] Running docker cleanup`,
3536
);
@@ -45,7 +46,7 @@ export const initCronJobs = async () => {
4546
for (const server of servers) {
4647
const { serverId, enableDockerCleanup, name } = server;
4748
if (enableDockerCleanup) {
48-
scheduleJob(serverId, "0 0 * * *", async () => {
49+
scheduleJob(serverId, CLEANUP_CRON_JOB, async () => {
4950
console.log(
5051
`SERVER-BACKUP[${new Date().toLocaleString()}] Running Cleanup ${name}`,
5152
);

‎packages/server/src/utils/volume-backups/backup.ts‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export const backupVolume = async (
1010
const { serviceType, volumeName, turnOff, prefix } = volumeBackup;
1111
const serverId =
1212
volumeBackup.application?.serverId || volumeBackup.compose?.serverId;
13-
const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
13+
const { VOLUME_BACKUPS_PATH, VOLUME_BACKUP_LOCK_PATH } = paths(!!serverId);
1414
const destination = volumeBackup.destination;
1515
const backupFileName = `${volumeName}-${new Date().toISOString()}.tar`;
1616
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
@@ -45,16 +45,56 @@ export const backupVolume = async (
4545
return baseCommand;
4646
}
4747

48+
const serviceLockId =
49+
serviceType === "application"
50+
? volumeBackup.application?.appName
51+
: `${volumeBackup.compose?.appName}_${volumeBackup.serviceName}`;
52+
53+
const lockPath = `${VOLUME_BACKUP_LOCK_PATH}-${serviceLockId}`;
54+
55+
const lockWrapper = (body: string) => `
56+
set -e
57+
58+
LOCK_PATH="${lockPath}"
59+
60+
echo "Waiting for volume backup lock: $LOCK_PATH"
61+
62+
if command -v flock >/dev/null 2>&1; then
63+
exec 9>"$LOCK_PATH"
64+
flock 9
65+
else
66+
LOCK_DIR="$LOCK_PATH.dir"
67+
while ! mkdir "$LOCK_DIR" 2>/dev/null; do
68+
echo "Waiting for volume backup lock: $LOCK_PATH"
69+
sleep 5
70+
done
71+
trap 'rm -rf "$LOCK_DIR"' EXIT
72+
fi
73+
74+
echo "Volume backup lock acquired"
75+
76+
${body}
77+
78+
echo "Volume backup lock released"
79+
`;
80+
81+
console.log(
82+
lockWrapper(`
83+
echo "Volume backup lock acquired"
84+
echo "Volume backup lock released"
85+
`),
86+
);
87+
4888
if (serviceType === "application") {
49-
return `
89+
return lockWrapper(`
5090
echo "Stopping application to 0 replicas"
5191
ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
5292
echo "Actual replicas: $ACTUAL_REPLICAS"
5393
docker service update --replicas=0 ${volumeBackup.application?.appName}
5494
${baseCommand}
5595
echo "Starting application to $ACTUAL_REPLICAS replicas"
5696
docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}
57-
`;
97+
`);
5898
}
5999
if (serviceType === "compose") {
60100
const compose = await findComposeById(
@@ -70,6 +110,7 @@ export const backupVolume = async (
70110
ACTUAL_REPLICAS=$(docker service inspect ${compose.appName}_${volumeBackup.serviceName} --format "{{.Spec.Mode.Replicated.Replicas}}")
71111
echo "Actual replicas: $ACTUAL_REPLICAS"
72112
docker service update --replicas=0 ${compose.appName}_${volumeBackup.serviceName}`;
113+
73114
startCommand = `
74115
echo "Starting compose to $ACTUAL_REPLICAS replicas"
75116
docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${compose.appName}_${volumeBackup.serviceName}`;
@@ -78,16 +119,17 @@ export const backupVolume = async (
78119
echo "Stopping compose container"
79120
ID=$(docker ps -q --filter "label=com.docker.compose.project=${compose.appName}" --filter "label=com.docker.compose.service=${volumeBackup.serviceName}")
80121
docker stop $ID`;
122+
81123
startCommand = `
82124
echo "Starting compose container"
83125
docker start $ID
84126
echo "Compose container started"
85127
`;
86128
}
87-
return `
129+
return lockWrapper(`
88130
${stopCommand}
89131
${baseCommand}
90132
${startCommand}
91-
`;
133+
`);
92134
}
93135
};

0 commit comments

Comments
 (0)