From b9e700243e43048b7fc0fcb4fba90c1faab15f1b Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Sat, 7 Feb 2026 22:03:16 -0600 Subject: [PATCH] feat(volume-backups): implement volume backup locking mechanism - Added a locking mechanism to prevent concurrent volume backups, ensuring data integrity during backup operations. - Introduced a `lockWrapper` function that manages the locking process using either `flock` or directory-based locking. - Updated the `backupVolume` function to utilize the locking mechanism for both application and compose service types, enhancing the reliability of backup processes. --- .../server/src/utils/volume-backups/backup.ts | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/packages/server/src/utils/volume-backups/backup.ts b/packages/server/src/utils/volume-backups/backup.ts index 68f721752..3d229ef64 100644 --- a/packages/server/src/utils/volume-backups/backup.ts +++ b/packages/server/src/utils/volume-backups/backup.ts @@ -10,7 +10,7 @@ export const backupVolume = async ( const { serviceType, volumeName, turnOff, prefix } = volumeBackup; const serverId = volumeBackup.application?.serverId || volumeBackup.compose?.serverId; - const { VOLUME_BACKUPS_PATH } = paths(!!serverId); + const { VOLUME_BACKUPS_PATH, VOLUME_BACKUP_LOCK_PATH } = paths(!!serverId); const destination = volumeBackup.destination; const backupFileName = `${volumeName}-${new Date().toISOString()}.tar`; const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`; @@ -45,8 +45,48 @@ export const backupVolume = async ( return baseCommand; } + const serviceLockId = + serviceType === "application" + ? volumeBackup.application?.appName + : `${volumeBackup.compose?.appName}_${volumeBackup.serviceName}`; + + const lockPath = `${VOLUME_BACKUP_LOCK_PATH}-${serviceLockId}`; + + const lockWrapper = (body: string) => ` + set -e + + LOCK_PATH="${lockPath}" + + echo "Waiting for volume backup lock: $LOCK_PATH" + + if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_PATH" + flock 9 + else + LOCK_DIR="$LOCK_PATH.dir" + while ! mkdir "$LOCK_DIR" 2>/dev/null; do + echo "Waiting for volume backup lock: $LOCK_PATH" + sleep 5 + done + trap 'rm -rf "$LOCK_DIR"' EXIT + fi + + echo "Volume backup lock acquired" + + ${body} + + echo "Volume backup lock released" + `; + + console.log( + lockWrapper(` + echo "Volume backup lock acquired" + echo "Volume backup lock released" + `), + ); + if (serviceType === "application") { - return ` + return lockWrapper(` echo "Stopping application to 0 replicas" ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") echo "Actual replicas: $ACTUAL_REPLICAS" @@ -54,7 +94,7 @@ export const backupVolume = async ( ${baseCommand} echo "Starting application to $ACTUAL_REPLICAS replicas" docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName} - `; + `); } if (serviceType === "compose") { const compose = await findComposeById( @@ -70,6 +110,7 @@ export const backupVolume = async ( ACTUAL_REPLICAS=$(docker service inspect ${compose.appName}_${volumeBackup.serviceName} --format "{{.Spec.Mode.Replicated.Replicas}}") echo "Actual replicas: $ACTUAL_REPLICAS" docker service update --replicas=0 ${compose.appName}_${volumeBackup.serviceName}`; + startCommand = ` echo "Starting compose to $ACTUAL_REPLICAS replicas" docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${compose.appName}_${volumeBackup.serviceName}`; @@ -78,16 +119,17 @@ export const backupVolume = async ( echo "Stopping compose container" ID=$(docker ps -q --filter "label=com.docker.compose.project=${compose.appName}" --filter "label=com.docker.compose.service=${volumeBackup.serviceName}") docker stop $ID`; + startCommand = ` echo "Starting compose container" docker start $ID echo "Compose container started" `; } - return ` + return lockWrapper(` ${stopCommand} ${baseCommand} ${startCommand} - `; + `); } };