diff --git a/apps/dokploy/server/api/routers/volume-backups.ts b/apps/dokploy/server/api/routers/volume-backups.ts index 3fdfe7de3..a6912f618 100644 --- a/apps/dokploy/server/api/routers/volume-backups.ts +++ b/apps/dokploy/server/api/routers/volume-backups.ts @@ -5,6 +5,7 @@ import { createVolumeBackup, runVolumeBackup, findVolumeBackupById, + restoreVolume, } from "@dokploy/server"; import { createVolumeBackupSchema, @@ -16,7 +17,6 @@ import { createTRPCRouter, protectedProcedure } from "../trpc"; import { db } from "@dokploy/server/db"; import { eq } from "drizzle-orm"; import { observable } from "@trpc/server/observable"; -import { restoreVolume } from "@dokploy/server/utils/volume-backups/utils"; import { execAsyncRemote, execAsyncStream, diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index dc86c8268..73586004d 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -63,7 +63,7 @@ export * from "./utils/notifications/utils"; export * from "./utils/notifications/docker-cleanup"; export * from "./utils/notifications/server-threshold"; -export * from "./utils/volume-backups/utils"; +export * from "./utils/volume-backups/index"; export * from "./utils/builders/index"; export * from "./utils/builders/compose"; diff --git a/packages/server/src/services/volume-backups.ts b/packages/server/src/services/volume-backups.ts index 0ae9717ca..887d71258 100644 --- a/packages/server/src/services/volume-backups.ts +++ b/packages/server/src/services/volume-backups.ts @@ -7,6 +7,7 @@ import { import { db } from "../db"; import { TRPCError } from "@trpc/server"; import type { z } from "zod"; +import { scheduleBackup } from "../utils/backups/utils"; export const findVolumeBackupById = async (volumeBackupId: string) => { const volumeBackup = await db.query.volumeBackups.findFirst({ @@ -39,7 +40,16 @@ export const createVolumeBackup = async ( const newVolumeBackup = await db .insert(volumeBackups) .values(volumeBackup) - .returning(); + .returning() + .then((e) => e[0]); + + await schedule({ + cronSchedule: backup.schedule, + backupId: backup.backupId, + type: "backup", + }); + + scheduleBackup(backup); return newVolumeBackup; }; diff --git a/packages/server/src/utils/volume-backups/backup.ts b/packages/server/src/utils/volume-backups/backup.ts new file mode 100644 index 000000000..401a47f06 --- /dev/null +++ b/packages/server/src/utils/volume-backups/backup.ts @@ -0,0 +1,90 @@ +import { type findVolumeBackupById, paths } from "@dokploy/server"; +import { normalizeS3Path } from "../backups/utils"; +import { getS3Credentials } from "../backups/utils"; +import path from "node:path"; +import { findComposeById } from "@dokploy/server"; + +export const backupVolume = async ( + volumeBackup: Awaited>, +) => { + const { serviceType, volumeName, turnOff, prefix } = volumeBackup; + const serverId = + volumeBackup.application?.serverId || volumeBackup.compose?.serverId; + const { VOLUME_BACKUPS_PATH } = paths(!!serverId); + const destination = volumeBackup.destination; + const backupFileName = `${volumeName}-${new Date().toISOString()}.tar`; + const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`; + const rcloneFlags = getS3Credentials(volumeBackup.destination); + const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeBackup.appName); + + const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} "${volumeBackupPath}/${backupFileName}" "${rcloneDestination}"`; + + const baseCommand = ` + set -e + echo "Volume name: ${volumeName}" + echo "Backup file name: ${backupFileName}" + echo "Turning off volume backup: ${turnOff ? "Yes" : "No"}" + echo "Starting volume backup" + echo "Dir: ${volumeBackupPath}" + docker run --rm \ + -v ${volumeName}:/volume_data \ + -v ${volumeBackupPath}:/backup \ + ubuntu \ + bash -c "cd /volume_data && tar cvf /backup/${backupFileName} ." + echo "Volume backup done ✅" + echo "Starting upload to S3..." + ${rcloneCommand} + echo "Upload to S3 done ✅" + `; + + if (!turnOff) { + return baseCommand; + } + + if (serviceType === "application") { + return ` + echo "Stopping application to 0 replicas" + ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") + echo "Actual replicas: $ACTUAL_REPLICAS" + docker service scale ${volumeBackup.application?.appName}=0 + ${baseCommand} + echo "Starting application to $ACTUAL_REPLICAS replicas" + docker service scale ${volumeBackup.application?.appName}=$ACTUAL_REPLICAS + `; + } + if (serviceType === "compose") { + const compose = await findComposeById( + volumeBackup.compose?.composeId || "", + ); + let stopCommand = ""; + let startCommand = ""; + + if (compose.composeType === "stack") { + stopCommand = ` + echo "Stopping compose to 0 replicas" + echo "Service name: ${compose.appName}_${volumeBackup.serviceName}" + ACTUAL_REPLICAS=$(docker service inspect ${compose.appName}_${volumeBackup.serviceName} --format "{{.Spec.Mode.Replicated.Replicas}}") + echo "Actual replicas: $ACTUAL_REPLICAS" + docker service scale ${compose.appName}_${volumeBackup.serviceName}=0`; + startCommand = ` + echo "Starting compose to $ACTUAL_REPLICAS replicas" + docker service scale ${compose.appName}_${volumeBackup.serviceName}=$ACTUAL_REPLICAS`; + } else { + stopCommand = ` + echo "Stopping compose container" + ID=$(docker ps -q --filter "label=com.docker.compose.project=${compose.appName}" --filter "label=com.docker.compose.service=${volumeBackup.serviceName}") + docker stop $ID`; + startCommand = ` + echo "Starting compose container" + docker start $ID + echo "Compose container started" + `; + } + return ` + ${stopCommand} + ${baseCommand} + ${startCommand} + `; + } +}; diff --git a/packages/server/src/utils/volume-backups/index.ts b/packages/server/src/utils/volume-backups/index.ts new file mode 100644 index 000000000..d35162f5e --- /dev/null +++ b/packages/server/src/utils/volume-backups/index.ts @@ -0,0 +1,3 @@ +export * from "./backup"; +export * from "./restore"; +export * from "./utils"; diff --git a/packages/server/src/utils/volume-backups/restore.ts b/packages/server/src/utils/volume-backups/restore.ts new file mode 100644 index 000000000..b30e585ea --- /dev/null +++ b/packages/server/src/utils/volume-backups/restore.ts @@ -0,0 +1,126 @@ +import { + findApplicationById, + findComposeById, + findDestinationById, + getS3Credentials, + paths, +} from "../.."; +import path from "node:path"; + +export const restoreVolume = async ( + id: string, + destinationId: string, + volumeName: string, + backupFileName: string, + serverId: string, + serviceType: "application" | "compose", +) => { + const destination = await findDestinationById(destinationId); + const { VOLUME_BACKUPS_PATH } = paths(!!serverId); + const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeName); + const rcloneFlags = getS3Credentials(destination); + const bucketPath = `:s3:${destination.bucket}`; + const backupPath = `${bucketPath}/${backupFileName}`; + + // Command to download backup file from S3 + const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${backupPath}" "${volumeBackupPath}/${backupFileName}"`; + + // Base restore command that creates the volume and restores data + const baseRestoreCommand = ` + set -e + echo "Volume name: ${volumeName}" + echo "Backup file name: ${backupFileName}" + echo "Volume backup path: ${volumeBackupPath}" + echo "Downloading backup from S3..." + mkdir -p ${volumeBackupPath} + ${downloadCommand} + echo "Download completed ✅" + echo "Creating new volume and restoring data..." + docker run --rm \ + -v ${volumeName}:/volume_data \ + -v ${volumeBackupPath}:/backup \ + ubuntu \ + bash -c "cd /volume_data && tar xvf /backup/${backupFileName} ." + echo "Volume restore completed ✅" + `; + + // Function to check if volume exists and get containers using it + const checkVolumeCommand = ` + # Check if volume exists + VOLUME_EXISTS=$(docker volume ls -q --filter name="^${volumeName}$" | wc -l) + echo "Volume exists: $VOLUME_EXISTS" + + if [ "$VOLUME_EXISTS" = "0" ]; then + echo "Volume doesn't exist, proceeding with direct restore" + ${baseRestoreCommand} + else + echo "Volume exists, checking for containers using it (including stopped ones)..." + + # Get ALL containers (running and stopped) using this volume - much simpler with native filter! + CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${volumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}") + + if [ -z "$CONTAINERS_USING_VOLUME" ]; then + echo "Volume exists but no containers are using it" + echo "Removing existing volume and proceeding with restore" + docker volume rm ${volumeName} --force + ${baseRestoreCommand} + else + echo "" + echo "⚠️ WARNING: Cannot restore volume as it is currently in use!" + echo "" + echo "📋 The following containers are using volume '${volumeName}':" + echo "" + + echo "$CONTAINERS_USING_VOLUME" | while IFS='|' read container_id container_name container_state labels; do + echo " 🐳 Container: $container_name ($container_id)" + echo " Status: $container_state" + + # Determine container type + if echo "$labels" | grep -q "com.docker.swarm.service.name="; then + SERVICE_NAME=$(echo "$labels" | grep -o "com.docker.swarm.service.name=[^,]*" | cut -d'=' -f2) + echo " Type: Docker Swarm Service ($SERVICE_NAME)" + elif echo "$labels" | grep -q "com.docker.compose.project="; then + PROJECT_NAME=$(echo "$labels" | grep -o "com.docker.compose.project=[^,]*" | cut -d'=' -f2) + echo " Type: Docker Compose ($PROJECT_NAME)" + else + echo " Type: Regular Container" + fi + echo "" + done + + echo "" + echo "🔧 To restore this volume, please:" + echo " 1. Stop all containers/services using this volume" + echo " 2. Remove the existing volume: docker volume rm ${volumeName}" + echo " 3. Run the restore operation again" + echo "" + echo "❌ Volume restore aborted - volume is in use" + + exit 1 + fi + fi + `; + + if (serviceType === "application") { + const application = await findApplicationById(id); + return ` + echo "=== VOLUME RESTORE FOR APPLICATION ===" + echo "Application: ${application.appName}" + ${checkVolumeCommand} + `; + } + + if (serviceType === "compose") { + const compose = await findComposeById(id); + + return ` + echo "=== VOLUME RESTORE FOR COMPOSE ===" + echo "Compose: ${compose.appName}" + echo "Compose Type: ${compose.composeType}" + ${checkVolumeCommand} + `; + } + + // Fallback for unknown service types + return checkVolumeCommand; +}; diff --git a/packages/server/src/utils/volume-backups/utils.ts b/packages/server/src/utils/volume-backups/utils.ts index fe138bccf..4185dd83f 100644 --- a/packages/server/src/utils/volume-backups/utils.ts +++ b/packages/server/src/utils/volume-backups/utils.ts @@ -3,15 +3,9 @@ import { createDeploymentVolumeBackup, execAsync, execAsyncRemote, - findApplicationById, - findComposeById, - findDestinationById, - getS3Credentials, - normalizeS3Path, - paths, updateDeploymentStatus, } from "../.."; -import path from "node:path"; +import { backupVolume } from "./backup"; export const runVolumeBackup = async (volumeBackupId: string) => { const volumeBackup = await findVolumeBackupById(volumeBackupId); @@ -39,206 +33,3 @@ export const runVolumeBackup = async (volumeBackupId: string) => { console.error(error); } }; - -const backupVolume = async ( - volumeBackup: Awaited>, -) => { - const { serviceType, volumeName, turnOff, prefix } = volumeBackup; - const serverId = - volumeBackup.application?.serverId || volumeBackup.compose?.serverId; - const { VOLUME_BACKUPS_PATH } = paths(!!serverId); - const destination = volumeBackup.destination; - const backupFileName = `${volumeName}-${new Date().toISOString()}.tar`; - const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`; - const rcloneFlags = getS3Credentials(volumeBackup.destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeBackup.appName); - - const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} "${volumeBackupPath}/${backupFileName}" "${rcloneDestination}"`; - - const baseCommand = ` - set -e - echo "Volume name: ${volumeName}" - echo "Backup file name: ${backupFileName}" - echo "Turning off volume backup: ${turnOff ? "Yes" : "No"}" - echo "Starting volume backup" - echo "Dir: ${volumeBackupPath}" - docker run --rm \ - -v ${volumeName}:/volume_data \ - -v ${volumeBackupPath}:/backup \ - ubuntu \ - bash -c "cd /volume_data && tar cvf /backup/${backupFileName} ." - echo "Volume backup done ✅" - echo "Starting upload to S3..." - ${rcloneCommand} - echo "Upload to S3 done ✅" - `; - - if (!turnOff) { - return baseCommand; - } - - if (serviceType === "application") { - return ` - echo "Stopping application to 0 replicas" - ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") - echo "Actual replicas: $ACTUAL_REPLICAS" - docker service scale ${volumeBackup.application?.appName}=0 - ${baseCommand} - echo "Starting application to $ACTUAL_REPLICAS replicas" - docker service scale ${volumeBackup.application?.appName}=$ACTUAL_REPLICAS - `; - } - if (serviceType === "compose") { - const compose = await findComposeById( - volumeBackup.compose?.composeId || "", - ); - let stopCommand = ""; - let startCommand = ""; - - if (compose.composeType === "stack") { - stopCommand = ` - echo "Stopping compose to 0 replicas" - echo "Service name: ${compose.appName}_${volumeBackup.serviceName}" - ACTUAL_REPLICAS=$(docker service inspect ${compose.appName}_${volumeBackup.serviceName} --format "{{.Spec.Mode.Replicated.Replicas}}") - echo "Actual replicas: $ACTUAL_REPLICAS" - docker service scale ${compose.appName}_${volumeBackup.serviceName}=0`; - startCommand = ` - echo "Starting compose to $ACTUAL_REPLICAS replicas" - docker service scale ${compose.appName}_${volumeBackup.serviceName}=$ACTUAL_REPLICAS`; - } else { - stopCommand = ` - echo "Stopping compose container" - ID=$(docker ps -q --filter "label=com.docker.compose.project=${compose.appName}" --filter "label=com.docker.compose.service=${volumeBackup.serviceName}") - docker stop $ID`; - startCommand = ` - echo "Starting compose container" - docker start $ID - echo "Compose container started" - `; - } - return ` - ${stopCommand} - ${baseCommand} - ${startCommand} - `; - } -}; - -export const restoreVolume = async ( - id: string, - destinationId: string, - volumeName: string, - backupFileName: string, - serverId: string, - serviceType: "application" | "compose", -) => { - const destination = await findDestinationById(destinationId); - const { VOLUME_BACKUPS_PATH } = paths(!!serverId); - const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeName); - const rcloneFlags = getS3Credentials(destination); - const bucketPath = `:s3:${destination.bucket}`; - const backupPath = `${bucketPath}/${backupFileName}`; - - // Command to download backup file from S3 - const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${backupPath}" "${volumeBackupPath}/${backupFileName}"`; - - // Base restore command that creates the volume and restores data - const baseRestoreCommand = ` - set -e - echo "Volume name: ${volumeName}" - echo "Backup file name: ${backupFileName}" - echo "Volume backup path: ${volumeBackupPath}" - echo "Downloading backup from S3..." - mkdir -p ${volumeBackupPath} - ${downloadCommand} - echo "Download completed ✅" - echo "Creating new volume and restoring data..." - docker run --rm \ - -v ${volumeName}:/volume_data \ - -v ${volumeBackupPath}:/backup \ - ubuntu \ - bash -c "cd /volume_data && tar xvf /backup/${backupFileName} ." - echo "Volume restore completed ✅" - `; - - // Function to check if volume exists and get containers using it - const checkVolumeCommand = ` - # Check if volume exists - VOLUME_EXISTS=$(docker volume ls -q --filter name="^${volumeName}$" | wc -l) - echo "Volume exists: $VOLUME_EXISTS" - - if [ "$VOLUME_EXISTS" = "0" ]; then - echo "Volume doesn't exist, proceeding with direct restore" - ${baseRestoreCommand} - else - echo "Volume exists, checking for containers using it (including stopped ones)..." - - # Get ALL containers (running and stopped) using this volume - much simpler with native filter! - CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${volumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}") - - if [ -z "$CONTAINERS_USING_VOLUME" ]; then - echo "Volume exists but no containers are using it" - echo "Removing existing volume and proceeding with restore" - docker volume rm ${volumeName} --force - ${baseRestoreCommand} - else - echo "" - echo "⚠️ WARNING: Cannot restore volume as it is currently in use!" - echo "" - echo "📋 The following containers are using volume '${volumeName}':" - echo "" - - echo "$CONTAINERS_USING_VOLUME" | while IFS='|' read container_id container_name container_state labels; do - echo " 🐳 Container: $container_name ($container_id)" - echo " Status: $container_state" - - # Determine container type - if echo "$labels" | grep -q "com.docker.swarm.service.name="; then - SERVICE_NAME=$(echo "$labels" | grep -o "com.docker.swarm.service.name=[^,]*" | cut -d'=' -f2) - echo " Type: Docker Swarm Service ($SERVICE_NAME)" - elif echo "$labels" | grep -q "com.docker.compose.project="; then - PROJECT_NAME=$(echo "$labels" | grep -o "com.docker.compose.project=[^,]*" | cut -d'=' -f2) - echo " Type: Docker Compose ($PROJECT_NAME)" - else - echo " Type: Regular Container" - fi - echo "" - done - - echo "" - echo "🔧 To restore this volume, please:" - echo " 1. Stop all containers/services using this volume" - echo " 2. Remove the existing volume: docker volume rm ${volumeName}" - echo " 3. Run the restore operation again" - echo "" - echo "❌ Volume restore aborted - volume is in use" - - exit 1 - fi - fi - `; - - if (serviceType === "application") { - const application = await findApplicationById(id); - return ` - echo "=== VOLUME RESTORE FOR APPLICATION ===" - echo "Application: ${application.appName}" - ${checkVolumeCommand} - `; - } - - if (serviceType === "compose") { - const compose = await findComposeById(id); - - return ` - echo "=== VOLUME RESTORE FOR COMPOSE ===" - echo "Compose: ${compose.appName}" - echo "Compose Type: ${compose.composeType}" - ${checkVolumeCommand} - `; - } - - // Fallback for unknown service types - return checkVolumeCommand; -};