diff --git a/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx b/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx index 243201e16..d7323e5b8 100644 --- a/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx +++ b/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx @@ -32,25 +32,76 @@ import { PopoverTrigger, } from "@/components/ui/popover"; import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; import { zodResolver } from "@hookform/resolvers/zod"; import copy from "copy-to-clipboard"; import { debounce } from "lodash"; -import { CheckIcon, ChevronsUpDown, Copy, RotateCcw } from "lucide-react"; +import { + CheckIcon, + ChevronsUpDown, + Copy, + RotateCcw, + RefreshCw, + DatabaseZap, +} from "lucide-react"; import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; import type { ServiceType } from "../../application/advanced/show-resources"; import { type LogLine, parseLogs } from "../../docker/logs/utils"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; interface Props { id: string; databaseType: Exclude | "web-server"; serverId?: string | null; + backupType?: "database" | "compose"; } +const getMetadataSchema = ( + backupType: "database" | "compose", + databaseType: string, +) => { + if (backupType !== "compose") return z.object({}).optional(); + + const schemas = { + postgres: z.object({ + databaseUser: z.string().min(1, "Database user is required"), + }), + mariadb: z.object({ + databaseUser: z.string().min(1, "Database user is required"), + databasePassword: z.string().min(1, "Database password is required"), + }), + mongo: z.object({ + databaseUser: z.string().min(1, "Database user is required"), + databasePassword: z.string().min(1, "Database password is required"), + }), + mysql: z.object({ + databaseRootPassword: z.string().min(1, "Root password is required"), + }), + "web-server": z.object({}), + }; + + return z.object({ + [databaseType]: schemas[databaseType as keyof typeof schemas], + serviceName: z.string().min(1, "Service name is required"), + }); +}; + const RestoreBackupSchema = z.object({ destinationId: z .string({ @@ -73,10 +124,16 @@ const RestoreBackupSchema = z.object({ .min(1, { message: "Database name is required", }), + databaseType: z + .string({ + required_error: "Please select a database type", + }) + .min(1, { + message: "Database type is required", + }), + metadata: z.object({}).optional(), }); -type RestoreBackup = z.infer; - const formatBytes = (bytes: number): string => { if (bytes === 0) return "0 Bytes"; const k = 1024; @@ -85,24 +142,41 @@ const formatBytes = (bytes: number): string => { return `${Number.parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`; }; -export const RestoreBackup = ({ id, databaseType, serverId }: Props) => { +export const RestoreBackup = ({ + id, + databaseType, + serverId, + backupType = "database", +}: Props) => { const [isOpen, setIsOpen] = useState(false); const [search, setSearch] = useState(""); const [debouncedSearchTerm, setDebouncedSearchTerm] = useState(""); + const [selectedDatabaseType, setSelectedDatabaseType] = useState( + backupType === "compose" ? "" : databaseType, + ); const { data: destinations = [] } = api.destination.all.useQuery(); - const form = useForm({ + const schema = RestoreBackupSchema.extend({ + metadata: getMetadataSchema(backupType, selectedDatabaseType), + }); + + const form = useForm>({ defaultValues: { destinationId: "", backupFile: "", databaseName: databaseType === "web-server" ? "dokploy" : "", + databaseType: backupType === "compose" ? "" : databaseType, + metadata: {}, }, - resolver: zodResolver(RestoreBackupSchema), + resolver: zodResolver(schema), }); const destionationId = form.watch("destinationId"); + const metadata = form.watch("metadata"); + // console.log({ metadata }); + const debouncedSetSearch = debounce((value: string) => { setDebouncedSearchTerm(value); }, 350); @@ -127,16 +201,15 @@ export const RestoreBackup = ({ id, databaseType, serverId }: Props) => { const [filteredLogs, setFilteredLogs] = useState([]); const [isDeploying, setIsDeploying] = useState(false); - // const { mutateAsync: restore, isLoading: isRestoring } = - // api.backup.restoreBackup.useMutation(); - api.backup.restoreBackupWithLogs.useSubscription( { databaseId: id, - databaseType, + databaseType: form.watch("databaseType"), databaseName: form.watch("databaseName"), backupFile: form.watch("backupFile"), destinationId: form.watch("destinationId"), + backupType: backupType, + metadata: metadata, }, { enabled: isDeploying, @@ -158,10 +231,32 @@ export const RestoreBackup = ({ id, databaseType, serverId }: Props) => { }, ); - const onSubmit = async (_data: RestoreBackup) => { + const onSubmit = async (data: z.infer) => { + if (backupType === "compose" && !data.databaseType) { + toast.error("Please select a database type"); + return; + } + console.log({ data }); setIsDeploying(true); }; + const [cacheType, setCacheType] = useState<"fetch" | "cache">("cache"); + const { + data: services = [], + isLoading: isLoadingServices, + refetch: refetchServices, + } = api.compose.loadServices.useQuery( + { + composeId: id, + type: cacheType, + }, + { + retry: false, + refetchOnWindowFocus: false, + enabled: backupType === "compose", + }, + ); + return ( @@ -170,7 +265,7 @@ export const RestoreBackup = ({ id, databaseType, serverId }: Props) => { Restore Backup - + @@ -373,25 +468,270 @@ export const RestoreBackup = ({ id, databaseType, serverId }: Props) => { control={form.control} name="databaseName" render={({ field }) => ( - + Database Name - + )} /> + + {backupType === "compose" && ( + <> + ( + + Database Type + + + + )} + /> + + ( + + Service Name +
+ + + + + + + +

+ Fetch: Will clone the repository and load the + services +

+ + + + + + + + + +

+ Cache: If you previously deployed this compose, + it will read the services from the last + deployment/fetch from the repository +

+ + + +
+ + + + )} + /> + + {selectedDatabaseType === "postgres" && ( + ( + + Database User + + + + + + )} + /> + )} + + {selectedDatabaseType === "mariadb" && ( + <> + ( + + Database User + + + + + + )} + /> + ( + + Database Password + + + + + + )} + /> + + )} + + {selectedDatabaseType === "mongo" && ( + <> + ( + + Database User + + + + + + )} + /> + ( + + Database Password + + + + + + )} + /> + + )} + + {selectedDatabaseType === "mysql" && ( + ( + + Root Password + + + + + + )} + /> + )} + + )} + diff --git a/apps/dokploy/components/dashboard/database/backups/show-backups.tsx b/apps/dokploy/components/dashboard/database/backups/show-backups.tsx index e219c3e30..062e7be7d 100644 --- a/apps/dokploy/components/dashboard/database/backups/show-backups.tsx +++ b/apps/dokploy/components/dashboard/database/backups/show-backups.tsx @@ -110,6 +110,7 @@ export const ShowBackups = ({ diff --git a/apps/dokploy/components/dashboard/database/backups/update-backup.tsx b/apps/dokploy/components/dashboard/database/backups/update-backup.tsx index a57cf01d9..8a12c535c 100644 --- a/apps/dokploy/components/dashboard/database/backups/update-backup.tsx +++ b/apps/dokploy/components/dashboard/database/backups/update-backup.tsx @@ -192,7 +192,6 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => { form.reset( { ...currentValues, - metadata: {}, }, { keepDefaultValues: true }, ); diff --git a/apps/dokploy/server/api/routers/backup.ts b/apps/dokploy/server/api/routers/backup.ts index 50368d164..32d011d67 100644 --- a/apps/dokploy/server/api/routers/backup.ts +++ b/apps/dokploy/server/api/routers/backup.ts @@ -11,6 +11,7 @@ import { createBackup, findBackupById, findComposeByBackupId, + findComposeById, findMariadbByBackupId, findMariadbById, findMongoByBackupId, @@ -42,6 +43,7 @@ import { execAsyncRemote, } from "@dokploy/server/utils/process/execAsync"; import { + restoreComposeBackup, restoreMariadbBackup, restoreMongoBackup, restoreMySqlBackup, @@ -129,6 +131,7 @@ export const backupRouter = createTRPCRouter({ .input(apiUpdateBackup) .mutation(async ({ input }) => { try { + console.log(input); await updateBackupById(input.backupId, input); const backup = await findBackupById(input.backupId); @@ -374,78 +377,96 @@ export const backupRouter = createTRPCRouter({ "mongo", "web-server", ]), + backupType: z.enum(["database", "compose"]), databaseName: z.string().min(1), backupFile: z.string().min(1), destinationId: z.string().min(1), + metadata: z.any(), }), ) .subscription(async ({ input }) => { const destination = await findDestinationById(input.destinationId); - if (input.databaseType === "postgres") { - const postgres = await findPostgresById(input.databaseId); + if (input.backupType === "database") { + if (input.databaseType === "postgres") { + const postgres = await findPostgresById(input.databaseId); - return observable((emit) => { - restorePostgresBackup( - postgres, - destination, - input.databaseName, - input.backupFile, - (log) => { - emit.next(log); - }, - ); - }); - } - if (input.databaseType === "mysql") { - const mysql = await findMySqlById(input.databaseId); - return observable((emit) => { - restoreMySqlBackup( - mysql, - destination, - input.databaseName, - input.backupFile, - (log) => { - emit.next(log); - }, - ); - }); - } - if (input.databaseType === "mariadb") { - const mariadb = await findMariadbById(input.databaseId); - return observable((emit) => { - restoreMariadbBackup( - mariadb, - destination, - input.databaseName, - input.backupFile, - (log) => { - emit.next(log); - }, - ); - }); - } - if (input.databaseType === "mongo") { - const mongo = await findMongoById(input.databaseId); - return observable((emit) => { - restoreMongoBackup( - mongo, - destination, - input.databaseName, - input.backupFile, - (log) => { - emit.next(log); - }, - ); - }); - } - if (input.databaseType === "web-server") { - return observable((emit) => { - restoreWebServerBackup(destination, input.backupFile, (log) => { - emit.next(log); + return observable((emit) => { + restorePostgresBackup( + postgres, + destination, + input.databaseName, + input.backupFile, + (log) => { + emit.next(log); + }, + ); }); + } + if (input.databaseType === "mysql") { + const mysql = await findMySqlById(input.databaseId); + return observable((emit) => { + restoreMySqlBackup( + mysql, + destination, + input.databaseName, + input.backupFile, + (log) => { + emit.next(log); + }, + ); + }); + } + if (input.databaseType === "mariadb") { + const mariadb = await findMariadbById(input.databaseId); + return observable((emit) => { + restoreMariadbBackup( + mariadb, + destination, + input.databaseName, + input.backupFile, + (log) => { + emit.next(log); + }, + ); + }); + } + if (input.databaseType === "mongo") { + const mongo = await findMongoById(input.databaseId); + return observable((emit) => { + restoreMongoBackup( + mongo, + destination, + input.databaseName, + input.backupFile, + (log) => { + emit.next(log); + }, + ); + }); + } + if (input.databaseType === "web-server") { + return observable((emit) => { + restoreWebServerBackup(destination, input.backupFile, (log) => { + emit.next(log); + }); + }); + } + } + if (input.backupType === "compose") { + const compose = await findComposeById(input.databaseId); + return observable((emit) => { + restoreComposeBackup( + compose, + destination, + input.databaseName, + input.backupFile, + input.metadata, + (log) => { + emit.next(log); + }, + ); }); } - return true; }), }); diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index ad9f2086b..c4f9b3177 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -27,6 +27,7 @@ import { createMount, deleteMount, findComposeById, + findDomainsByComposeId, findProjectById, findServerById, findUserById, @@ -267,7 +268,8 @@ export const composeRouter = createTRPCRouter({ message: "You are not authorized to get this compose", }); } - const composeFile = await addDomainToCompose(compose); + const domains = await findDomainsByComposeId(input.composeId); + const composeFile = await addDomainToCompose(compose, domains); return dump(composeFile, { lineWidth: 1000, }); diff --git a/packages/server/src/db/schema/backups.ts b/packages/server/src/db/schema/backups.ts index 6dca5caab..67d8698c7 100644 --- a/packages/server/src/db/schema/backups.ts +++ b/packages/server/src/db/schema/backups.ts @@ -137,7 +137,7 @@ const createSchema = createInsertSchema(backups, { mysqlId: z.string().optional(), mongoId: z.string().optional(), userId: z.string().optional(), - metadata: z.object({}).optional(), + metadata: z.any().optional(), }); export const apiCreateBackup = createSchema.pick({ diff --git a/packages/server/src/utils/backups/compose.ts b/packages/server/src/utils/backups/compose.ts index 06d0465ff..85b9669eb 100644 --- a/packages/server/src/utils/backups/compose.ts +++ b/packages/server/src/utils/backups/compose.ts @@ -21,7 +21,7 @@ export const runComposeBackup = async ( const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; - const command = `docker ps --filter "status=running" --filter "label=dokploy.backup.id=${backup.backupId}" --format "{{.ID}}" | head -n 1`; + const command = getFindContainerCommand(compose, backup.serviceName || ""); if (compose.serverId) { const { stdout } = await execAsyncRemote(compose.serverId, command); if (!stdout) { @@ -88,3 +88,26 @@ export const runComposeBackup = async ( throw error; } }; + +export const getFindContainerCommand = ( + compose: Compose, + serviceName: string, +) => { + const { appName, composeType } = compose; + const labels = + composeType === "stack" + ? { + namespace: `label=com.docker.stack.namespace=${appName}`, + service: `label=com.docker.swarm.service.name=${appName}_${serviceName}`, + } + : { + project: `label=com.docker.compose.project=${appName}`, + service: `label=com.docker.compose.service=${serviceName}`, + }; + + const command = `docker ps --filter "status=running" \ + --filter "${Object.values(labels).join('" --filter "')}" \ + --format "{{.ID}}" | head -n 1`; + + return command.trim(); +}; diff --git a/packages/server/src/utils/builders/compose.ts b/packages/server/src/utils/builders/compose.ts index 47d4c71e6..19e7d1529 100644 --- a/packages/server/src/utils/builders/compose.ts +++ b/packages/server/src/utils/builders/compose.ts @@ -22,15 +22,15 @@ import { spawnAsync } from "../process/spawnAsync"; export type ComposeNested = InferResultType< "compose", - { project: true; mounts: true; domains: true; backups: true } + { project: true; mounts: true; domains: true } >; export const buildCompose = async (compose: ComposeNested, logPath: string) => { const writeStream = createWriteStream(logPath, { flags: "a" }); - const { sourceType, appName, mounts, composeType } = compose; + const { sourceType, appName, mounts, composeType, domains } = compose; try { const { COMPOSE_PATH } = paths(); const command = createCommand(compose); - await writeDomainsToCompose(compose); + await writeDomainsToCompose(compose, domains); createEnvFile(compose); if (compose.isolatedDeployment) { diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index 8d3143640..4f0083979 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -38,8 +38,6 @@ import type { PropertiesNetworks, } from "./types"; import { encodeBase64 } from "./utils"; -import type { Backup } from "@dokploy/server/services/backup"; -import { createBackupLabels } from "./backup"; export const cloneCompose = async (compose: Compose) => { if (compose.sourceType === "github") { @@ -134,13 +132,13 @@ export const readComposeFile = async (compose: Compose) => { }; export const writeDomainsToCompose = async ( - compose: Compose & { domains: Domain[]; backups: Backup[] }, + compose: Compose, + domains: Domain[], ) => { - const { domains, backups } = compose; - if (!domains.length && !backups.length) { + if (!domains.length) { return; } - const composeConverted = await addDomainToCompose(compose); + const composeConverted = await addDomainToCompose(compose, domains); const path = getComposePath(compose); const composeString = dump(composeConverted, { lineWidth: 1000 }); @@ -152,7 +150,7 @@ export const writeDomainsToCompose = async ( }; export const writeDomainsToComposeRemote = async ( - compose: Compose & { domains: Domain[]; backups: Backup[] }, + compose: Compose, domains: Domain[], logPath: string, ) => { @@ -161,7 +159,7 @@ export const writeDomainsToComposeRemote = async ( } try { - const composeConverted = await addDomainToCompose(compose); + const composeConverted = await addDomainToCompose(compose, domains); const path = getComposePath(compose); if (!composeConverted) { @@ -182,20 +180,22 @@ exit 1; `; } }; +// (node:59875) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 SIGTERM listeners added to [process]. Use emitter.setMaxListeners() to increase limit export const addDomainToCompose = async ( - compose: Compose & { domains: Domain[]; backups: Backup[] }, + compose: Compose, + domains: Domain[], ) => { - const { appName, domains, backups } = compose; + const { appName } = compose; let result: ComposeSpecification | null; if (compose.serverId) { - result = await loadDockerComposeRemote(compose); + result = await loadDockerComposeRemote(compose); // aca hay que ir al servidor e ir a traer el compose file al servidor } else { result = await loadDockerCompose(compose); } - if (!result || (domains.length === 0 && backups.length === 0)) { + if (!result || domains.length === 0) { return null; } @@ -210,7 +210,6 @@ export const addDomainToCompose = async ( result = randomized; } - // Add domains to the compose for (const domain of domains) { const { serviceName, https } = domain; if (!serviceName) { @@ -265,38 +264,6 @@ export const addDomainToCompose = async ( } } - // Add backups to the compose - for (const backup of backups) { - const { backupId, serviceName, enabled } = backup; - - if (!enabled) { - continue; - } - - if (!serviceName) { - throw new Error( - "Service name not found, please check the backups to use a valid service name", - ); - } - - if (!result?.services?.[serviceName]) { - throw new Error(`The service ${serviceName} not found in the compose`); - } - - const backupLabels = createBackupLabels(backupId); - - if (!result.services[serviceName].labels) { - result.services[serviceName].labels = []; - } - - result.services[serviceName].labels = [ - ...(Array.isArray(result.services[serviceName].labels) - ? result.services[serviceName].labels - : []), - ...backupLabels, - ]; - } - // Add dokploy-network to the root of the compose file if (!compose.isolatedDeployment) { result.networks = addDokployNetworkToRoot(result.networks); diff --git a/packages/server/src/utils/restore/compose.ts b/packages/server/src/utils/restore/compose.ts new file mode 100644 index 000000000..f9deb52bd --- /dev/null +++ b/packages/server/src/utils/restore/compose.ts @@ -0,0 +1,99 @@ +import type { Destination } from "@dokploy/server/services/destination"; +import type { Compose } from "@dokploy/server/services/compose"; +import { getS3Credentials } from "../backups/utils"; +import { execAsync, execAsyncRemote } from "../process/execAsync"; +import type { Backup } from "@dokploy/server/services/backup"; +import { getFindContainerCommand } from "../backups/compose"; + +export const restoreComposeBackup = async ( + compose: Compose, + destination: Destination, + database: string, + backupFile: string, + metadata: Backup["metadata"] & { serviceName: string }, + emit: (log: string) => void, +) => { + try { + console.log({ metadata }); + const { serverId } = compose; + + const rcloneFlags = getS3Credentials(destination); + const bucketPath = `:s3:${destination.bucket}`; + const backupPath = `${bucketPath}/${backupFile}`; + + const command = getFindContainerCommand(compose, metadata.serviceName); + + console.log("command", command); + let containerId = ""; + if (serverId) { + const { stdout, stderr } = await execAsyncRemote(serverId, command); + emit(stdout); + emit(stderr); + containerId = stdout.trim(); + } else { + const { stdout, stderr } = await execAsync(command); + console.log("stdout", stdout); + console.log("stderr", stderr); + emit(stdout); + emit(stderr); + containerId = stdout.trim(); + } + let restoreCommand = ""; + + if (metadata.postgres) { + restoreCommand = `rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} pg_restore -U ${metadata.postgres.databaseUser} -d ${database} --clean --if-exists`; + } else if (metadata.mariadb) { + restoreCommand = ` + rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} mariadb -u ${metadata.mariadb.databaseUser} -p${metadata.mariadb.databasePassword} ${database} + `; + } else if (metadata.mysql) { + restoreCommand = ` + rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} mysql -u root -p${metadata.mysql.databaseRootPassword} ${database} + `; + } else if (metadata.mongo) { + const tempDir = "/tmp/dokploy-restore"; + const fileName = backupFile.split("/").pop() || "backup.dump.gz"; + const decompressedName = fileName.replace(".gz", ""); + restoreCommand = `\ + rm -rf ${tempDir} && \ + mkdir -p ${tempDir} && \ + rclone copy ${rcloneFlags.join(" ")} "${backupPath}" ${tempDir} && \ + cd ${tempDir} && \ + gunzip -f "${fileName}" && \ + docker exec -i ${containerId} mongorestore --username ${metadata.mongo.databaseUser} --password ${metadata.mongo.databasePassword} --authenticationDatabase admin --db ${database} --archive < "${decompressedName}" && \ + rm -rf ${tempDir}`; + } + + emit("Starting restore..."); + emit(`Backup path: ${backupPath}`); + + emit(`Executing command: ${restoreCommand}`); + + if (serverId) { + const { stdout, stderr } = await execAsyncRemote( + serverId, + restoreCommand, + ); + emit(stdout); + emit(stderr); + } else { + const { stdout, stderr } = await execAsync(restoreCommand); + console.log("stdout", stdout); + console.log("stderr", stderr); + emit(stdout); + emit(stderr); + } + + emit("Restore completed successfully!"); + } catch (error) { + console.error(error); + emit( + `Error: ${ + error instanceof Error ? error.message : "Error restoring mongo backup" + }`, + ); + throw new Error( + error instanceof Error ? error.message : "Error restoring mongo backup", + ); + } +}; diff --git a/packages/server/src/utils/restore/index.ts b/packages/server/src/utils/restore/index.ts index 972e1d1d7..615e53d35 100644 --- a/packages/server/src/utils/restore/index.ts +++ b/packages/server/src/utils/restore/index.ts @@ -3,3 +3,4 @@ export { restoreMySqlBackup } from "./mysql"; export { restoreMariadbBackup } from "./mariadb"; export { restoreMongoBackup } from "./mongo"; export { restoreWebServerBackup } from "./web-server"; +export { restoreComposeBackup } from "./compose";