From a46e7759b2f33a576af49f83931134fb2d1065a4 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Wed, 18 Sep 2024 00:40:52 -0600 Subject: [PATCH] refactor(multi-server): add rclone to multi server --- .../dashboard/project/add-database.tsx | 8 +- .../settings/servers/show-servers.tsx | 9 +- .../settings/servers/update-server.tsx | 269 ++++++++++++++++++ apps/dokploy/server/api/routers/compose.ts | 8 +- apps/dokploy/server/api/services/compose.ts | 11 +- apps/dokploy/server/api/services/postgres.ts | 3 +- apps/dokploy/server/db/schema/mariadb.ts | 1 + apps/dokploy/server/db/schema/mongo.ts | 1 + apps/dokploy/server/db/schema/mysql.ts | 1 + apps/dokploy/server/db/schema/postgres.ts | 1 + apps/dokploy/server/db/schema/redis.ts | 1 + apps/dokploy/server/db/schema/server.ts | 4 + apps/dokploy/server/utils/backups/postgres.ts | 56 +++- apps/dokploy/server/utils/databases/mongo.ts | 2 - .../dokploy/server/utils/process/execAsync.ts | 8 +- .../server/utils/servers/setup-server.ts | 5 + apps/dokploy/server/wss/listen-deployment.ts | 6 +- pnpm-lock.yaml | 6 +- 18 files changed, 368 insertions(+), 32 deletions(-) create mode 100644 apps/dokploy/components/dashboard/settings/servers/update-server.tsx diff --git a/apps/dokploy/components/dashboard/project/add-database.tsx b/apps/dokploy/components/dashboard/project/add-database.tsx index 7db399e96..c56b1c179 100644 --- a/apps/dokploy/components/dashboard/project/add-database.tsx +++ b/apps/dokploy/components/dashboard/project/add-database.tsx @@ -80,9 +80,7 @@ const baseDatabaseSchema = z.object({ databasePassword: z.string(), dockerImage: z.string(), description: z.string().nullable(), - serverId: z.string().min(1, { - message: "Server is required", - }), + serverId: z.string().nullable(), }); const mySchema = z.discriminatedUnion("type", [ @@ -174,6 +172,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => { description: "", databaseName: "", databaseUser: "", + serverId: null, }, resolver: zodResolver(mySchema), }); @@ -222,6 +221,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => { promise = redisMutation.mutateAsync({ ...commonParams, databasePassword: data.databasePassword, + serverId: data.serverId, projectId, }); } else if (data.type === "mariadb") { @@ -379,7 +379,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => { Select a Server diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index 147ebfc75..ecdf1456e 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -25,6 +25,7 @@ import { toast } from "sonner"; import { TerminalModal } from "../web-server/terminal-modal"; import { AddServer } from "./add-server"; import { SetupServer } from "./setup-server"; +import { UpdateServer } from "./update-server"; export const ShowServers = () => { const { data, refetch } = api.server.all.useQuery(); const { mutateAsync } = api.server.remove.useMutation(); @@ -83,6 +84,7 @@ export const ShowServers = () => { IP Address Port Username + SSH Key Created Actions @@ -101,6 +103,11 @@ export const ShowServers = () => { {server.username} + + + {server.sshKeyId ? "Yes" : "No"} + + {format(new Date(server.createdAt), "PPpp")} @@ -121,7 +128,7 @@ export const ShowServers = () => { Enter the terminal - + ; + +interface Props { + serverId: string; +} + +export const UpdateServer = ({ serverId }: Props) => { + const utils = api.useUtils(); + const [isOpen, setIsOpen] = useState(false); + const { data, isLoading } = api.server.one.useQuery( + { + serverId, + }, + { + enabled: !!serverId, + }, + ); + const { data: sshKeys } = api.sshKey.all.useQuery(); + const { mutateAsync, error, isError } = api.server.update.useMutation(); + const form = useForm({ + defaultValues: { + description: "", + name: "", + ipAddress: "", + port: 22, + username: "root", + sshKeyId: "", + }, + resolver: zodResolver(Schema), + }); + + useEffect(() => { + form.reset({ + description: data?.description || "", + name: data?.name || "", + ipAddress: data?.ipAddress || "", + port: data?.port || 22, + username: data?.username || "root", + sshKeyId: data?.sshKeyId || "", + }); + }, [form, form.reset, form.formState.isSubmitSuccessful, data]); + + const onSubmit = async (formData: Schema) => { + await mutateAsync({ + name: formData.name, + description: formData.description || "", + ipAddress: formData.ipAddress || "", + port: formData.port || 22, + username: formData.username || "root", + sshKeyId: formData.sshKeyId || "", + serverId: serverId, + }) + .then(async (data) => { + await utils.server.all.invalidate(); + toast.success("Server Updated"); + setIsOpen(false); + }) + .catch(() => { + toast.error("Error to update a server"); + }); + }; + + return ( + + + e.preventDefault()} + > + Edit Server + + + + + Update Server + + Update a server to deploy your applications remotely. + + + {isError && {error?.message}} + + + + ( + + Name + + + + + + + )} + /> + + ( + + Description + + + + + + + )} + /> + ( + + Select a SSH Key + + + + + + + {sshKeys?.map((sshKey) => ( + + {sshKey.name} + + ))} + + Registries ({sshKeys?.length}) + + + + + + + )} + /> + + ( + + IP Address + + + + + + + )} + /> + ( + + Port + + + + + + + )} + /> + + + ( + + Username + + + + + + + )} + /> + + + + + Update + + + + + + ); +}; diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index 433b53455..4892bc3a1 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -15,10 +15,7 @@ import { } from "@/server/queues/deployments-queue"; import { myQueue } from "@/server/queues/queueSetup"; import { createCommand } from "@/server/utils/builders/compose"; -import { - randomizeComposeFile, - randomizeSpecificationFile, -} from "@/server/utils/docker/compose"; +import { randomizeComposeFile } from "@/server/utils/docker/compose"; import { addDomainToCompose, cloneCompose, @@ -252,8 +249,9 @@ export const composeRouter = createTRPCRouter({ if (input.serverId) { const server = await findServerById(input.serverId); serverIp = server.ipAddress; + } else if (process.env.NODE_ENV === "development") { + serverIp = "127.0.0.1"; } - const projectName = slugify(`${project.name} ${input.id}`); const { envs, mounts, domains } = generate({ serverIp: serverIp || "", diff --git a/apps/dokploy/server/api/services/compose.ts b/apps/dokploy/server/api/services/compose.ts index 5557c05e1..1ac00893d 100644 --- a/apps/dokploy/server/api/services/compose.ts +++ b/apps/dokploy/server/api/services/compose.ts @@ -335,15 +335,8 @@ export const deployRemoteCompose = async ({ command += getCreateComposeFileCommand(compose, deployment.logPath); } - async function* sequentialSteps() { - yield execAsyncRemote(compose.serverId, command); - yield getBuildComposeCommand(compose, deployment.logPath); - } - - const steps = sequentialSteps(); - for await (const step of steps) { - step; - } + await execAsyncRemote(compose.serverId, command); + await getBuildComposeCommand(compose, deployment.logPath); console.log(" ---- done ----"); } diff --git a/apps/dokploy/server/api/services/postgres.ts b/apps/dokploy/server/api/services/postgres.ts index a355da9ac..56c91149c 100644 --- a/apps/dokploy/server/api/services/postgres.ts +++ b/apps/dokploy/server/api/services/postgres.ts @@ -116,8 +116,9 @@ export const removePostgresById = async (postgresId: string) => { export const deployPostgres = async (postgresId: string) => { const postgres = await findPostgresById(postgresId); try { + const promises = []; if (postgres.serverId) { - await execAsyncRemote( + const result = await execAsyncRemote( postgres.serverId, `docker pull ${postgres.dockerImage}`, ); diff --git a/apps/dokploy/server/db/schema/mariadb.ts b/apps/dokploy/server/db/schema/mariadb.ts index 7a3c71711..3bc37185e 100644 --- a/apps/dokploy/server/db/schema/mariadb.ts +++ b/apps/dokploy/server/db/schema/mariadb.ts @@ -83,6 +83,7 @@ const createSchema = createInsertSchema(mariadb, { applicationStatus: z.enum(["idle", "running", "done", "error"]), externalPort: z.number(), description: z.string().optional(), + serverId: z.string().optional(), }); export const apiCreateMariaDB = createSchema diff --git a/apps/dokploy/server/db/schema/mongo.ts b/apps/dokploy/server/db/schema/mongo.ts index f86b9fcfe..8c8af166c 100644 --- a/apps/dokploy/server/db/schema/mongo.ts +++ b/apps/dokploy/server/db/schema/mongo.ts @@ -77,6 +77,7 @@ const createSchema = createInsertSchema(mongo, { applicationStatus: z.enum(["idle", "running", "done", "error"]), externalPort: z.number(), description: z.string().optional(), + serverId: z.string().optional(), }); export const apiCreateMongo = createSchema diff --git a/apps/dokploy/server/db/schema/mysql.ts b/apps/dokploy/server/db/schema/mysql.ts index 1443c35a0..d4c119557 100644 --- a/apps/dokploy/server/db/schema/mysql.ts +++ b/apps/dokploy/server/db/schema/mysql.ts @@ -80,6 +80,7 @@ const createSchema = createInsertSchema(mysql, { applicationStatus: z.enum(["idle", "running", "done", "error"]), externalPort: z.number(), description: z.string().optional(), + serverId: z.string().optional(), }); export const apiCreateMySql = createSchema diff --git a/apps/dokploy/server/db/schema/postgres.ts b/apps/dokploy/server/db/schema/postgres.ts index f90ff1701..7d7844049 100644 --- a/apps/dokploy/server/db/schema/postgres.ts +++ b/apps/dokploy/server/db/schema/postgres.ts @@ -78,6 +78,7 @@ const createSchema = createInsertSchema(postgres, { externalPort: z.number(), createdAt: z.string(), description: z.string().optional(), + serverId: z.string().optional(), }); export const apiCreatePostgres = createSchema diff --git a/apps/dokploy/server/db/schema/redis.ts b/apps/dokploy/server/db/schema/redis.ts index 12210672d..1fc646970 100644 --- a/apps/dokploy/server/db/schema/redis.ts +++ b/apps/dokploy/server/db/schema/redis.ts @@ -72,6 +72,7 @@ const createSchema = createInsertSchema(redis, { applicationStatus: z.enum(["idle", "running", "done", "error"]), externalPort: z.number(), description: z.string().optional(), + serverId: z.string().optional(), }); export const apiCreateRedis = createSchema diff --git a/apps/dokploy/server/db/schema/server.ts b/apps/dokploy/server/db/schema/server.ts index 82dd92129..4bce9a960 100644 --- a/apps/dokploy/server/db/schema/server.ts +++ b/apps/dokploy/server/db/schema/server.ts @@ -95,5 +95,9 @@ export const apiUpdateServer = createSchema name: true, description: true, serverId: true, + ipAddress: true, + port: true, + username: true, + sshKeyId: true, }) .required(); diff --git a/apps/dokploy/server/utils/backups/postgres.ts b/apps/dokploy/server/utils/backups/postgres.ts index ec7af81fe..48be6a437 100644 --- a/apps/dokploy/server/utils/backups/postgres.ts +++ b/apps/dokploy/server/utils/backups/postgres.ts @@ -5,7 +5,7 @@ import type { Postgres } from "@/server/api/services/postgres"; import { findProjectById } from "@/server/api/services/project"; import { getServiceContainer } from "../docker/utils"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync } from "../process/execAsync"; +import { execAsync, execAsyncRemote } from "../process/execAsync"; import { uploadToS3 } from "./utils"; export const runPostgresBackup = async ( @@ -57,3 +57,57 @@ export const runPostgresBackup = async ( // Restore // /Applications/pgAdmin 4.app/Contents/SharedSupport/pg_restore --host "localhost" --port "5432" --username "mauricio" --no-password --dbname "postgres" --verbose "/Users/mauricio/Downloads/_databases_2024-04-12T07_02_05.234Z.sql" + +export const runRemotePostgresBackup = async ( + postgres: Postgres, + backup: BackupSchedule, +) => { + const { appName, databaseUser, name, projectId } = postgres; + const project = await findProjectById(projectId); + + const { prefix, database } = backup; + const destination = backup.destination; + const backupFileName = `${new Date().toISOString()}.sql.gz`; + const bucketDestination = path.join(prefix, backupFileName); + const containerPath = `/backup/${backupFileName}`; + const hostPath = `./${backupFileName}`; + try { + const { Id: containerId } = await getServiceContainer(appName); + const pgDumpCommand = `pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip`; + // const rcloneCommand = `rclone rcat --buffer-size 16M ${rcloneDestination}`; + + // const command = ` + // // docker exec ${containerId} /bin/bash -c "${pgDumpCommand} | ${rcloneCommand}" + // `; + + await execAsyncRemote( + postgres.serverId, + `docker exec ${containerId} /bin/bash -c "rm -rf /backup && mkdir -p /backup"`, + ); + // await execAsync( + // `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip > ${containerPath}"`, + // ); + await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`); + + await uploadToS3(destination, bucketDestination, hostPath); + await sendDatabaseBackupNotifications({ + applicationName: name, + projectName: project.name, + databaseType: "postgres", + type: "success", + }); + } catch (error) { + await sendDatabaseBackupNotifications({ + applicationName: name, + projectName: project.name, + databaseType: "postgres", + type: "error", + // @ts-ignore + errorMessage: error?.message || "Error message not provided", + }); + + throw error; + } finally { + await unlink(hostPath); + } +}; diff --git a/apps/dokploy/server/utils/databases/mongo.ts b/apps/dokploy/server/utils/databases/mongo.ts index c4d517533..5e2e2bf54 100644 --- a/apps/dokploy/server/utils/databases/mongo.ts +++ b/apps/dokploy/server/utils/databases/mongo.ts @@ -1,5 +1,3 @@ -import type { Mongo } from "@/server/api/services/mongo"; -import type { Mount } from "@/server/api/services/mount"; import type { CreateServiceOptions } from "dockerode"; import { calculateResources, diff --git a/apps/dokploy/server/utils/process/execAsync.ts b/apps/dokploy/server/utils/process/execAsync.ts index 12d93c5c5..8c8fdc199 100644 --- a/apps/dokploy/server/utils/process/execAsync.ts +++ b/apps/dokploy/server/utils/process/execAsync.ts @@ -15,10 +15,12 @@ export const execAsyncRemote = async ( const keys = await readSSHKey(server.sshKeyId); - const conn = new Client(); let stdout = ""; let stderr = ""; return new Promise((resolve, reject) => { + const conn = new Client(); + + sleep(1000); conn .once("ready", () => { console.log("Client :: ready"); @@ -57,3 +59,7 @@ export const execAsyncRemote = async ( }); }); }; + +export const sleep = (ms: number) => { + return new Promise((resolve) => setTimeout(resolve, ms)); +}; diff --git a/apps/dokploy/server/utils/servers/setup-server.ts b/apps/dokploy/server/utils/servers/setup-server.ts index 72bd32ade..72d5c61ad 100644 --- a/apps/dokploy/server/utils/servers/setup-server.ts +++ b/apps/dokploy/server/utils/servers/setup-server.ts @@ -74,6 +74,7 @@ const connectToServer = async (serverId: string, logPath: string) => { command_exists() { command -v "$@" > /dev/null 2>&1 } + ${installRClone()} ${installDocker()} ${setupSwarm()} ${setupNetwork()} @@ -235,6 +236,10 @@ export const createDefaultMiddlewares = () => { return command; }; +export const installRClone = () => ` +curl https://rclone.org/install.sh | sudo bash +`; + export const createTraefikInstance = () => { const command = ` # Check if dokpyloy-traefik exists diff --git a/apps/dokploy/server/wss/listen-deployment.ts b/apps/dokploy/server/wss/listen-deployment.ts index 00fca3eaf..e32e1b682 100644 --- a/apps/dokploy/server/wss/listen-deployment.ts +++ b/apps/dokploy/server/wss/listen-deployment.ts @@ -45,9 +45,7 @@ export const setupDeploymentLogsWebSocketServer = ( } try { - console.log(serverId); if (serverId) { - console.log("Entre aca"); const server = await findServerById(serverId); if (!server.sshKeyId) return; @@ -90,8 +88,6 @@ export const setupDeploymentLogsWebSocketServer = ( }); }); } else { - console.log("Entre aca2"); - console.log(logPath); const tail = spawn("tail", ["-n", "+1", "-f", logPath]); tail.stdout.on("data", (data) => { @@ -105,7 +101,7 @@ export const setupDeploymentLogsWebSocketServer = ( } catch (error) { // @ts-ignore // const errorMessage = error?.message as unknown as string; - // ws.send(errorMessage); + ws.send(errorMessage); } }); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d407fcea7..142d85d09 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14110,7 +14110,7 @@ snapshots: eslint: 8.45.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0) eslint-plugin-jsx-a11y: 6.9.0(eslint@8.45.0) eslint-plugin-react: 7.35.0(eslint@8.45.0) eslint-plugin-react-hooks: 5.0.0-canary-7118f5dd7-20230705(eslint@8.45.0) @@ -14134,7 +14134,7 @@ snapshots: enhanced-resolve: 5.17.1 eslint: 8.45.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0) fast-glob: 3.3.2 get-tsconfig: 4.7.5 is-core-module: 2.15.0 @@ -14156,7 +14156,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5