From 6da122eab7d5f86fc6ef4c7b7cdcf970e811fc67 Mon Sep 17 00:00:00 2001 From: Vlad Vladov Date: Wed, 3 Sep 2025 17:57:44 +0300 Subject: [PATCH 01/23] feat(tags): Add support for tags from Github Packages --- .../pages/api/deploy/[refreshToken].ts | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/apps/dokploy/pages/api/deploy/[refreshToken].ts b/apps/dokploy/pages/api/deploy/[refreshToken].ts index 3e515b182..22fabb39d 100644 --- a/apps/dokploy/pages/api/deploy/[refreshToken].ts +++ b/apps/dokploy/pages/api/deploy/[refreshToken].ts @@ -43,17 +43,19 @@ export default async function handler( if (sourceType === "docker") { const applicationDockerTag = extractImageTag(application.dockerImage); - const webhookDockerTag = extractImageTagFromRequest( + const webhookDockerTags = extractImageTagFromRequest( req.headers, req.body, ); - if ( + const isMismatch = applicationDockerTag && - webhookDockerTag && - webhookDockerTag !== applicationDockerTag - ) { + webhookDockerTags && + webhookDockerTags.length > 0 && + !webhookDockerTags.includes(applicationDockerTag); + + if (isMismatch) { res.status(301).json({ - message: `Application Image Tag (${applicationDockerTag}) doesn't match request event payload Image Tag (${webhookDockerTag}).`, + message: `Application Image Tag (${applicationDockerTag}) doesn't match request event payload Image Tag(s) (${webhookDockerTags.join(", ")}).`, }); return; } @@ -236,10 +238,38 @@ function extractImageTag(dockerImage: string | null) { export const extractImageTagFromRequest = ( headers: any, body: any, -): string | null => { +): string[] | null => { if (headers["user-agent"]?.includes("Go-http-client")) { if (body.push_data && body.repository) { - return body.push_data.tag; + return [body.push_data.tag] as string[]; + } + } + // GitHub Packages: package or registry_package events (container tags) + // See: https://docs.github.com/en/webhooks/webhook-events-and-payloads#package + const githubEvent = headers["x-github-event"]; + + if (githubEvent === "package" || githubEvent === "registry_package") { + const pkg = body?.package ?? body?.registry_package?.package ?? null; + const packageVersion = + body?.package_version ?? body?.registry_package?.package_version ?? null; + const packageType = pkg?.package_type; + + if (packageType === "container" && packageVersion) { + const tags = + packageVersion?.metadata?.container?.tags ?? + packageVersion?.container?.tags ?? + null; + if (Array.isArray(tags) && tags.length > 0) { + return tags as string[]; + } + const singleTag = + packageVersion?.metadata?.container?.tag ?? + packageVersion?.metadata?.tag ?? + packageVersion?.tag ?? + null; + if (typeof singleTag === "string") { + return [singleTag] as string[]; + } } } return null; From 7b398939f72a37b2a47b95b0e9dfe464af62dec8 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:12:49 -0600 Subject: [PATCH 02/23] Refactor compose and deployment services: streamline cloning and execution commands, remove redundant remote functions, and enhance error handling. Update database schema to include application build server ID for better tracking of deployments. --- apps/dokploy/server/api/routers/compose.ts | 9 +- .../server/queues/deployments-queue.ts | 85 ++-- packages/server/src/db/schema/deployment.ts | 6 + packages/server/src/services/application.ts | 260 +++--------- packages/server/src/services/compose.ts | 235 ++--------- packages/server/src/services/deployment.ts | 16 +- packages/server/src/utils/builders/compose.ts | 28 +- .../server/src/utils/builders/docker-file.ts | 17 +- packages/server/src/utils/builders/heroku.ts | 13 +- packages/server/src/utils/builders/index.ts | 19 +- .../server/src/utils/builders/nixpacks.ts | 51 ++- packages/server/src/utils/builders/paketo.ts | 13 +- .../server/src/utils/builders/railpack.ts | 21 +- packages/server/src/utils/builders/static.ts | 18 +- packages/server/src/utils/cluster/upload.ts | 25 +- packages/server/src/utils/docker/collision.ts | 7 +- packages/server/src/utils/docker/domain.ts | 69 +--- .../server/src/utils/process/execAsync.ts | 6 +- .../server/src/utils/providers/bitbucket.ts | 205 +-------- packages/server/src/utils/providers/git.ts | 391 +++--------------- packages/server/src/utils/providers/gitea.ts | 235 +---------- packages/server/src/utils/providers/github.ts | 232 ++--------- packages/server/src/utils/providers/gitlab.ts | 235 +---------- packages/server/src/utils/providers/raw.ts | 31 +- 24 files changed, 386 insertions(+), 1841 deletions(-) diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index e2f25b763..026b6e8ad 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -3,13 +3,14 @@ import { addNewService, checkServiceAccess, cloneCompose, - cloneComposeRemote, createCommand, createCompose, createComposeByTemplate, createDomain, createMount, deleteMount, + execAsync, + execAsyncRemote, findComposeById, findDomainsByComposeId, findEnvironmentById, @@ -302,10 +303,12 @@ export const composeRouter = createTRPCRouter({ message: "You are not authorized to fetch this compose", }); } + + const command = await cloneCompose(compose); if (compose.serverId) { - await cloneComposeRemote(compose); + await execAsyncRemote(compose.serverId, command); } else { - await cloneCompose(compose); + await execAsync(command); } return compose.sourceType; } catch (err) { diff --git a/apps/dokploy/server/queues/deployments-queue.ts b/apps/dokploy/server/queues/deployments-queue.ts index b8dfb8cd0..a1d9d29f1 100644 --- a/apps/dokploy/server/queues/deployments-queue.ts +++ b/apps/dokploy/server/queues/deployments-queue.ts @@ -2,13 +2,9 @@ import { deployApplication, deployCompose, deployPreviewApplication, - deployRemoteApplication, - deployRemoteCompose, deployRemotePreviewApplication, rebuildApplication, rebuildCompose, - rebuildRemoteApplication, - rebuildRemoteCompose, updateApplicationStatus, updateCompose, updatePreviewDeployment, @@ -24,68 +20,35 @@ export const deploymentWorker = new Worker( if (job.data.applicationType === "application") { await updateApplicationStatus(job.data.applicationId, "running"); - if (job.data.server) { - if (job.data.type === "redeploy") { - await rebuildRemoteApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "deploy") { - await deployRemoteApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } - } else { - if (job.data.type === "redeploy") { - await rebuildApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "deploy") { - await deployApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } + if (job.data.type === "redeploy") { + await rebuildApplication({ + applicationId: job.data.applicationId, + titleLog: job.data.titleLog, + descriptionLog: job.data.descriptionLog, + }); + } else if (job.data.type === "deploy") { + await deployApplication({ + applicationId: job.data.applicationId, + titleLog: job.data.titleLog, + descriptionLog: job.data.descriptionLog, + }); } } else if (job.data.applicationType === "compose") { await updateCompose(job.data.composeId, { composeStatus: "running", }); - - if (job.data.server) { - if (job.data.type === "redeploy") { - await rebuildRemoteCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "deploy") { - await deployRemoteCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } - } else { - if (job.data.type === "deploy") { - await deployCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "redeploy") { - await rebuildCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } + if (job.data.type === "deploy") { + await deployCompose({ + composeId: job.data.composeId, + titleLog: job.data.titleLog, + descriptionLog: job.data.descriptionLog, + }); + } else if (job.data.type === "redeploy") { + await rebuildCompose({ + composeId: job.data.composeId, + titleLog: job.data.titleLog, + descriptionLog: job.data.descriptionLog, + }); } } else if (job.data.applicationType === "application-preview") { await updatePreviewDeployment(job.data.previewDeploymentId, { diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index d6b0ddbcc..c49bb0760 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -34,6 +34,12 @@ export const deployments = pgTable("deployment", { status: deploymentStatus("status").default("running"), logPath: text("logPath").notNull(), pid: text("pid"), + applicationBuildServerId: text("applicationBuildServerId").references( + () => server.serverId, + { + onDelete: "cascade", + }, + ), applicationId: text("applicationId").references( () => applications.applicationId, { onDelete: "cascade" }, diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index 1891f9b6b..e2c38a9fa 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -13,35 +13,23 @@ import { } from "@dokploy/server/utils/builders"; import { sendBuildErrorNotifications } from "@dokploy/server/utils/notifications/build-error"; import { sendBuildSuccessNotifications } from "@dokploy/server/utils/notifications/build-success"; -import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { - cloneBitbucketRepository, - getBitbucketCloneCommand, -} from "@dokploy/server/utils/providers/bitbucket"; + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { cloneBitbucketRepository } from "@dokploy/server/utils/providers/bitbucket"; import { buildDocker, buildRemoteDocker, } from "@dokploy/server/utils/providers/docker"; -import { - cloneGitRepository, - getCustomGitCloneCommand, -} from "@dokploy/server/utils/providers/git"; -import { - cloneGiteaRepository, - getGiteaCloneCommand, -} from "@dokploy/server/utils/providers/gitea"; -import { - cloneGithubRepository, - getGithubCloneCommand, -} from "@dokploy/server/utils/providers/github"; -import { - cloneGitlabRepository, - getGitlabCloneCommand, -} from "@dokploy/server/utils/providers/gitlab"; +import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; +import { cloneGiteaRepository } from "@dokploy/server/utils/providers/gitea"; +import { cloneGithubRepository } from "@dokploy/server/utils/providers/github"; +import { cloneGitlabRepository } from "@dokploy/server/utils/providers/gitlab"; import { createTraefikConfig } from "@dokploy/server/utils/traefik/application"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; -import { encodeBase64 } from "../utils/docker/utils"; +import { cleanUpSystemPrune, encodeBase64 } from "../utils/docker/utils"; import { getDokployUrl } from "./admin"; import { createDeployment, @@ -192,43 +180,43 @@ export const deployApplication = async ({ }); try { + let command = "set -e;"; if (application.sourceType === "github") { - await cloneGithubRepository({ - ...application, - logPath: deployment.logPath, - }); - await buildApplication(application, deployment.logPath); + command += await cloneGithubRepository(application); } else if (application.sourceType === "gitlab") { - await cloneGitlabRepository(application, deployment.logPath); - await buildApplication(application, deployment.logPath); + command += await cloneGitlabRepository(application); } else if (application.sourceType === "gitea") { - await cloneGiteaRepository(application, deployment.logPath); - await buildApplication(application, deployment.logPath); + command += await cloneGiteaRepository(application); } else if (application.sourceType === "bitbucket") { - await cloneBitbucketRepository(application, deployment.logPath); - await buildApplication(application, deployment.logPath); - } else if (application.sourceType === "docker") { - await buildDocker(application, deployment.logPath); + command += await cloneBitbucketRepository(application); } else if (application.sourceType === "git") { - await cloneGitRepository(application, deployment.logPath); - await buildApplication(application, deployment.logPath); + command += await cloneGitRepository(application); } else if (application.sourceType === "drop") { await buildApplication(application, deployment.logPath); } + command += getBuildCommand(application); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + if (application.serverId) { + await execAsyncRemote(application.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); + } + + await mechanizeDockerContainer(application); await updateDeploymentStatus(deployment.deploymentId, "done"); await updateApplicationStatus(applicationId, "done"); - if (application.rollbackActive) { - const tagImage = - application.sourceType === "docker" - ? application.dockerImage - : application.appName; - await createRollback({ - appName: tagImage || "", - deploymentId: deployment.deploymentId, - }); - } + // if (application.rollbackActive) { + // const tagImage = + // application.sourceType === "docker" + // ? application.dockerImage + // : application.appName; + // await createRollback({ + // appName: tagImage || "", + // deploymentId: deployment.deploymentId, + // }); + // } await sendBuildSuccessNotifications({ projectName: application.environment.project.name, @@ -239,6 +227,12 @@ export const deployApplication = async ({ domains: application.domains, }); } catch (error) { + const command = `echo "Error occurred ❌, check the logs for details." >> ${deployment.logPath};`; + if (application.serverId) { + await execAsyncRemote(application.serverId, command); + } else { + await execAsync(command); + } await updateDeploymentStatus(deployment.deploymentId, "error"); await updateApplicationStatus(applicationId, "error"); @@ -254,7 +248,6 @@ export const deployApplication = async ({ throw error; } - return true; }; @@ -276,129 +269,21 @@ export const rebuildApplication = async ({ }); try { - if (application.sourceType === "github") { - await buildApplication(application, deployment.logPath); - } else if (application.sourceType === "gitlab") { - await buildApplication(application, deployment.logPath); - } else if (application.sourceType === "bitbucket") { - await buildApplication(application, deployment.logPath); - } else if (application.sourceType === "docker") { - await buildDocker(application, deployment.logPath); - } else if (application.sourceType === "git") { - await buildApplication(application, deployment.logPath); - } else if (application.sourceType === "drop") { - await buildApplication(application, deployment.logPath); - } - await updateDeploymentStatus(deployment.deploymentId, "done"); - await updateApplicationStatus(applicationId, "done"); - } catch (error) { - await updateDeploymentStatus(deployment.deploymentId, "error"); - await updateApplicationStatus(applicationId, "error"); - throw error; - } - - return true; -}; - -export const deployRemoteApplication = async ({ - applicationId, - titleLog = "Manual deployment", - descriptionLog = "", -}: { - applicationId: string; - titleLog: string; - descriptionLog: string; -}) => { - const application = await findApplicationById(applicationId); - - const buildLink = `${await getDokployUrl()}/dashboard/project/${application.environment.projectId}/environment/${application.environmentId}/services/application/${application.applicationId}?tab=deployments`; - const deployment = await createDeployment({ - applicationId: applicationId, - title: titleLog, - description: descriptionLog, - }); - - try { + let command = "set -e;"; + // Check case for docker only + command += getBuildCommand(application); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (application.serverId) { - let command = "set -e;"; - if (application.sourceType === "github") { - command += await getGithubCloneCommand({ - ...application, - serverId: application.serverId, - logPath: deployment.logPath, - }); - } else if (application.sourceType === "gitlab") { - command += await getGitlabCloneCommand(application, deployment.logPath); - } else if (application.sourceType === "bitbucket") { - command += await getBitbucketCloneCommand( - application, - deployment.logPath, - ); - } else if (application.sourceType === "gitea") { - command += await getGiteaCloneCommand(application, deployment.logPath); - } else if (application.sourceType === "git") { - command += await getCustomGitCloneCommand( - application, - deployment.logPath, - ); - } else if (application.sourceType === "docker") { - command += await buildRemoteDocker(application, deployment.logPath); - } - - if (application.sourceType !== "docker") { - command += getBuildCommand(application, deployment.logPath); - } - await execAsyncRemote(application.serverId, command); - await mechanizeDockerContainer(application); + await execAsyncRemote(application.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); } - + await mechanizeDockerContainer(application); await updateDeploymentStatus(deployment.deploymentId, "done"); await updateApplicationStatus(applicationId, "done"); - - if (application.rollbackActive) { - const tagImage = - application.sourceType === "docker" - ? application.dockerImage - : application.appName; - await createRollback({ - appName: tagImage || "", - deploymentId: deployment.deploymentId, - }); - } - - await sendBuildSuccessNotifications({ - projectName: application.environment.project.name, - applicationName: application.name, - applicationType: "application", - buildLink, - organizationId: application.environment.project.organizationId, - domains: application.domains, - }); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - - const encodedContent = encodeBase64(errorMessage); - - await execAsyncRemote( - application.serverId, - ` - echo "\n\n===================================EXTRA LOGS============================================" >> ${deployment.logPath}; - echo "Error occurred ❌, check the logs for details." >> ${deployment.logPath}; - echo "${encodedContent}" | base64 -d >> "${deployment.logPath}";`, - ); - await updateDeploymentStatus(deployment.deploymentId, "error"); await updateApplicationStatus(applicationId, "error"); - - await sendBuildErrorNotifications({ - projectName: application.environment.project.name, - applicationName: application.name, - applicationType: "application", - errorMessage: `Please check the logs for details: ${errorMessage}`, - buildLink, - organizationId: application.environment.project.organizationId, - }); - throw error; } @@ -586,7 +471,7 @@ export const deployRemotePreviewApplication = async ({ if (application.serverId) { let command = "set -e;"; if (application.sourceType === "github") { - command += await getGithubCloneCommand({ + command += await cloneGithubRepository({ ...application, appName: previewDeployment.appName, branch: previewDeployment.branch, @@ -629,53 +514,6 @@ export const deployRemotePreviewApplication = async ({ return true; }; -export const rebuildRemoteApplication = async ({ - applicationId, - titleLog = "Rebuild deployment", - descriptionLog = "", -}: { - applicationId: string; - titleLog: string; - descriptionLog: string; -}) => { - const application = await findApplicationById(applicationId); - - const deployment = await createDeployment({ - applicationId: applicationId, - title: titleLog, - description: descriptionLog, - }); - - try { - if (application.serverId) { - if (application.sourceType !== "docker") { - let command = "set -e;"; - command += getBuildCommand(application, deployment.logPath); - await execAsyncRemote(application.serverId, command); - } - await mechanizeDockerContainer(application); - } - await updateDeploymentStatus(deployment.deploymentId, "done"); - await updateApplicationStatus(applicationId, "done"); - } catch (error) { - // @ts-ignore - const encodedContent = encodeBase64(error?.message); - - await execAsyncRemote( - application.serverId, - ` - echo "\n\n===================================EXTRA LOGS============================================" >> ${deployment.logPath}; - echo "Error occurred ❌, check the logs for details." >> ${deployment.logPath}; - echo "${encodedContent}" | base64 -d >> "${deployment.logPath}";`, - ); - await updateDeploymentStatus(deployment.deploymentId, "error"); - await updateApplicationStatus(applicationId, "error"); - throw error; - } - - return true; -}; - export const getApplicationStats = async (appName: string) => { const filter = { status: ["running"], diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index 1436c52cc..4b96f443d 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -7,14 +7,10 @@ import { cleanAppName, compose, } from "@dokploy/server/db/schema"; -import { - buildCompose, - getBuildComposeCommand, -} from "@dokploy/server/utils/builders/compose"; +import { getBuildComposeCommand } from "@dokploy/server/utils/builders/compose"; import { randomizeSpecificationFile } from "@dokploy/server/utils/docker/compose"; import { cloneCompose, - cloneComposeRemote, loadDockerCompose, loadDockerComposeRemote, } from "@dokploy/server/utils/docker/domain"; @@ -25,33 +21,14 @@ import { execAsync, execAsyncRemote, } from "@dokploy/server/utils/process/execAsync"; -import { - cloneBitbucketRepository, - getBitbucketCloneCommand, -} from "@dokploy/server/utils/providers/bitbucket"; -import { - cloneGitRepository, - getCustomGitCloneCommand, -} from "@dokploy/server/utils/providers/git"; -import { - cloneGiteaRepository, - getGiteaCloneCommand, -} from "@dokploy/server/utils/providers/gitea"; -import { - cloneGithubRepository, - getGithubCloneCommand, -} from "@dokploy/server/utils/providers/github"; -import { - cloneGitlabRepository, - getGitlabCloneCommand, -} from "@dokploy/server/utils/providers/gitlab"; -import { - createComposeFile, - getCreateComposeFileCommand, -} from "@dokploy/server/utils/providers/raw"; +import { cloneBitbucketRepository } from "@dokploy/server/utils/providers/bitbucket"; +import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; +import { cloneGiteaRepository } from "@dokploy/server/utils/providers/gitea"; +import { cloneGithubRepository } from "@dokploy/server/utils/providers/github"; +import { cloneGitlabRepository } from "@dokploy/server/utils/providers/gitlab"; +import { getCreateComposeFileCommand } from "@dokploy/server/utils/providers/raw"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; -import { encodeBase64 } from "../utils/docker/utils"; import { getDokployUrl } from "./admin"; import { createDeploymentCompose, updateDeploymentStatus } from "./deployment"; import { validUniqueServerAppName } from "./project"; @@ -163,10 +140,11 @@ export const loadServices = async ( const compose = await findComposeById(composeId); if (type === "fetch") { + const command = await cloneCompose(compose); if (compose.serverId) { - await cloneComposeRemote(compose); + await execAsyncRemote(compose.serverId, command); } else { - await cloneCompose(compose); + await execAsync(command); } } @@ -235,24 +213,33 @@ export const deployCompose = async ({ }); try { + const entity = { + ...compose, + type: "compose" as const, + }; + let command = "set -e;"; if (compose.sourceType === "github") { - await cloneGithubRepository({ - ...compose, - logPath: deployment.logPath, - type: "compose", - }); + command += await cloneGithubRepository(entity); } else if (compose.sourceType === "gitlab") { - await cloneGitlabRepository(compose, deployment.logPath, true); + command += await cloneGitlabRepository(entity); } else if (compose.sourceType === "bitbucket") { - await cloneBitbucketRepository(compose, deployment.logPath, true); + command += await cloneBitbucketRepository(entity); } else if (compose.sourceType === "git") { - await cloneGitRepository(compose, deployment.logPath, true); + command += await cloneGitRepository(entity); } else if (compose.sourceType === "gitea") { - await cloneGiteaRepository(compose, deployment.logPath, true); + command += await cloneGiteaRepository(entity); } else if (compose.sourceType === "raw") { - await createComposeFile(compose, deployment.logPath); + command += getCreateComposeFileCommand(entity); } - await buildCompose(compose, deployment.logPath); + + command += await getBuildComposeCommand(entity); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + if (compose.serverId) { + await execAsyncRemote(compose.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); + } + await updateDeploymentStatus(deployment.deploymentId, "done"); await updateCompose(composeId, { composeStatus: "done", @@ -302,154 +289,16 @@ export const rebuildCompose = async ({ }); try { + let command = "set -e;"; if (compose.sourceType === "raw") { - await createComposeFile(compose, deployment.logPath); + command += getCreateComposeFileCommand(compose); } - await buildCompose(compose, deployment.logPath); - - await updateDeploymentStatus(deployment.deploymentId, "done"); - await updateCompose(composeId, { - composeStatus: "done", - }); - } catch (error) { - await updateDeploymentStatus(deployment.deploymentId, "error"); - await updateCompose(composeId, { - composeStatus: "error", - }); - throw error; - } - - return true; -}; - -export const deployRemoteCompose = async ({ - composeId, - titleLog = "Manual deployment", - descriptionLog = "", -}: { - composeId: string; - titleLog: string; - descriptionLog: string; -}) => { - const compose = await findComposeById(composeId); - - const buildLink = `${await getDokployUrl()}/dashboard/project/${ - compose.environment.projectId - }/environment/${compose.environmentId}/services/compose/${compose.composeId}?tab=deployments`; - const deployment = await createDeploymentCompose({ - composeId: composeId, - title: titleLog, - description: descriptionLog, - }); - try { + command += await getBuildComposeCommand(compose); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (compose.serverId) { - let command = "set -e;"; - - if (compose.sourceType === "github") { - command += await getGithubCloneCommand({ - ...compose, - logPath: deployment.logPath, - type: "compose", - serverId: compose.serverId, - }); - } else if (compose.sourceType === "gitlab") { - command += await getGitlabCloneCommand( - compose, - deployment.logPath, - true, - ); - } else if (compose.sourceType === "bitbucket") { - command += await getBitbucketCloneCommand( - compose, - deployment.logPath, - true, - ); - } else if (compose.sourceType === "git") { - command += await getCustomGitCloneCommand( - compose, - deployment.logPath, - true, - ); - console.log(command); - } else if (compose.sourceType === "raw") { - command += getCreateComposeFileCommand(compose, deployment.logPath); - } else if (compose.sourceType === "gitea") { - command += await getGiteaCloneCommand( - compose, - deployment.logPath, - true, - ); - } - - await execAsyncRemote(compose.serverId, command); - await getBuildComposeCommand(compose, deployment.logPath); - } - - await updateDeploymentStatus(deployment.deploymentId, "done"); - await updateCompose(composeId, { - composeStatus: "done", - }); - - await sendBuildSuccessNotifications({ - projectName: compose.environment.project.name, - applicationName: compose.name, - applicationType: "compose", - buildLink, - organizationId: compose.environment.project.organizationId, - domains: compose.domains, - }); - } catch (error) { - // @ts-ignore - const encodedContent = encodeBase64(error?.message); - - await execAsyncRemote( - compose.serverId, - ` - echo "\n\n===================================EXTRA LOGS============================================" >> ${deployment.logPath}; - echo "Error occurred ❌, check the logs for details." >> ${deployment.logPath}; - echo "${encodedContent}" | base64 -d >> "${deployment.logPath}";`, - ); - await updateDeploymentStatus(deployment.deploymentId, "error"); - await updateCompose(composeId, { - composeStatus: "error", - }); - await sendBuildErrorNotifications({ - projectName: compose.environment.project.name, - applicationName: compose.name, - applicationType: "compose", - // @ts-ignore - errorMessage: error?.message || "Error building", - buildLink, - organizationId: compose.environment.project.organizationId, - }); - throw error; - } -}; - -export const rebuildRemoteCompose = async ({ - composeId, - titleLog = "Rebuild deployment", - descriptionLog = "", -}: { - composeId: string; - titleLog: string; - descriptionLog: string; -}) => { - const compose = await findComposeById(composeId); - - const deployment = await createDeploymentCompose({ - composeId: composeId, - title: titleLog, - description: descriptionLog, - }); - - try { - if (compose.sourceType === "raw") { - const command = getCreateComposeFileCommand(compose, deployment.logPath); - await execAsyncRemote(compose.serverId, command); - } - if (compose.serverId) { - await getBuildComposeCommand(compose, deployment.logPath); + await execAsyncRemote(compose.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); } await updateDeploymentStatus(deployment.deploymentId, "done"); @@ -457,16 +306,6 @@ export const rebuildRemoteCompose = async ({ composeStatus: "done", }); } catch (error) { - // @ts-ignore - const encodedContent = encodeBase64(error?.message); - - await execAsyncRemote( - compose.serverId, - ` - echo "\n\n===================================EXTRA LOGS============================================" >> ${deployment.logPath}; - echo "Error occurred ❌, check the logs for details." >> ${deployment.logPath}; - echo "${encodedContent}" | base64 -d >> "${deployment.logPath}";`, - ); await updateDeploymentStatus(deployment.deploymentId, "error"); await updateCompose(composeId, { composeStatus: "error", diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index ed03b32fc..95d543dd0 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -74,20 +74,21 @@ export const createDeployment = async ( >, ) => { const application = await findApplicationById(deployment.applicationId); - try { await removeLastTenDeployments( deployment.applicationId, "application", application.serverId, ); - const { LOGS_PATH } = paths(!!application.serverId); + const serverId = application.serverId; + + const { LOGS_PATH } = paths(!!serverId); const formattedDateTime = format(new Date(), "yyyy-MM-dd:HH:mm:ss"); const fileName = `${application.appName}-${formattedDateTime}.log`; const logFilePath = path.join(LOGS_PATH, application.appName, fileName); - if (application.serverId) { - const server = await findServerById(application.serverId); + if (serverId) { + const server = await findServerById(serverId); const command = ` mkdir -p ${LOGS_PATH}/${application.appName}; @@ -99,7 +100,7 @@ export const createDeployment = async ( await fsPromises.mkdir(path.join(LOGS_PATH, application.appName), { recursive: true, }); - await fsPromises.writeFile(logFilePath, "Initializing deployment"); + await fsPromises.writeFile(logFilePath, "Initializing deployment\n"); } const deploymentCreate = await db @@ -111,6 +112,7 @@ export const createDeployment = async ( logPath: logFilePath, description: deployment.description || "", startedAt: new Date().toISOString(), + // applicationBuildServerId: application.serverBuildId, }) .returning(); if (deploymentCreate.length === 0 || !deploymentCreate[0]) { @@ -249,7 +251,7 @@ export const createDeploymentCompose = async ( const command = ` mkdir -p ${LOGS_PATH}/${compose.appName}; -echo "Initializing deployment" >> ${logFilePath}; +echo "Initializing deployment\n" >> ${logFilePath}; `; await execAsyncRemote(server.serverId, command); @@ -257,7 +259,7 @@ echo "Initializing deployment" >> ${logFilePath}; await fsPromises.mkdir(path.join(LOGS_PATH, compose.appName), { recursive: true, }); - await fsPromises.writeFile(logFilePath, "Initializing deployment"); + await fsPromises.writeFile(logFilePath, "Initializing deployment\n"); } const deploymentCreate = await db diff --git a/packages/server/src/utils/builders/compose.ts b/packages/server/src/utils/builders/compose.ts index 667b46b74..6b26a0d1c 100644 --- a/packages/server/src/utils/builders/compose.ts +++ b/packages/server/src/utils/builders/compose.ts @@ -17,7 +17,7 @@ import { getEnviromentVariablesObject, prepareEnvironmentVariables, } from "../docker/utils"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; +import { execAsync } from "../process/execAsync"; import { spawnAsync } from "../process/spawnAsync"; export type ComposeNested = InferResultType< @@ -96,22 +96,15 @@ export const buildCompose = async (compose: ComposeNested, logPath: string) => { } }; -export const getBuildComposeCommand = async ( - compose: ComposeNested, - logPath: string, -) => { - const { COMPOSE_PATH } = paths(true); +export const getBuildComposeCommand = async (compose: ComposeNested) => { + const { COMPOSE_PATH } = paths(!!compose.serverId); const { sourceType, appName, mounts, composeType, domains } = compose; const command = createCommand(compose); const envCommand = getCreateEnvFileCommand(compose); const projectPath = join(COMPOSE_PATH, compose.appName, "code"); const exportEnvCommand = getExportEnvCommand(compose); - const newCompose = await writeDomainsToComposeRemote( - compose, - domains, - logPath, - ); + const newCompose = await writeDomainsToComposeRemote(compose, domains); const logContent = ` App Name: ${appName} Build Compose 🐳 @@ -133,7 +126,7 @@ Compose Type: ${composeType} ✅`; const bashCommand = ` set -e { - echo "${logBox}" >> "${logPath}" + echo "${logBox}"; ${newCompose} @@ -143,17 +136,18 @@ Compose Type: ${composeType} ✅`; ${exportEnvCommand} ${compose.isolatedDeployment ? `docker network inspect ${compose.appName} >/dev/null 2>&1 || docker network create --attachable ${compose.appName}` : ""} - docker ${command.split(" ").join(" ")} >> "${logPath}" 2>&1 || { echo "Error: ❌ Docker command failed" >> "${logPath}"; exit 1; } + docker ${command.split(" ").join(" ")} 2>&1 || { echo "Error: ❌ Docker command failed"; exit 1; } ${compose.isolatedDeployment ? `docker network connect ${compose.appName} $(docker ps --filter "name=dokploy-traefik" -q) >/dev/null 2>&1` : ""} - echo "Docker Compose Deployed: ✅" >> "${logPath}" + echo "Docker Compose Deployed: ✅"; } || { - echo "Error: ❌ Script execution failed" >> "${logPath}" + echo "Error: ❌ Script execution failed"; exit 1 } `; - return await execAsyncRemote(compose.serverId, bashCommand); + return bashCommand; + // return await execAsyncRemote(compose.serverId, bashCommand); }; const sanitizeCommand = (command: string) => { @@ -216,7 +210,7 @@ const createEnvFile = (compose: ComposeNested) => { }; export const getCreateEnvFileCommand = (compose: ComposeNested) => { - const { COMPOSE_PATH } = paths(true); + const { COMPOSE_PATH } = paths(!!compose.serverId); const { env, composePath, appName } = compose; const composeFilePath = join(COMPOSE_PATH, appName, "code", composePath) || diff --git a/packages/server/src/utils/builders/docker-file.ts b/packages/server/src/utils/builders/docker-file.ts index b5c2b59c3..4636cb22c 100644 --- a/packages/server/src/utils/builders/docker-file.ts +++ b/packages/server/src/utils/builders/docker-file.ts @@ -100,10 +100,7 @@ export const buildCustomDocker = async ( } }; -export const getDockerCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getDockerCommand = (application: ApplicationNested) => { const { appName, env, @@ -176,17 +173,17 @@ export const getDockerCommand = ( } command += ` -echo "Building ${appName}" >> ${logPath}; -cd ${dockerContextPath} >> ${logPath} 2>> ${logPath} || { - echo "❌ The path ${dockerContextPath} does not exist" >> ${logPath}; +echo "Building ${appName}" ; +cd ${dockerContextPath} || { + echo "❌ The path ${dockerContextPath} does not exist" ; exit 1; } -${joinedSecrets} docker ${commandArgs.join(" ")} >> ${logPath} 2>> ${logPath} || { - echo "❌ Docker build failed" >> ${logPath}; +${joinedSecrets} docker ${commandArgs.join(" ")} || { + echo "❌ Docker build failed" ; exit 1; } -echo "✅ Docker build completed." >> ${logPath}; +echo "✅ Docker build completed." ; `; return command; diff --git a/packages/server/src/utils/builders/heroku.ts b/packages/server/src/utils/builders/heroku.ts index 3306f2fc2..a0a8da153 100644 --- a/packages/server/src/utils/builders/heroku.ts +++ b/packages/server/src/utils/builders/heroku.ts @@ -45,10 +45,7 @@ export const buildHeroku = async ( } }; -export const getHerokuCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getHerokuCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; const buildAppDirectory = getBuildAppDirectory(application); @@ -77,12 +74,12 @@ export const getHerokuCommand = ( const command = `pack ${args.join(" ")}`; const bashCommand = ` -echo "Starting heroku build..." >> ${logPath}; -${command} >> ${logPath} 2>> ${logPath} || { - echo "❌ Heroku build failed" >> ${logPath}; +echo "Starting heroku build..." ; +${command} || { + echo "❌ Heroku build failed" ; exit 1; } -echo "✅ Heroku build completed." >> ${logPath}; +echo "✅ Heroku build completed." ; `; return bashCommand; diff --git a/packages/server/src/utils/builders/index.ts b/packages/server/src/utils/builders/index.ts index 5ae0704c5..35172789c 100644 --- a/packages/server/src/utils/builders/index.ts +++ b/packages/server/src/utils/builders/index.ts @@ -76,34 +76,31 @@ export const buildApplication = async ( } }; -export const getBuildCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getBuildCommand = (application: ApplicationNested) => { let command = ""; const { buildType, registry } = application; switch (buildType) { case "nixpacks": - command = getNixpacksCommand(application, logPath); + command = getNixpacksCommand(application); break; case "heroku_buildpacks": - command = getHerokuCommand(application, logPath); + command = getHerokuCommand(application); break; case "paketo_buildpacks": - command = getPaketoCommand(application, logPath); + command = getPaketoCommand(application); break; case "static": - command = getStaticCommand(application, logPath); + command = getStaticCommand(application); break; case "dockerfile": - command = getDockerCommand(application, logPath); + command = getDockerCommand(application); break; case "railpack": - command = getRailpackCommand(application, logPath); + command = getRailpackCommand(application); break; } if (registry) { - command += uploadImageRemoteCommand(application, logPath); + command += uploadImageRemoteCommand(application); } return command; diff --git a/packages/server/src/utils/builders/nixpacks.ts b/packages/server/src/utils/builders/nixpacks.ts index 76905d0e7..705539f06 100644 --- a/packages/server/src/utils/builders/nixpacks.ts +++ b/packages/server/src/utils/builders/nixpacks.ts @@ -92,10 +92,7 @@ export const buildNixpacks = async ( } }; -export const getNixpacksCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getNixpacksCommand = (application: ApplicationNested) => { const { env, appName, publishDirectory, cleanCache } = application; const buildAppDirectory = getBuildAppDirectory(application); @@ -121,13 +118,13 @@ export const getNixpacksCommand = ( args.push("--no-error-without-start"); } const command = `nixpacks ${args.join(" ")}`; - let bashCommand = ` -echo "Starting nixpacks build..." >> ${logPath}; -${command} >> ${logPath} 2>> ${logPath} || { - echo "❌ Nixpacks build failed" >> ${logPath}; - exit 1; -} -echo "✅ Nixpacks build completed." >> ${logPath}; + const bashCommand = ` + echo "Starting nixpacks build..." ; + ${command} || { + echo "❌ Nixpacks build failed" ; + exit 1; + } + echo "✅ Nixpacks build completed." ; `; /* @@ -135,23 +132,23 @@ echo "✅ Nixpacks build completed." >> ${logPath}; and copy the artifacts on the host filesystem. Then, remove the container and create a static build. */ - if (publishDirectory) { - const localPath = path.join(buildAppDirectory, publishDirectory); - const isDirectory = - publishDirectory.endsWith("/") || !path.extname(publishDirectory); + // if (publishDirectory) { + // const localPath = path.join(buildAppDirectory, publishDirectory); + // const isDirectory = + // publishDirectory.endsWith("/") || !path.extname(publishDirectory); - bashCommand += ` -docker create --name ${buildContainerId} ${appName} -mkdir -p ${localPath} -docker cp ${buildContainerId}:/app/${publishDirectory}${isDirectory ? "/." : ""} ${path.join(buildAppDirectory, publishDirectory)} >> ${logPath} 2>> ${logPath} || { - docker rm ${buildContainerId} - echo "❌ Copying ${publishDirectory} to ${path.join(buildAppDirectory, publishDirectory)} failed" >> ${logPath}; - exit 1; -} -docker rm ${buildContainerId} -${getStaticCommand(application, logPath)} - `; - } + // bashCommand += ` + // docker create --name ${buildContainerId} ${appName} + // mkdir -p ${localPath} + // docker cp ${buildContainerId}:/app/${publishDirectory}${isDirectory ? "/." : ""} ${path.join(buildAppDirectory, publishDirectory)} || { + // docker rm ${buildContainerId} + // echo "❌ Copying ${publishDirectory} to ${path.join(buildAppDirectory, publishDirectory)} failed" ; + // exit 1; + // } + // docker rm ${buildContainerId} + // ${getStaticCommand(application)} + // `; + // } return bashCommand; }; diff --git a/packages/server/src/utils/builders/paketo.ts b/packages/server/src/utils/builders/paketo.ts index b95a1bb31..51e2301f9 100644 --- a/packages/server/src/utils/builders/paketo.ts +++ b/packages/server/src/utils/builders/paketo.ts @@ -44,10 +44,7 @@ export const buildPaketo = async ( } }; -export const getPaketoCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getPaketoCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; const buildAppDirectory = getBuildAppDirectory(application); @@ -76,12 +73,12 @@ export const getPaketoCommand = ( const command = `pack ${args.join(" ")}`; const bashCommand = ` -echo "Starting Paketo build..." >> ${logPath}; -${command} >> ${logPath} 2>> ${logPath} || { - echo "❌ Paketo build failed" >> ${logPath}; +echo "Starting Paketo build..." ; +${command} || { + echo "❌ Paketo build failed" ; exit 1; } -echo "✅ Paketo build completed." >> ${logPath}; +echo "✅ Paketo build completed." ; `; return bashCommand; diff --git a/packages/server/src/utils/builders/railpack.ts b/packages/server/src/utils/builders/railpack.ts index 4adc9ca1c..822257773 100644 --- a/packages/server/src/utils/builders/railpack.ts +++ b/packages/server/src/utils/builders/railpack.ts @@ -116,10 +116,7 @@ export const buildRailpack = async ( } }; -export const getRailpackCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getRailpackCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; const buildAppDirectory = getBuildAppDirectory(application); const envVariables = prepareEnvironmentVariables( @@ -183,21 +180,21 @@ export const getRailpackCommand = ( docker buildx create --use --name builder-containerd --driver docker-container || true docker buildx use builder-containerd -echo "Preparing Railpack build plan..." >> "${logPath}"; -railpack ${prepareArgs.join(" ")} >> ${logPath} 2>> ${logPath} || { - echo "❌ Railpack prepare failed" >> ${logPath}; +echo "Preparing Railpack build plan..." ; +railpack ${prepareArgs.join(" ")} || { + echo "❌ Railpack prepare failed" ; exit 1; } -echo "✅ Railpack prepare completed." >> ${logPath}; +echo "✅ Railpack prepare completed." ; -echo "Building with Railpack frontend..." >> "${logPath}"; +echo "Building with Railpack frontend..." ; # Export environment variables for secrets ${exportEnvs.join("\n")} -docker ${buildArgs.join(" ")} >> ${logPath} 2>> ${logPath} || { - echo "❌ Railpack build failed" >> ${logPath}; +docker ${buildArgs.join(" ")} || { + echo "❌ Railpack build failed" ; exit 1; } -echo "✅ Railpack build completed." >> ${logPath}; +echo "✅ Railpack build completed." ; docker buildx rm builder-containerd `; diff --git a/packages/server/src/utils/builders/static.ts b/packages/server/src/utils/builders/static.ts index e59faa711..5e1f10cc3 100644 --- a/packages/server/src/utils/builders/static.ts +++ b/packages/server/src/utils/builders/static.ts @@ -83,10 +83,7 @@ export const buildStatic = async ( } }; -export const getStaticCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const getStaticCommand = (application: ApplicationNested) => { const { publishDirectory } = application; const buildAppDirectory = getBuildAppDirectory(application); @@ -100,13 +97,10 @@ export const getStaticCommand = ( ].join("\n"), ); - command += getDockerCommand( - { - ...application, - buildType: "dockerfile", - dockerfile: "Dockerfile", - }, - logPath, - ); + command += getDockerCommand({ + ...application, + buildType: "dockerfile", + dockerfile: "Dockerfile", + }); return command; }; diff --git a/packages/server/src/utils/cluster/upload.ts b/packages/server/src/utils/cluster/upload.ts index c13a2701c..6a2dfd66f 100644 --- a/packages/server/src/utils/cluster/upload.ts +++ b/packages/server/src/utils/cluster/upload.ts @@ -59,10 +59,7 @@ export const uploadImage = async ( } }; -export const uploadImageRemoteCommand = ( - application: ApplicationNested, - logPath: string, -) => { +export const uploadImageRemoteCommand = (application: ApplicationNested) => { const registry = application.registry; if (!registry) { @@ -82,22 +79,22 @@ export const uploadImageRemoteCommand = ( try { const command = ` - echo "📦 [Enabled Registry] Uploading image to '${registry.registryType}' | '${registryTag}'" >> ${logPath}; - echo "${registry.password}" | docker login ${finalURL} -u ${registry.username} --password-stdin >> ${logPath} 2>> ${logPath} || { - echo "❌ DockerHub Failed" >> ${logPath}; + echo "📦 [Enabled Registry] Uploading image to '${registry.registryType}' | '${registryTag}'" ; + echo "${registry.password}" | docker login ${finalURL} -u ${registry.username} --password-stdin || { + echo "❌ DockerHub Failed" ; exit 1; } - echo "✅ Registry Login Success" >> ${logPath}; - docker tag ${imageName} ${registryTag} >> ${logPath} 2>> ${logPath} || { - echo "❌ Error tagging image" >> ${logPath}; + echo "✅ Registry Login Success" ; + docker tag ${imageName} ${registryTag} || { + echo "❌ Error tagging image" ; exit 1; } - echo "✅ Image Tagged" >> ${logPath}; - docker push ${registryTag} 2>> ${logPath} || { - echo "❌ Error pushing image" >> ${logPath}; + echo "✅ Image Tagged" ; + docker push ${registryTag} || { + echo "❌ Error pushing image" ; exit 1; } - echo "✅ Image Pushed" >> ${logPath}; + echo "✅ Image Pushed" ; `; return command; } catch (error) { diff --git a/packages/server/src/utils/docker/collision.ts b/packages/server/src/utils/docker/collision.ts index 9752100ca..88d20d4d8 100644 --- a/packages/server/src/utils/docker/collision.ts +++ b/packages/server/src/utils/docker/collision.ts @@ -1,11 +1,11 @@ import { findComposeById } from "@dokploy/server/services/compose"; import { stringify } from "yaml"; +import { execAsync, execAsyncRemote } from "../process/execAsync"; import { addAppNameToAllServiceNames } from "./collision/root-network"; import { generateRandomHash } from "./compose"; import { addSuffixToAllVolumes } from "./compose/volume"; import { cloneCompose, - cloneComposeRemote, loadDockerCompose, loadDockerComposeRemote, } from "./domain"; @@ -31,10 +31,11 @@ export const randomizeIsolatedDeploymentComposeFile = async ( ) => { const compose = await findComposeById(composeId); + const command = await cloneCompose(compose); if (compose.serverId) { - await cloneComposeRemote(compose); + await execAsyncRemote(compose.serverId, command); } else { - await cloneCompose(compose); + await execAsync(command); } let composeData: ComposeSpecification | null; diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index 7a9521d1d..e8c66e697 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -6,30 +6,12 @@ import type { Compose } from "@dokploy/server/services/compose"; import type { Domain } from "@dokploy/server/services/domain"; import { parse, stringify } from "yaml"; import { execAsyncRemote } from "../process/execAsync"; -import { - cloneRawBitbucketRepository, - cloneRawBitbucketRepositoryRemote, -} from "../providers/bitbucket"; -import { - cloneGitRawRepository, - cloneRawGitRepositoryRemote, -} from "../providers/git"; -import { - cloneRawGiteaRepository, - cloneRawGiteaRepositoryRemote, -} from "../providers/gitea"; -import { - cloneRawGithubRepository, - cloneRawGithubRepositoryRemote, -} from "../providers/github"; -import { - cloneRawGitlabRepository, - cloneRawGitlabRepositoryRemote, -} from "../providers/gitlab"; -import { - createComposeFileRaw, - createComposeFileRawRemote, -} from "../providers/raw"; +import { cloneBitbucketRepository } from "../providers/bitbucket"; +import { cloneGitRepository } from "../providers/git"; +import { cloneGiteaRepository } from "../providers/gitea"; +import { cloneGithubRepository } from "../providers/github"; +import { cloneGitlabRepository } from "../providers/gitlab"; +import { getCreateComposeFileCommand } from "../providers/raw"; import { randomizeDeployableSpecificationFile } from "./collision"; import { randomizeSpecificationFile } from "./compose"; import type { @@ -40,35 +22,25 @@ import type { import { encodeBase64 } from "./utils"; export const cloneCompose = async (compose: Compose) => { + let command = "set -e;"; + const entity = { + ...compose, + type: "compose" as const, + }; if (compose.sourceType === "github") { - await cloneRawGithubRepository(compose); + command += await cloneGithubRepository(entity); } else if (compose.sourceType === "gitlab") { - await cloneRawGitlabRepository(compose); + command += await cloneGitlabRepository(entity); } else if (compose.sourceType === "bitbucket") { - await cloneRawBitbucketRepository(compose); + command += await cloneBitbucketRepository(entity); } else if (compose.sourceType === "git") { - await cloneGitRawRepository(compose); + command += await cloneGitRepository(entity); } else if (compose.sourceType === "gitea") { - await cloneRawGiteaRepository(compose); + command += await cloneGiteaRepository(entity); } else if (compose.sourceType === "raw") { - await createComposeFileRaw(compose); - } -}; - -export const cloneComposeRemote = async (compose: Compose) => { - if (compose.sourceType === "github") { - await cloneRawGithubRepositoryRemote(compose); - } else if (compose.sourceType === "gitlab") { - await cloneRawGitlabRepositoryRemote(compose); - } else if (compose.sourceType === "bitbucket") { - await cloneRawBitbucketRepositoryRemote(compose); - } else if (compose.sourceType === "git") { - await cloneRawGitRepositoryRemote(compose); - } else if (compose.sourceType === "gitea") { - await cloneRawGiteaRepositoryRemote(compose); - } else if (compose.sourceType === "raw") { - await createComposeFileRawRemote(compose); + command += getCreateComposeFileCommand(compose); } + return command; }; export const getComposePath = (compose: Compose) => { @@ -152,7 +124,6 @@ export const writeDomainsToCompose = async ( export const writeDomainsToComposeRemote = async ( compose: Compose, domains: Domain[], - logPath: string, ) => { if (!domains.length) { return ""; @@ -164,7 +135,7 @@ export const writeDomainsToComposeRemote = async ( if (!composeConverted) { return ` -echo "❌ Error: Compose file not found" >> ${logPath}; +echo "❌ Error: Compose file not found"; exit 1; `; } @@ -175,7 +146,7 @@ exit 1; } } catch (error) { // @ts-ignore - return `echo "❌ Has occured an error: ${error?.message || error}" >> ${logPath}; + return `echo "❌ Has occured an error: ${error?.message || error}"; exit 1; `; } diff --git a/packages/server/src/utils/process/execAsync.ts b/packages/server/src/utils/process/execAsync.ts index 84f0701d9..13b06c6c4 100644 --- a/packages/server/src/utils/process/execAsync.ts +++ b/packages/server/src/utils/process/execAsync.ts @@ -116,11 +116,7 @@ export const execAsyncRemote = async ( if (code === 0) { resolve({ stdout, stderr }); } else { - reject( - new Error( - `Command exited with code ${code}. Stderr: ${stderr}, command: ${command}`, - ), - ); + reject(new Error(`Error occurred ❌: ${stderr}`)); } }) .on("data", (data: string) => { diff --git a/packages/server/src/utils/providers/bitbucket.ts b/packages/server/src/utils/providers/bitbucket.ts index ed6cd8c31..0267ec9b3 100644 --- a/packages/server/src/utils/providers/bitbucket.ts +++ b/packages/server/src/utils/providers/bitbucket.ts @@ -1,4 +1,3 @@ -import { createWriteStream } from "node:fs"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { @@ -9,12 +8,8 @@ import { type Bitbucket, findBitbucketById, } from "@dokploy/server/services/bitbucket"; -import type { Compose } from "@dokploy/server/services/compose"; import type { InferResultType } from "@dokploy/server/types/with"; import { TRPCError } from "@trpc/server"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsyncRemote } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; export type ApplicationWithBitbucket = InferResultType< "applications", @@ -81,13 +76,16 @@ export const getBitbucketHeaders = (bitbucketProvider: Bitbucket) => { }; }; -export const cloneBitbucketRepository = async ( - entity: ApplicationWithBitbucket | ComposeWithBitbucket, - logPath: string, - isCompose = false, -) => { - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(); - const writeStream = createWriteStream(logPath, { flags: "a" }); +type BitbucketClone = (ApplicationWithBitbucket | ComposeWithBitbucket) & { + serverId: string | null; + type?: "application" | "compose"; +}; + +export const cloneBitbucketRepository = async ({ + type = "application", + ...entity +}: BitbucketClone) => { + let command = "set -e;"; const { appName, bitbucketRepository, @@ -96,187 +94,24 @@ export const cloneBitbucketRepository = async ( bitbucketId, bitbucket, enableSubmodules, + serverId, } = entity; + const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(!!serverId); if (!bitbucketId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Bitbucket Provider not found", - }); + command += `echo "Error: ❌ Bitbucket Provider not found"; exit 1;`; + return command; } - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); + command += `rm -rf ${outputPath};`; + command += `mkdir -p ${outputPath};`; const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`; const cloneUrl = getBitbucketCloneUrl(bitbucket, repoclone); - try { - writeStream.write(`\nCloning Repo ${repoclone} to ${outputPath}: ✅\n`); - const cloneArgs = [ - "clone", - "--branch", - bitbucketBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; - - await spawnAsync("git", cloneArgs, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - writeStream.write(`\nCloned ${repoclone} to ${outputPath}: ✅\n`); - } catch (error) { - writeStream.write(`ERROR Cloning: ${error}: ❌`); - throw error; - } finally { - writeStream.end(); - } -}; - -export const cloneRawBitbucketRepository = async (entity: Compose) => { - const { COMPOSE_PATH } = paths(); - const { - appName, - bitbucketRepository, - bitbucketOwner, - bitbucketBranch, - bitbucketId, - enableSubmodules, - } = entity; - - if (!bitbucketId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Bitbucket Provider not found", - }); - } - - const bitbucketProvider = await findBitbucketById(bitbucketId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); - const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`; - const cloneUrl = getBitbucketCloneUrl(bitbucketProvider, repoclone); - - try { - const cloneArgs = [ - "clone", - "--branch", - bitbucketBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; - - await spawnAsync("git", cloneArgs); - } catch (error) { - throw error; - } -}; - -export const cloneRawBitbucketRepositoryRemote = async (compose: Compose) => { - const { COMPOSE_PATH } = paths(true); - const { - appName, - bitbucketRepository, - bitbucketOwner, - bitbucketBranch, - bitbucketId, - serverId, - enableSubmodules, - } = compose; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!bitbucketId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Bitbucket Provider not found", - }); - } - - const bitbucketProvider = await findBitbucketById(bitbucketId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`; - const cloneUrl = getBitbucketCloneUrl(bitbucketProvider, repoclone); - - try { - const cloneCommand = ` - rm -rf ${outputPath}; - git clone --branch ${bitbucketBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} - `; - await execAsyncRemote(serverId, cloneCommand); - } catch (error) { - throw error; - } -}; - -export const getBitbucketCloneCommand = async ( - entity: ApplicationWithBitbucket | ComposeWithBitbucket, - logPath: string, - isCompose = false, -) => { - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - const { - appName, - bitbucketRepository, - bitbucketOwner, - bitbucketBranch, - bitbucketId, - serverId, - enableSubmodules, - } = entity; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - - if (!bitbucketId) { - const command = ` - echo "Error: ❌ Bitbucket Provider not found" >> ${logPath}; - exit 1; - `; - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "NOT_FOUND", - message: "Bitbucket Provider not found", - }); - } - - const bitbucketProvider = await findBitbucketById(bitbucketId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); - const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`; - const cloneUrl = getBitbucketCloneUrl(bitbucketProvider, repoclone); - - const cloneCommand = ` -rm -rf ${outputPath}; -mkdir -p ${outputPath}; -if ! git clone --branch ${bitbucketBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Fail to clone the repository ${repoclone}" >> ${logPath}; - exit 1; -fi -echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath}; - `; - - return cloneCommand; + command += `echo "Cloning Repo ${repoclone} to ${outputPath}: ✅";`; + command += `git clone --branch ${bitbucketBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} --progress;`; + return command; }; export const getBitbucketRepositories = async (bitbucketId?: string) => { diff --git a/packages/server/src/utils/providers/git.ts b/packages/server/src/utils/providers/git.ts index 5779252db..19c8ab8a0 100644 --- a/packages/server/src/utils/providers/git.ts +++ b/packages/server/src/utils/providers/git.ts @@ -1,159 +1,64 @@ -import { createWriteStream } from "node:fs"; import path, { join } from "node:path"; import { paths } from "@dokploy/server/constants"; -import type { Compose } from "@dokploy/server/services/compose"; import { findSSHKeyById, updateSSHKeyById, } from "@dokploy/server/services/ssh-key"; -import { TRPCError } from "@trpc/server"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; -export const cloneGitRepository = async ( - entity: { - appName: string; - customGitUrl?: string | null; - customGitBranch?: string | null; - customGitSSHKeyId?: string | null; - enableSubmodules?: boolean; - }, - logPath: string, - isCompose = false, -) => { - const { SSH_PATH, COMPOSE_PATH, APPLICATIONS_PATH } = paths(); +interface CloneGitRepository { + appName: string; + customGitUrl?: string | null; + customGitBranch?: string | null; + customGitSSHKeyId?: string | null; + enableSubmodules?: boolean; + serverId: string | null; + type?: "application" | "compose"; +} + +export const cloneGitRepository = async ({ + type = "application", + ...entity +}: CloneGitRepository) => { + let command = "set -e;"; const { appName, customGitUrl, customGitBranch, customGitSSHKeyId, enableSubmodules, + serverId, } = entity; + const { SSH_PATH, COMPOSE_PATH, APPLICATIONS_PATH } = paths(!!serverId); if (!customGitUrl || !customGitBranch) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: Repository not found", - }); + command += `echo "Error: ❌ Repository not found"; exit 1;`; + return command; } - const writeStream = createWriteStream(logPath, { flags: "a" }); const temporalKeyPath = path.join("/tmp", "id_rsa"); if (customGitSSHKeyId) { const sshKey = await findSSHKeyById(customGitSSHKeyId); - await execAsync(` + command += ` echo "${sshKey.privateKey}" > ${temporalKeyPath} - chmod 600 ${temporalKeyPath} - `); + chmod 600 ${temporalKeyPath}; + `; } - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; const outputPath = join(basePath, appName, "code"); const knownHostsPath = path.join(SSH_PATH, "known_hosts"); - try { - if (!isHttpOrHttps(customGitUrl)) { - if (!customGitSSHKeyId) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: - "Error: you are trying to clone a ssh repository without a ssh key, please set a ssh key", - }); - } - await addHostToKnownHosts(customGitUrl); + if (!isHttpOrHttps(customGitUrl)) { + if (!customGitSSHKeyId) { + command += `echo "Error: ❌ You are trying to clone a ssh repository without a ssh key, please set a ssh key"; exit 1;`; + return command; } - await recreateDirectory(outputPath); - writeStream.write( - `\nCloning Repo Custom ${customGitUrl} to ${outputPath}: ✅\n`, - ); - - if (customGitSSHKeyId) { - await updateSSHKeyById({ - sshKeyId: customGitSSHKeyId, - lastUsedAt: new Date().toISOString(), - }); - } - - const { port } = sanitizeRepoPathSSH(customGitUrl); - const cloneArgs = [ - "clone", - "--branch", - customGitBranch, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - customGitUrl, - outputPath, - "--progress", - ]; - - await spawnAsync( - "git", - cloneArgs, - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - { - env: { - ...process.env, - ...(customGitSSHKeyId && { - GIT_SSH_COMMAND: `ssh -i ${temporalKeyPath}${port ? ` -p ${port}` : ""} -o UserKnownHostsFile=${knownHostsPath}`, - }), - }, - }, - ); - - writeStream.write(`\nCloned Custom Git ${customGitUrl}: ✅\n`); - } catch (error) { - writeStream.write(`\nERROR Cloning Custom Git: ${error}: ❌\n`); - throw error; - } finally { - writeStream.end(); + command += addHostToKnownHostsCommand(customGitUrl); } -}; - -export const getCustomGitCloneCommand = async ( - entity: { - appName: string; - customGitUrl?: string | null; - customGitBranch?: string | null; - customGitSSHKeyId?: string | null; - serverId: string | null; - enableSubmodules: boolean; - }, - logPath: string, - isCompose = false, -) => { - const { SSH_PATH, COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - const { - appName, - customGitUrl, - customGitBranch, - customGitSSHKeyId, - serverId, - enableSubmodules, - } = entity; - - if (!customGitUrl || !customGitBranch) { - const command = ` - echo "Error: ❌ Repository not found" >> ${logPath}; - exit 1; - `; - - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: Repository not found", - }); - } - - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - const knownHostsPath = path.join(SSH_PATH, "known_hosts"); + command += `rm -rf ${outputPath};`; + command += `mkdir -p ${outputPath};`; + command += `echo "Cloning Repo Custom ${customGitUrl} to ${outputPath}: ✅";`; if (customGitSSHKeyId) { await updateSSHKeyById({ @@ -161,48 +66,22 @@ export const getCustomGitCloneCommand = async ( lastUsedAt: new Date().toISOString(), }); } - try { - const command = []; - if (!isHttpOrHttps(customGitUrl)) { - if (!customGitSSHKeyId) { - command.push( - `echo "Error: you are trying to clone a ssh repository without a ssh key, please set a ssh key ❌" >> ${logPath}; - exit 1; - `, - ); - } - command.push(addHostToKnownHostsCommand(customGitUrl)); - } - command.push(`rm -rf ${outputPath};`); - command.push(`mkdir -p ${outputPath};`); - command.push( - `echo "Cloning Custom Git ${customGitUrl}" to ${outputPath}: ✅ >> ${logPath};`, - ); - if (customGitSSHKeyId) { - const sshKey = await findSSHKeyById(customGitSSHKeyId); - const { port } = sanitizeRepoPathSSH(customGitUrl); - const gitSshCommand = `ssh -i /tmp/id_rsa${port ? ` -p ${port}` : ""} -o UserKnownHostsFile=${knownHostsPath}`; - command.push( - ` - echo "${sshKey.privateKey}" > /tmp/id_rsa - chmod 600 /tmp/id_rsa - export GIT_SSH_COMMAND="${gitSshCommand}" - `, - ); - } - command.push( - `if ! git clone --branch ${customGitBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${customGitUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Fail to clone the repository ${customGitUrl}" >> ${logPath}; + if (customGitSSHKeyId) { + const sshKey = await findSSHKeyById(customGitSSHKeyId); + const { port } = sanitizeRepoPathSSH(customGitUrl); + const gitSshCommand = `ssh -i /tmp/id_rsa${port ? ` -p ${port}` : ""} -o UserKnownHostsFile=${knownHostsPath}`; + command += `echo "${sshKey.privateKey}" > /tmp/id_rsa;`; + command += "chmod 600 /tmp/id_rsa;"; + command += `export GIT_SSH_COMMAND="${gitSshCommand}";`; + } + command += `if ! git clone --branch ${customGitBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${customGitUrl} ${outputPath}; then + echo "❌ [ERROR] Fail to clone the repository ${customGitUrl}"; exit 1; fi - `, - ); - command.push(`echo "Cloned Custom Git ${customGitUrl}: ✅" >> ${logPath};`); - return command.join("\n"); - } catch (error) { - throw error; - } + `; + + return command; }; const isHttpOrHttps = (url: string): boolean => { @@ -210,19 +89,19 @@ const isHttpOrHttps = (url: string): boolean => { return regex.test(url); }; -const addHostToKnownHosts = async (repositoryURL: string) => { - const { SSH_PATH } = paths(); - const { domain, port } = sanitizeRepoPathSSH(repositoryURL); - const knownHostsPath = path.join(SSH_PATH, "known_hosts"); +// const addHostToKnownHosts = async (repositoryURL: string) => { +// const { SSH_PATH } = paths(); +// const { domain, port } = sanitizeRepoPathSSH(repositoryURL); +// const knownHostsPath = path.join(SSH_PATH, "known_hosts"); - const command = `ssh-keyscan -p ${port} ${domain} >> ${knownHostsPath}`; - try { - await execAsync(command); - } catch (error) { - console.error(`Error adding host to known_hosts: ${error}`); - throw error; - } -}; +// const command = `ssh-keyscan -p ${port} ${domain} >> ${knownHostsPath}`; +// try { +// await execAsync(command); +// } catch (error) { +// console.error(`Error adding host to known_hosts: ${error}`); +// throw error; +// } +// }; const addHostToKnownHostsCommand = (repositoryURL: string) => { const { SSH_PATH } = paths(true); @@ -266,161 +145,3 @@ const sanitizeRepoPathSSH = (input: string) => { }, }; }; - -export const cloneGitRawRepository = async (entity: { - appName: string; - customGitUrl?: string | null; - customGitBranch?: string | null; - customGitSSHKeyId?: string | null; - enableSubmodules?: boolean; -}) => { - const { - appName, - customGitUrl, - customGitBranch, - customGitSSHKeyId, - enableSubmodules, - } = entity; - - if (!customGitUrl || !customGitBranch) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: Repository not found", - }); - } - - const { SSH_PATH, COMPOSE_PATH } = paths(); - const temporalKeyPath = path.join("/tmp", "id_rsa"); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const knownHostsPath = path.join(SSH_PATH, "known_hosts"); - - if (customGitSSHKeyId) { - const sshKey = await findSSHKeyById(customGitSSHKeyId); - - await execAsync(` - echo "${sshKey.privateKey}" > ${temporalKeyPath} - chmod 600 ${temporalKeyPath} - `); - } - - try { - if (!isHttpOrHttps(customGitUrl)) { - if (!customGitSSHKeyId) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: - "Error: you are trying to clone a ssh repository without a ssh key, please set a ssh key", - }); - } - await addHostToKnownHosts(customGitUrl); - } - await recreateDirectory(outputPath); - - if (customGitSSHKeyId) { - await updateSSHKeyById({ - sshKeyId: customGitSSHKeyId, - lastUsedAt: new Date().toISOString(), - }); - } - - const { port } = sanitizeRepoPathSSH(customGitUrl); - const cloneArgs = [ - "clone", - "--branch", - customGitBranch, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - customGitUrl, - outputPath, - "--progress", - ]; - - await spawnAsync("git", cloneArgs, (_data) => {}, { - env: { - ...process.env, - ...(customGitSSHKeyId && { - GIT_SSH_COMMAND: `ssh -i ${temporalKeyPath}${port ? ` -p ${port}` : ""} -o UserKnownHostsFile=${knownHostsPath}`, - }), - }, - }); - } catch (error) { - throw error; - } -}; - -export const cloneRawGitRepositoryRemote = async (compose: Compose) => { - const { - appName, - customGitBranch, - customGitUrl, - customGitSSHKeyId, - serverId, - enableSubmodules, - } = compose; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!customGitUrl) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Git Provider not found", - }); - } - - const { SSH_PATH, COMPOSE_PATH } = paths(true); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const knownHostsPath = path.join(SSH_PATH, "known_hosts"); - - if (customGitSSHKeyId) { - await updateSSHKeyById({ - sshKeyId: customGitSSHKeyId, - lastUsedAt: new Date().toISOString(), - }); - } - try { - const command = []; - if (!isHttpOrHttps(customGitUrl)) { - if (!customGitSSHKeyId) { - command.push( - `echo "Error: you are trying to clone a ssh repository without a ssh key, please set a ssh key ❌" ; - exit 1; - `, - ); - } - command.push(addHostToKnownHostsCommand(customGitUrl)); - } - command.push(`rm -rf ${outputPath};`); - command.push(`mkdir -p ${outputPath};`); - if (customGitSSHKeyId) { - const sshKey = await findSSHKeyById(customGitSSHKeyId); - const { port } = sanitizeRepoPathSSH(customGitUrl); - const gitSshCommand = `ssh -i /tmp/id_rsa${port ? ` -p ${port}` : ""} -o UserKnownHostsFile=${knownHostsPath}`; - command.push( - ` - echo "${sshKey.privateKey}" > /tmp/id_rsa - chmod 600 /tmp/id_rsa - export GIT_SSH_COMMAND="${gitSshCommand}" - `, - ); - } - - command.push( - `if ! git clone --branch ${customGitBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${customGitUrl} ${outputPath} ; then - echo "[ERROR] Fail to clone the repository "; - exit 1; - fi - `, - ); - - await execAsyncRemote(serverId, command.join("\n")); - } catch (error) { - throw error; - } -}; diff --git a/packages/server/src/utils/providers/gitea.ts b/packages/server/src/utils/providers/gitea.ts index db6dcbb78..5b69f6826 100644 --- a/packages/server/src/utils/providers/gitea.ts +++ b/packages/server/src/utils/providers/gitea.ts @@ -1,7 +1,5 @@ -import { createWriteStream } from "node:fs"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; -import type { Compose } from "@dokploy/server/services/compose"; import { findGiteaById, type Gitea, @@ -9,9 +7,6 @@ import { } from "@dokploy/server/services/gitea"; import type { InferResultType } from "@dokploy/server/types/with"; import { TRPCError } from "@trpc/server"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsyncRemote } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; export const getErrorCloneRequirements = (entity: { giteaRepository?: string | null; @@ -119,79 +114,16 @@ export type ApplicationWithGitea = InferResultType< export type ComposeWithGitea = InferResultType<"compose", { gitea: true }>; -export const getGiteaCloneCommand = async ( - entity: ApplicationWithGitea | ComposeWithGitea, - logPath: string, - isCompose = false, -) => { - const { - appName, - giteaBranch, - giteaId, - giteaOwner, - giteaRepository, - serverId, - enableSubmodules, - } = entity; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - - if (!giteaId) { - const command = ` - echo "Error: ❌ Gitlab Provider not found" >> ${logPath}; - exit 1; - `; - - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea Provider not found", - }); - } - - // Use paths(true) for remote operations - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - await refreshGiteaToken(giteaId); - const gitea = await findGiteaById(giteaId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - - const repoClone = `${giteaOwner}/${giteaRepository}.git`; - const cloneUrl = buildGiteaCloneUrl( - gitea?.giteaUrl!, - gitea?.accessToken!, - giteaOwner!, - giteaRepository!, - ); - - const cloneCommand = ` - rm -rf ${outputPath}; - mkdir -p ${outputPath}; - - if ! git clone --branch ${giteaBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Failed to clone the repository ${repoClone}" >> ${logPath}; - exit 1; - fi - - echo "Cloned ${repoClone} to ${outputPath}: ✅" >> ${logPath}; - `; - - return cloneCommand; +type GiteaClone = (ApplicationWithGitea | ComposeWithGitea) & { + serverId: string | null; + type?: "application" | "compose"; }; -export const cloneGiteaRepository = async ( - entity: ApplicationWithGitea | ComposeWithGitea, - logPath: string, - isCompose = false, -) => { - const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(); - - const writeStream = createWriteStream(logPath, { flags: "a" }); +export const cloneGiteaRepository = async ({ + type = "application", + ...entity +}: GiteaClone) => { + let command = "set -e;"; const { appName, giteaBranch, @@ -199,27 +131,27 @@ export const cloneGiteaRepository = async ( giteaOwner, giteaRepository, enableSubmodules, + serverId, } = entity; + const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(!!serverId); if (!giteaId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea Provider not found", - }); + command += `echo "Error: ❌ Gitea Provider not found"; exit 1;`; + return command; } await refreshGiteaToken(giteaId); const giteaProvider = await findGiteaById(giteaId); + if (!giteaProvider) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea provider not found in the database", - }); + command += `echo "❌ [ERROR] Gitea provider not found in the database"; exit 1;`; + return command; } - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); + command += `rm -rf ${outputPath};`; + command += `mkdir -p ${outputPath};`; const repoClone = `${giteaOwner}/${giteaRepository}.git`; const cloneUrl = buildGiteaCloneUrl( @@ -229,134 +161,9 @@ export const cloneGiteaRepository = async ( giteaRepository!, ); - writeStream.write(`\nCloning Repo ${repoClone} to ${outputPath}...\n`); - - try { - await spawnAsync( - "git", - [ - "clone", - "--branch", - giteaBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ], - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - ); - writeStream.write(`\nCloned ${repoClone}: ✅\n`); - } catch (error) { - writeStream.write(`ERROR Cloning: ${error}: ❌`); - throw error; - } finally { - writeStream.end(); - } -}; - -export const cloneRawGiteaRepository = async (entity: Compose) => { - const { - appName, - giteaRepository, - giteaOwner, - giteaBranch, - giteaId, - enableSubmodules, - } = entity; - const { COMPOSE_PATH } = paths(); - - if (!giteaId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea Provider not found", - }); - } - await refreshGiteaToken(giteaId); - const giteaProvider = await findGiteaById(giteaId); - if (!giteaProvider) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea provider not found in the database", - }); - } - - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); - - const cloneUrl = buildGiteaCloneUrl( - giteaProvider.giteaUrl, - giteaProvider.accessToken!, - giteaOwner!, - giteaRepository!, - ); - - try { - await spawnAsync("git", [ - "clone", - "--branch", - giteaBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]); - } catch (error) { - throw error; - } -}; - -export const cloneRawGiteaRepositoryRemote = async (compose: Compose) => { - const { - appName, - giteaRepository, - giteaOwner, - giteaBranch, - giteaId, - serverId, - enableSubmodules, - } = compose; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!giteaId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitea Provider not found", - }); - } - const { COMPOSE_PATH } = paths(true); - const giteaProvider = await findGiteaById(giteaId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const cloneUrl = buildGiteaCloneUrl( - giteaProvider.giteaUrl, - giteaProvider.accessToken!, - giteaOwner!, - giteaRepository!, - ); - - try { - const command = ` - rm -rf ${outputPath}; - git clone --branch ${giteaBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} - `; - await execAsyncRemote(serverId, command); - } catch (error) { - throw error; - } + command += `echo "Cloning Repo ${repoClone} to ${outputPath}: ✅";`; + command += `git clone --branch ${giteaBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} --progress;`; + return command; }; export const haveGiteaRequirements = (giteaProvider: Gitea) => { diff --git a/packages/server/src/utils/providers/github.ts b/packages/server/src/utils/providers/github.ts index 30125db8b..c0f1b651a 100644 --- a/packages/server/src/utils/providers/github.ts +++ b/packages/server/src/utils/providers/github.ts @@ -1,16 +1,11 @@ -import { createWriteStream } from "node:fs"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { apiFindGithubBranches } from "@dokploy/server/db/schema"; -import type { Compose } from "@dokploy/server/services/compose"; import { findGithubById, type Github } from "@dokploy/server/services/github"; import type { InferResultType } from "@dokploy/server/types/with"; import { createAppAuth } from "@octokit/auth-app"; import { TRPCError } from "@trpc/server"; import { Octokit } from "octokit"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsyncRemote } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; export const authGithub = (githubProvider: Github): Octokit => { if (!haveGithubRequirements(githubProvider)) { @@ -123,42 +118,39 @@ interface CloneGithubRepository { branch: string | null; githubId: string | null; repository: string | null; - logPath: string; type?: "application" | "compose"; enableSubmodules: boolean; + serverId: string | null; } export const cloneGithubRepository = async ({ - logPath, type = "application", ...entity }: CloneGithubRepository) => { + let command = "set -e;"; const isCompose = type === "compose"; - const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(); - const writeStream = createWriteStream(logPath, { flags: "a" }); - const { appName, repository, owner, branch, githubId, enableSubmodules } = - entity; + const { + appName, + repository, + owner, + branch, + githubId, + enableSubmodules, + serverId, + } = entity; + const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(!!serverId); if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); + command += `echo "Error: ❌ Github Provider not found"; exit 1;`; + + return command; } const requirements = getErrorCloneRequirements(entity); // Check if requirements are met if (requirements.length > 0) { - writeStream.write( - `\nGitHub Repository configuration failed for application: ${appName}\n`, - ); - writeStream.write("Reasons:\n"); - writeStream.write(requirements.join("\n")); - writeStream.end(); - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: GitHub repository information is incomplete.", - }); + command += `echo "GitHub Repository configuration failed for application: ${appName}"; echo "Reasons:"; echo "${requirements.join("\n")}"; exit 1;`; + return command; } const githubProvider = await findGithubById(githubId); @@ -167,193 +159,15 @@ export const cloneGithubRepository = async ({ const octokit = authGithub(githubProvider); const token = await getGithubToken(octokit); const repoclone = `github.com/${owner}/${repository}.git`; - await recreateDirectory(outputPath); + // await recreateDirectory(outputPath); + command += `rm -rf ${outputPath};`; + command += `mkdir -p ${outputPath};`; const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - writeStream.write(`\nCloning Repo ${repoclone} to ${outputPath}: ✅\n`); - const cloneArgs = [ - "clone", - "--branch", - branch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; + command += `echo "Cloning Repo ${repoclone} to ${outputPath}: ✅";`; + command += `git clone --branch ${branch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} --progress;`; - await spawnAsync("git", cloneArgs, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - writeStream.write(`\nCloned ${repoclone}: ✅\n`); - } catch (error) { - writeStream.write(`ERROR Cloning: ${error}: ❌`); - throw error; - } finally { - writeStream.end(); - } -}; - -export const getGithubCloneCommand = async ({ - logPath, - type = "application", - ...entity -}: CloneGithubRepository & { serverId: string }) => { - const { - appName, - repository, - owner, - branch, - githubId, - serverId, - enableSubmodules, - } = entity; - const isCompose = type === "compose"; - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - - if (!githubId) { - const command = ` - echo "Error: ❌ Github Provider not found" >> ${logPath}; - exit 1; - `; - - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } - - const requirements = getErrorCloneRequirements(entity); - - // Build log messages - let logMessages = ""; - if (requirements.length > 0) { - logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`; - logMessages += "Reasons:\n"; - logMessages += requirements.join("\n"); - const escapedLogMessages = logMessages - .replace(/\\/g, "\\\\") - .replace(/"/g, '\\"') - .replace(/\n/g, "\\n"); - - const bashCommand = ` - echo "${escapedLogMessages}" >> ${logPath}; - exit 1; # Exit with error code - `; - - await execAsyncRemote(serverId, bashCommand); - return; - } - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - const githubProvider = await findGithubById(githubId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - const cloneUrl = `https://oauth2:${token}@${repoclone}`; - - const cloneCommand = ` -rm -rf ${outputPath}; -mkdir -p ${outputPath}; -if ! git clone --branch ${branch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Fail to clone repository ${repoclone}" >> ${logPath}; - exit 1; -fi -echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath}; - `; - - return cloneCommand; -}; - -export const cloneRawGithubRepository = async (entity: Compose) => { - const { appName, repository, owner, branch, githubId, enableSubmodules } = - entity; - - if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } - const { COMPOSE_PATH } = paths(); - const githubProvider = await findGithubById(githubId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - await recreateDirectory(outputPath); - const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - const cloneArgs = [ - "clone", - "--branch", - branch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; - await spawnAsync("git", cloneArgs); - } catch (error) { - throw error; - } -}; - -export const cloneRawGithubRepositoryRemote = async (compose: Compose) => { - const { - appName, - repository, - owner, - branch, - githubId, - serverId, - enableSubmodules, - } = compose; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } - - const { COMPOSE_PATH } = paths(true); - const githubProvider = await findGithubById(githubId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - const command = ` - rm -rf ${outputPath}; - git clone --branch ${branch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} - `; - await execAsyncRemote(serverId, command); - } catch (error) { - throw error; - } + return command; }; export const getGithubRepositories = async (githubId?: string) => { diff --git a/packages/server/src/utils/providers/gitlab.ts b/packages/server/src/utils/providers/gitlab.ts index 840347fdb..9c75cd239 100644 --- a/packages/server/src/utils/providers/gitlab.ts +++ b/packages/server/src/utils/providers/gitlab.ts @@ -1,8 +1,6 @@ -import { createWriteStream } from "node:fs"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { apiGitlabTestConnection } from "@dokploy/server/db/schema"; -import type { Compose } from "@dokploy/server/services/compose"; import { findGitlabById, type Gitlab, @@ -10,9 +8,6 @@ import { } from "@dokploy/server/services/gitlab"; import type { InferResultType } from "@dokploy/server/types/with"; import { TRPCError } from "@trpc/server"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsyncRemote } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; export const refreshGitlabToken = async (gitlabProviderId: string) => { const gitlabProvider = await findGitlabById(gitlabProviderId); @@ -102,25 +97,29 @@ const getGitlabCloneUrl = (gitlab: GitlabInfo, repoClone: string) => { return cloneUrl; }; -export const cloneGitlabRepository = async ( - entity: ApplicationWithGitlab | ComposeWithGitlab, - logPath: string, - isCompose = false, -) => { - const writeStream = createWriteStream(logPath, { flags: "a" }); +type GitlabClone = (ApplicationWithGitlab | ComposeWithGitlab) & { + serverId: string | null; + type?: "application" | "compose"; +}; + +export const cloneGitlabRepository = async ({ + type = "application", + ...entity +}: GitlabClone) => { + let command = "set -e;"; const { appName, gitlabBranch, gitlabId, gitlabPathNamespace, enableSubmodules, + serverId, } = entity; + const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(!!serverId); if (!gitlabId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitlab Provider not found", - }); + command += `echo "Error: ❌ Gitlab Provider not found"; exit 1;`; + return command; } await refreshGitlabToken(gitlabId); @@ -130,127 +129,19 @@ export const cloneGitlabRepository = async ( // Check if requirements are met if (requirements.length > 0) { - writeStream.write( - `\nGitLab Repository configuration failed for application: ${appName}\n`, - ); - writeStream.write("Reasons:\n"); - writeStream.write(requirements.join("\n")); - writeStream.end(); - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: GitLab repository information is incomplete.", - }); + command += `echo "❌ [ERROR] GitLab Repository configuration failed for application: ${appName}"; echo "Reasons:"; echo "${requirements.join("\n")}"; exit 1;`; + return command; } - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); + command += `rm -rf ${outputPath};`; + command += `mkdir -p ${outputPath};`; const repoClone = getGitlabRepoClone(gitlab, gitlabPathNamespace); const cloneUrl = getGitlabCloneUrl(gitlab, repoClone); - try { - writeStream.write(`\nCloning Repo ${repoClone} to ${outputPath}: ✅\n`); - const cloneArgs = [ - "clone", - "--branch", - gitlabBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; - - await spawnAsync("git", cloneArgs, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - writeStream.write(`\nCloned ${repoClone}: ✅\n`); - } catch (error) { - writeStream.write(`ERROR Cloning: ${error}: ❌`); - throw error; - } finally { - writeStream.end(); - } -}; - -export const getGitlabCloneCommand = async ( - entity: ApplicationWithGitlab | ComposeWithGitlab, - logPath: string, - isCompose = false, -) => { - const { - appName, - gitlabPathNamespace, - gitlabBranch, - gitlabId, - serverId, - enableSubmodules, - } = entity; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - - if (!gitlabId) { - const command = ` - echo "Error: ❌ Gitlab Provider not found" >> ${logPath}; - exit 1; - `; - - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitlab Provider not found", - }); - } - - const requirements = getErrorCloneRequirements(entity); - - // Build log messages - let logMessages = ""; - if (requirements.length > 0) { - logMessages += `\nGitLab Repository configuration failed for application: ${appName}\n`; - logMessages += "Reasons:\n"; - logMessages += requirements.join("\n"); - const escapedLogMessages = logMessages - .replace(/\\/g, "\\\\") - .replace(/"/g, '\\"') - .replace(/\n/g, "\\n"); - - const bashCommand = ` - echo "${escapedLogMessages}" >> ${logPath}; - exit 1; # Exit with error code - `; - - await execAsyncRemote(serverId, bashCommand); - return; - } - - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - await refreshGitlabToken(gitlabId); - const gitlab = await findGitlabById(gitlabId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); - const repoClone = getGitlabRepoClone(gitlab, gitlabPathNamespace); - const cloneUrl = getGitlabCloneUrl(gitlab, repoClone); - const cloneCommand = ` -rm -rf ${outputPath}; -mkdir -p ${outputPath}; -if ! git clone --branch ${gitlabBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Fail to clone the repository ${repoClone}" >> ${logPath}; - exit 1; -fi -echo "Cloned ${repoClone} to ${outputPath}: ✅" >> ${logPath}; - `; - - return cloneCommand; + command += `echo "Cloning Repo ${repoClone} to ${outputPath}: ✅";`; + command += `git clone --branch ${gitlabBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} --progress;`; + return command; }; export const getGitlabRepositories = async (gitlabId?: string) => { @@ -355,88 +246,6 @@ export const getGitlabBranches = async (input: { }[]; }; -export const cloneRawGitlabRepository = async (entity: Compose) => { - const { - appName, - gitlabBranch, - gitlabId, - gitlabPathNamespace, - enableSubmodules, - } = entity; - - if (!gitlabId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitlab Provider not found", - }); - } - - const { COMPOSE_PATH } = paths(); - await refreshGitlabToken(gitlabId); - const gitlabProvider = await findGitlabById(gitlabId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - await recreateDirectory(outputPath); - const repoClone = getGitlabRepoClone(gitlabProvider, gitlabPathNamespace); - const cloneUrl = getGitlabCloneUrl(gitlabProvider, repoClone); - try { - const cloneArgs = [ - "clone", - "--branch", - gitlabBranch!, - "--depth", - "1", - ...(enableSubmodules ? ["--recurse-submodules"] : []), - cloneUrl, - outputPath, - "--progress", - ]; - await spawnAsync("git", cloneArgs); - } catch (error) { - throw error; - } -}; - -export const cloneRawGitlabRepositoryRemote = async (compose: Compose) => { - const { - appName, - gitlabPathNamespace, - gitlabBranch, - gitlabId, - serverId, - enableSubmodules, - } = compose; - - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!gitlabId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Gitlab Provider not found", - }); - } - const { COMPOSE_PATH } = paths(true); - await refreshGitlabToken(gitlabId); - const gitlabProvider = await findGitlabById(gitlabId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const repoClone = getGitlabRepoClone(gitlabProvider, gitlabPathNamespace); - const cloneUrl = getGitlabCloneUrl(gitlabProvider, repoClone); - try { - const command = ` - rm -rf ${outputPath}; - git clone --branch ${gitlabBranch} --depth 1 ${enableSubmodules ? "--recurse-submodules" : ""} ${cloneUrl} ${outputPath} - `; - await execAsyncRemote(serverId, command); - } catch (error) { - throw error; - } -}; - export const testGitlabConnection = async ( input: typeof apiGitlabTestConnection._type, ) => { diff --git a/packages/server/src/utils/providers/raw.ts b/packages/server/src/utils/providers/raw.ts index 34ba0012a..7b541867d 100644 --- a/packages/server/src/utils/providers/raw.ts +++ b/packages/server/src/utils/providers/raw.ts @@ -1,4 +1,3 @@ -import { createWriteStream } from "node:fs"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; @@ -7,33 +6,7 @@ import { encodeBase64 } from "../docker/utils"; import { recreateDirectory } from "../filesystem/directory"; import { execAsyncRemote } from "../process/execAsync"; -export const createComposeFile = async (compose: Compose, logPath: string) => { - const { COMPOSE_PATH } = paths(); - const { appName, composeFile } = compose; - const writeStream = createWriteStream(logPath, { flags: "a" }); - const outputPath = join(COMPOSE_PATH, appName, "code"); - - try { - await recreateDirectory(outputPath); - writeStream.write( - `\nCreating File 'docker-compose.yml' to ${outputPath}: ✅\n`, - ); - - await writeFile(join(outputPath, "docker-compose.yml"), composeFile); - - writeStream.write(`\nFile 'docker-compose.yml' created: ✅\n`); - } catch (error) { - writeStream.write(`\nERROR Creating Compose File: ${error}: ❌\n`); - throw error; - } finally { - writeStream.end(); - } -}; - -export const getCreateComposeFileCommand = ( - compose: Compose, - logPath: string, -) => { +export const getCreateComposeFileCommand = (compose: Compose) => { const { COMPOSE_PATH } = paths(true); const { appName, composeFile } = compose; const outputPath = join(COMPOSE_PATH, appName, "code"); @@ -43,7 +16,7 @@ export const getCreateComposeFileCommand = ( rm -rf ${outputPath}; mkdir -p ${outputPath}; echo "${encodedContent}" | base64 -d > "${filePath}"; - echo "File 'docker-compose.yml' created: ✅" >> ${logPath}; + echo "File 'docker-compose.yml' created: ✅"; `; return bashCommand; }; From 5ac32f9f24a6b56b20a0397042e799424b68391e Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:16:18 -0600 Subject: [PATCH 03/23] Refactor repository cloning interfaces: standardize parameters for Bitbucket, Gitea, and GitLab repository cloning functions to improve consistency and maintainability across the codebase. --- packages/server/src/services/application.ts | 1 - packages/server/src/utils/providers/bitbucket.ts | 14 +++++++++++++- packages/server/src/utils/providers/gitea.ts | 13 ++++++++++++- packages/server/src/utils/providers/gitlab.ts | 11 ++++++++--- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index e2c38a9fa..c3508352c 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -29,7 +29,6 @@ import { cloneGitlabRepository } from "@dokploy/server/utils/providers/gitlab"; import { createTraefikConfig } from "@dokploy/server/utils/traefik/application"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; -import { cleanUpSystemPrune, encodeBase64 } from "../utils/docker/utils"; import { getDokployUrl } from "./admin"; import { createDeployment, diff --git a/packages/server/src/utils/providers/bitbucket.ts b/packages/server/src/utils/providers/bitbucket.ts index 0267ec9b3..8a62c5d22 100644 --- a/packages/server/src/utils/providers/bitbucket.ts +++ b/packages/server/src/utils/providers/bitbucket.ts @@ -81,10 +81,22 @@ type BitbucketClone = (ApplicationWithBitbucket | ComposeWithBitbucket) & { type?: "application" | "compose"; }; +interface CloneBitbucketRepository { + appName: string; + bitbucketRepository: string | null; + bitbucketOwner: string | null; + bitbucketBranch: string | null; + bitbucketId: string | null; + bitbucket: Bitbucket | null; + enableSubmodules: boolean; + serverId: string | null; + type?: "application" | "compose"; +} + export const cloneBitbucketRepository = async ({ type = "application", ...entity -}: BitbucketClone) => { +}: CloneBitbucketRepository) => { let command = "set -e;"; const { appName, diff --git a/packages/server/src/utils/providers/gitea.ts b/packages/server/src/utils/providers/gitea.ts index 5b69f6826..ec8946ab3 100644 --- a/packages/server/src/utils/providers/gitea.ts +++ b/packages/server/src/utils/providers/gitea.ts @@ -119,10 +119,21 @@ type GiteaClone = (ApplicationWithGitea | ComposeWithGitea) & { type?: "application" | "compose"; }; +interface CloneGiteaRepository { + appName: string; + giteaBranch: string | null; + giteaId: string | null; + giteaOwner: string | null; + giteaRepository: string | null; + enableSubmodules: boolean; + serverId: string | null; + type?: "application" | "compose"; +} + export const cloneGiteaRepository = async ({ type = "application", ...entity -}: GiteaClone) => { +}: CloneGiteaRepository) => { let command = "set -e;"; const { appName, diff --git a/packages/server/src/utils/providers/gitlab.ts b/packages/server/src/utils/providers/gitlab.ts index 9c75cd239..a3106b8c0 100644 --- a/packages/server/src/utils/providers/gitlab.ts +++ b/packages/server/src/utils/providers/gitlab.ts @@ -97,15 +97,20 @@ const getGitlabCloneUrl = (gitlab: GitlabInfo, repoClone: string) => { return cloneUrl; }; -type GitlabClone = (ApplicationWithGitlab | ComposeWithGitlab) & { +interface CloneGitlabRepository { + appName: string; + gitlabBranch: string | null; + gitlabId: string | null; + gitlabPathNamespace: string | null; + enableSubmodules: boolean; serverId: string | null; type?: "application" | "compose"; -}; +} export const cloneGitlabRepository = async ({ type = "application", ...entity -}: GitlabClone) => { +}: CloneGitlabRepository) => { let command = "set -e;"; const { appName, From f96114ad80b97b2fef9b9e3e8d3b1b0c6f38e5d0 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:18:07 -0600 Subject: [PATCH 04/23] Refactor Bitbucket repository cloning logic: remove unused parameter and enhance error handling by retrieving Bitbucket provider directly within the function. --- packages/server/src/utils/providers/bitbucket.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utils/providers/bitbucket.ts b/packages/server/src/utils/providers/bitbucket.ts index 8a62c5d22..217dba271 100644 --- a/packages/server/src/utils/providers/bitbucket.ts +++ b/packages/server/src/utils/providers/bitbucket.ts @@ -87,7 +87,6 @@ interface CloneBitbucketRepository { bitbucketOwner: string | null; bitbucketBranch: string | null; bitbucketId: string | null; - bitbucket: Bitbucket | null; enableSubmodules: boolean; serverId: string | null; type?: "application" | "compose"; @@ -104,7 +103,6 @@ export const cloneBitbucketRepository = async ({ bitbucketOwner, bitbucketBranch, bitbucketId, - bitbucket, enableSubmodules, serverId, } = entity; @@ -114,7 +112,12 @@ export const cloneBitbucketRepository = async ({ command += `echo "Error: ❌ Bitbucket Provider not found"; exit 1;`; return command; } + const bitbucket = await findBitbucketById(bitbucketId); + if (!bitbucket) { + command += `echo "Error: ❌ Bitbucket Provider not found"; exit 1;`; + return command; + } const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; const outputPath = join(basePath, appName, "code"); command += `rm -rf ${outputPath};`; From a05b75fc6792e0c0ad5e752df6d0e813cf53b5a1 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:24:13 -0600 Subject: [PATCH 05/23] Refactor deployment logic: remove unused remote preview deployment function, streamline deployment commands, and enhance error handling for Docker image pulling. Update build command generation for Docker source type. --- .../server/queues/deployments-queue.ts | 27 +--- packages/server/src/services/application.ts | 152 +++--------------- packages/server/src/utils/builders/index.ts | 4 + packages/server/src/utils/providers/docker.ts | 70 +------- 4 files changed, 42 insertions(+), 211 deletions(-) diff --git a/apps/dokploy/server/queues/deployments-queue.ts b/apps/dokploy/server/queues/deployments-queue.ts index a1d9d29f1..4c117e7e3 100644 --- a/apps/dokploy/server/queues/deployments-queue.ts +++ b/apps/dokploy/server/queues/deployments-queue.ts @@ -2,7 +2,6 @@ import { deployApplication, deployCompose, deployPreviewApplication, - deployRemotePreviewApplication, rebuildApplication, rebuildCompose, updateApplicationStatus, @@ -54,24 +53,14 @@ export const deploymentWorker = new Worker( await updatePreviewDeployment(job.data.previewDeploymentId, { previewStatus: "running", }); - if (job.data.server) { - if (job.data.type === "deploy") { - await deployRemotePreviewApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - previewDeploymentId: job.data.previewDeploymentId, - }); - } - } else { - if (job.data.type === "deploy") { - await deployPreviewApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - previewDeploymentId: job.data.previewDeploymentId, - }); - } + + if (job.data.type === "deploy") { + await deployPreviewApplication({ + applicationId: job.data.applicationId, + titleLog: job.data.titleLog, + descriptionLog: job.data.descriptionLog, + previewDeploymentId: job.data.previewDeploymentId, + }); } } } catch (error) { diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index c3508352c..181467a76 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -18,10 +18,7 @@ import { execAsyncRemote, } from "@dokploy/server/utils/process/execAsync"; import { cloneBitbucketRepository } from "@dokploy/server/utils/providers/bitbucket"; -import { - buildDocker, - buildRemoteDocker, -} from "@dokploy/server/utils/providers/docker"; +import { buildRemoteDocker } from "@dokploy/server/utils/providers/docker"; import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; import { cloneGiteaRepository } from "@dokploy/server/utils/providers/gitea"; import { cloneGithubRepository } from "@dokploy/server/utils/providers/github"; @@ -192,9 +189,12 @@ export const deployApplication = async ({ command += await cloneGitRepository(application); } else if (application.sourceType === "drop") { await buildApplication(application, deployment.logPath); + } else if (application.sourceType === "docker") { + command += await buildRemoteDocker(application); } command += getBuildCommand(application); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (application.serverId) { await execAsyncRemote(application.serverId, commandWithLog); @@ -206,16 +206,16 @@ export const deployApplication = async ({ await updateDeploymentStatus(deployment.deploymentId, "done"); await updateApplicationStatus(applicationId, "done"); - // if (application.rollbackActive) { - // const tagImage = - // application.sourceType === "docker" - // ? application.dockerImage - // : application.appName; - // await createRollback({ - // appName: tagImage || "", - // deploymentId: deployment.deploymentId, - // }); - // } + if (application.rollbackActive) { + const tagImage = + application.sourceType === "docker" + ? application.dockerImage + : application.appName; + await createRollback({ + appName: tagImage || "", + deploymentId: deployment.deploymentId, + }); + } await sendBuildSuccessNotifications({ projectName: application.environment.project.name, @@ -359,131 +359,23 @@ export const deployPreviewApplication = async ({ application.buildArgs = `${application.previewBuildArgs}\nDOKPLOY_DEPLOY_URL=${previewDeployment?.domain?.host}`; application.buildSecrets = `${application.previewBuildSecrets}\nDOKPLOY_DEPLOY_URL=${previewDeployment?.domain?.host}`; + let command = "set -e;"; if (application.sourceType === "github") { - await cloneGithubRepository({ + command += await cloneGithubRepository({ ...application, appName: previewDeployment.appName, branch: previewDeployment.branch, - logPath: deployment.logPath, }); - await buildApplication(application, deployment.logPath); - } - const successComment = getIssueComment( - application.name, - "success", - previewDomain, - ); - await updateIssueComment({ - ...issueParams, - body: `### Dokploy Preview Deployment\n\n${successComment}`, - }); - await updateDeploymentStatus(deployment.deploymentId, "done"); - await updatePreviewDeployment(previewDeploymentId, { - previewStatus: "done", - }); - } catch (error) { - const comment = getIssueComment(application.name, "error", previewDomain); - await updateIssueComment({ - ...issueParams, - body: `### Dokploy Preview Deployment\n\n${comment}`, - }); - await updateDeploymentStatus(deployment.deploymentId, "error"); - await updatePreviewDeployment(previewDeploymentId, { - previewStatus: "error", - }); - throw error; - } + command += getBuildCommand(application); - return true; -}; - -export const deployRemotePreviewApplication = async ({ - applicationId, - titleLog = "Preview Deployment", - descriptionLog = "", - previewDeploymentId, -}: { - applicationId: string; - titleLog: string; - descriptionLog: string; - previewDeploymentId: string; -}) => { - const application = await findApplicationById(applicationId); - - const deployment = await createDeploymentPreview({ - title: titleLog, - description: descriptionLog, - previewDeploymentId: previewDeploymentId, - }); - - const previewDeployment = - await findPreviewDeploymentById(previewDeploymentId); - - await updatePreviewDeployment(previewDeploymentId, { - createdAt: new Date().toISOString(), - }); - - const previewDomain = getDomainHost(previewDeployment?.domain as Domain); - const issueParams = { - owner: application?.owner || "", - repository: application?.repository || "", - issue_number: previewDeployment.pullRequestNumber, - comment_id: Number.parseInt(previewDeployment.pullRequestCommentId), - githubId: application?.githubId || "", - }; - try { - const commentExists = await issueCommentExists({ - ...issueParams, - }); - if (!commentExists) { - const result = await createPreviewDeploymentComment({ - ...issueParams, - previewDomain, - appName: previewDeployment.appName, - githubId: application?.githubId || "", - previewDeploymentId, - }); - - if (!result) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Pull request comment not found", - }); + const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + if (application.serverId) { + await execAsyncRemote(application.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); } - - issueParams.comment_id = Number.parseInt(result?.pullRequestCommentId); - } - const buildingComment = getIssueComment( - application.name, - "running", - previewDomain, - ); - await updateIssueComment({ - ...issueParams, - body: `### Dokploy Preview Deployment\n\n${buildingComment}`, - }); - application.appName = previewDeployment.appName; - application.env = `${application.previewEnv}\nDOKPLOY_DEPLOY_URL=${previewDeployment?.domain?.host}`; - application.buildArgs = `${application.previewBuildArgs}\nDOKPLOY_DEPLOY_URL=${previewDeployment?.domain?.host}`; - application.buildSecrets = `${application.previewBuildSecrets}\nDOKPLOY_DEPLOY_URL=${previewDeployment?.domain?.host}`; - - if (application.serverId) { - let command = "set -e;"; - if (application.sourceType === "github") { - command += await cloneGithubRepository({ - ...application, - appName: previewDeployment.appName, - branch: previewDeployment.branch, - serverId: application.serverId, - logPath: deployment.logPath, - }); - } - - command += getBuildCommand(application, deployment.logPath); - await execAsyncRemote(application.serverId, command); await mechanizeDockerContainer(application); } - const successComment = getIssueComment( application.name, "success", diff --git a/packages/server/src/utils/builders/index.ts b/packages/server/src/utils/builders/index.ts index 35172789c..d3637ca6b 100644 --- a/packages/server/src/utils/builders/index.ts +++ b/packages/server/src/utils/builders/index.ts @@ -79,6 +79,10 @@ export const buildApplication = async ( export const getBuildCommand = (application: ApplicationNested) => { let command = ""; const { buildType, registry } = application; + + if (application.sourceType === "docker") { + return ""; + } switch (buildType) { case "nixpacks": command = getNixpacksCommand(application); diff --git a/packages/server/src/utils/providers/docker.ts b/packages/server/src/utils/providers/docker.ts index 56341b7d6..06f962dc7 100644 --- a/packages/server/src/utils/providers/docker.ts +++ b/packages/server/src/utils/providers/docker.ts @@ -1,60 +1,6 @@ -import { createWriteStream } from "node:fs"; -import { type ApplicationNested, mechanizeDockerContainer } from "../builders"; -import { pullImage } from "../docker/utils"; +import type { ApplicationNested } from "../builders"; -interface RegistryAuth { - username: string; - password: string; - registryUrl: string; -} - -export const buildDocker = async ( - application: ApplicationNested, - logPath: string, -): Promise => { - const { buildType, dockerImage, username, password } = application; - const authConfig: Partial = { - username: username || "", - password: password || "", - registryUrl: application.registryUrl || "", - }; - - const writeStream = createWriteStream(logPath, { flags: "a" }); - - writeStream.write(`\nBuild ${buildType}\n`); - - writeStream.write(`Pulling ${dockerImage}: ✅\n`); - - try { - if (!dockerImage) { - throw new Error("Docker image not found"); - } - - await pullImage( - dockerImage, - (data) => { - if (writeStream.writable) { - writeStream.write(`${data}\n`); - } - }, - authConfig, - ); - await mechanizeDockerContainer(application); - writeStream.write("\nDocker Deployed: ✅\n"); - } catch (error) { - writeStream.write( - `❌ Error: ${error instanceof Error ? error.message : String(error)}`, - ); - throw error; - } finally { - writeStream.end(); - } -}; - -export const buildRemoteDocker = async ( - application: ApplicationNested, - logPath: string, -) => { +export const buildRemoteDocker = async (application: ApplicationNested) => { const { registryUrl, dockerImage, username, password } = application; try { @@ -62,25 +8,25 @@ export const buildRemoteDocker = async ( throw new Error("Docker image not found"); } let command = ` -echo "Pulling ${dockerImage}" >> ${logPath}; +echo "Pulling ${dockerImage}"; `; if (username && password) { command += ` -if ! echo "${password}" | docker login --username "${username}" --password-stdin "${registryUrl || ""}" >> ${logPath} 2>&1; then - echo "❌ Login failed" >> ${logPath}; +if ! echo "${password}" | docker login --username "${username}" --password-stdin "${registryUrl || ""}" 2>&1; then + echo "❌ Login failed"; exit 1; fi `; } command += ` -docker pull ${dockerImage} >> ${logPath} 2>> ${logPath} || { - echo "❌ Pulling image failed" >> ${logPath}; +docker pull ${dockerImage} 2>&1 || { + echo "❌ Pulling image failed"; exit 1; } -echo "✅ Pulling image completed." >> ${logPath}; +echo "✅ Pulling image completed."; `; return command; } catch (error) { From ef10996dd86279f65f915148240a26ae2f515b69 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:28:32 -0600 Subject: [PATCH 06/23] Refactor builder utilities: remove unused build functions for Docker, Heroku, Nixpacks, Paketo, and Railpack, streamlining the codebase. Update static command generation to enhance clarity and maintainability. --- .../server/src/utils/builders/docker-file.ts | 93 +------------ packages/server/src/utils/builders/heroku.ts | 43 ------ .../server/src/utils/builders/nixpacks.ts | 123 +++--------------- packages/server/src/utils/builders/paketo.ts | 42 ------ .../server/src/utils/builders/railpack.ts | 101 -------------- packages/server/src/utils/builders/static.ts | 59 +-------- 6 files changed, 21 insertions(+), 440 deletions(-) diff --git a/packages/server/src/utils/builders/docker-file.ts b/packages/server/src/utils/builders/docker-file.ts index 4636cb22c..a0acf5e6c 100644 --- a/packages/server/src/utils/builders/docker-file.ts +++ b/packages/server/src/utils/builders/docker-file.ts @@ -1,4 +1,3 @@ -import type { WriteStream } from "node:fs"; import { getEnviromentVariablesObject, prepareEnvironmentVariables, @@ -7,98 +6,8 @@ import { getBuildAppDirectory, getDockerContextPath, } from "../filesystem/directory"; -import { spawnAsync } from "../process/spawnAsync"; import type { ApplicationNested } from "."; -import { createEnvFile, createEnvFileCommand } from "./utils"; - -export const buildCustomDocker = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { - appName, - env, - publishDirectory, - buildArgs, - buildSecrets, - dockerBuildStage, - cleanCache, - } = application; - const dockerFilePath = getBuildAppDirectory(application); - try { - const image = `${appName}`; - - const defaultContextPath = - dockerFilePath.substring(0, dockerFilePath.lastIndexOf("/") + 1) || "."; - - const dockerContextPath = getDockerContextPath(application); - - const commandArgs = ["build", "-t", image, "-f", dockerFilePath, "."]; - - if (cleanCache) { - commandArgs.push("--no-cache"); - } - - if (dockerBuildStage) { - commandArgs.push("--target", dockerBuildStage); - } - - const args = prepareEnvironmentVariables( - buildArgs, - application.environment.project.env, - application.environment.env, - ); - - for (const arg of args) { - commandArgs.push("--build-arg", arg); - } - - const secrets = getEnviromentVariablesObject( - buildSecrets, - application.environment.project.env, - application.environment.env, - ); - - for (const key in secrets) { - // Although buildx is smart enough to know we may be referring to an environment variable name, - // we still make sure it doesn't fall back to type=file. - // See: https://docs.docker.com/reference/cli/docker/buildx/build/#secret - commandArgs.push("--secret", `type=env,id=${key}`); - } - - /* - Do not generate an environment file when publishDirectory is specified, - as it could be publicly exposed. - */ - if (!publishDirectory) { - createEnvFile( - dockerFilePath, - env, - application.environment.project.env, - application.environment.env, - ); - } - - await spawnAsync( - "docker", - commandArgs, - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - { - cwd: dockerContextPath || defaultContextPath, - env: { - ...process.env, - ...secrets, - }, - }, - ); - } catch (error) { - throw error; - } -}; +import { createEnvFileCommand } from "./utils"; export const getDockerCommand = (application: ApplicationNested) => { const { diff --git a/packages/server/src/utils/builders/heroku.ts b/packages/server/src/utils/builders/heroku.ts index a0a8da153..e1ab4dff4 100644 --- a/packages/server/src/utils/builders/heroku.ts +++ b/packages/server/src/utils/builders/heroku.ts @@ -1,50 +1,7 @@ -import type { WriteStream } from "node:fs"; import { prepareEnvironmentVariables } from "../docker/utils"; import { getBuildAppDirectory } from "../filesystem/directory"; -import { spawnAsync } from "../process/spawnAsync"; import type { ApplicationNested } from "."; -// TODO: integrate in the vps sudo chown -R $(whoami) ~/.docker -export const buildHeroku = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { env, appName, cleanCache } = application; - const buildAppDirectory = getBuildAppDirectory(application); - const envVariables = prepareEnvironmentVariables( - env, - application.environment.project.env, - application.environment.env, - ); - try { - const args = [ - "build", - appName, - "--path", - buildAppDirectory, - "--builder", - `heroku/builder:${application.herokuVersion || "24"}`, - ]; - - for (const env of envVariables) { - args.push("--env", env); - } - - if (cleanCache) { - args.push("--clear-cache"); - } - - await spawnAsync("pack", args, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - return true; - } catch (e) { - throw e; - } -}; - export const getHerokuCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; diff --git a/packages/server/src/utils/builders/nixpacks.ts b/packages/server/src/utils/builders/nixpacks.ts index 705539f06..37f1953a4 100644 --- a/packages/server/src/utils/builders/nixpacks.ts +++ b/packages/server/src/utils/builders/nixpacks.ts @@ -1,97 +1,10 @@ -import { existsSync, mkdirSync, type WriteStream } from "node:fs"; import path from "node:path"; -import { - buildStatic, - getStaticCommand, -} from "@dokploy/server/utils/builders/static"; +import { getStaticCommand } from "@dokploy/server/utils/builders/static"; import { nanoid } from "nanoid"; import { prepareEnvironmentVariables } from "../docker/utils"; import { getBuildAppDirectory } from "../filesystem/directory"; -import { spawnAsync } from "../process/spawnAsync"; import type { ApplicationNested } from "."; -export const buildNixpacks = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { env, appName, publishDirectory, cleanCache } = application; - - const buildAppDirectory = getBuildAppDirectory(application); - const buildContainerId = `${appName}-${nanoid(10)}`; - const envVariables = prepareEnvironmentVariables( - env, - application.environment.project.env, - application.environment.env, - ); - - const writeToStream = (data: string) => { - if (writeStream.writable) { - writeStream.write(data); - } - }; - - try { - const args = ["build", buildAppDirectory, "--name", appName]; - - if (cleanCache) { - args.push("--no-cache"); - } - - for (const env of envVariables) { - args.push("--env", env); - } - - if (publishDirectory) { - /* No need for any start command, since we'll use nginx later on */ - args.push("--no-error-without-start"); - } - - await spawnAsync("nixpacks", args, writeToStream); - - /* - Run the container with the image created by nixpacks, - and copy the artifacts on the host filesystem. - Then, remove the container and create a static build. - */ - if (publishDirectory) { - await spawnAsync( - "docker", - ["create", "--name", buildContainerId, appName], - writeToStream, - ); - - const localPath = path.join(buildAppDirectory, publishDirectory); - - if (!existsSync(path.dirname(localPath))) { - mkdirSync(path.dirname(localPath), { recursive: true }); - } - - // https://docs.docker.com/reference/cli/docker/container/cp/ - const isDirectory = - publishDirectory.endsWith("/") || !path.extname(publishDirectory); - - await spawnAsync( - "docker", - [ - "cp", - `${buildContainerId}:/app/${publishDirectory}${isDirectory ? "/." : ""}`, - localPath, - ], - writeToStream, - ); - - await spawnAsync("docker", ["rm", buildContainerId], writeToStream); - - await buildStatic(application, writeStream); - } - return true; - } catch (e) { - await spawnAsync("docker", ["rm", buildContainerId], writeToStream); - - throw e; - } -}; - export const getNixpacksCommand = (application: ApplicationNested) => { const { env, appName, publishDirectory, cleanCache } = application; @@ -118,7 +31,7 @@ export const getNixpacksCommand = (application: ApplicationNested) => { args.push("--no-error-without-start"); } const command = `nixpacks ${args.join(" ")}`; - const bashCommand = ` + let bashCommand = ` echo "Starting nixpacks build..." ; ${command} || { echo "❌ Nixpacks build failed" ; @@ -132,23 +45,23 @@ export const getNixpacksCommand = (application: ApplicationNested) => { and copy the artifacts on the host filesystem. Then, remove the container and create a static build. */ - // if (publishDirectory) { - // const localPath = path.join(buildAppDirectory, publishDirectory); - // const isDirectory = - // publishDirectory.endsWith("/") || !path.extname(publishDirectory); + if (publishDirectory) { + const localPath = path.join(buildAppDirectory, publishDirectory); + const isDirectory = + publishDirectory.endsWith("/") || !path.extname(publishDirectory); - // bashCommand += ` - // docker create --name ${buildContainerId} ${appName} - // mkdir -p ${localPath} - // docker cp ${buildContainerId}:/app/${publishDirectory}${isDirectory ? "/." : ""} ${path.join(buildAppDirectory, publishDirectory)} || { - // docker rm ${buildContainerId} - // echo "❌ Copying ${publishDirectory} to ${path.join(buildAppDirectory, publishDirectory)} failed" ; - // exit 1; - // } - // docker rm ${buildContainerId} - // ${getStaticCommand(application)} - // `; - // } + bashCommand += ` + docker create --name ${buildContainerId} ${appName} + mkdir -p ${localPath} + docker cp ${buildContainerId}:/app/${publishDirectory}${isDirectory ? "/." : ""} ${path.join(buildAppDirectory, publishDirectory)} || { + docker rm ${buildContainerId} + echo "❌ Copying ${publishDirectory} to ${path.join(buildAppDirectory, publishDirectory)} failed" ; + exit 1; + } + docker rm ${buildContainerId} + ${getStaticCommand(application)} + `; + } return bashCommand; }; diff --git a/packages/server/src/utils/builders/paketo.ts b/packages/server/src/utils/builders/paketo.ts index 51e2301f9..eb9767e7f 100644 --- a/packages/server/src/utils/builders/paketo.ts +++ b/packages/server/src/utils/builders/paketo.ts @@ -1,49 +1,7 @@ -import type { WriteStream } from "node:fs"; import { prepareEnvironmentVariables } from "../docker/utils"; import { getBuildAppDirectory } from "../filesystem/directory"; -import { spawnAsync } from "../process/spawnAsync"; import type { ApplicationNested } from "."; -export const buildPaketo = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { env, appName, cleanCache } = application; - const buildAppDirectory = getBuildAppDirectory(application); - const envVariables = prepareEnvironmentVariables( - env, - application.environment.project.env, - application.environment.env, - ); - try { - const args = [ - "build", - appName, - "--path", - buildAppDirectory, - "--builder", - "paketobuildpacks/builder-jammy-full", - ]; - - if (cleanCache) { - args.push("--clear-cache"); - } - - for (const env of envVariables) { - args.push("--env", env); - } - - await spawnAsync("pack", args, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - return true; - } catch (e) { - throw e; - } -}; - export const getPaketoCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; diff --git a/packages/server/src/utils/builders/railpack.ts b/packages/server/src/utils/builders/railpack.ts index 822257773..cb188fd09 100644 --- a/packages/server/src/utils/builders/railpack.ts +++ b/packages/server/src/utils/builders/railpack.ts @@ -1,13 +1,10 @@ import { createHash } from "node:crypto"; -import type { WriteStream } from "node:fs"; import { nanoid } from "nanoid"; import { parseEnvironmentKeyValuePair, prepareEnvironmentVariables, } from "../docker/utils"; import { getBuildAppDirectory } from "../filesystem/directory"; -import { execAsync } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; import type { ApplicationNested } from "."; const calculateSecretsHash = (envVariables: string[]): string => { @@ -18,104 +15,6 @@ const calculateSecretsHash = (envVariables: string[]): string => { return hash.digest("hex"); }; -export const buildRailpack = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { env, appName, cleanCache } = application; - const buildAppDirectory = getBuildAppDirectory(application); - const envVariables = prepareEnvironmentVariables( - env, - application.environment.project.env, - application.environment.env, - ); - - try { - await execAsync( - "docker buildx create --use --name builder-containerd --driver docker-container || true", - ); - - await execAsync("docker buildx use builder-containerd"); - - // First prepare the build plan and info - const prepareArgs = [ - "prepare", - buildAppDirectory, - "--plan-out", - `${buildAppDirectory}/railpack-plan.json`, - "--info-out", - `${buildAppDirectory}/railpack-info.json`, - ]; - - // Add environment variables to prepare command - for (const env of envVariables) { - prepareArgs.push("--env", env); - } - - // Run prepare command - await spawnAsync("railpack", prepareArgs, (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - - // Calculate secrets hash for layer invalidation - const secretsHash = calculateSecretsHash(envVariables); - - // Build with BuildKit using the Railpack frontend - const cacheKey = cleanCache ? nanoid(10) : undefined; - const buildArgs = [ - "buildx", - "build", - ...(cacheKey - ? [ - "--build-arg", - `secrets-hash=${secretsHash}`, - "--build-arg", - `cache-key=${cacheKey}`, - ] - : []), - "--build-arg", - `BUILDKIT_SYNTAX=ghcr.io/railwayapp/railpack-frontend:v${application.railpackVersion}`, - "-f", - `${buildAppDirectory}/railpack-plan.json`, - "--output", - `type=docker,name=${appName}`, - ]; - - // Add secrets properly formatted - const env: { [key: string]: string } = {}; - for (const pair of envVariables) { - const [key, value] = parseEnvironmentKeyValuePair(pair); - if (key && value) { - buildArgs.push("--secret", `id=${key},env=${key}`); - env[key] = value; - } - } - - buildArgs.push(buildAppDirectory); - - await spawnAsync( - "docker", - buildArgs, - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - { - env: { ...process.env, ...env }, - }, - ); - - return true; - } catch (e) { - throw e; - } finally { - await execAsync("docker buildx rm builder-containerd"); - } -}; - export const getRailpackCommand = (application: ApplicationNested) => { const { env, appName, cleanCache } = application; const buildAppDirectory = getBuildAppDirectory(application); diff --git a/packages/server/src/utils/builders/static.ts b/packages/server/src/utils/builders/static.ts index 5e1f10cc3..4ddb290be 100644 --- a/packages/server/src/utils/builders/static.ts +++ b/packages/server/src/utils/builders/static.ts @@ -1,9 +1,5 @@ -import type { WriteStream } from "node:fs"; -import { - buildCustomDocker, - getDockerCommand, -} from "@dokploy/server/utils/builders/docker-file"; -import { createFile, getCreateFileCommand } from "../docker/utils"; +import { getDockerCommand } from "@dokploy/server/utils/builders/docker-file"; +import { getCreateFileCommand } from "../docker/utils"; import { getBuildAppDirectory } from "../filesystem/directory"; import type { ApplicationNested } from "."; @@ -32,57 +28,6 @@ http { } `; -export const buildStatic = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const { publishDirectory, isStaticSpa } = application; - const buildAppDirectory = getBuildAppDirectory(application); - - try { - if (isStaticSpa) { - createFile(buildAppDirectory, "nginx.conf", nginxSpaConfig); - } - - createFile( - buildAppDirectory, - ".dockerignore", - [".git", ".env", "Dockerfile", ".dockerignore"].join("\n"), - ); - - createFile( - buildAppDirectory, - "Dockerfile", - [ - "FROM nginx:alpine", - "WORKDIR /usr/share/nginx/html/", - isStaticSpa ? "COPY nginx.conf /etc/nginx/nginx.conf" : "", - `COPY ${publishDirectory || "."} .`, - 'CMD ["nginx", "-g", "daemon off;"]', - ].join("\n"), - ); - - createFile( - buildAppDirectory, - ".dockerignore", - [".git", ".env", "Dockerfile", ".dockerignore"].join("\n"), - ); - - await buildCustomDocker( - { - ...application, - buildType: "dockerfile", - dockerfile: "Dockerfile", - }, - writeStream, - ); - - return true; - } catch (e) { - throw e; - } -}; - export const getStaticCommand = (application: ApplicationNested) => { const { publishDirectory } = application; const buildAppDirectory = getBuildAppDirectory(application); From 668aaf9a91b17fe118455dc585e4c64da408c334 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:29:40 -0600 Subject: [PATCH 07/23] Refactor deployment utilities: rename remote deployment functions for clarity and consistency, enhancing the deployment logic in the application. Streamline the build application function by commenting out unused build types to improve maintainability. --- apps/api/src/utils.ts | 20 +++++------ packages/server/src/utils/builders/index.ts | 38 ++++++++++----------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/apps/api/src/utils.ts b/apps/api/src/utils.ts index ee2ac3e50..0d0b574fc 100644 --- a/apps/api/src/utils.ts +++ b/apps/api/src/utils.ts @@ -1,9 +1,9 @@ import { - deployRemoteApplication, - deployRemoteCompose, - deployRemotePreviewApplication, - rebuildRemoteApplication, - rebuildRemoteCompose, + deployApplication, + deployCompose, + deployPreviewApplication, + rebuildApplication, + rebuildCompose, updateApplicationStatus, updateCompose, updatePreviewDeployment, @@ -16,13 +16,13 @@ export const deploy = async (job: DeployJob) => { await updateApplicationStatus(job.applicationId, "running"); if (job.server) { if (job.type === "redeploy") { - await rebuildRemoteApplication({ + await rebuildApplication({ applicationId: job.applicationId, titleLog: job.titleLog || "Rebuild deployment", descriptionLog: job.descriptionLog || "", }); } else if (job.type === "deploy") { - await deployRemoteApplication({ + await deployApplication({ applicationId: job.applicationId, titleLog: job.titleLog || "Manual deployment", descriptionLog: job.descriptionLog || "", @@ -36,13 +36,13 @@ export const deploy = async (job: DeployJob) => { if (job.server) { if (job.type === "redeploy") { - await rebuildRemoteCompose({ + await rebuildCompose({ composeId: job.composeId, titleLog: job.titleLog || "Rebuild deployment", descriptionLog: job.descriptionLog || "", }); } else if (job.type === "deploy") { - await deployRemoteCompose({ + await deployCompose({ composeId: job.composeId, titleLog: job.titleLog || "Manual deployment", descriptionLog: job.descriptionLog || "", @@ -55,7 +55,7 @@ export const deploy = async (job: DeployJob) => { }); if (job.server) { if (job.type === "deploy") { - await deployRemotePreviewApplication({ + await deployPreviewApplication({ applicationId: job.applicationId, titleLog: job.titleLog || "Preview Deployment", descriptionLog: job.descriptionLog || "", diff --git a/packages/server/src/utils/builders/index.ts b/packages/server/src/utils/builders/index.ts index d3637ca6b..207238ca5 100644 --- a/packages/server/src/utils/builders/index.ts +++ b/packages/server/src/utils/builders/index.ts @@ -11,12 +11,12 @@ import { prepareEnvironmentVariables, } from "../docker/utils"; import { getRemoteDocker } from "../servers/remote-docker"; -import { buildCustomDocker, getDockerCommand } from "./docker-file"; -import { buildHeroku, getHerokuCommand } from "./heroku"; -import { buildNixpacks, getNixpacksCommand } from "./nixpacks"; -import { buildPaketo, getPaketoCommand } from "./paketo"; -import { buildRailpack, getRailpackCommand } from "./railpack"; -import { buildStatic, getStaticCommand } from "./static"; +import { getDockerCommand } from "./docker-file"; +import { getHerokuCommand } from "./heroku"; +import { getNixpacksCommand } from "./nixpacks"; +import { getPaketoCommand } from "./paketo"; +import { getRailpackCommand } from "./railpack"; +import { getStaticCommand } from "./static"; // NIXPACKS codeDirectory = where is the path of the code directory // HEROKU codeDirectory = where is the path of the code directory @@ -45,19 +45,19 @@ export const buildApplication = async ( `\nBuild ${buildType}: ✅\nSource Type: ${sourceType}: ✅\n`, ); console.log(`Build ${buildType}: ✅`); - if (buildType === "nixpacks") { - await buildNixpacks(application, writeStream); - } else if (buildType === "heroku_buildpacks") { - await buildHeroku(application, writeStream); - } else if (buildType === "paketo_buildpacks") { - await buildPaketo(application, writeStream); - } else if (buildType === "dockerfile") { - await buildCustomDocker(application, writeStream); - } else if (buildType === "static") { - await buildStatic(application, writeStream); - } else if (buildType === "railpack") { - await buildRailpack(application, writeStream); - } + // if (buildType === "nixpacks") { + // await buildNixpacks(application, writeStream); + // } else if (buildType === "heroku_buildpacks") { + // await buildHeroku(application, writeStream); + // } else if (buildType === "paketo_buildpacks") { + // await buildPaketo(application, writeStream); + // } else if (buildType === "dockerfile") { + // await buildCustomDocker(application, writeStream); + // } else if (buildType === "static") { + // await buildStatic(application, writeStream); + // } else if (buildType === "railpack") { + // await buildRailpack(application, writeStream); + // } if (application.registryId) { await uploadImage(application, writeStream); From f718ab334e2ebb9f63b2090ca612b484ed8ddd57 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:42:43 -0600 Subject: [PATCH 08/23] Refactor compose utilities: remove unused functions and streamline the buildCompose logic for improved maintainability. Update domain handling by retaining only the necessary remote function, enhancing clarity in the codebase. --- packages/server/src/utils/builders/compose.ts | 114 +----------------- packages/server/src/utils/docker/domain.ts | 19 --- packages/server/src/utils/providers/raw.ts | 37 +----- 3 files changed, 2 insertions(+), 168 deletions(-) diff --git a/packages/server/src/utils/builders/compose.ts b/packages/server/src/utils/builders/compose.ts index 6b26a0d1c..7792ed11c 100644 --- a/packages/server/src/utils/builders/compose.ts +++ b/packages/server/src/utils/builders/compose.ts @@ -1,100 +1,18 @@ -import { - createWriteStream, - existsSync, - mkdirSync, - writeFileSync, -} from "node:fs"; import { dirname, join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { InferResultType } from "@dokploy/server/types/with"; import boxen from "boxen"; -import { - writeDomainsToCompose, - writeDomainsToComposeRemote, -} from "../docker/domain"; +import { writeDomainsToComposeRemote } from "../docker/domain"; import { encodeBase64, getEnviromentVariablesObject, prepareEnvironmentVariables, } from "../docker/utils"; -import { execAsync } from "../process/execAsync"; -import { spawnAsync } from "../process/spawnAsync"; export type ComposeNested = InferResultType< "compose", { environment: { with: { project: true } }; mounts: true; domains: true } >; -export const buildCompose = async (compose: ComposeNested, logPath: string) => { - const writeStream = createWriteStream(logPath, { flags: "a" }); - const { sourceType, appName, mounts, composeType, domains } = compose; - try { - const { COMPOSE_PATH } = paths(); - const command = createCommand(compose); - await writeDomainsToCompose(compose, domains); - createEnvFile(compose); - - if (compose.isolatedDeployment) { - await execAsync( - `docker network inspect ${compose.appName} >/dev/null 2>&1 || docker network create ${composeType === "stack" ? "--driver overlay" : ""} --attachable ${compose.appName}`, - ); - } - - const logContent = ` - App Name: ${appName} - Build Compose 🐳 - Detected: ${mounts.length} mounts 📂 - Command: docker ${command} - Source Type: docker ${sourceType} ✅ - Compose Type: ${composeType} ✅`; - const logBox = boxen(logContent, { - padding: { - left: 1, - right: 1, - bottom: 1, - }, - width: 80, - borderStyle: "double", - }); - writeStream.write(`\n${logBox}\n`); - const projectPath = join(COMPOSE_PATH, compose.appName, "code"); - - await spawnAsync( - "docker", - [...command.split(" ")], - (data) => { - if (writeStream.writable) { - writeStream.write(data.toString()); - } - }, - { - cwd: projectPath, - env: { - NODE_ENV: process.env.NODE_ENV, - PATH: process.env.PATH, - ...(composeType === "stack" && { - ...getEnviromentVariablesObject( - compose.env, - compose.environment.project.env, - ), - }), - }, - }, - ); - - if (compose.isolatedDeployment) { - await execAsync( - `docker network connect ${compose.appName} $(docker ps --filter "name=dokploy-traefik" -q) >/dev/null 2>&1`, - ).catch(() => {}); - } - - writeStream.write("Docker Compose Deployed: ✅"); - } catch (error) { - writeStream.write(`Error ❌ ${(error as Error).message}`); - throw error; - } finally { - writeStream.end(); - } -}; export const getBuildComposeCommand = async (compose: ComposeNested) => { const { COMPOSE_PATH } = paths(!!compose.serverId); @@ -179,36 +97,6 @@ export const createCommand = (compose: ComposeNested) => { return command; }; -const createEnvFile = (compose: ComposeNested) => { - const { COMPOSE_PATH } = paths(); - const { env, composePath, appName } = compose; - const composeFilePath = - join(COMPOSE_PATH, appName, "code", composePath) || - join(COMPOSE_PATH, appName, "code", "docker-compose.yml"); - - const envFilePath = join(dirname(composeFilePath), ".env"); - let envContent = `APP_NAME=${appName}\n`; - envContent += env || ""; - if (!envContent.includes("DOCKER_CONFIG")) { - envContent += "\nDOCKER_CONFIG=/root/.docker"; - } - - if (compose.randomize) { - envContent += `\nCOMPOSE_PREFIX=${compose.suffix}`; - } - - const envFileContent = prepareEnvironmentVariables( - envContent, - compose.environment.project.env, - compose.environment.env, - ).join("\n"); - - if (!existsSync(dirname(envFilePath))) { - mkdirSync(dirname(envFilePath), { recursive: true }); - } - writeFileSync(envFilePath, envFileContent); -}; - export const getCreateEnvFileCommand = (compose: ComposeNested) => { const { COMPOSE_PATH } = paths(!!compose.serverId); const { env, composePath, appName } = compose; diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index e8c66e697..3d4167d2e 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -1,5 +1,4 @@ import fs, { existsSync, readFileSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { Compose } from "@dokploy/server/services/compose"; @@ -103,24 +102,6 @@ export const readComposeFile = async (compose: Compose) => { return null; }; -export const writeDomainsToCompose = async ( - compose: Compose, - domains: Domain[], -) => { - if (!domains.length) { - return; - } - const composeConverted = await addDomainToCompose(compose, domains); - - const path = getComposePath(compose); - const composeString = stringify(composeConverted, { lineWidth: 1000 }); - try { - await writeFile(path, composeString, "utf8"); - } catch (error) { - throw error; - } -}; - export const writeDomainsToComposeRemote = async ( compose: Compose, domains: Domain[], diff --git a/packages/server/src/utils/providers/raw.ts b/packages/server/src/utils/providers/raw.ts index 7b541867d..508df86ed 100644 --- a/packages/server/src/utils/providers/raw.ts +++ b/packages/server/src/utils/providers/raw.ts @@ -1,13 +1,10 @@ -import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { paths } from "@dokploy/server/constants"; import type { Compose } from "@dokploy/server/services/compose"; import { encodeBase64 } from "../docker/utils"; -import { recreateDirectory } from "../filesystem/directory"; -import { execAsyncRemote } from "../process/execAsync"; export const getCreateComposeFileCommand = (compose: Compose) => { - const { COMPOSE_PATH } = paths(true); + const { COMPOSE_PATH } = paths(!!compose.serverId); const { appName, composeFile } = compose; const outputPath = join(COMPOSE_PATH, appName, "code"); const filePath = join(outputPath, "docker-compose.yml"); @@ -20,35 +17,3 @@ export const getCreateComposeFileCommand = (compose: Compose) => { `; return bashCommand; }; - -export const createComposeFileRaw = async (compose: Compose) => { - const { COMPOSE_PATH } = paths(); - const { appName, composeFile } = compose; - const outputPath = join(COMPOSE_PATH, appName, "code"); - const filePath = join(outputPath, "docker-compose.yml"); - try { - await recreateDirectory(outputPath); - await writeFile(filePath, composeFile); - } catch (error) { - throw error; - } -}; - -export const createComposeFileRawRemote = async (compose: Compose) => { - const { COMPOSE_PATH } = paths(true); - const { appName, composeFile, serverId } = compose; - const outputPath = join(COMPOSE_PATH, appName, "code"); - const filePath = join(outputPath, "docker-compose.yml"); - - try { - const encodedContent = encodeBase64(composeFile); - const command = ` - rm -rf ${outputPath}; - mkdir -p ${outputPath}; - echo "${encodedContent}" | base64 -d > "${filePath}"; - `; - await execAsyncRemote(serverId, command); - } catch (error) { - throw error; - } -}; From d8ab7a59ffcebc267ed667a8772e1b05341d9a44 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 03:43:54 -0600 Subject: [PATCH 09/23] Refactor Bitbucket header utility: remove unused BitbucketClone type definition to streamline the code and enhance maintainability. --- packages/server/src/utils/providers/bitbucket.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/server/src/utils/providers/bitbucket.ts b/packages/server/src/utils/providers/bitbucket.ts index 217dba271..2248baaaf 100644 --- a/packages/server/src/utils/providers/bitbucket.ts +++ b/packages/server/src/utils/providers/bitbucket.ts @@ -76,11 +76,6 @@ export const getBitbucketHeaders = (bitbucketProvider: Bitbucket) => { }; }; -type BitbucketClone = (ApplicationWithBitbucket | ComposeWithBitbucket) & { - serverId: string | null; - type?: "application" | "compose"; -}; - interface CloneBitbucketRepository { appName: string; bitbucketRepository: string | null; From b814bdc612299643a392e59440e903c5e6e6f1ec Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sun, 9 Nov 2025 11:13:39 -0600 Subject: [PATCH 10/23] Refactor application and compose deployment logic: remove unused buildApplication function, streamline command logging for deployment, and enhance static command generation for improved maintainability and clarity in the codebase. --- packages/server/src/services/application.ts | 3 - packages/server/src/services/compose.ts | 18 +++++- packages/server/src/utils/builders/index.ts | 45 +------------- packages/server/src/utils/builders/static.ts | 20 ++++++- packages/server/src/utils/cluster/upload.ts | 63 +------------------- packages/server/src/utils/docker/domain.ts | 3 +- 6 files changed, 38 insertions(+), 114 deletions(-) diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index 181467a76..8dc67ddb6 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -7,7 +7,6 @@ import { } from "@dokploy/server/db/schema"; import { getAdvancedStats } from "@dokploy/server/monitoring/utils"; import { - buildApplication, getBuildCommand, mechanizeDockerContainer, } from "@dokploy/server/utils/builders"; @@ -187,8 +186,6 @@ export const deployApplication = async ({ command += await cloneBitbucketRepository(application); } else if (application.sourceType === "git") { command += await cloneGitRepository(application); - } else if (application.sourceType === "drop") { - await buildApplication(application, deployment.logPath); } else if (application.sourceType === "docker") { command += await buildRemoteDocker(application); } diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index 4b96f443d..2e2a2fc59 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -232,8 +232,15 @@ export const deployCompose = async ({ command += getCreateComposeFileCommand(entity); } + let commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + if (compose.serverId) { + await execAsyncRemote(compose.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); + } + command += await getBuildComposeCommand(entity); - const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (compose.serverId) { await execAsyncRemote(compose.serverId, commandWithLog); } else { @@ -293,8 +300,15 @@ export const rebuildCompose = async ({ if (compose.sourceType === "raw") { command += getCreateComposeFileCommand(compose); } + + let commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + if (compose.serverId) { + await execAsyncRemote(compose.serverId, commandWithLog); + } else { + await execAsync(commandWithLog); + } command += await getBuildComposeCommand(compose); - const commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; + commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (compose.serverId) { await execAsyncRemote(compose.serverId, commandWithLog); } else { diff --git a/packages/server/src/utils/builders/index.ts b/packages/server/src/utils/builders/index.ts index 207238ca5..bfc3e894a 100644 --- a/packages/server/src/utils/builders/index.ts +++ b/packages/server/src/utils/builders/index.ts @@ -1,7 +1,6 @@ -import { createWriteStream } from "node:fs"; import type { InferResultType } from "@dokploy/server/types/with"; import type { CreateServiceOptions } from "dockerode"; -import { uploadImage, uploadImageRemoteCommand } from "../cluster/upload"; +import { uploadImageRemoteCommand } from "../cluster/upload"; import { calculateResources, generateBindMounts, @@ -34,48 +33,6 @@ export type ApplicationNested = InferResultType< } >; -export const buildApplication = async ( - application: ApplicationNested, - logPath: string, -) => { - const writeStream = createWriteStream(logPath, { flags: "a" }); - const { buildType, sourceType } = application; - try { - writeStream.write( - `\nBuild ${buildType}: ✅\nSource Type: ${sourceType}: ✅\n`, - ); - console.log(`Build ${buildType}: ✅`); - // if (buildType === "nixpacks") { - // await buildNixpacks(application, writeStream); - // } else if (buildType === "heroku_buildpacks") { - // await buildHeroku(application, writeStream); - // } else if (buildType === "paketo_buildpacks") { - // await buildPaketo(application, writeStream); - // } else if (buildType === "dockerfile") { - // await buildCustomDocker(application, writeStream); - // } else if (buildType === "static") { - // await buildStatic(application, writeStream); - // } else if (buildType === "railpack") { - // await buildRailpack(application, writeStream); - // } - - if (application.registryId) { - await uploadImage(application, writeStream); - } - await mechanizeDockerContainer(application); - writeStream.write("Docker Deployed: ✅"); - } catch (error) { - if (error instanceof Error) { - writeStream.write(`Error ❌\n${error?.message}`); - } else { - writeStream.write("Error ❌"); - } - throw error; - } finally { - writeStream.end(); - } -}; - export const getBuildCommand = (application: ApplicationNested) => { let command = ""; const { buildType, registry } = application; diff --git a/packages/server/src/utils/builders/static.ts b/packages/server/src/utils/builders/static.ts index 4ddb290be..99fa25285 100644 --- a/packages/server/src/utils/builders/static.ts +++ b/packages/server/src/utils/builders/static.ts @@ -29,16 +29,32 @@ http { `; export const getStaticCommand = (application: ApplicationNested) => { - const { publishDirectory } = application; + const { publishDirectory, isStaticSpa } = application; const buildAppDirectory = getBuildAppDirectory(application); + let command = ""; + if (isStaticSpa) { + command += getCreateFileCommand( + buildAppDirectory, + "nginx.conf", + nginxSpaConfig, + ); + } - let command = getCreateFileCommand( + command += getCreateFileCommand( + buildAppDirectory, + ".dockerignore", + [".git", ".env", "Dockerfile", ".dockerignore"].join("\n"), + ); + + command += getCreateFileCommand( buildAppDirectory, "Dockerfile", [ "FROM nginx:alpine", "WORKDIR /usr/share/nginx/html/", + isStaticSpa ? "COPY nginx.conf /etc/nginx/nginx.conf" : "", `COPY ${publishDirectory || "."} .`, + 'CMD ["nginx", "-g", "daemon off;"]', ].join("\n"), ); diff --git a/packages/server/src/utils/cluster/upload.ts b/packages/server/src/utils/cluster/upload.ts index 6a2dfd66f..33be287e1 100644 --- a/packages/server/src/utils/cluster/upload.ts +++ b/packages/server/src/utils/cluster/upload.ts @@ -1,63 +1,4 @@ -import type { WriteStream } from "node:fs"; import type { ApplicationNested } from "../builders"; -import { spawnAsync } from "../process/spawnAsync"; - -export const uploadImage = async ( - application: ApplicationNested, - writeStream: WriteStream, -) => { - const registry = application.registry; - - if (!registry) { - throw new Error("Registry not found"); - } - - const { registryUrl, imagePrefix, username } = registry; - const { appName } = application; - const imageName = `${appName}:latest`; - - const finalURL = registryUrl; - - // Build registry tag in correct format: registry.com/owner/image:tag - // For ghcr.io: ghcr.io/username/image:tag - // For docker.io: docker.io/username/image:tag - const registryTag = imagePrefix - ? `${registryUrl ? `${registryUrl}/` : ""}${imagePrefix}/${imageName}` - : `${registryUrl ? `${registryUrl}/` : ""}${username}/${imageName}`; - - try { - writeStream.write( - `📦 [Enabled Registry] Uploading image to ${registry.registryType} | ${imageName} | ${finalURL} | ${registryTag}\n`, - ); - const loginCommand = spawnAsync( - "docker", - ["login", finalURL, "-u", registry.username, "--password-stdin"], - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - ); - loginCommand.child?.stdin?.write(registry.password); - loginCommand.child?.stdin?.end(); - await loginCommand; - - await spawnAsync("docker", ["tag", imageName, registryTag], (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - - await spawnAsync("docker", ["push", registryTag], (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }); - } catch (error) { - console.log(error); - throw error; - } -}; export const uploadImageRemoteCommand = (application: ApplicationNested) => { const registry = application.registry; @@ -74,8 +15,8 @@ export const uploadImageRemoteCommand = (application: ApplicationNested) => { // Build registry tag in correct format: registry.com/owner/image:tag const registryTag = imagePrefix - ? `${registryUrl}/${imagePrefix}/${imageName}` - : `${registryUrl}/${username}/${imageName}`; + ? `${registryUrl ? `${registryUrl}/` : ""}${imagePrefix}/${imageName}` + : `${registryUrl ? `${registryUrl}/` : ""}${username}/${imageName}`; try { const command = ` diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index 3d4167d2e..ffe900302 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -132,7 +132,6 @@ exit 1; `; } }; -// (node:59875) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 SIGTERM listeners added to [process]. Use emitter.setMaxListeners() to increase limit export const addDomainToCompose = async ( compose: Compose, domains: Domain[], @@ -142,7 +141,7 @@ export const addDomainToCompose = async ( let result: ComposeSpecification | null; if (compose.serverId) { - result = await loadDockerComposeRemote(compose); // aca hay que ir al servidor e ir a traer el compose file al servidor + result = await loadDockerComposeRemote(compose); } else { result = await loadDockerCompose(compose); } From b202974a7d78a9f3152ffc17cdf6681b52584b3d Mon Sep 17 00:00:00 2001 From: HarikrishnanD Date: Tue, 11 Nov 2025 11:34:10 +0530 Subject: [PATCH 11/23] fix: add protocol prefix to invitation links --- apps/dokploy/server/api/routers/user.ts | 4 ++-- packages/server/src/services/admin.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/dokploy/server/api/routers/user.ts b/apps/dokploy/server/api/routers/user.ts index d30b99b3a..baec4d6ac 100644 --- a/apps/dokploy/server/api/routers/user.ts +++ b/apps/dokploy/server/api/routers/user.ts @@ -4,6 +4,7 @@ import { findNotificationById, findOrganizationById, findUserById, + getDokployUrl, getUserByToken, IS_CLOUD, removeUserById, @@ -419,11 +420,10 @@ export const userRouter = createTRPCRouter({ }); } - const admin = await findAdmin(); const host = process.env.NODE_ENV === "development" ? "http://localhost:3000" - : admin.user.host; + : await getDokployUrl(); const inviteLink = `${host}/invitation?token=${input.invitationId}`; const organization = await findOrganizationById( diff --git a/packages/server/src/services/admin.ts b/packages/server/src/services/admin.ts index 55ee5caee..0cbb20785 100644 --- a/packages/server/src/services/admin.ts +++ b/packages/server/src/services/admin.ts @@ -110,7 +110,8 @@ export const getDokployUrl = async () => { const admin = await findAdmin(); if (admin.user.host) { - return `https://${admin.user.host}`; + const protocol = admin.user.https ? "https" : "http"; + return `${protocol}://${admin.user.host}`; } return `http://${admin.user.serverIp}:${process.env.PORT}`; }; From d22aa0583c330ead589b4d9ac7415f4f52ca6c7b Mon Sep 17 00:00:00 2001 From: Bima42 Date: Thu, 13 Nov 2025 16:17:21 +0100 Subject: [PATCH 12/23] chore: bump traefik to 3.6.1 --- apps/dokploy/setup.ts | 2 +- packages/server/src/setup/traefik-setup.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dokploy/setup.ts b/apps/dokploy/setup.ts index 55e1da87c..e0ccb86d8 100644 --- a/apps/dokploy/setup.ts +++ b/apps/dokploy/setup.ts @@ -22,7 +22,7 @@ import { await initializeNetwork(); createDefaultTraefikConfig(); createDefaultServerTraefikConfig(); - await execAsync("docker pull traefik:v3.5.0"); + await execAsync("docker pull traefik:v3.6.1"); await initializeStandaloneTraefik(); await initializeRedis(); await initializePostgres(); diff --git a/packages/server/src/setup/traefik-setup.ts b/packages/server/src/setup/traefik-setup.ts index fa9bf78d0..73cff0b1c 100644 --- a/packages/server/src/setup/traefik-setup.ts +++ b/packages/server/src/setup/traefik-setup.ts @@ -20,7 +20,7 @@ export const TRAEFIK_PORT = Number.parseInt(process.env.TRAEFIK_PORT!, 10) || 80; export const TRAEFIK_HTTP3_PORT = Number.parseInt(process.env.TRAEFIK_HTTP3_PORT!, 10) || 443; -export const TRAEFIK_VERSION = process.env.TRAEFIK_VERSION || "3.5.0"; +export const TRAEFIK_VERSION = process.env.TRAEFIK_VERSION || "3.6.1"; export interface TraefikOptions { env?: string[]; From d549aa6a623de978e0f14cf3b45da80e36a09dd8 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Thu, 13 Nov 2025 22:35:16 -0600 Subject: [PATCH 13/23] feat: add last deployment date to services and update sorting logic - Introduced `lastDeployDate` property to track the most recent deployment for applications and compose services. - Updated the `extractServicesFromEnvironment` function to calculate and include the last deployment date. - Modified sorting logic to allow sorting by last deployment date, enhancing the user experience on the environment dashboard. - Adjusted local storage default sort preference to prioritize last deployment date. --- .../environment/[environmentId].tsx | 121 ++++++++++++++---- packages/server/src/services/environment.ts | 12 +- 2 files changed, 108 insertions(+), 25 deletions(-) diff --git a/apps/dokploy/pages/dashboard/project/[projectId]/environment/[environmentId].tsx b/apps/dokploy/pages/dashboard/project/[projectId]/environment/[environmentId].tsx index c09111d20..886756ab2 100644 --- a/apps/dokploy/pages/dashboard/project/[projectId]/environment/[environmentId].tsx +++ b/apps/dokploy/pages/dashboard/project/[projectId]/environment/[environmentId].tsx @@ -115,6 +115,7 @@ export type Services = { id: string; createdAt: string; status?: "idle" | "running" | "done" | "error"; + lastDeployDate?: Date | null; }; type Project = Awaited>; @@ -128,16 +129,34 @@ export const extractServicesFromEnvironment = ( const allServices: Services[] = []; const applications: Services[] = - environment.applications?.map((item) => ({ - appName: item.appName, - name: item.name, - type: "application", - id: item.applicationId, - createdAt: item.createdAt, - status: item.applicationStatus, - description: item.description, - serverId: item.serverId, - })) || []; + environment.applications?.map((item) => { + // Get the most recent deployment date + let lastDeployDate: Date | null = null; + const deployments = (item as any).deployments; + if (deployments && deployments.length > 0) { + for (const deployment of deployments) { + const deployDate = new Date( + deployment.finishedAt || + deployment.startedAt || + deployment.createdAt, + ); + if (!lastDeployDate || deployDate > lastDeployDate) { + lastDeployDate = deployDate; + } + } + } + return { + appName: item.appName, + name: item.name, + type: "application", + id: item.applicationId, + createdAt: item.createdAt, + status: item.applicationStatus, + description: item.description, + serverId: item.serverId, + lastDeployDate, + }; + }) || []; const mariadb: Services[] = environment.mariadb?.map((item) => ({ @@ -200,16 +219,34 @@ export const extractServicesFromEnvironment = ( })) || []; const compose: Services[] = - environment.compose?.map((item) => ({ - appName: item.appName, - name: item.name, - type: "compose", - id: item.composeId, - createdAt: item.createdAt, - status: item.composeStatus, - description: item.description, - serverId: item.serverId, - })) || []; + environment.compose?.map((item) => { + // Get the most recent deployment date + let lastDeployDate: Date | null = null; + const deployments = (item as any).deployments; + if (deployments && deployments.length > 0) { + for (const deployment of deployments) { + const deployDate = new Date( + deployment.finishedAt || + deployment.startedAt || + deployment.createdAt, + ); + if (!lastDeployDate || deployDate > lastDeployDate) { + lastDeployDate = deployDate; + } + } + } + return { + appName: item.appName, + name: item.name, + type: "compose", + id: item.composeId, + createdAt: item.createdAt, + status: item.composeStatus, + description: item.description, + serverId: item.serverId, + lastDeployDate, + }; + }) || []; allServices.push( ...applications, @@ -237,9 +274,9 @@ const EnvironmentPage = ( const { data: auth } = api.user.get.useQuery(); const [sortBy, setSortBy] = useState(() => { if (typeof window !== "undefined") { - return localStorage.getItem("servicesSort") || "createdAt-desc"; + return localStorage.getItem("servicesSort") || "lastDeploy-desc"; } - return "createdAt-desc"; + return "lastDeploy-desc"; }); useEffect(() => { @@ -261,10 +298,45 @@ const EnvironmentPage = ( comparison = new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(); break; + case "lastDeploy": { + const aLastDeploy = a.lastDeployDate; + const bLastDeploy = b.lastDeployDate; + + if (direction === "desc") { + // For "desc" (newest first): services with deployments first, then those without + if (!aLastDeploy && !bLastDeploy) { + comparison = 0; + } else if (!aLastDeploy) { + comparison = 1; // a (no deploy) goes after b (has deploy) + } else if (!bLastDeploy) { + comparison = -1; // a (has deploy) goes before b (no deploy) + } else { + // Both have deployments: newest first (negative if a is newer) + comparison = bLastDeploy.getTime() - aLastDeploy.getTime(); + } + } else { + // For "asc" (oldest first): services with deployments first, then those without + if (!aLastDeploy && !bLastDeploy) { + comparison = 0; + } else if (!aLastDeploy) { + comparison = 1; // a (no deploy) goes after b (has deploy) + } else if (!bLastDeploy) { + comparison = -1; // a (has deploy) goes before b (no deploy) + } else { + // Both have deployments: oldest first + comparison = aLastDeploy.getTime() - bLastDeploy.getTime(); + } + } + break; + } default: comparison = 0; } - return direction === "asc" ? comparison : -comparison; + // For other fields, apply direction normally + if (field !== "lastDeploy") { + return direction === "asc" ? comparison : -comparison; + } + return comparison; }); }; @@ -1217,6 +1289,9 @@ const EnvironmentPage = ( + + Recently deployed + Newest first diff --git a/packages/server/src/services/environment.ts b/packages/server/src/services/environment.ts index 1d77510be..c35862714 100644 --- a/packages/server/src/services/environment.ts +++ b/packages/server/src/services/environment.ts @@ -34,13 +34,21 @@ export const findEnvironmentById = async (environmentId: string) => { const environment = await db.query.environments.findFirst({ where: eq(environments.environmentId, environmentId), with: { - applications: true, + applications: { + with: { + deployments: true, + }, + }, mariadb: true, mongo: true, mysql: true, postgres: true, redis: true, - compose: true, + compose: { + with: { + deployments: true, + }, + }, project: true, }, }); From c35fe0d457790093c96baecac99ab73f438a2f87 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 01:10:49 -0600 Subject: [PATCH 14/23] feat: enhance Docker image handling in deployment logic - Added functions to extract image name and tag from Docker images and webhook requests. - Implemented validation for Docker image names and tags during deployment. - Expanded test coverage for image tag extraction and commit message generation for GitHub Packages events. - Improved error handling for missing image names and tags in deployment requests. --- apps/dokploy/__test__/deploy/github.test.ts | 312 +++++++++++++++++- .../pages/api/deploy/[refreshToken].ts | 212 +++++++++--- 2 files changed, 483 insertions(+), 41 deletions(-) diff --git a/apps/dokploy/__test__/deploy/github.test.ts b/apps/dokploy/__test__/deploy/github.test.ts index 03805b08d..46be44883 100644 --- a/apps/dokploy/__test__/deploy/github.test.ts +++ b/apps/dokploy/__test__/deploy/github.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { extractCommitMessage } from "@/pages/api/deploy/[refreshToken]"; +import { + extractCommitMessage, + extractImageName, + extractImageTag, + extractImageTagFromRequest, +} from "@/pages/api/deploy/[refreshToken]"; describe("GitHub Webhook Skip CI", () => { const mockGithubHeaders = { @@ -96,3 +101,308 @@ describe("GitHub Webhook Skip CI", () => { ); }); }); + +describe("GitHub Packages Docker Image Tag Extraction", () => { + it("should extract tag from container_metadata", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + container_metadata: { + tag: { + name: "v1.0.0", + digest: "sha256:abc123...", + }, + }, + package_url: "ghcr.io/owner/repo:v1.0.0", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBe("v1.0.0"); + }); + + it("should extract tag from package_url when container_metadata tag matches version", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + container_metadata: { + tag: { + name: "sha256:abc123...", + digest: "sha256:abc123...", + }, + }, + package_url: "ghcr.io/owner/repo:latest", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBe("latest"); + }); + + it("should extract tag from package_url when container_metadata is missing", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + package_url: "ghcr.io/owner/repo:1.2.3", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBe("1.2.3"); + }); + + it("should handle different tag formats in package_url", () => { + const headers = { "x-github-event": "registry_package" }; + const testCases = [ + { url: "ghcr.io/owner/repo:latest", expected: "latest" }, + { url: "ghcr.io/owner/repo:v1.0.0", expected: "v1.0.0" }, + { url: "ghcr.io/owner/repo:1.2.3", expected: "1.2.3" }, + { url: "ghcr.io/owner/repo:dev", expected: "dev" }, + ]; + + for (const testCase of testCases) { + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + package_url: testCase.url, + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBe(testCase.expected); + } + }); + + it("should return null for non-registry_package events", () => { + const headers = { "x-github-event": "push" }; + const body = { + registry_package: { + package_version: { + package_url: "ghcr.io/owner/repo:latest", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBeNull(); + }); + + it("should return null when package_version is missing", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: {}, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBeNull(); + }); + + it("should return null when package_url has no tag", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + package_url: "ghcr.io/owner/repo", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBeNull(); + }); + + it("should return null when package_url ends with colon (no tag)", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + package_url: "ghcr.io/owner/repo:", + container_metadata: { + tag: { + name: "", + digest: "sha256:abc123...", + }, + }, + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBeNull(); + }); + + it("should return null when tag name is empty string", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + container_metadata: { + tag: { + name: "", + digest: "sha256:abc123...", + }, + }, + package_url: "ghcr.io/owner/repo:", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBeNull(); + }); + + it("should ignore tag if it matches the version (digest)", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + container_metadata: { + tag: { + name: "sha256:abc123...", + digest: "sha256:abc123...", + }, + }, + package_url: "ghcr.io/owner/repo:latest", + }, + }, + }; + + const tag = extractImageTagFromRequest(headers, body); + expect(tag).toBe("latest"); + }); + + it("should handle registry_package commit message with package_url", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + package_url: "ghcr.io/owner/repo:latest", + }, + }, + }; + + const message = extractCommitMessage(headers, body); + expect(message).toBe("Docker GHCR image pushed: ghcr.io/owner/repo:latest"); + }); + + it("should handle registry_package commit message when package_url is missing", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: { + package_version: { + version: "sha256:abc123...", + }, + }, + }; + + const message = extractCommitMessage(headers, body); + expect(message).toBe("Docker GHCR image pushed"); + }); + + it("should handle registry_package commit message when package_version is missing", () => { + const headers = { "x-github-event": "registry_package" }; + const body = { + registry_package: {}, + }; + + const message = extractCommitMessage(headers, body); + expect(message).toBe("NEW COMMIT"); + }); +}); + +describe("Docker Image Name and Tag Extraction", () => { + describe("extractImageName", () => { + it("should return image name without tag", () => { + expect(extractImageName("my-image:latest")).toBe("my-image"); + expect(extractImageName("my-image:1.0.0")).toBe("my-image"); + expect(extractImageName("ghcr.io/owner/repo:latest")).toBe( + "ghcr.io/owner/repo", + ); + }); + + it("should return full image name when no tag is present", () => { + expect(extractImageName("my-image")).toBe("my-image"); + expect(extractImageName("ghcr.io/owner/repo")).toBe("ghcr.io/owner/repo"); + }); + + it("should handle images with port numbers correctly", () => { + expect(extractImageName("registry:5000/image:tag")).toBe( + "registry:5000/image", + ); + expect(extractImageName("localhost:5000/my-app:latest")).toBe( + "localhost:5000/my-app", + ); + }); + + it("should handle complex image paths", () => { + expect( + extractImageName("myregistryhost:5000/fedora/httpd:version1.0"), + ).toBe("myregistryhost:5000/fedora/httpd"); + expect(extractImageName("registry.example.com:8080/ns/app:v1.2.3")).toBe( + "registry.example.com:8080/ns/app", + ); + }); + + it("should return null for invalid inputs", () => { + expect(extractImageName(null)).toBeNull(); + expect(extractImageName("")).toBeNull(); + }); + + it("should handle edge cases with multiple colons", () => { + expect(extractImageName("image:tag:extra")).toBe("image:tag"); + expect(extractImageName("registry:5000:invalid")).toBe("registry:5000"); + }); + }); + + describe("extractImageTag", () => { + it("should extract tag from image with tag", () => { + expect(extractImageTag("my-image:latest")).toBe("latest"); + expect(extractImageTag("my-image:1.0.0")).toBe("1.0.0"); + expect(extractImageTag("ghcr.io/owner/repo:v1.2.3")).toBe("v1.2.3"); + }); + + it("should return 'latest' when no tag is present", () => { + expect(extractImageTag("my-image")).toBe("latest"); + expect(extractImageTag("ghcr.io/owner/repo")).toBe("latest"); + }); + + it("should handle complex image paths with tags", () => { + expect( + extractImageTag("myregistryhost:5000/fedora/httpd:version1.0"), + ).toBe("version1.0"); + expect(extractImageTag("registry.example.com:8080/ns/app:v1.2.3")).toBe( + "v1.2.3", + ); + }); + + it("should return null for invalid inputs", () => { + expect(extractImageTag(null)).toBeNull(); + expect(extractImageTag("")).toBeNull(); + }); + + it("should handle edge cases with multiple colons", () => { + expect(extractImageTag("image:tag:extra")).toBe("extra"); + expect(extractImageTag("registry:5000/image:tag")).toBe("tag"); + }); + + it("should handle numeric tags", () => { + expect(extractImageTag("my-image:123")).toBe("123"); + expect(extractImageTag("my-image:1")).toBe("1"); + }); + }); +}); diff --git a/apps/dokploy/pages/api/deploy/[refreshToken].ts b/apps/dokploy/pages/api/deploy/[refreshToken].ts index 797f13802..1441d9776 100644 --- a/apps/dokploy/pages/api/deploy/[refreshToken].ts +++ b/apps/dokploy/pages/api/deploy/[refreshToken].ts @@ -12,6 +12,17 @@ import type { DeploymentJob } from "@/server/queues/queue-types"; import { myQueue } from "@/server/queues/queueSetup"; import { deploy } from "@/server/utils/deploy"; +/** + * Helper function to get package_version from registry_package events + */ +const getPackageVersion = (headers: any, body: any) => { + const event = headers["x-github-event"]; + if (event === "registry_package") { + return body.registry_package?.package_version; + } + return null; +}; + export default async function handler( req: NextApiRequest, res: NextApiResponse, @@ -46,28 +57,66 @@ export default async function handler( } const deploymentTitle = extractCommitMessage(req.headers, req.body); - const deploymentHash = extractHash(req.headers, req.body); + const deploymentHash = extractHash(req.headers, req.body); const sourceType = application.sourceType; if (sourceType === "docker") { + const applicationImageName = extractImageName(application.dockerImage); const applicationDockerTag = extractImageTag(application.dockerImage); - const webhookDockerTags = extractImageTagFromRequest( + + const webhookImageName = extractImageNameFromRequest( + req.headers, + req.body, + ); + const webhookDockerTag = extractImageTagFromRequest( req.headers, req.body, ); - const isMismatch = - applicationDockerTag && - webhookDockerTags && - webhookDockerTags.length > 0 && - !webhookDockerTags.includes(applicationDockerTag); - if (isMismatch) { + if (!applicationImageName) { res.status(301).json({ - message: `Application Image Tag (${applicationDockerTag}) doesn't match request event payload Image Tag(s) (${webhookDockerTags.join(", ")}).`, + message: "Application Docker Image Name Not Found", }); return; } + + if (!webhookImageName) { + res.status(301).json({ + message: "Webhook Docker Image Name Not Found", + }); + return; + } + + // Validate image name matches + if (webhookImageName !== applicationImageName) { + res.status(301).json({ + message: `Application Image Name (${applicationImageName}) doesn't match request event payload Image Name (${webhookImageName}).`, + }); + return; + } + + if (!applicationDockerTag) { + res.status(301).json({ + message: "Application Docker Tag Not Found", + }); + return; + } + + if (!webhookDockerTag) { + res.status(301).json({ + message: "Webhook Docker Tag Not Found", + }); + return; + } + + if (webhookDockerTag !== applicationDockerTag) { + res.status(301).json({ + message: `Application Image Tag (${applicationDockerTag}) doesn't match request event payload Image Tag (${webhookDockerTag}).`, + }); + return; + } + console.log("[END] Docker Deploy Validation"); } else if (sourceType === "github") { const normalizedCommits = req.body?.commits?.flatMap( (commit: any) => commit.modified, @@ -224,6 +273,39 @@ export default async function handler( } } +/** + * Return the image name without the tag + * Example: "my-image" => "my-image" + * Example: "my-image:latest" => "my-image" + * Example: "my-image:1.0.0" => "my-image" + * Example: "myregistryhost:5000/fedora/httpd:version1.0" => "myregistryhost:5000/fedora/httpd" + * @link https://docs.docker.com/reference/cli/docker/image/tag/ + */ +export function extractImageName(dockerImage: string | null): string | null { + if (!dockerImage || typeof dockerImage !== "string") { + return null; + } + + // Handle case where there's no tag (no colon or colon is part of port number) + const lastColonIndex = dockerImage.lastIndexOf(":"); + if (lastColonIndex === -1) { + return dockerImage; + } + + // Check if the part after the last colon looks like a tag (not a port number) + // Port numbers are typically 1-5 digits, tags are usually longer or contain letters + const afterColon = dockerImage.substring(lastColonIndex + 1); + const isPortNumber = /^\d{1,5}$/.test(afterColon); + + // If it's a port number (like registry:5000/image), don't split + if (isPortNumber) { + return dockerImage; + } + + // Otherwise, split at the last colon to get image name + return dockerImage.substring(0, lastColonIndex); +} + /** * Return the last part of the image name, which is the tag * Example: "my-image" => null @@ -232,7 +314,7 @@ export default async function handler( * Example: "myregistryhost:5000/fedora/httpd:version1.0" => "version1.0" * @link https://docs.docker.com/reference/cli/docker/image/tag/ */ -function extractImageTag(dockerImage: string | null) { +export function extractImageTag(dockerImage: string | null) { if (!dockerImage || typeof dockerImage !== "string") { return null; } @@ -242,49 +324,99 @@ function extractImageTag(dockerImage: string | null) { } /** + * Extract the image name (without tag) from webhook request * @link https://docs.docker.com/docker-hub/webhooks/#example-webhook-payload + * @link https://docs.github.com/en/webhooks/webhook-events-and-payloads#registry_package + */ +export const extractImageNameFromRequest = ( + headers: any, + body: any, +): string | null => { + // GitHub Packages: registry_package events (container registry) + const packageVersion = getPackageVersion(headers, body); + if (packageVersion?.package_url) { + const packageUrl = packageVersion.package_url; + // Remove tag if present (everything after the last colon) + if (packageUrl.includes(":")) { + const lastColonIndex = packageUrl.lastIndexOf(":"); + // Check if it's a port number (like registry:5000/image) + const afterColon = packageUrl.substring(lastColonIndex + 1); + const isPortNumber = /^\d{1,5}$/.test(afterColon); + if (isPortNumber) { + return packageUrl; + } + return packageUrl.substring(0, lastColonIndex); + } + return packageUrl; + } + + // Docker Hub + if (headers["user-agent"]?.includes("Go-http-client")) { + if (body.repository) { + const repoName = body.repository.repo_name; + return `${repoName}`; + } + } + return null; +}; + +/** + * @link https://docs.docker.com/docker-hub/webhooks/#example-webhook-payload + * @link https://docs.github.com/en/webhooks/webhook-events-and-payloads#registry_package */ export const extractImageTagFromRequest = ( headers: any, body: any, -): string[] | null => { - if (headers["user-agent"]?.includes("Go-http-client")) { - if (body.push_data && body.repository) { - return [body.push_data.tag] as string[]; +): string | null => { + // GitHub Packages: registry_package events (container registry) + const packageVersion = getPackageVersion(headers, body); + if (packageVersion) { + // Try to get tag from container_metadata first (most reliable) + // Only use it if it's not empty and not the same as the version (digest) + const tagName = packageVersion.container_metadata?.tag?.name?.trim() || ""; + if ( + tagName && + tagName !== packageVersion.version && + !tagName.startsWith("sha256:") + ) { + return tagName; + } + // Fallback: extract tag from package_url (e.g., "ghcr.io/owner/repo:tag") + if (packageVersion.package_url) { + const packageUrl = packageVersion.package_url; + // Handle case where package_url ends with colon (no tag) + if (packageUrl.endsWith(":")) { + return null; + } + const tagMatch = packageUrl.match(/:([^:]+)$/); + if (tagMatch?.[1]?.trim()) { + return tagMatch[1].trim(); + } } } - // GitHub Packages: package or registry_package events (container tags) - // See: https://docs.github.com/en/webhooks/webhook-events-and-payloads#package - const githubEvent = headers["x-github-event"]; - if (githubEvent === "package" || githubEvent === "registry_package") { - const pkg = body?.package ?? body?.registry_package?.package ?? null; - const packageVersion = - body?.package_version ?? body?.registry_package?.package_version ?? null; - const packageType = pkg?.package_type; - - if (packageType === "container" && packageVersion) { - const tags = - packageVersion?.metadata?.container?.tags ?? - packageVersion?.container?.tags ?? - null; - if (Array.isArray(tags) && tags.length > 0) { - return tags as string[]; - } - const singleTag = - packageVersion?.metadata?.container?.tag ?? - packageVersion?.metadata?.tag ?? - packageVersion?.tag ?? - null; - if (typeof singleTag === "string") { - return [singleTag] as string[]; - } + // Docker Hub + if (headers["user-agent"]?.includes("Go-http-client")) { + if (body.push_data && body.repository) { + return body.push_data.tag; } } return null; }; export const extractCommitMessage = (headers: any, body: any) => { + // GitHub Packages: registry_package events (container tags) + const githubEvent = headers["x-github-event"]; + if (githubEvent === "registry_package") { + const packageVersion = getPackageVersion(headers, body); + if (packageVersion) { + if (packageVersion.package_url) { + return `Docker GHCR image pushed: ${packageVersion.package_url}`; + } + return "Docker GHCR image pushed"; + } + // If package_version is missing, fall through to default behavior + } // GitHub if (headers["x-github-event"]) { return body.head_commit ? body.head_commit.message : "NEW COMMIT"; @@ -313,7 +445,7 @@ export const extractCommitMessage = (headers: any, body: any) => { if (headers["user-agent"]?.includes("Go-http-client")) { if (body.push_data && body.repository) { - return `Docker image pushed: ${body.repository.repo_name}:${body.push_data.tag} by ${body.push_data.pusher}`; + return `DockerHub image pushed: ${body.repository.repo_name}:${body.push_data.tag} by ${body.push_data.pusher}`; } } From fbb1f1f266f222db2081f4eff578935714ede9af Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 01:11:52 -0600 Subject: [PATCH 15/23] fix: remove unnecessary log statement in Docker deploy validation --- apps/dokploy/pages/api/deploy/[refreshToken].ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/dokploy/pages/api/deploy/[refreshToken].ts b/apps/dokploy/pages/api/deploy/[refreshToken].ts index 1441d9776..4d4258cb6 100644 --- a/apps/dokploy/pages/api/deploy/[refreshToken].ts +++ b/apps/dokploy/pages/api/deploy/[refreshToken].ts @@ -116,7 +116,6 @@ export default async function handler( }); return; } - console.log("[END] Docker Deploy Validation"); } else if (sourceType === "github") { const normalizedCommits = req.body?.commits?.flatMap( (commit: any) => commit.modified, From a9b9dd4b66710c6df9f08a647527c6044a7de4d4 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 01:14:35 -0600 Subject: [PATCH 16/23] fix: conditionally include deployment hash in job data logging --- apps/dokploy/pages/api/deploy/[refreshToken].ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dokploy/pages/api/deploy/[refreshToken].ts b/apps/dokploy/pages/api/deploy/[refreshToken].ts index 4d4258cb6..2ab607736 100644 --- a/apps/dokploy/pages/api/deploy/[refreshToken].ts +++ b/apps/dokploy/pages/api/deploy/[refreshToken].ts @@ -241,7 +241,7 @@ export default async function handler( const jobData: DeploymentJob = { applicationId: application.applicationId as string, titleLog: deploymentTitle, - descriptionLog: `Hash: ${deploymentHash}`, + ...(deploymentHash && { descriptionLog: `Hash: ${deploymentHash}` }), type: "deploy", applicationType: "application", server: !!application.serverId, From 4d36741e50d9c6e52019ee4881e6f6b36ab2210a Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 01:33:07 -0600 Subject: [PATCH 17/23] refactor: streamline service extraction logic in add-permissions component - Updated type definitions for Environment and Project to improve clarity and maintainability. - Refactored the extractServices function to use optional chaining and nullish coalescing for safer data handling. - Enhanced type safety by casting the mapped services to the Services type. --- .../settings/users/add-permissions.tsx | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/apps/dokploy/components/dashboard/settings/users/add-permissions.tsx b/apps/dokploy/components/dashboard/settings/users/add-permissions.tsx index fb4d01547..7c6ef8b84 100644 --- a/apps/dokploy/components/dashboard/settings/users/add-permissions.tsx +++ b/apps/dokploy/components/dashboard/settings/users/add-permissions.tsx @@ -1,4 +1,3 @@ -import type { findEnvironmentById } from "@dokploy/server/index"; import { zodResolver } from "@hookform/resolvers/zod"; import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; @@ -27,12 +26,10 @@ import { FormMessage, } from "@/components/ui/form"; import { Switch } from "@/components/ui/switch"; -import { api } from "@/utils/api"; +import { api, type RouterOutputs } from "@/utils/api"; -type Environment = Omit< - Awaited>, - "project" ->; +type Project = RouterOutputs["project"]["all"][number]; +type Environment = Project["environments"][number]; export type Services = { appName: string; @@ -53,17 +50,16 @@ export type Services = { }; export const extractServices = (data: Environment | undefined) => { - const applications: Services[] = - data?.applications.map((item) => ({ - appName: item.appName, - name: item.name, - type: "application", - id: item.applicationId, - createdAt: item.createdAt, - status: item.applicationStatus, - description: item.description, - serverId: item.serverId, - })) || []; + const applications: Services[] = (data?.applications?.map((item) => ({ + appName: item.appName, + name: item.name, + type: "application", + id: item.applicationId, + createdAt: item.createdAt, + status: item.applicationStatus, + description: item.description, + serverId: item.serverId, + })) ?? []) as Services[]; const mariadb: Services[] = data?.mariadb.map((item) => ({ @@ -125,17 +121,16 @@ export const extractServices = (data: Environment | undefined) => { serverId: item.serverId, })) || []; - const compose: Services[] = - data?.compose.map((item) => ({ - appName: item.appName, - name: item.name, - type: "compose", - id: item.composeId, - createdAt: item.createdAt, - status: item.composeStatus, - description: item.description, - serverId: item.serverId, - })) || []; + const compose: Services[] = (data?.compose?.map((item) => ({ + appName: item.appName, + name: item.name, + type: "compose", + id: item.composeId, + createdAt: item.createdAt, + status: item.composeStatus, + description: item.description, + serverId: item.serverId, + })) ?? []) as Services[]; applications.push( ...mysql, From 61d9ae397adb0a2704626322f3b9ede40f717ded Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 22:27:38 -0600 Subject: [PATCH 18/23] feat: add git commit info extraction to deployment logic - Integrated `getGitCommitInfo` function to retrieve the latest commit message and hash for applications and compose services. - Updated deployment logic to conditionally include commit information in deployment updates, enhancing traceability. - Refactored import statements for better organization and clarity. --- packages/server/src/services/application.ts | 18 ++++++++- packages/server/src/services/compose.ts | 25 +++++++++++- packages/server/src/utils/docker/domain.ts | 2 + packages/server/src/utils/providers/git.ts | 42 +++++++++++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index 8dc67ddb6..a3eb959b4 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -18,7 +18,10 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { cloneBitbucketRepository } from "@dokploy/server/utils/providers/bitbucket"; import { buildRemoteDocker } from "@dokploy/server/utils/providers/docker"; -import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; +import { + cloneGitRepository, + getGitCommitInfo, +} from "@dokploy/server/utils/providers/git"; import { cloneGiteaRepository } from "@dokploy/server/utils/providers/gitea"; import { cloneGithubRepository } from "@dokploy/server/utils/providers/github"; import { cloneGitlabRepository } from "@dokploy/server/utils/providers/gitlab"; @@ -29,6 +32,7 @@ import { getDokployUrl } from "./admin"; import { createDeployment, createDeploymentPreview, + updateDeployment, updateDeploymentStatus, } from "./deployment"; import { type Domain, getDomainHost } from "./domain"; @@ -243,6 +247,18 @@ export const deployApplication = async ({ }); throw error; + } finally { + // Only extract commit info for non-docker sources + if (application.sourceType !== "docker") { + const commitInfo = await getGitCommitInfo(application); + + if (commitInfo) { + await updateDeployment(deployment.deploymentId, { + title: commitInfo.message, + description: `Commit: ${commitInfo.hash}`, + }); + } + } } return true; }; diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index 2e2a2fc59..519a0c404 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -22,7 +22,10 @@ import { execAsyncRemote, } from "@dokploy/server/utils/process/execAsync"; import { cloneBitbucketRepository } from "@dokploy/server/utils/providers/bitbucket"; -import { cloneGitRepository } from "@dokploy/server/utils/providers/git"; +import { + cloneGitRepository, + getGitCommitInfo, +} from "@dokploy/server/utils/providers/git"; import { cloneGiteaRepository } from "@dokploy/server/utils/providers/gitea"; import { cloneGithubRepository } from "@dokploy/server/utils/providers/github"; import { cloneGitlabRepository } from "@dokploy/server/utils/providers/gitlab"; @@ -30,7 +33,11 @@ import { getCreateComposeFileCommand } from "@dokploy/server/utils/providers/raw import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import { getDokployUrl } from "./admin"; -import { createDeploymentCompose, updateDeploymentStatus } from "./deployment"; +import { + createDeploymentCompose, + updateDeployment, + updateDeploymentStatus, +} from "./deployment"; import { validUniqueServerAppName } from "./project"; export type Compose = typeof compose.$inferSelect; @@ -239,6 +246,7 @@ export const deployCompose = async ({ await execAsync(commandWithLog); } + command = "set -e;"; command += await getBuildComposeCommand(entity); commandWithLog = `(${command}) >> ${deployment.logPath} 2>&1`; if (compose.serverId) { @@ -275,6 +283,19 @@ export const deployCompose = async ({ organizationId: compose.environment.project.organizationId, }); throw error; + } finally { + if (compose.sourceType !== "raw") { + const commitInfo = await getGitCommitInfo({ + ...compose, + type: "compose", + }); + if (commitInfo) { + await updateDeployment(deployment.deploymentId, { + title: commitInfo.message, + description: `Commit: ${commitInfo.hash}`, + }); + } + } } }; diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index ffe900302..a176a4560 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -131,6 +131,8 @@ exit 1; exit 1; `; } + + return ""; }; export const addDomainToCompose = async ( compose: Compose, diff --git a/packages/server/src/utils/providers/git.ts b/packages/server/src/utils/providers/git.ts index 19c8ab8a0..8e640892d 100644 --- a/packages/server/src/utils/providers/git.ts +++ b/packages/server/src/utils/providers/git.ts @@ -4,6 +4,7 @@ import { findSSHKeyById, updateSSHKeyById, } from "@dokploy/server/services/ssh-key"; +import { execAsync, execAsyncRemote } from "../process/execAsync"; interface CloneGitRepository { appName: string; @@ -145,3 +146,44 @@ const sanitizeRepoPathSSH = (input: string) => { }, }; }; + +interface Props { + appName: string; + type?: "application" | "compose"; + serverId: string | null; +} + +export const getGitCommitInfo = async ({ + appName, + type = "application", + serverId, +}: Props) => { + const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(!!serverId); + const basePath = type === "compose" ? COMPOSE_PATH : APPLICATIONS_PATH; + const outputPath = join(basePath, appName, "code"); + let stdoutResult = ""; + const result = { + message: "", + hash: "", + }; + try { + const gitCommand = `git -C ${outputPath} log -1 --pretty=format:"%H---DELIMITER---%B"`; + if (serverId) { + const { stdout } = await execAsyncRemote(serverId, gitCommand); + stdoutResult = stdout.trim(); + } else { + const { stdout } = await execAsync(gitCommand); + stdoutResult = stdout.trim(); + } + + const parts = stdoutResult.split("---DELIMITER---"); + if (parts && parts.length === 2) { + result.hash = parts[0]?.trim() || ""; + result.message = parts[1]?.trim() || ""; + } + } catch (error) { + console.error(`Error getting git commit info: ${error}`); + return null; + } + return result; +}; From 04a1a84077eb768cdd65e336c26e68889f829ab5 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Fri, 14 Nov 2025 23:09:02 -0600 Subject: [PATCH 19/23] fix: ensure proper cleanup of Docker buildx builder container - Added commands to remove the builder container after Railpack build and prepare failures to prevent resource leaks. - Improved bash command structure for better readability and maintenance. --- packages/server/src/utils/builders/railpack.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/server/src/utils/builders/railpack.ts b/packages/server/src/utils/builders/railpack.ts index cb188fd09..305ff20e8 100644 --- a/packages/server/src/utils/builders/railpack.ts +++ b/packages/server/src/utils/builders/railpack.ts @@ -75,6 +75,7 @@ export const getRailpackCommand = (application: ApplicationNested) => { buildArgs.push(buildAppDirectory); const bashCommand = ` + # Ensure we have a builder with containerd docker buildx create --use --name builder-containerd --driver docker-container || true docker buildx use builder-containerd @@ -82,6 +83,7 @@ docker buildx use builder-containerd echo "Preparing Railpack build plan..." ; railpack ${prepareArgs.join(" ")} || { echo "❌ Railpack prepare failed" ; + docker buildx rm builder-containerd || true exit 1; } echo "✅ Railpack prepare completed." ; @@ -91,6 +93,7 @@ echo "Building with Railpack frontend..." ; ${exportEnvs.join("\n")} docker ${buildArgs.join(" ")} || { echo "❌ Railpack build failed" ; + docker buildx rm builder-containerd || true exit 1; } echo "✅ Railpack build completed." ; From 69b7777db4a050376d64f88a677c335f5a6e5d2b Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Sat, 15 Nov 2025 00:28:44 -0600 Subject: [PATCH 20/23] chore: update node-os-utils to version 2.0.1 and refactor lodash imports - Upgraded `node-os-utils` from version 1.3.7 to 2.0.1 across multiple package.json files. - Removed deprecated `@types/node-os-utils` dependency. - Refactored lodash imports to use a single import statement for consistency. - Enhanced Docker stats monitoring by integrating new features from `node-os-utils` version 2.0.1. --- .../database/backups/restore-backup.tsx | 4 +- .../dashboard/docker/logs/terminal-line.tsx | 4 +- apps/dokploy/package.json | 3 +- apps/dokploy/server/wss/docker-stats.ts | 79 +++++++++++++++++++ packages/server/package.json | 3 +- packages/server/src/monitoring/utils.ts | 29 ++++--- packages/server/src/services/application.ts | 3 + pnpm-lock.yaml | 26 ++---- 8 files changed, 112 insertions(+), 39 deletions(-) diff --git a/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx b/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx index 6a0fb030a..01f6944e1 100644 --- a/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx +++ b/apps/dokploy/components/dashboard/database/backups/restore-backup.tsx @@ -1,6 +1,6 @@ import { zodResolver } from "@hookform/resolvers/zod"; import copy from "copy-to-clipboard"; -import { debounce } from "lodash"; +import _ from "lodash"; import { CheckIcon, ChevronsUpDown, @@ -236,7 +236,7 @@ export const RestoreBackup = ({ const currentDatabaseType = form.watch("databaseType"); const metadata = form.watch("metadata"); - const debouncedSetSearch = debounce((value: string) => { + const debouncedSetSearch = _.debounce((value: string) => { setDebouncedSearchTerm(value); }, 350); diff --git a/apps/dokploy/components/dashboard/docker/logs/terminal-line.tsx b/apps/dokploy/components/dashboard/docker/logs/terminal-line.tsx index 5b929f3b6..a75f50386 100644 --- a/apps/dokploy/components/dashboard/docker/logs/terminal-line.tsx +++ b/apps/dokploy/components/dashboard/docker/logs/terminal-line.tsx @@ -1,5 +1,5 @@ import { FancyAnsi } from "fancy-ansi"; -import { escapeRegExp } from "lodash"; +import _ from "lodash"; import { Badge } from "@/components/ui/badge"; import { Tooltip, @@ -47,7 +47,7 @@ export function TerminalLine({ log, noTimestamp, searchTerm }: LogLineProps) { } const htmlContent = fancyAnsi.toHtml(text); - const searchRegex = new RegExp(`(${escapeRegExp(term)})`, "gi"); + const searchRegex = new RegExp(`(${_.escapeRegExp(term)})`, "gi"); const modifiedContent = htmlContent.replace( searchRegex, diff --git a/apps/dokploy/package.json b/apps/dokploy/package.json index 183771f79..c9addf8de 100644 --- a/apps/dokploy/package.json +++ b/apps/dokploy/package.json @@ -120,7 +120,7 @@ "next": "^15.3.2", "next-i18next": "^15.4.2", "next-themes": "^0.2.1", - "node-os-utils": "1.3.7", + "node-os-utils": "2.0.1", "node-pty": "1.0.0", "node-schedule": "2.1.1", "nodemailer": "6.9.14", @@ -163,7 +163,6 @@ "@types/lodash": "4.17.4", "@types/micromatch": "4.0.9", "@types/node": "^18.19.104", - "@types/node-os-utils": "1.3.4", "@types/node-schedule": "2.1.6", "@types/nodemailer": "^6.4.17", "@types/qrcode": "^1.5.5", diff --git a/apps/dokploy/server/wss/docker-stats.ts b/apps/dokploy/server/wss/docker-stats.ts index 99e993dce..ebd53e93f 100644 --- a/apps/dokploy/server/wss/docker-stats.ts +++ b/apps/dokploy/server/wss/docker-stats.ts @@ -6,7 +6,9 @@ import { recordAdvancedStats, validateRequest, } from "@dokploy/server"; +import { OSUtils } from "node-os-utils"; import { WebSocketServer } from "ws"; +import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; export const setupDockerStatsMonitoringSocketServer = ( server: http.Server, @@ -49,6 +51,83 @@ export const setupDockerStatsMonitoringSocketServer = ( } const intervalId = setInterval(async () => { try { + // Special case: when monitoring "dokploy", get host system stats instead of container stats + if (appName === "dokploy") { + const osutils = new OSUtils(); + + // Get CPU usage + const cpuResult = await osutils.cpu.usage(); + const cpuUsage = cpuResult.success ? cpuResult.data : 0; + + // Get memory info + const memResult = await osutils.memory.info(); + let memUsedGB = 0; + let memTotalGB = 0; + let memUsedPercent = 0; + if (memResult.success) { + memTotalGB = memResult.data.total.toGB(); + memUsedGB = memResult.data.used.toGB(); + memUsedPercent = memResult.data.usagePercentage; + } + + // Get network stats from network.overview() or network.statsAsync() + let netInputBytes = 0; + let netOutputBytes = 0; + const networkOverview = await osutils.network.overview(); + if (networkOverview.success) { + netInputBytes = networkOverview.data.totalRxBytes.toBytes(); + netOutputBytes = networkOverview.data.totalTxBytes.toBytes(); + } + + // Get Block I/O from disk.stats() (available in v2.0!) + // If disk.stats() doesn't work in container, fallback to /proc/diskstats + let blockReadBytes = 0; + let blockWriteBytes = 0; + const diskStats = await osutils.disk.stats(); + if (diskStats.success && diskStats.data.length > 0) { + for (const stat of diskStats.data) { + blockReadBytes += stat.readBytes.toBytes(); + blockWriteBytes += stat.writeBytes.toBytes(); + } + } + + // Format memory usage similar to docker stats format: "used / total" + const memUsedFormatted = `${memUsedGB.toFixed(2)}GiB`; + const memTotalFormatted = `${memTotalGB.toFixed(2)}GiB`; + const memUsageFormatted = `${memUsedFormatted} / ${memTotalFormatted}`; + + // Format network I/O + const netInputMb = netInputBytes / (1024 * 1024); + const netOutputMb = netOutputBytes / (1024 * 1024); + const netIOFormatted = `${netInputMb.toFixed(2)}MB / ${netOutputMb.toFixed(2)}MB`; + + // Format Block I/O + const blockIOFormatted = `${formatBytes(blockReadBytes)} / ${formatBytes(blockWriteBytes)}`; + + // Create a stat object compatible with recordAdvancedStats + const stat = { + CPUPerc: `${cpuUsage.toFixed(2)}%`, + MemPerc: `${memUsedPercent.toFixed(2)}%`, + MemUsage: memUsageFormatted, + BlockIO: blockIOFormatted, + NetIO: netIOFormatted, + Container: "dokploy", + ID: "host-system", + Name: "dokploy", + }; + + await recordAdvancedStats(stat, appName); + const data = await getLastAdvancedStatsFile(appName); + console.log(data); + + ws.send( + JSON.stringify({ + data, + }), + ); + return; + } + const filter = { status: ["running"], ...(appType === "application" && { diff --git a/packages/server/package.json b/packages/server/package.json index 4d0f2e804..077ee3d5d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -61,7 +61,7 @@ "lodash": "4.17.21", "micromatch": "4.0.8", "nanoid": "3.3.11", - "node-os-utils": "1.3.7", + "node-os-utils": "2.0.1", "node-pty": "1.0.0", "node-schedule": "2.1.1", "nodemailer": "6.9.14", @@ -88,7 +88,6 @@ "@types/lodash": "4.17.4", "@types/micromatch": "4.0.9", "@types/node": "^18.19.104", - "@types/node-os-utils": "1.3.4", "@types/node-schedule": "2.1.6", "@types/nodemailer": "^6.4.17", "@types/qrcode": "^1.5.5", diff --git a/packages/server/src/monitoring/utils.ts b/packages/server/src/monitoring/utils.ts index 11ebb6169..23cb63f56 100644 --- a/packages/server/src/monitoring/utils.ts +++ b/packages/server/src/monitoring/utils.ts @@ -1,7 +1,6 @@ import { promises } from "node:fs"; -import osUtils from "node-os-utils"; +import { OSUtils } from "node-os-utils"; import { paths } from "../constants"; - export interface Container { BlockIO: string; CPUPerc: string; @@ -38,19 +37,23 @@ export const recordAdvancedStats = async ( }); if (appName === "dokploy") { - const disk = await osUtils.drive.info("/"); + const osutils = new OSUtils(); + const diskResult = await osutils.disk.usageByMountPoint("/"); - const diskUsage = disk.usedGb; - const diskTotal = disk.totalGb; - const diskUsedPercentage = disk.usedPercentage; - const diskFree = disk.freeGb; + if (diskResult.success && diskResult.data) { + const disk = diskResult.data; + const diskUsage = disk.used.toGB().toFixed(2); + const diskTotal = disk.total.toGB().toFixed(2); + const diskUsedPercentage = disk.usagePercentage; + const diskFree = disk.available.toGB().toFixed(2); - await updateStatsFile(appName, "disk", { - diskTotal: +diskTotal, - diskUsedPercentage: +diskUsedPercentage, - diskUsage: +diskUsage, - diskFree: +diskFree, - }); + await updateStatsFile(appName, "disk", { + diskTotal: +diskTotal, + diskUsedPercentage: +diskUsedPercentage, + diskUsage: +diskUsage, + diskFree: +diskFree, + }); + } } }; diff --git a/packages/server/src/services/application.ts b/packages/server/src/services/application.ts index a3eb959b4..c10babe56 100644 --- a/packages/server/src/services/application.ts +++ b/packages/server/src/services/application.ts @@ -419,6 +419,9 @@ export const deployPreviewApplication = async ({ }; export const getApplicationStats = async (appName: string) => { + if (appName === "dokploy") { + return await getAdvancedStats(appName); + } const filter = { status: ["running"], label: [`com.docker.swarm.service.name=${appName}`], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1aae074f7..ba76d1b73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -347,8 +347,8 @@ importers: specifier: ^0.2.1 version: 0.2.1(next@15.3.2(@opentelemetry/api@1.9.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(react-dom@18.2.0(react@18.2.0))(react@18.2.0) node-os-utils: - specifier: 1.3.7 - version: 1.3.7 + specifier: 2.0.1 + version: 2.0.1 node-pty: specifier: 1.0.0 version: 1.0.0 @@ -473,9 +473,6 @@ importers: '@types/node': specifier: ^18.19.104 version: 18.19.104 - '@types/node-os-utils': - specifier: 1.3.4 - version: 1.3.4 '@types/node-schedule': specifier: 2.1.6 version: 2.1.6 @@ -688,8 +685,8 @@ importers: specifier: 3.3.11 version: 3.3.11 node-os-utils: - specifier: 1.3.7 - version: 1.3.7 + specifier: 2.0.1 + version: 2.0.1 node-pty: specifier: 1.0.0 version: 1.0.0 @@ -766,9 +763,6 @@ importers: '@types/node': specifier: ^18.19.104 version: 18.19.104 - '@types/node-os-utils': - specifier: 1.3.4 - version: 1.3.4 '@types/node-schedule': specifier: 2.1.6 version: 2.1.6 @@ -4000,9 +3994,6 @@ packages: '@types/mysql@2.15.26': resolution: {integrity: sha512-DSLCOXhkvfS5WNNPbfn2KdICAmk8lLc+/PNvnPnF7gOdMZCxopXduqv0OQ13y/yA/zXTSikZZqVgybUxOEg6YQ==} - '@types/node-os-utils@1.3.4': - resolution: {integrity: sha512-BCUYrbdoO4FUbx6MB9atLNFnkxdliFaxdiTJMIPPiecXIApc5zf4NIqV5G1jWv/ReZvtYyHLs40RkBjHX+vykA==} - '@types/node-schedule@2.1.6': resolution: {integrity: sha512-6AlZSUiNTdaVmH5jXYxX9YgmF1zfOlbjUqw0EllTBmZCnN1R5RR/m/u3No1OiWR05bnQ4jM4/+w4FcGvkAtnKQ==} @@ -6312,8 +6303,9 @@ packages: '@types/node': optional: true - node-os-utils@1.3.7: - resolution: {integrity: sha512-fvnX9tZbR7WfCG5BAy3yO/nCLyjVWD6MghEq0z5FDfN+ZXpLWNITBdbifxQkQ25ebr16G0N7eRWJisOcMEHG3Q==} + node-os-utils@2.0.1: + resolution: {integrity: sha512-rH2N3qHZETLhdgTGhMMCE8zU3gsWO4we1MFtrSiAI7tYWrnJRc6dk2PseV4co3Lb0v/MbRONLQI2biHQYbpTpg==} + engines: {node: '>=18.0.0'} node-pty@1.0.0: resolution: {integrity: sha512-wtBMWWS7dFZm/VgqElrTvtfMq4GzJ6+edFI0Y0zyzygUSZMgZdraDUMUhCIvkjhJjme15qWmbyJbtAx4ot4uZA==} @@ -11338,8 +11330,6 @@ snapshots: dependencies: '@types/node': 20.17.51 - '@types/node-os-utils@1.3.4': {} - '@types/node-schedule@2.1.6': dependencies: '@types/node': 20.17.51 @@ -13852,7 +13842,7 @@ snapshots: optionalDependencies: '@types/node': 18.19.104 - node-os-utils@1.3.7: {} + node-os-utils@2.0.1: {} node-pty@1.0.0: dependencies: From 969147cd59fc6bc783bf2eabb808b4dc4a6dbd59 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Sat, 15 Nov 2025 00:56:05 -0600 Subject: [PATCH 21/23] feat: enhance Docker stats monitoring with disk I/O statistics - Updated OSUtils instantiation to include disk I/O statistics. - Implemented filtering to exclude virtual devices from disk stats, ensuring only real disk devices are monitored. --- apps/dokploy/server/wss/docker-stats.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/dokploy/server/wss/docker-stats.ts b/apps/dokploy/server/wss/docker-stats.ts index ebd53e93f..f54ff7f69 100644 --- a/apps/dokploy/server/wss/docker-stats.ts +++ b/apps/dokploy/server/wss/docker-stats.ts @@ -53,7 +53,11 @@ export const setupDockerStatsMonitoringSocketServer = ( try { // Special case: when monitoring "dokploy", get host system stats instead of container stats if (appName === "dokploy") { - const osutils = new OSUtils(); + const osutils = new OSUtils({ + disk: { + includeStats: true, // Enable disk I/O statistics + }, + }); // Get CPU usage const cpuResult = await osutils.cpu.usage(); @@ -85,7 +89,17 @@ export const setupDockerStatsMonitoringSocketServer = ( let blockWriteBytes = 0; const diskStats = await osutils.disk.stats(); if (diskStats.success && diskStats.data.length > 0) { + // Filter out virtual devices (loop, ram, sr, etc.) - only include real disk devices + const excludePatterns = [/^loop/, /^ram/, /^sr\d+$/, /^fd\d+$/]; for (const stat of diskStats.data) { + // Skip virtual devices + if ( + stat.device && + excludePatterns.some((pattern) => pattern.test(stat.device)) + ) { + continue; + } + // readBytes and writeBytes are DataSize objects with .toBytes() method blockReadBytes += stat.readBytes.toBytes(); blockWriteBytes += stat.writeBytes.toBytes(); } From a4caa47e106f6762876625d1555816ef3905591e Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Sat, 15 Nov 2025 00:59:00 -0600 Subject: [PATCH 22/23] feat: implement host system stats retrieval for Docker monitoring - Added a new function `getHostSystemStats` to encapsulate the logic for retrieving host system statistics using `node-os-utils`. - Refactored Docker stats monitoring to utilize the new function, improving code clarity and maintainability. - Removed redundant OSUtils instantiation from the Docker stats monitoring logic. --- apps/dokploy/server/wss/docker-stats.ts | 80 +------------------- packages/server/src/monitoring/utils.ts | 97 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 78 deletions(-) diff --git a/apps/dokploy/server/wss/docker-stats.ts b/apps/dokploy/server/wss/docker-stats.ts index f54ff7f69..02c834789 100644 --- a/apps/dokploy/server/wss/docker-stats.ts +++ b/apps/dokploy/server/wss/docker-stats.ts @@ -2,13 +2,12 @@ import type http from "node:http"; import { docker, execAsync, + getHostSystemStats, getLastAdvancedStatsFile, recordAdvancedStats, validateRequest, } from "@dokploy/server"; -import { OSUtils } from "node-os-utils"; import { WebSocketServer } from "ws"; -import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; export const setupDockerStatsMonitoringSocketServer = ( server: http.Server, @@ -53,82 +52,7 @@ export const setupDockerStatsMonitoringSocketServer = ( try { // Special case: when monitoring "dokploy", get host system stats instead of container stats if (appName === "dokploy") { - const osutils = new OSUtils({ - disk: { - includeStats: true, // Enable disk I/O statistics - }, - }); - - // Get CPU usage - const cpuResult = await osutils.cpu.usage(); - const cpuUsage = cpuResult.success ? cpuResult.data : 0; - - // Get memory info - const memResult = await osutils.memory.info(); - let memUsedGB = 0; - let memTotalGB = 0; - let memUsedPercent = 0; - if (memResult.success) { - memTotalGB = memResult.data.total.toGB(); - memUsedGB = memResult.data.used.toGB(); - memUsedPercent = memResult.data.usagePercentage; - } - - // Get network stats from network.overview() or network.statsAsync() - let netInputBytes = 0; - let netOutputBytes = 0; - const networkOverview = await osutils.network.overview(); - if (networkOverview.success) { - netInputBytes = networkOverview.data.totalRxBytes.toBytes(); - netOutputBytes = networkOverview.data.totalTxBytes.toBytes(); - } - - // Get Block I/O from disk.stats() (available in v2.0!) - // If disk.stats() doesn't work in container, fallback to /proc/diskstats - let blockReadBytes = 0; - let blockWriteBytes = 0; - const diskStats = await osutils.disk.stats(); - if (diskStats.success && diskStats.data.length > 0) { - // Filter out virtual devices (loop, ram, sr, etc.) - only include real disk devices - const excludePatterns = [/^loop/, /^ram/, /^sr\d+$/, /^fd\d+$/]; - for (const stat of diskStats.data) { - // Skip virtual devices - if ( - stat.device && - excludePatterns.some((pattern) => pattern.test(stat.device)) - ) { - continue; - } - // readBytes and writeBytes are DataSize objects with .toBytes() method - blockReadBytes += stat.readBytes.toBytes(); - blockWriteBytes += stat.writeBytes.toBytes(); - } - } - - // Format memory usage similar to docker stats format: "used / total" - const memUsedFormatted = `${memUsedGB.toFixed(2)}GiB`; - const memTotalFormatted = `${memTotalGB.toFixed(2)}GiB`; - const memUsageFormatted = `${memUsedFormatted} / ${memTotalFormatted}`; - - // Format network I/O - const netInputMb = netInputBytes / (1024 * 1024); - const netOutputMb = netOutputBytes / (1024 * 1024); - const netIOFormatted = `${netInputMb.toFixed(2)}MB / ${netOutputMb.toFixed(2)}MB`; - - // Format Block I/O - const blockIOFormatted = `${formatBytes(blockReadBytes)} / ${formatBytes(blockWriteBytes)}`; - - // Create a stat object compatible with recordAdvancedStats - const stat = { - CPUPerc: `${cpuUsage.toFixed(2)}%`, - MemPerc: `${memUsedPercent.toFixed(2)}%`, - MemUsage: memUsageFormatted, - BlockIO: blockIOFormatted, - NetIO: netIOFormatted, - Container: "dokploy", - ID: "host-system", - Name: "dokploy", - }; + const stat = await getHostSystemStats(); await recordAdvancedStats(stat, appName); const data = await getLastAdvancedStatsFile(appName); diff --git a/packages/server/src/monitoring/utils.ts b/packages/server/src/monitoring/utils.ts index 23cb63f56..2c42b99a6 100644 --- a/packages/server/src/monitoring/utils.ts +++ b/packages/server/src/monitoring/utils.ts @@ -1,6 +1,7 @@ import { promises } from "node:fs"; import { OSUtils } from "node-os-utils"; import { paths } from "../constants"; + export interface Container { BlockIO: string; CPUPerc: string; @@ -57,6 +58,102 @@ export const recordAdvancedStats = async ( } }; +/** + * Get host system statistics using node-os-utils + * This is used when monitoring "dokploy" to show host stats instead of container stats + */ +export const getHostSystemStats = async (): Promise => { + const osutils = new OSUtils({ + disk: { + includeStats: true, // Enable disk I/O statistics + }, + }); + + // Get CPU usage + const cpuResult = await osutils.cpu.usage(); + const cpuUsage = cpuResult.success ? cpuResult.data : 0; + + // Get memory info + const memResult = await osutils.memory.info(); + let memUsedGB = 0; + let memTotalGB = 0; + let memUsedPercent = 0; + if (memResult.success) { + memTotalGB = memResult.data.total.toGB(); + memUsedGB = memResult.data.used.toGB(); + memUsedPercent = memResult.data.usagePercentage; + } + + // Get network stats from network.overview() + let netInputBytes = 0; + let netOutputBytes = 0; + const networkOverview = await osutils.network.overview(); + if (networkOverview.success) { + netInputBytes = networkOverview.data.totalRxBytes.toBytes(); + netOutputBytes = networkOverview.data.totalTxBytes.toBytes(); + } + + // Get Block I/O from disk.stats() + let blockReadBytes = 0; + let blockWriteBytes = 0; + const diskStats = await osutils.disk.stats(); + if (diskStats.success && diskStats.data.length > 0) { + // Filter out virtual devices (loop, ram, sr, etc.) - only include real disk devices + const excludePatterns = [/^loop/, /^ram/, /^sr\d+$/, /^fd\d+$/]; + for (const stat of diskStats.data) { + // Skip virtual devices + if ( + stat.device && + excludePatterns.some((pattern) => pattern.test(stat.device)) + ) { + continue; + } + // readBytes and writeBytes are DataSize objects with .toBytes() method + blockReadBytes += stat.readBytes.toBytes(); + blockWriteBytes += stat.writeBytes.toBytes(); + } + } + + // Format values similar to docker stats + const formatBytes = (bytes: number): string => { + if (bytes >= 1024 * 1024 * 1024) { + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)}GiB`; + } + if (bytes >= 1024 * 1024) { + return `${(bytes / (1024 * 1024)).toFixed(2)}MiB`; + } + if (bytes >= 1024) { + return `${(bytes / 1024).toFixed(2)}KiB`; + } + return `${bytes}B`; + }; + + // Format memory usage similar to docker stats format: "used / total" + const memUsedFormatted = `${memUsedGB.toFixed(2)}GiB`; + const memTotalFormatted = `${memTotalGB.toFixed(2)}GiB`; + const memUsageFormatted = `${memUsedFormatted} / ${memTotalFormatted}`; + + // Format network I/O + const netInputMb = netInputBytes / (1024 * 1024); + const netOutputMb = netOutputBytes / (1024 * 1024); + const netIOFormatted = `${netInputMb.toFixed(2)}MB / ${netOutputMb.toFixed(2)}MB`; + + // Format Block I/O + const blockIOFormatted = `${formatBytes(blockReadBytes)} / ${formatBytes(blockWriteBytes)}`; + + // Create a stat object compatible with recordAdvancedStats + return { + CPUPerc: `${cpuUsage.toFixed(2)}%`, + MemPerc: `${memUsedPercent.toFixed(2)}%`, + MemUsage: memUsageFormatted, + BlockIO: blockIOFormatted, + NetIO: netIOFormatted, + Container: "dokploy", + ID: "host-system", + Name: "dokploy", + }; +}; + export const getAdvancedStats = async (appName: string) => { return { cpu: await readStatsFile(appName, "cpu"), From 09a98a29e033d47f64648a439f784040e31d801c Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Sat, 15 Nov 2025 00:59:36 -0600 Subject: [PATCH 23/23] fix: remove unnecessary console log from Docker stats monitoring --- apps/dokploy/server/wss/docker-stats.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/dokploy/server/wss/docker-stats.ts b/apps/dokploy/server/wss/docker-stats.ts index 02c834789..bd740e976 100644 --- a/apps/dokploy/server/wss/docker-stats.ts +++ b/apps/dokploy/server/wss/docker-stats.ts @@ -56,7 +56,6 @@ export const setupDockerStatsMonitoringSocketServer = ( await recordAdvancedStats(stat, appName); const data = await getLastAdvancedStatsFile(appName); - console.log(data); ws.send( JSON.stringify({