feat: add multi server compose

apps/dokploy/server/utils/builders/compose.ts

@@ -8,7 +8,10 @@ import { dirname, join } from "node:path";
 import { COMPOSE_PATH } from "@/server/constants";
 import type { InferResultType } from "@/server/types/with";
 import boxen from "boxen";
-import { writeDomainsToCompose } from "../docker/domain";
+import {
+	writeDomainsToCompose,
+	writeDomainsToComposeRemote,
+} from "../docker/domain";
 import { prepareEnvironmentVariables } from "../docker/utils";
 import { spawnAsync } from "../process/spawnAsync";
 
@@ -65,13 +68,16 @@ Compose Type: ${composeType} ✅`;
 	}
 };
 
-export const getBuildComposeCommand = (
+export const getBuildComposeCommand = async (
 	compose: ComposeNested,
 	logPath: string,
 ) => {
 	const { sourceType, appName, mounts, composeType, domains } = compose;
 	const command = createCommand(compose);
+	const envCommand = getCreateEnvFileCommand(compose);
 	const projectPath = join(COMPOSE_PATH, compose.appName, "code");
+
+	const newCompose = await writeDomainsToComposeRemote(compose, domains);
 	const logContent = `
 App Name: ${appName}
 Build Compose 🐳
@@ -80,8 +86,19 @@ Command: docker ${command}
 Source Type: docker ${sourceType} ✅
 Compose Type: ${composeType} ✅`;
 
+	const logBox = boxen(logContent, {
+		padding: {
+			left: 1,
+			right: 1,
+			bottom: 1,
+		},
+		width: 80,
+		borderStyle: "double",
+	});
+
 	const bashCommand = `
-echo "${logContent}" >> ${logPath};
+${newCompose}
+echo "${logBox}" >> ${logPath};
 cd ${projectPath} || exit 1;
 docker ${command.split(" ").join(" ")} >> ${logPath} 2>&1;
 echo "Docker Compose Deployed: ✅" >> ${logPath};
@@ -144,3 +161,26 @@ const createEnvFile = (compose: ComposeNested) => {
 	}
 	writeFileSync(envFilePath, envFileContent);
 };
+
+export const getCreateEnvFileCommand = (compose: ComposeNested) => {
+	const { env, composePath, appName } = compose;
+	const composeFilePath =
+		join(COMPOSE_PATH, appName, "code", composePath) ||
+		join(COMPOSE_PATH, appName, "code", "docker-compose.yml");
+
+	const envFilePath = join(dirname(composeFilePath), ".env");
+	let envContent = env || "";
+	if (!envContent.includes("DOCKER_CONFIG")) {
+		envContent += "\nDOCKER_CONFIG=/root/.docker/config.json";
+	}
+
+	if (compose.randomize) {
+		envContent += `\nCOMPOSE_PREFIX=${compose.suffix}`;
+	}
+
+	const envFileContent = prepareEnvironmentVariables(envContent).join("\n");
+	return `
+	mkdir -p ${envFilePath};
+	echo "${envFileContent}" > ${envFilePath} 2>/dev/null;
+	`;
+};

apps/dokploy/server/utils/docker/domain.ts

@@ -5,17 +5,33 @@ import type { Compose } from "@/server/api/services/compose";
 import type { Domain } from "@/server/api/services/domain";
 import { COMPOSE_PATH } from "@/server/constants";
 import { dump, load } from "js-yaml";
-import { cloneRawBitbucketRepository } from "../providers/bitbucket";
-import { cloneGitRawRepository } from "../providers/git";
-import { cloneRawGithubRepository } from "../providers/github";
-import { cloneRawGitlabRepository } from "../providers/gitlab";
-import { createComposeFileRaw } from "../providers/raw";
+import {
+	cloneRawBitbucketRepository,
+	cloneRawBitbucketRepositoryRemote,
+} from "../providers/bitbucket";
+import {
+	cloneGitRawRepository,
+	cloneRawGitRepositoryRemote,
+} from "../providers/git";
+import {
+	cloneRawGithubRepository,
+	cloneRawGithubRepositoryRemote,
+} from "../providers/github";
+import {
+	cloneRawGitlabRepository,
+	cloneRawGitlabRepositoryRemote,
+} from "../providers/gitlab";
+import {
+	createComposeFileRaw,
+	createComposeFileRawRemote,
+} from "../providers/raw";
 import { randomizeSpecificationFile } from "./compose";
 import type {
 	ComposeSpecification,
 	DefinitionsService,
 	PropertiesNetworks,
 } from "./types";
+import { execAsyncRemote } from "../process/execAsync";
 
 export const cloneCompose = async (compose: Compose) => {
 	if (compose.sourceType === "github") {
@@ -31,6 +47,20 @@ export const cloneCompose = async (compose: Compose) => {
 	}
 };
 
+export const cloneComposeRemote = async (compose: Compose) => {
+	if (compose.sourceType === "github") {
+		await cloneRawGithubRepositoryRemote(compose);
+	} else if (compose.sourceType === "gitlab") {
+		await cloneRawGitlabRepositoryRemote(compose);
+	} else if (compose.sourceType === "bitbucket") {
+		await cloneRawBitbucketRepositoryRemote(compose);
+	} else if (compose.sourceType === "git") {
+		await cloneRawGitRepositoryRemote(compose);
+	} else if (compose.sourceType === "raw") {
+		await createComposeFileRawRemote(compose);
+	}
+};
+
 export const getComposePath = (compose: Compose) => {
 	const { appName, sourceType, composePath } = compose;
 	let path = "";
@@ -57,6 +87,23 @@ export const loadDockerCompose = async (
 	return null;
 };
 
+export const loadDockerComposeRemote = async (
+	compose: Compose,
+): Promise<ComposeSpecification | null> => {
+	const path = getComposePath(compose);
+	try {
+		if (!compose.serverId) {
+			return null;
+		}
+		const { stdout } = await execAsyncRemote(compose.serverId, `cat ${path}`);
+		if (!stdout) return null;
+		const parsedConfig = load(stdout) as ComposeSpecification;
+		return parsedConfig;
+	} catch (err) {
+		return null;
+	}
+};
+
 export const readComposeFile = async (compose: Compose) => {
 	const path = getComposePath(compose);
 	if (existsSync(path)) {
@@ -84,12 +131,39 @@ export const writeDomainsToCompose = async (
 	}
 };
 
+export const writeDomainsToComposeRemote = async (
+	compose: Compose,
+	domains: Domain[],
+) => {
+	if (!domains.length) {
+		return;
+	}
+	const composeConverted = await addDomainToCompose(compose, domains);
+	const path = getComposePath(compose);
+
+	try {
+		if (compose.serverId) {
+			const composeString = dump(composeConverted, { lineWidth: 1000 });
+			return `echo "${composeString}" >> ${path};`;
+		}
+	} catch (error) {
+		throw error;
+	}
+};
+
 export const addDomainToCompose = async (
 	compose: Compose,
 	domains: Domain[],
 ) => {
 	const { appName } = compose;
-	let result = await loadDockerCompose(compose);
+
+	let result: ComposeSpecification | null;
+
+	if (compose.serverId) {
+		result = await loadDockerComposeRemote(compose);
+	} else {
+		result = await loadDockerCompose(compose);
+	}
 
 	if (!result || domains.length === 0) {
 		return null;

apps/dokploy/server/utils/providers/bitbucket.ts

@@ -11,6 +11,7 @@ import type { InferResultType } from "@/server/types/with";
 import { TRPCError } from "@trpc/server";
 import { recreateDirectory } from "../filesystem/directory";
 import { spawnAsync } from "../process/spawnAsync";
+import { execAsyncRemote } from "../process/execAsync";
 
 export type ApplicationWithBitbucket = InferResultType<
 	"applications",
@@ -117,6 +118,46 @@ export const cloneRawBitbucketRepository = async (entity: Compose) => {
 	}
 };
 
+export const cloneRawBitbucketRepositoryRemote = async (compose: Compose) => {
+	const {
+		appName,
+		bitbucketRepository,
+		bitbucketOwner,
+		bitbucketBranch,
+		bitbucketId,
+		serverId,
+	} = compose;
+
+	if (!serverId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Server not found",
+		});
+	}
+	if (!bitbucketId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Bitbucket Provider not found",
+		});
+	}
+
+	const bitbucketProvider = await findBitbucketById(bitbucketId);
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`;
+	const cloneUrl = `https://${bitbucketProvider?.bitbucketUsername}:${bitbucketProvider?.appPassword}@${repoclone}`;
+
+	try {
+		await execAsyncRemote(
+			serverId,
+			`git clone --branch ${bitbucketBranch} --depth 1 ${cloneUrl} ${outputPath}`,
+		);
+	} catch (error) {
+		throw error;
+	}
+};
+
 export const getBitbucketCloneCommand = async (
 	entity: ApplicationWithBitbucket | ComposeWithBitbucket,
 	logPath: string,

apps/dokploy/server/utils/providers/git.ts

@@ -4,8 +4,9 @@ import { updateSSHKeyById } from "@/server/api/services/ssh-key";
 import { APPLICATIONS_PATH, COMPOSE_PATH, SSH_PATH } from "@/server/constants";
 import { TRPCError } from "@trpc/server";
 import { recreateDirectory } from "../filesystem/directory";
-import { execAsync } from "../process/execAsync";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
 import { spawnAsync } from "../process/spawnAsync";
+import { Compose } from "@/server/api/services/compose";
 
 export const cloneGitRepository = async (
 	entity: {
@@ -143,6 +144,7 @@ export const getCustomGitCloneCommand = async (
 		command.push(`echo "Cloned Custom Git ${customGitUrl}: ✅" >> ${logPath}`);
 		return command.join("\n");
 	} catch (error) {
+		console.log(error);
 		throw error;
 	}
 };
@@ -264,3 +266,63 @@ export const cloneGitRawRepository = async (entity: {
 		throw error;
 	}
 };
+
+export const cloneRawGitRepositoryRemote = async (compose: Compose) => {
+	const {
+		appName,
+		customGitBranch,
+		customGitUrl,
+		customGitSSHKeyId,
+		serverId,
+	} = compose;
+
+	if (!serverId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Server not found",
+		});
+	}
+	if (!customGitUrl) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Git Provider not found",
+		});
+	}
+
+	const keyPath = path.join(SSH_PATH, `${customGitSSHKeyId}_rsa`);
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	const knownHostsPath = path.join(SSH_PATH, "known_hosts");
+
+	if (customGitSSHKeyId) {
+		await updateSSHKeyById({
+			sshKeyId: customGitSSHKeyId,
+			lastUsedAt: new Date().toISOString(),
+		});
+	}
+	try {
+		const command = [];
+		if (!isHttpOrHttps(customGitUrl)) {
+			command.push(addHostToKnownHostsCommand(customGitUrl));
+		}
+		command.push(`rm -rf ${outputPath};`);
+		command.push(`mkdir -p ${outputPath};`);
+		if (customGitSSHKeyId) {
+			command.push(
+				`GIT_SSH_COMMAND="ssh -i ${keyPath} -o UserKnownHostsFile=${knownHostsPath}"`,
+			);
+		}
+
+		command.push(
+			`if ! git clone --branch ${customGitBranch} --depth 1 --progress ${customGitUrl} ${outputPath} ; then
+				echo "[ERROR] Fail to clone the repository ";
+				exit 1;
+			fi
+			`,
+		);
+
+		await execAsyncRemote(serverId, command.join("\n"));
+	} catch (error) {
+		throw error;
+	}
+};

apps/dokploy/server/utils/providers/github.ts

@@ -12,6 +12,7 @@ import type { Compose } from "@/server/api/services/compose";
 import { type Github, findGithubById } from "@/server/api/services/github";
 import type { apiFindGithubBranches } from "@/server/db/schema";
 import { executeCommand } from "../servers/command";
+import { execAsyncRemote } from "../process/execAsync";
 
 export const authGithub = (githubProvider: Github) => {
 	if (!haveGithubRequirements(githubProvider)) {
@@ -233,6 +234,39 @@ export const cloneRawGithubRepository = async (entity: Compose) => {
 	}
 };
 
+export const cloneRawGithubRepositoryRemote = async (compose: Compose) => {
+	const { appName, repository, owner, branch, githubId, serverId } = compose;
+
+	if (!serverId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Server not found",
+		});
+	}
+	if (!githubId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "GitHub Provider not found",
+		});
+	}
+	const githubProvider = await findGithubById(githubId);
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	const octokit = authGithub(githubProvider);
+	const token = await getGithubToken(octokit);
+	const repoclone = `github.com/${owner}/${repository}.git`;
+	await recreateDirectory(outputPath);
+	const cloneUrl = `https://oauth2:${token}@${repoclone}`;
+	try {
+		await execAsyncRemote(
+			serverId,
+			`git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath}`,
+		);
+	} catch (error) {
+		throw error;
+	}
+};
+
 export const getGithubRepositories = async (githubId?: string) => {
 	if (!githubId) {
 		return [];

apps/dokploy/server/utils/providers/gitlab.ts

@@ -13,6 +13,7 @@ import { TRPCError } from "@trpc/server";
 import { recreateDirectory } from "../filesystem/directory";
 import { spawnAsync } from "../process/spawnAsync";
 import { executeCommand } from "../servers/command";
+import { execAsyncRemote } from "../process/execAsync";
 
 export const refreshGitlabToken = async (gitlabProviderId: string) => {
 	const gitlabProvider = await findGitlabById(gitlabProviderId);
@@ -361,6 +362,39 @@ export const cloneRawGitlabRepository = async (entity: Compose) => {
 	}
 };
 
+export const cloneRawGitlabRepositoryRemote = async (compose: Compose) => {
+	const { appName, gitlabPathNamespace, branch, gitlabId, serverId } = compose;
+
+	if (!serverId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Server not found",
+		});
+	}
+	if (!gitlabId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Gitlab Provider not found",
+		});
+	}
+	const gitlabProvider = await findGitlabById(gitlabId);
+
+	await refreshGitlabToken(gitlabId);
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `gitlab.com/${gitlabPathNamespace}.git`;
+	const cloneUrl = `https://oauth2:${gitlabProvider?.accessToken}@${repoclone}`;
+	try {
+		await execAsyncRemote(
+			serverId,
+			`git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath}`,
+		);
+	} catch (error) {
+		throw error;
+	}
+};
+
 export const testGitlabConnection = async (
 	input: typeof apiGitlabTestConnection._type,
 ) => {

apps/dokploy/server/utils/providers/raw.ts

@@ -4,6 +4,7 @@ import { join } from "node:path";
 import type { Compose } from "@/server/api/services/compose";
 import { COMPOSE_PATH } from "@/server/constants";
 import { recreateDirectory } from "../filesystem/directory";
+import { execAsyncRemote } from "../process/execAsync";
 
 export const createComposeFile = async (compose: Compose, logPath: string) => {
 	const { appName, composeFile } = compose;
@@ -45,3 +46,19 @@ export const createComposeFileRaw = async (compose: Compose) => {
 		throw error;
 	}
 };
+
+export const createComposeFileRawRemote = async (compose: Compose) => {
+	const { appName, composeFile, serverId } = compose;
+	const outputPath = join(COMPOSE_PATH, appName, "code");
+	const filePath = join(outputPath, "docker-compose.yml");
+
+	try {
+		const command = `
+			mkdir -p ${outputPath};
+			echo "${composeFile}" > ${filePath};
+		`;
+		await execAsyncRemote(serverId, command);
+	} catch (error) {
+		throw error;
+	}
+};