From 9aff4bc10bb2e5070b9aa52c4d459155fe63ff6a Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sat, 1 Mar 2025 03:11:29 -0600 Subject: [PATCH] refactor: update template system with new configuration structure and processing --- .../dashboard/project/add-template.tsx | 78 ++- apps/dokploy/server/api/routers/compose.ts | 136 +++-- .../templates/pocketbase/docker-compose.yml | 9 - apps/dokploy/templates/pocketbase/index.ts | 22 - apps/dokploy/templates/templates.ts | 15 - apps/dokploy/templates/utils/index.ts | 17 +- packages/server/src/templates/utils/github.ts | 504 +++++++++--------- packages/server/src/templates/utils/index.ts | 108 ++-- .../server/src/templates/utils/metadata.yaml | 44 ++ .../server/src/templates/utils/template.json | 42 ++ packages/server/src/types/template.ts | 44 ++ 11 files changed, 585 insertions(+), 434 deletions(-) delete mode 100644 apps/dokploy/templates/pocketbase/docker-compose.yml delete mode 100644 apps/dokploy/templates/pocketbase/index.ts create mode 100644 packages/server/src/templates/utils/metadata.yaml create mode 100644 packages/server/src/templates/utils/template.json create mode 100644 packages/server/src/types/template.ts diff --git a/apps/dokploy/components/dashboard/project/add-template.tsx b/apps/dokploy/components/dashboard/project/add-template.tsx index cc6962aab..c28e2332b 100644 --- a/apps/dokploy/components/dashboard/project/add-template.tsx +++ b/apps/dokploy/components/dashboard/project/add-template.tsx @@ -70,6 +70,38 @@ import Link from "next/link"; import { useState } from "react"; import { toast } from "sonner"; +interface TemplateData { + metadata: { + id: string; + name: string; + description: string; + version: string; + logo: string; + links: { + github: string; + website?: string; + docs?: string; + }; + tags: string[]; + }; + variables: { + [key: string]: string; + }; + config: { + domains: Array<{ + serviceName: string; + port: number; + path?: string; + host?: string; + }>; + env: Record; + mounts?: Array<{ + filePath: string; + content: string; + }>; + }; +} + interface Props { projectId: string; } @@ -94,11 +126,13 @@ export const AddTemplate = ({ projectId }: Props) => { data?.filter((template) => { const matchesTags = selectedTags.length === 0 || - template.tags.some((tag) => selectedTags.includes(tag)); + template.metadata.tags.some((tag) => selectedTags.includes(tag)); const matchesQuery = query === "" || - template.name.toLowerCase().includes(query.toLowerCase()) || - template.description.toLowerCase().includes(query.toLowerCase()); + template.metadata.name.toLowerCase().includes(query.toLowerCase()) || + template.metadata.description + .toLowerCase() + .includes(query.toLowerCase()); return matchesTags && matchesQuery; }) || []; @@ -249,9 +283,9 @@ export const AddTemplate = ({ projectId }: Props) => { : "grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 xl:grid-cols-5 2xl:grid-cols-6", )} > - {templates?.map((template, index) => ( + {templates?.map((template) => (
{ )} > - {template.version} + {template.metadata.version} {/* Template Header */}
{ )} > {template.name}
- {template.name} + {template.metadata.name} {viewMode === "detailed" && - template.tags.length > 0 && ( + template.metadata.tags.length > 0 && (
- {template.tags.map((tag) => ( + {template.metadata.tags.map((tag) => ( { {viewMode === "detailed" && (
- {template.description} + {template.metadata.description}
)} @@ -318,24 +352,24 @@ export const AddTemplate = ({ projectId }: Props) => { {viewMode === "detailed" && (
- {template.links.website && ( + {template.metadata.links.website && ( )} - {template.links.docs && ( + {template.metadata.links.docs && ( @@ -364,8 +398,8 @@ export const AddTemplate = ({ projectId }: Props) => { This will create an application from the{" "} - {template.name} template and add it to your - project. + {template.metadata.name} template and add it to + your project.
@@ -431,7 +465,7 @@ export const AddTemplate = ({ projectId }: Props) => { const promise = mutateAsync({ projectId, serverId: serverId || undefined, - id: template.id, + id: template.metadata.id, }); toast.promise(promise, { loading: "Setting up...", @@ -440,10 +474,10 @@ export const AddTemplate = ({ projectId }: Props) => { projectId, }); setOpen(false); - return `${template.name} template created successfully`; + return `${template.metadata.name} template created successfully`; }, error: (err) => { - return `An error ocurred deploying ${template.name} template`; + return `An error occurred deploying ${template.metadata.name} template`; }, }); }} diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index f8deea0d8..f668c2818 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -13,12 +13,13 @@ import { import { cleanQueuesByCompose, myQueue } from "@/server/queues/queueSetup"; import { templates } from "@/templates/templates"; import type { TemplatesKeys } from "@/templates/types/templates-data.type"; +import { generatePassword } from "@/templates/utils"; import { - generatePassword, - loadTemplateModule, - readTemplateComposeFile, -} from "@/templates/utils"; -import { fetchTemplatesList } from "@dokploy/server/templates/utils/github"; + fetchTemplateFiles, + fetchTemplatesList, + processTemplate, +} from "@dokploy/server/templates/utils/github"; +import { readTemplateComposeFile } from "@dokploy/server/templates/utils"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import { dump } from "js-yaml"; @@ -56,6 +57,29 @@ import { updateCompose, } from "@dokploy/server"; +import { z } from "zod"; +import type { TemplateConfig } from "@dokploy/server/types/template"; + +// Add the template config schema +const templateConfigSchema = z.object({ + variables: z.record(z.string()), + domains: z.array( + z.object({ + serviceName: z.string(), + port: z.number(), + path: z.string().optional(), + host: z.string().optional(), + }), + ), + env: z.record(z.string()), + mounts: z.array( + z.object({ + filePath: z.string(), + content: z.string(), + }), + ), +}) satisfies z.ZodType; + export const composeRouter = createTRPCRouter({ create: protectedProcedure .input(apiCreateCompose) @@ -374,7 +398,13 @@ export const composeRouter = createTRPCRouter({ return true; }), deployTemplate: protectedProcedure - .input(apiCreateComposeByTemplate) + .input( + z.object({ + projectId: z.string(), + serverId: z.string().optional(), + id: z.string(), + }), + ) .mutation(async ({ ctx, input }) => { if (ctx.user.rol === "user") { await checkServiceAccess(ctx.user.authId, input.projectId, "create"); @@ -387,9 +417,7 @@ export const composeRouter = createTRPCRouter({ }); } - const composeFile = await readTemplateComposeFile(input.id); - - const generate = await loadTemplateModule(input.id); + const template = await fetchTemplateFiles(input.id); const admin = await findAdminById(ctx.user.adminId); let serverIp = admin.serverIp || "127.0.0.1"; @@ -402,50 +430,60 @@ export const composeRouter = createTRPCRouter({ } else if (process.env.NODE_ENV === "development") { serverIp = "127.0.0.1"; } - const projectName = slugify(`${project.name} ${input.id}`); - const { envs, mounts, domains } = await generate({ - serverIp: serverIp || "", - projectName: projectName, + + const generate = processTemplate(template.config, { + serverIp: serverIp, + projectName: project.name, }); - const compose = await createComposeByTemplate({ - ...input, - composeFile: composeFile, - env: envs?.join("\n"), - serverId: input.serverId, - name: input.id, - sourceType: "raw", - appName: `${projectName}-${generatePassword(6)}`, - isolatedDeployment: true, - }); + console.log(generate.domains); + console.log(generate.envs); + console.log(generate.mounts); - if (ctx.user.rol === "user") { - await addNewService(ctx.user.authId, compose.composeId); - } + // const projectName = slugify(`${project.name} ${input.id}`); + // const { envs, mounts, domains } = await generate({ + // serverIp: serverIp || "", + // projectName: projectName, + // }); - if (mounts && mounts?.length > 0) { - for (const mount of mounts) { - await createMount({ - filePath: mount.filePath, - mountPath: "", - content: mount.content, - serviceId: compose.composeId, - serviceType: "compose", - type: "file", - }); - } - } + // const compose = await createComposeByTemplate({ + // ...input, + // composeFile: composeFile, + // env: envs?.join("\n"), + // serverId: input.serverId, + // name: input.id, + // sourceType: "raw", + // appName: `${projectName}-${generatePassword(6)}`, + // isolatedDeployment: true, + // }); - if (domains && domains?.length > 0) { - for (const domain of domains) { - await createDomain({ - ...domain, - domainType: "compose", - certificateType: "none", - composeId: compose.composeId, - }); - } - } + // if (ctx.user.rol === "user") { + // await addNewService(ctx.user.authId, compose.composeId); + // } + + // if (mounts && mounts?.length > 0) { + // for (const mount of mounts) { + // await createMount({ + // filePath: mount.filePath, + // mountPath: "", + // content: mount.content, + // serviceId: compose.composeId, + // serviceType: "compose", + // type: "file", + // }); + // } + // } + + // if (domains && domains?.length > 0) { + // for (const domain of domains) { + // await createDomain({ + // ...domain, + // domainType: "compose", + // certificateType: "none", + // composeId: compose.composeId, + // }); + // } + // } return null; }), @@ -465,7 +503,7 @@ export const composeRouter = createTRPCRouter({ } // Fall back to local templates if GitHub fetch fails - return templates; + return []; }), getTags: protectedProcedure.query(async ({ input }) => { diff --git a/apps/dokploy/templates/pocketbase/docker-compose.yml b/apps/dokploy/templates/pocketbase/docker-compose.yml deleted file mode 100644 index cf5e94f1b..000000000 --- a/apps/dokploy/templates/pocketbase/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.8" -services: - pocketbase: - image: spectado/pocketbase:0.23.3 - restart: unless-stopped - volumes: - - /etc/dokploy/templates/${HASH}/data:/pb_data - - /etc/dokploy/templates/${HASH}/public:/pb_public - - /etc/dokploy/templates/${HASH}/migrations:/pb_migrations diff --git a/apps/dokploy/templates/pocketbase/index.ts b/apps/dokploy/templates/pocketbase/index.ts deleted file mode 100644 index f9fc7f8f0..000000000 --- a/apps/dokploy/templates/pocketbase/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { - type DomainSchema, - type Schema, - type Template, - generateRandomDomain, -} from "../utils"; - -export function generate(schema: Schema): Template { - const mainDomain = generateRandomDomain(schema); - - const domains: DomainSchema[] = [ - { - host: mainDomain, - port: 80, - serviceName: "pocketbase", - }, - ]; - - return { - domains, - }; -} diff --git a/apps/dokploy/templates/templates.ts b/apps/dokploy/templates/templates.ts index d39465a8e..e1fff365e 100644 --- a/apps/dokploy/templates/templates.ts +++ b/apps/dokploy/templates/templates.ts @@ -47,21 +47,6 @@ export const templates: TemplateData[] = [ load: () => import("./supabase/index").then((m) => m.generate), tags: ["database", "firebase", "postgres"], }, - { - id: "pocketbase", - name: "Pocketbase", - version: "v0.22.12", - description: - "Pocketbase is a self-hosted alternative to Firebase that allows you to build and host your own backend services.", - links: { - github: "https://github.com/pocketbase/pocketbase", - website: "https://pocketbase.io/", - docs: "https://pocketbase.io/docs/", - }, - logo: "pocketbase.svg", - load: () => import("./pocketbase/index").then((m) => m.generate), - tags: ["database", "cms", "headless"], - }, { id: "plausible", name: "Plausible", diff --git a/apps/dokploy/templates/utils/index.ts b/apps/dokploy/templates/utils/index.ts index 941afc806..0c7f69774 100644 --- a/apps/dokploy/templates/utils/index.ts +++ b/apps/dokploy/templates/utils/index.ts @@ -28,7 +28,10 @@ export interface Template { export const generateRandomDomain = ({ serverIp, projectName, -}: Schema): string => { +}: { + serverIp: string; + projectName: string; +}): string => { const hash = randomBytes(3).toString("hex"); const slugIp = serverIp.replaceAll(".", "-"); @@ -41,9 +44,15 @@ export const generateHash = (projectName: string, quantity = 3): string => { }; export const generatePassword = (quantity = 16): string => { - return randomBytes(Math.ceil(quantity / 2)) - .toString("hex") - .slice(0, quantity); + const characters = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let password = ""; + for (let i = 0; i < quantity; i++) { + password += characters.charAt( + Math.floor(Math.random() * characters.length), + ); + } + return password.toLowerCase(); }; export const generateBase64 = (bytes = 32): string => { diff --git a/packages/server/src/templates/utils/github.ts b/packages/server/src/templates/utils/github.ts index ebbbbdb2d..a062f9e2b 100644 --- a/packages/server/src/templates/utils/github.ts +++ b/packages/server/src/templates/utils/github.ts @@ -1,13 +1,11 @@ -import { execSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { Octokit } from "@octokit/rest"; -import * as esbuild from "esbuild"; import { load } from "js-yaml"; import { templateConfig } from "../config"; -import type { Template } from "./index"; +import type { Schema, Template, DomainSchema } from "./index"; import { generateBase64, generateHash, @@ -21,20 +19,179 @@ const octokit = new Octokit({ }); /** - * Interface for template metadata + * Complete template interface that includes both metadata and configuration */ -export interface TemplateMetadata { - id: string; - name: string; - version: string; - description: string; - logo: string; - links: { - github?: string; - website?: string; - docs?: string; +export interface CompleteTemplate { + metadata: { + id: string; + name: string; + description: string; + tags: string[]; + version: string; + logo: string; + links: { + github: string; + website?: string; + docs?: string; + }; }; - tags: string[]; + variables: { + [key: string]: string; + }; + config: { + domains: Array<{ + serviceName: string; + port: number; + path?: string; + host?: string; + }>; + env: Record; + mounts?: Array<{ + filePath: string; + content: string; + }>; + }; +} + +/** + * Utility functions that can be used in template values + */ +const TEMPLATE_FUNCTIONS = { + $randomDomain: () => true, + $password: (length = 16) => `$password(${length})`, + $base64: (bytes = 32) => `$base64(${bytes})`, + $base32: (bytes = 32) => `$base32(${bytes})`, + $hash: (length = 8) => `$hash(${length})`, +} as const; + +/** + * Process a string value and replace variables + */ +function processValue( + value: string, + variables: Record, + schema: Schema, +): string { + // First replace utility functions + let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => { + // Handle utility functions + if (varName === "randomDomain") { + return generateRandomDomain(schema); + } + if (varName.startsWith("base64:")) { + const length = Number.parseInt(varName.split(":")[1], 10) || 32; + return generateBase64(length); + } + if (varName.startsWith("base32:")) { + const length = Number.parseInt(varName.split(":")[1], 10) || 32; + return Buffer.from(randomBytes(length)) + .toString("base64") + .substring(0, length); + } + if (varName.startsWith("password:")) { + const length = Number.parseInt(varName.split(":")[1], 10) || 16; + return generatePassword(length); + } + if (varName.startsWith("hash:")) { + const length = Number.parseInt(varName.split(":")[1], 10) || 8; + return generateHash(length); + } + // If not a utility function, try to get from variables + return variables[varName] || match; + }); + + // Then replace any remaining ${var} with their values from variables + processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => { + return variables[varName] || match; + }); + + return processedValue; +} + +/** + * Processes a template configuration and returns the generated template + */ +export function processTemplate( + config: CompleteTemplate, + schema: Schema, +): Template { + const result: Template = { + envs: [], + domains: [], + mounts: [], + }; + + // First pass: Process variables that don't depend on domains + const variables: Record = {}; + for (const [key, value] of Object.entries(config.variables)) { + if (value === "${randomDomain}") { + variables[key] = generateRandomDomain(schema); + } else if (value.startsWith("${base64:")) { + const match = value.match(/\${base64:(\d+)}/); + const length = match?.[1] ? Number.parseInt(match[1], 10) : 32; + variables[key] = generateBase64(length); + } else if (value.startsWith("${base32:")) { + const match = value.match(/\${base32:(\d+)}/); + const length = match?.[1] ? Number.parseInt(match[1], 10) : 32; + variables[key] = Buffer.from(randomBytes(length)) + .toString("base64") + .substring(0, length); + } else if (value.startsWith("${password:")) { + const match = value.match(/\${password:(\d+)}/); + const length = match?.[1] ? Number.parseInt(match[1], 10) : 16; + variables[key] = generatePassword(length); + } else if (value.startsWith("${hash:")) { + const match = value.match(/\${hash:(\d+)}/); + const length = match?.[1] ? Number.parseInt(match[1], 10) : 8; + variables[key] = generateHash(length); + } else { + variables[key] = value; + } + } + + console.log(variables); + + // Process domains and add them to variables + for (const domain of config.config.domains) { + // If host is specified, process it with variables, otherwise generate random domain + const host = domain.host + ? processValue(domain.host, variables, schema) + : generateRandomDomain(schema); + + result.domains.push({ + host, + ...domain, + }); + // Add domain to variables for reference + variables[`domain:${domain.serviceName}`] = host; + } + + // Process environment variables with access to all variables + for (const [key, value] of Object.entries(config.config.env)) { + const processedValue = processValue(value, variables, schema); + result.envs.push(`${key}=${processedValue}`); + } + + // Process mounts with access to all variables + if (config.config.mounts) { + for (const mount of config.config.mounts) { + result.mounts.push({ + filePath: mount.filePath, + content: processValue(mount.content, variables, schema), + }); + } + } + + return result; +} + +/** + * GitHub tree item with required fields + */ +interface GitTreeItem { + path: string; + type: string; + sha: string; } /** @@ -44,62 +201,54 @@ export async function fetchTemplatesList( owner = templateConfig.owner, repo = templateConfig.repo, branch = templateConfig.branch, -): Promise { +): Promise { try { - // Fetch templates directory content - const { data: dirContent } = await octokit.repos.getContent({ + // First get the tree SHA for the branch + const { data: ref } = await octokit.git.getRef({ owner, repo, - path: "templates", - ref: branch, + ref: `heads/${branch}`, }); - console.log("DIR CONTENT", dirContent); + // Get the full tree recursively + const { data: tree } = await octokit.git.getTree({ + owner, + repo, + tree_sha: ref.object.sha, + recursive: "true", + }); - if (!Array.isArray(dirContent)) { - throw new Error("Templates directory not found or is not a directory"); - } + // Filter for template.yml files in the templates directory + const templateFiles = tree.tree.filter((item): item is GitTreeItem => { + return ( + item.type === "blob" && + typeof item.path === "string" && + typeof item.sha === "string" && + item.path.startsWith("templates/") && + item.path.endsWith("/template.yml") + ); + }); - // Filter for directories only (each directory is a template) - const templateDirs = dirContent.filter((item) => item.type === "dir"); - - // Fetch metadata for each template + // Fetch and parse each template.yml const templates = await Promise.all( - templateDirs.map(async (dir) => { + templateFiles.map(async (file) => { try { - // Try to fetch metadata.json for each template - const { data: metadataFile } = await octokit.repos.getContent({ + const { data: content } = await octokit.git.getBlob({ owner, repo, - path: `templates/${dir.name}/metadata.json`, - ref: branch, + file_sha: file.sha, }); - if ("content" in metadataFile && metadataFile.encoding === "base64") { - const content = Buffer.from( - metadataFile.content, - "base64", - ).toString(); - return JSON.parse(content) as TemplateMetadata; - } + const decoded = Buffer.from(content.content, "base64").toString(); + return load(decoded) as CompleteTemplate; } catch (error) { - // If metadata.json doesn't exist, create a basic metadata object - return { - id: dir.name, - name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1), - version: "latest", - description: `${dir.name} template`, - logo: "default.svg", - links: {}, - tags: [], - }; + console.warn(`Failed to load template from ${file.path}:`, error); + return null; } - - return null; }), ); - return templates.filter(Boolean) as TemplateMetadata[]; + return templates.filter(Boolean) as CompleteTemplate[]; } catch (error) { console.error("Error fetching templates list:", error); throw error; @@ -114,32 +263,73 @@ export async function fetchTemplateFiles( owner = templateConfig.owner, repo = templateConfig.repo, branch = templateConfig.branch, -): Promise<{ indexTs: string; dockerCompose: string }> { +): Promise<{ config: CompleteTemplate; dockerCompose: string }> { try { - // Fetch index.ts - const { data: indexFile } = await octokit.repos.getContent({ + // Get the tree SHA for the branch + const { data: ref } = await octokit.git.getRef({ owner, repo, - path: `templates/${templateId}/index.ts`, - ref: branch, + ref: `heads/${branch}`, }); - // Fetch docker-compose.yml - const { data: composeFile } = await octokit.repos.getContent({ + // Get the full tree recursively + const { data: tree } = await octokit.git.getTree({ owner, repo, - path: `templates/${templateId}/docker-compose.yml`, - ref: branch, + tree_sha: ref.object.sha, + recursive: "true", }); - if (!("content" in indexFile) || !("content" in composeFile)) { + // Find the template.yml and docker-compose.yml files + const templateYml = tree.tree + .filter((item): item is GitTreeItem => { + return ( + item.type === "blob" && + typeof item.path === "string" && + typeof item.sha === "string" + ); + }) + .find((item) => item.path === `templates/${templateId}/template.yml`); + + const dockerComposeYml = tree.tree + .filter((item): item is GitTreeItem => { + return ( + item.type === "blob" && + typeof item.path === "string" && + typeof item.sha === "string" + ); + }) + .find( + (item) => item.path === `templates/${templateId}/docker-compose.yml`, + ); + + if (!templateYml || !dockerComposeYml) { throw new Error("Template files not found"); } - const indexTs = Buffer.from(indexFile.content, "base64").toString(); - const dockerCompose = Buffer.from(composeFile.content, "base64").toString(); + // Fetch both files in parallel + const [templateContent, composeContent] = await Promise.all([ + octokit.git.getBlob({ + owner, + repo, + file_sha: templateYml.sha, + }), + octokit.git.getBlob({ + owner, + repo, + file_sha: dockerComposeYml.sha, + }), + ]); - return { indexTs, dockerCompose }; + const config = load( + Buffer.from(templateContent.data.content, "base64").toString(), + ) as CompleteTemplate; + const dockerCompose = Buffer.from( + composeContent.data.content, + "base64", + ).toString(); + + return { config, dockerCompose }; } catch (error) { console.error(`Error fetching template ${templateId}:`, error); throw error; @@ -147,179 +337,11 @@ export async function fetchTemplateFiles( } /** - * Executes the template's index.ts code dynamically - * Uses a template-based approach that's safer and more efficient + * Loads and processes a template */ -export async function executeTemplateCode( - indexTsCode: string, - schema: { serverIp: string; projectName: string }, -): Promise