feat: implement comprehensive environment variable resolution in preparation functions, enhancing flexibility and support for nested references across services and environments

This commit is contained in:
Mauricio Siu
2025-09-03 21:41:11 -06:00
parent 4c5771b55b
commit fb749cd862
18 changed files with 441 additions and 41 deletions

View File

@@ -0,0 +1,335 @@
import { prepareEnvironmentVariables } from "@dokploy/server/index";
import { describe, expect, it } from "vitest";
const projectEnv = `
ENVIRONMENT=staging
DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db
PORT=3000
`;
const environmentEnv = `
NODE_ENV=development
API_URL=https://api.dev.example.com
REDIS_URL=redis://localhost:6379
DATABASE_NAME=dev_database
SECRET_KEY=env-secret-123
`;
describe("prepareEnvironmentVariables (environment variables)", () => {
it("resolves environment variables correctly", () => {
const serviceWithEnvVars = `
NODE_ENV=\${{environment.NODE_ENV}}
API_URL=\${{environment.API_URL}}
SERVICE_PORT=4000
`;
const resolved = prepareEnvironmentVariables(
serviceWithEnvVars,
"",
environmentEnv,
);
expect(resolved).toEqual([
"NODE_ENV=development",
"API_URL=https://api.dev.example.com",
"SERVICE_PORT=4000",
]);
});
it("resolves both project and environment variables", () => {
const serviceWithBoth = `
ENVIRONMENT=\${{project.ENVIRONMENT}}
NODE_ENV=\${{environment.NODE_ENV}}
API_URL=\${{environment.API_URL}}
DATABASE_URL=\${{project.DATABASE_URL}}
SERVICE_PORT=4000
`;
const resolved = prepareEnvironmentVariables(
serviceWithBoth,
projectEnv,
environmentEnv,
);
expect(resolved).toEqual([
"ENVIRONMENT=staging",
"NODE_ENV=development",
"API_URL=https://api.dev.example.com",
"DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db",
"SERVICE_PORT=4000",
]);
});
it("handles undefined environment variables", () => {
const serviceWithUndefined = `
UNDEFINED_VAR=\${{environment.UNDEFINED_VAR}}
`;
expect(() =>
prepareEnvironmentVariables(serviceWithUndefined, "", environmentEnv),
).toThrow("Invalid environment variable: environment.UNDEFINED_VAR");
});
it("allows service variables to override environment variables", () => {
const serviceOverrideEnv = `
NODE_ENV=production
API_URL=\${{environment.API_URL}}
`;
const resolved = prepareEnvironmentVariables(
serviceOverrideEnv,
"",
environmentEnv,
);
expect(resolved).toEqual([
"NODE_ENV=production", // Overrides environment variable
"API_URL=https://api.dev.example.com",
]);
});
it("resolves complex references with project, environment, and service variables", () => {
const complexServiceEnv = `
FULL_DATABASE_URL=\${{project.DATABASE_URL}}/\${{environment.DATABASE_NAME}}
API_ENDPOINT=\${{environment.API_URL}}/\${{project.ENVIRONMENT}}/api
SERVICE_NAME=my-service
COMPLEX_VAR=\${{SERVICE_NAME}}-\${{environment.NODE_ENV}}-\${{project.ENVIRONMENT}}
`;
const resolved = prepareEnvironmentVariables(
complexServiceEnv,
projectEnv,
environmentEnv,
);
expect(resolved).toEqual([
"FULL_DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db/dev_database",
"API_ENDPOINT=https://api.dev.example.com/staging/api",
"SERVICE_NAME=my-service",
"COMPLEX_VAR=my-service-development-staging",
]);
});
it("handles environment variables with special characters", () => {
const specialEnvVars = `
SPECIAL_URL=https://special.com
COMPLEX_KEY="key-with-@#$%^&*()"
JWT_SECRET="secret-with-spaces and symbols!@#"
`;
const serviceWithSpecial = `
FULL_URL=\${{environment.SPECIAL_URL}}/path?key=\${{environment.COMPLEX_KEY}}
AUTH_SECRET=\${{environment.JWT_SECRET}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithSpecial,
"",
specialEnvVars,
);
expect(resolved).toEqual([
"FULL_URL=https://special.com/path?key=key-with-@#$%^&*()",
"AUTH_SECRET=secret-with-spaces and symbols!@#",
]);
});
it("maintains precedence: service > environment > project", () => {
const conflictingProjectEnv = `
NODE_ENV=production-project
API_URL=https://project.api.com
DATABASE_NAME=project_db
`;
const conflictingEnvironmentEnv = `
NODE_ENV=development-environment
API_URL=https://environment.api.com
DATABASE_NAME=env_db
`;
const serviceWithConflicts = `
NODE_ENV=service-override
PROJECT_ENV=\${{project.NODE_ENV}}
ENV_VAR=\${{environment.API_URL}}
DB_NAME=\${{environment.DATABASE_NAME}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithConflicts,
conflictingProjectEnv,
conflictingEnvironmentEnv,
);
expect(resolved).toEqual([
"NODE_ENV=service-override", // Service wins
"PROJECT_ENV=production-project", // Project reference
"ENV_VAR=https://environment.api.com", // Environment reference
"DB_NAME=env_db", // Environment reference
]);
});
it("handles empty environment variables", () => {
const serviceWithEmpty = `
SERVICE_VAR=test
PROJECT_VAR=\${{project.ENVIRONMENT}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithEmpty,
projectEnv,
"",
);
expect(resolved).toEqual(["SERVICE_VAR=test", "PROJECT_VAR=staging"]);
});
it("handles mixed quotes and environment variables", () => {
const envWithQuotes = `
QUOTED_VAR="development"
SINGLE_QUOTED='https://api.dev.example.com'
MIXED_VAR="value with 'single' quotes"
`;
const serviceWithQuotes = `
NODE_ENV=\${{environment.QUOTED_VAR}}
API_URL=\${{environment.SINGLE_QUOTED}}
COMPLEX="Prefix-\${{environment.MIXED_VAR}}-Suffix"
`;
const resolved = prepareEnvironmentVariables(
serviceWithQuotes,
"",
envWithQuotes,
);
expect(resolved).toEqual([
"NODE_ENV=development",
"API_URL=https://api.dev.example.com",
"COMPLEX=Prefix-value with 'single' quotes-Suffix",
]);
});
it("resolves multiple environment references in single value", () => {
const multiRefEnv = `
HOST=localhost
PORT=5432
USERNAME=postgres
PASSWORD=secret123
`;
const serviceWithMultiRefs = `
DATABASE_URL=postgresql://\${{environment.USERNAME}}:\${{environment.PASSWORD}}@\${{environment.HOST}}:\${{environment.PORT}}/mydb
CONNECTION_STRING=\${{environment.HOST}}:\${{environment.PORT}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithMultiRefs,
"",
multiRefEnv,
);
expect(resolved).toEqual([
"DATABASE_URL=postgresql://postgres:secret123@localhost:5432/mydb",
"CONNECTION_STRING=localhost:5432",
]);
});
it("handles nested references with environment and project variables", () => {
const nestedProjectEnv = `
BASE_DOMAIN=example.com
PROTOCOL=https
`;
const nestedEnvironmentEnv = `
SUBDOMAIN=api.dev
PATH_PREFIX=/v1
`;
const serviceWithNested = `
FULL_URL=\${{project.PROTOCOL}}://\${{environment.SUBDOMAIN}}.\${{project.BASE_DOMAIN}}\${{environment.PATH_PREFIX}}/endpoint
API_BASE=\${{project.PROTOCOL}}://\${{environment.SUBDOMAIN}}.\${{project.BASE_DOMAIN}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithNested,
nestedProjectEnv,
nestedEnvironmentEnv,
);
expect(resolved).toEqual([
"FULL_URL=https://api.dev.example.com/v1/endpoint",
"API_BASE=https://api.dev.example.com",
]);
});
it("throws error for malformed environment variable references", () => {
const serviceWithMalformed = `
MALFORMED1=\${{environment.}}
MALFORMED2=\${{environment}}
VALID=\${{environment.NODE_ENV}}
`;
// Should throw error for empty variable name after environment.
expect(() =>
prepareEnvironmentVariables(serviceWithMalformed, "", environmentEnv),
).toThrow("Invalid environment variable: environment.");
});
it("handles environment variables with numeric values", () => {
const numericEnv = `
PORT=8080
TIMEOUT=30
RETRY_COUNT=3
PERCENTAGE=99.5
`;
const serviceWithNumeric = `
SERVER_PORT=\${{environment.PORT}}
REQUEST_TIMEOUT=\${{environment.TIMEOUT}}
MAX_RETRIES=\${{environment.RETRY_COUNT}}
SUCCESS_RATE=\${{environment.PERCENTAGE}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithNumeric,
"",
numericEnv,
);
expect(resolved).toEqual([
"SERVER_PORT=8080",
"REQUEST_TIMEOUT=30",
"MAX_RETRIES=3",
"SUCCESS_RATE=99.5",
]);
});
it("handles boolean-like environment variables", () => {
const booleanEnv = `
DEBUG=true
ENABLED=false
PRODUCTION=1
DEVELOPMENT=0
`;
const serviceWithBoolean = `
DEBUG_MODE=\${{environment.DEBUG}}
FEATURE_ENABLED=\${{environment.ENABLED}}
IS_PROD=\${{environment.PRODUCTION}}
IS_DEV=\${{environment.DEVELOPMENT}}
`;
const resolved = prepareEnvironmentVariables(
serviceWithBoolean,
"",
booleanEnv,
);
expect(resolved).toEqual([
"DEBUG_MODE=true",
"FEATURE_ENABLED=false",
"IS_PROD=1",
"IS_DEV=0",
]);
});
});

View File

@@ -1,4 +1,4 @@
import type { findEnvironmentById } from "@dokploy/server";
import type { findEnvironmentsByProjectId } from "@dokploy/server";
import {
ChevronDownIcon,
PencilIcon,
@@ -33,10 +33,9 @@ import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
import { api } from "@/utils/api";
type Environment = Omit<
Awaited<ReturnType<typeof findEnvironmentById>>,
"project"
>;
type Environment = Awaited<
ReturnType<typeof findEnvironmentsByProjectId>
>[number];
interface AdvancedEnvironmentSelectorProps {
projectId: string;
currentEnvironmentId?: string;
@@ -53,13 +52,12 @@ export const AdvancedEnvironmentSelector = ({
const [selectedEnvironment, setSelectedEnvironment] =
useState<Environment | null>(null);
const { data: project } = api.project.one.useQuery(
{ projectId },
const { data: environments } = api.environment.byProjectId.useQuery(
{ projectId: projectId },
{
enabled: !!projectId,
},
);
const environments = project?.environments || [];
// Form states
const [name, setName] = useState("");
@@ -144,7 +142,7 @@ export const AdvancedEnvironmentSelector = ({
// Redirect to production if we deleted the current environment
if (selectedEnvironment.environmentId === currentEnvironmentId) {
const productionEnv = environments.find(
const productionEnv = environments?.find(
(env) => env.name === "production",
);
if (productionEnv) {
@@ -190,7 +188,7 @@ export const AdvancedEnvironmentSelector = ({
setIsDeleteDialogOpen(true);
};
const currentEnv = environments.find(
const currentEnv = environments?.find(
(env) => env.environmentId === currentEnvironmentId,
);
@@ -210,7 +208,7 @@ export const AdvancedEnvironmentSelector = ({
<DropdownMenuLabel>Environments</DropdownMenuLabel>
<DropdownMenuSeparator />
{environments.map((environment) => (
{environments?.map((environment) => (
<div key={environment.environmentId} className="flex items-center">
<DropdownMenuItem
className="flex-1 cursor-pointer"
@@ -229,6 +227,18 @@ export const AdvancedEnvironmentSelector = ({
</DropdownMenuItem>
{/* Action buttons for non-production environments */}
<EnvironmentVariables environmentId={environment.environmentId}>
<Button
variant="ghost"
size="sm"
className="h-6 w-6 p-0"
onClick={(e) => {
e.stopPropagation();
}}
>
<Terminal className="h-3 w-3" />
</Button>
</EnvironmentVariables>
{environment.name !== "production" && (
<div className="flex items-center gap-1 px-2">
<Button
@@ -242,20 +252,7 @@ export const AdvancedEnvironmentSelector = ({
>
<PencilIcon className="h-3 w-3" />
</Button>
<EnvironmentVariables
environmentId={environment.environmentId}
>
<Button
variant="ghost"
size="sm"
className="h-6 w-6 p-0"
onClick={(e) => {
e.stopPropagation();
}}
>
<Terminal className="h-3 w-3" />
</Button>
</EnvironmentVariables>
<Button
variant="ghost"
size="sm"

View File

@@ -1460,6 +1460,10 @@ export async function getServerSideProps(
environmentId: params.environmentId,
});
await helpers.environment.byProjectId.fetch({
projectId: params.projectId,
});
return {
props: {
trpcState: helpers.dehydrate(),

View File

@@ -5,7 +5,7 @@ import {
environments,
} from "@dokploy/server/db/schema";
import { TRPCError } from "@trpc/server";
import { eq } from "drizzle-orm";
import { asc, eq } from "drizzle-orm";
export type Environment = typeof environments.$inferSelect;
@@ -56,7 +56,17 @@ export const findEnvironmentById = async (environmentId: string) => {
export const findEnvironmentsByProjectId = async (projectId: string) => {
const projectEnvironments = await db.query.environments.findMany({
where: eq(environments.projectId, projectId),
orderBy: (environments, { asc }) => [asc(environments.createdAt)],
orderBy: asc(environments.createdAt),
with: {
applications: true,
mariadb: true,
mongo: true,
mysql: true,
postgres: true,
redis: true,
compose: true,
project: true,
},
});
return projectEnvironments;
};

View File

@@ -206,6 +206,7 @@ const createEnvFile = (compose: ComposeNested) => {
const envFileContent = prepareEnvironmentVariables(
envContent,
compose.environment.project.env,
compose.environment.env,
).join("\n");
if (!existsSync(dirname(envFilePath))) {
@@ -236,6 +237,7 @@ export const getCreateEnvFileCommand = (compose: ComposeNested) => {
const envFileContent = prepareEnvironmentVariables(
envContent,
compose.environment.project.env,
compose.environment.env,
).join("\n");
const encodedContent = encodeBase64(envFileContent);

View File

@@ -29,6 +29,7 @@ export const buildCustomDocker = async (
const args = prepareEnvironmentVariables(
buildArgs,
application.environment.project.env,
application.environment.env,
);
const dockerContextPath = getDockerContextPath(application);
@@ -51,7 +52,12 @@ export const buildCustomDocker = async (
as it could be publicly exposed.
*/
if (!publishDirectory) {
createEnvFile(dockerFilePath, env, application.environment.project.env);
createEnvFile(
dockerFilePath,
env,
application.environment.project.env,
application.environment.env,
);
}
await spawnAsync(
@@ -93,6 +99,7 @@ export const getDockerCommand = (
const args = prepareEnvironmentVariables(
buildArgs,
application.environment.project.env,
application.environment.env,
);
const dockerContextPath =
@@ -122,6 +129,7 @@ export const getDockerCommand = (
dockerFilePath,
env,
application.environment.project.env,
application.environment.env,
);
}

View File

@@ -14,6 +14,7 @@ export const buildHeroku = async (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
try {
const args = [
@@ -54,6 +55,7 @@ export const getHerokuCommand = (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
const args = [

View File

@@ -149,6 +149,7 @@ export const mechanizeDockerContainer = async (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
const image = getImageName(application);

View File

@@ -21,6 +21,7 @@ export const buildNixpacks = async (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
const writeToStream = (data: string) => {
@@ -102,6 +103,7 @@ export const getNixpacksCommand = (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
const args = ["build", buildAppDirectory, "--name", appName];

View File

@@ -13,6 +13,7 @@ export const buildPaketo = async (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
try {
const args = [
@@ -53,6 +54,7 @@ export const getPaketoCommand = (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
const args = [

View File

@@ -27,6 +27,7 @@ export const buildRailpack = async (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
try {
@@ -124,6 +125,7 @@ export const getRailpackCommand = (
const envVariables = prepareEnvironmentVariables(
env,
application.environment.project.env,
application.environment.env,
);
// Prepare command

View File

@@ -6,14 +6,17 @@ export const createEnvFile = (
directory: string,
env: string | null,
projectEnv?: string | null,
environmentEnv?: string | null,
) => {
const envFilePath = join(dirname(directory), ".env");
if (!existsSync(dirname(envFilePath))) {
mkdirSync(dirname(envFilePath), { recursive: true });
}
const envFileContent = prepareEnvironmentVariables(env, projectEnv).join(
"\n",
);
const envFileContent = prepareEnvironmentVariables(
env,
projectEnv,
environmentEnv,
).join("\n");
writeFileSync(envFilePath, envFileContent);
};
@@ -21,10 +24,13 @@ export const createEnvFileCommand = (
directory: string,
env: string | null,
projectEnv?: string | null,
environmentEnv?: string | null,
) => {
const envFileContent = prepareEnvironmentVariables(env, projectEnv).join(
"\n",
);
const envFileContent = prepareEnvironmentVariables(
env,
projectEnv,
environmentEnv,
).join("\n");
const encodedContent = encodeBase64(envFileContent || "");
const envFilePath = join(dirname(directory), ".env");

View File

@@ -55,6 +55,7 @@ export const buildMariadb = async (mariadb: MariadbNested) => {
const envVariables = prepareEnvironmentVariables(
defaultMariadbEnv,
mariadb.environment.project.env,
mariadb.environment.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);

View File

@@ -103,6 +103,7 @@ ${command ?? "wait $MONGOD_PID"}`;
const envVariables = prepareEnvironmentVariables(
defaultMongoEnv,
mongo.environment.project.env,
mongo.environment.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);

View File

@@ -61,6 +61,7 @@ export const buildMysql = async (mysql: MysqlNested) => {
const envVariables = prepareEnvironmentVariables(
defaultMysqlEnv,
mysql.environment.project.env,
mysql.environment.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);

View File

@@ -54,6 +54,7 @@ export const buildPostgres = async (postgres: PostgresNested) => {
const envVariables = prepareEnvironmentVariables(
defaultPostgresEnv,
postgres.environment.project.env,
postgres.environment.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);

View File

@@ -52,6 +52,7 @@ export const buildRedis = async (redis: RedisNested) => {
const envVariables = prepareEnvironmentVariables(
defaultRedisEnv,
redis.environment.project.env,
redis.environment.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);

View File

@@ -259,21 +259,44 @@ export const removeService = async (
export const prepareEnvironmentVariables = (
serviceEnv: string | null,
projectEnv?: string | null,
environmentEnv?: string | null,
) => {
const projectVars = parse(projectEnv ?? "");
const environmentVars = parse(environmentEnv ?? "");
const serviceVars = parse(serviceEnv ?? "");
const resolvedVars = Object.entries(serviceVars).map(([key, value]) => {
let resolvedValue = value;
// Replace project variables
if (projectVars) {
resolvedValue = value.replace(/\$\{\{project\.(.*?)\}\}/g, (_, ref) => {
if (projectVars[ref] !== undefined) {
return projectVars[ref];
}
throw new Error(`Invalid project environment variable: project.${ref}`);
});
resolvedValue = resolvedValue.replace(
/\$\{\{project\.(.*?)\}\}/g,
(_, ref) => {
if (projectVars[ref] !== undefined) {
return projectVars[ref];
}
throw new Error(
`Invalid project environment variable: project.${ref}`,
);
},
);
}
// Replace environment variables
if (environmentVars) {
resolvedValue = resolvedValue.replace(
/\$\{\{environment\.(.*?)\}\}/g,
(_, ref) => {
if (environmentVars[ref] !== undefined) {
return environmentVars[ref];
}
throw new Error(`Invalid environment variable: environment.${ref}`);
},
);
}
// Replace self-references (service variables)
resolvedValue = resolvedValue.replace(/\$\{\{(.*?)\}\}/g, (_, ref) => {
if (serviceVars[ref] !== undefined) {
return serviceVars[ref];
@@ -301,8 +324,9 @@ export const parseEnvironmentKeyValuePair = (
export const getEnviromentVariablesObject = (
input: string | null,
projectEnv?: string | null,
environmentEnv?: string | null,
) => {
const envs = prepareEnvironmentVariables(input, projectEnv);
const envs = prepareEnvironmentVariables(input, projectEnv, environmentEnv);
const jsonObject: Record<string, string> = {};