mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-07-09 07:55:25 +02:00
feat: add builds concurrency management for servers
- Introduced a new `BuildsConcurrency` component to manage the number of concurrent builds for both local and remote servers, gated by license validity. - Implemented backend logic to resolve effective builds concurrency based on server settings and organization licenses. - Added unit tests for concurrency resolution logic to ensure correct behavior under various licensing scenarios. - Updated database schema to include `buildsConcurrency` field for servers and web server settings. - Refactored deployment queue to support in-memory job processing with configurable concurrency limits. This feature enhances deployment flexibility and control for enterprise users.
This commit is contained in:
@@ -68,11 +68,9 @@ import {
|
||||
environments,
|
||||
projects,
|
||||
} from "@/server/db/schema";
|
||||
import { deploymentWorker } from "@/server/queues/deployments-queue";
|
||||
import type { DeploymentJob } from "@/server/queues/queue-types";
|
||||
import {
|
||||
cleanQueuesByApplication,
|
||||
getJobsByApplicationId,
|
||||
killDockerBuild,
|
||||
myQueue,
|
||||
} from "@/server/queues/queueSetup";
|
||||
@@ -242,12 +240,7 @@ export const applicationRouter = createTRPCRouter({
|
||||
.returning();
|
||||
|
||||
if (!IS_CLOUD) {
|
||||
const queueJobs = await getJobsByApplicationId(input.applicationId);
|
||||
for (const job of queueJobs) {
|
||||
if (job.id) {
|
||||
deploymentWorker.cancelJob(job.id, "User requested cancellation");
|
||||
}
|
||||
}
|
||||
await cleanQueuesByApplication(input.applicationId);
|
||||
}
|
||||
|
||||
const cleanupOperations = [
|
||||
@@ -339,10 +332,10 @@ export const applicationRouter = createTRPCRouter({
|
||||
type: "redeploy",
|
||||
applicationType: "application",
|
||||
server: !!application.serverId,
|
||||
serverId: application.serverId ?? undefined,
|
||||
};
|
||||
|
||||
if (IS_CLOUD && application.serverId) {
|
||||
jobData.serverId = application.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
@@ -707,9 +700,9 @@ export const applicationRouter = createTRPCRouter({
|
||||
type: "deploy",
|
||||
applicationType: "application",
|
||||
server: !!application.serverId,
|
||||
serverId: application.serverId ?? undefined,
|
||||
};
|
||||
if (IS_CLOUD && application.serverId) {
|
||||
jobData.serverId = application.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
@@ -826,9 +819,9 @@ export const applicationRouter = createTRPCRouter({
|
||||
type: "deploy",
|
||||
applicationType: "application",
|
||||
server: !!app.serverId,
|
||||
serverId: app.serverId ?? undefined,
|
||||
};
|
||||
if (IS_CLOUD && app.serverId) {
|
||||
jobData.serverId = app.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
|
||||
@@ -68,11 +68,9 @@ import {
|
||||
environments,
|
||||
projects,
|
||||
} from "@/server/db/schema";
|
||||
import { deploymentWorker } from "@/server/queues/deployments-queue";
|
||||
import type { DeploymentJob } from "@/server/queues/queue-types";
|
||||
import {
|
||||
cleanQueuesByCompose,
|
||||
getJobsByComposeId,
|
||||
killDockerBuild,
|
||||
myQueue,
|
||||
} from "@/server/queues/queueSetup";
|
||||
@@ -252,12 +250,7 @@ export const composeRouter = createTRPCRouter({
|
||||
.returning();
|
||||
|
||||
if (!IS_CLOUD) {
|
||||
const queueJobs = await getJobsByComposeId(input.composeId);
|
||||
for (const job of queueJobs) {
|
||||
if (job.id) {
|
||||
deploymentWorker.cancelJob(job.id, "User requested cancellation");
|
||||
}
|
||||
}
|
||||
await cleanQueuesByCompose(input.composeId);
|
||||
}
|
||||
|
||||
const cleanupOperations = [
|
||||
@@ -430,10 +423,10 @@ export const composeRouter = createTRPCRouter({
|
||||
applicationType: "compose",
|
||||
descriptionLog: input.description || "",
|
||||
server: !!compose.serverId,
|
||||
serverId: compose.serverId ?? undefined,
|
||||
};
|
||||
|
||||
if (IS_CLOUD && compose.serverId) {
|
||||
jobData.serverId = compose.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
@@ -479,9 +472,9 @@ export const composeRouter = createTRPCRouter({
|
||||
applicationType: "compose",
|
||||
descriptionLog: input.description || "",
|
||||
server: !!compose.serverId,
|
||||
serverId: compose.serverId ?? undefined,
|
||||
};
|
||||
if (IS_CLOUD && compose.serverId) {
|
||||
jobData.serverId = compose.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
|
||||
@@ -86,10 +86,10 @@ export const previewDeploymentRouter = createTRPCRouter({
|
||||
applicationType: "application-preview",
|
||||
previewDeploymentId: input.previewDeploymentId,
|
||||
server: !!application.serverId,
|
||||
serverId: application.serverId ?? undefined,
|
||||
};
|
||||
|
||||
if (IS_CLOUD && application.serverId) {
|
||||
jobData.serverId = application.serverId;
|
||||
deploy(jobData).catch((error) => {
|
||||
console.error("Background deployment failed:", error);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ import { z } from "zod";
|
||||
import { updateServersBasedOnQuantity } from "@/pages/api/stripe/webhook";
|
||||
import {
|
||||
createTRPCRouter,
|
||||
enterpriseProcedure,
|
||||
protectedProcedure,
|
||||
withPermission,
|
||||
} from "@/server/api/trpc";
|
||||
@@ -34,6 +35,7 @@ import {
|
||||
apiFindOneServer,
|
||||
apiRemoveServer,
|
||||
apiUpdateServer,
|
||||
apiUpdateServerBuildsConcurrency,
|
||||
apiUpdateServerMonitoring,
|
||||
applications,
|
||||
compose,
|
||||
@@ -479,6 +481,20 @@ export const serverRouter = createTRPCRouter({
|
||||
throw error;
|
||||
}
|
||||
}),
|
||||
updateBuildsConcurrency: enterpriseProcedure
|
||||
.input(apiUpdateServerBuildsConcurrency)
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const currentServer = await findServerById(input.serverId);
|
||||
if (currentServer.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not authorized to update this server",
|
||||
});
|
||||
}
|
||||
return await updateServerById(input.serverId, {
|
||||
buildsConcurrency: input.buildsConcurrency,
|
||||
});
|
||||
}),
|
||||
publicIp: protectedProcedure.query(async () => {
|
||||
if (IS_CLOUD) {
|
||||
return "";
|
||||
|
||||
@@ -67,6 +67,7 @@ import {
|
||||
apiServerSchema,
|
||||
apiTraefikConfig,
|
||||
apiUpdateDockerCleanup,
|
||||
apiUpdateWebServerBuildsConcurrency,
|
||||
projects,
|
||||
server,
|
||||
} from "@/server/db/schema";
|
||||
@@ -468,6 +469,28 @@ export const settingsRouter = createTRPCRouter({
|
||||
return true;
|
||||
}),
|
||||
|
||||
updateBuildsConcurrency: enterpriseProcedure
|
||||
.input(apiUpdateWebServerBuildsConcurrency)
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
if (IS_CLOUD) {
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
message: "This feature is only available for self-hosted instances",
|
||||
});
|
||||
}
|
||||
|
||||
await updateWebServerSettings({
|
||||
buildsConcurrency: input.buildsConcurrency,
|
||||
});
|
||||
|
||||
await audit(ctx, {
|
||||
action: "update",
|
||||
resourceType: "settings",
|
||||
resourceName: "builds-concurrency",
|
||||
});
|
||||
return true;
|
||||
}),
|
||||
|
||||
updateEnforceSSO: enterpriseProcedure
|
||||
.input(z.object({ enforceSSO: z.boolean() }))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
|
||||
66
apps/dokploy/server/queues/concurrency.ts
Normal file
66
apps/dokploy/server/queues/concurrency.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { db } from "@dokploy/server/db";
|
||||
import { organization, server } from "@dokploy/server/db/schema";
|
||||
import { hasValidLicense } from "@dokploy/server/services/proprietary/license-key";
|
||||
import { getWebServerSettings } from "@dokploy/server/services/web-server-settings";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { LOCAL_PARTITION } from "./in-memory-queue";
|
||||
|
||||
/**
|
||||
* Resolve the effective builds concurrency for a queue partition.
|
||||
*
|
||||
* Concurrent deployments (concurrency > 1) are an enterprise feature: without a
|
||||
* valid license the effective concurrency is always clamped to 1, so the
|
||||
* community experience is unchanged and an expired license degrades gracefully
|
||||
* back to sequential deployments instead of breaking anything.
|
||||
*
|
||||
* - `LOCAL_PARTITION` -> concurrency stored on the web server settings (the
|
||||
* local Dokploy web server), gated by the owner organization's license.
|
||||
* - any other partition -> concurrency stored on the matching `server` row,
|
||||
* gated by that server's organization license.
|
||||
*/
|
||||
export const resolveBuildsConcurrency = async (
|
||||
partition: string,
|
||||
): Promise<number> => {
|
||||
try {
|
||||
if (partition === LOCAL_PARTITION) {
|
||||
return await resolveLocalConcurrency();
|
||||
}
|
||||
return await resolveServerConcurrency(partition);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"Failed to resolve builds concurrency, defaulting to 1",
|
||||
error,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
};
|
||||
|
||||
const clamp = (value: number, licensed: boolean): number => {
|
||||
if (!licensed) return 1;
|
||||
return Math.min(20, Math.max(1, value));
|
||||
};
|
||||
|
||||
const resolveLocalConcurrency = async (): Promise<number> => {
|
||||
const settings = await getWebServerSettings();
|
||||
const buildsConcurrency = settings?.buildsConcurrency ?? 1;
|
||||
|
||||
// Self-hosted is single-tenant; gate on any organization's license.
|
||||
const anyOrg = await db.query.organization.findFirst({
|
||||
columns: { id: true },
|
||||
});
|
||||
const licensed = anyOrg ? await hasValidLicense(anyOrg.id) : false;
|
||||
|
||||
return clamp(buildsConcurrency, licensed);
|
||||
};
|
||||
|
||||
const resolveServerConcurrency = async (serverId: string): Promise<number> => {
|
||||
const currentServer = await db.query.server.findFirst({
|
||||
where: eq(server.serverId, serverId),
|
||||
columns: { buildsConcurrency: true, organizationId: true },
|
||||
});
|
||||
|
||||
if (!currentServer) return 1;
|
||||
|
||||
const licensed = await hasValidLicense(currentServer.organizationId);
|
||||
return clamp(currentServer.buildsConcurrency, licensed);
|
||||
};
|
||||
@@ -2,7 +2,6 @@ import {
|
||||
deployApplication,
|
||||
deployCompose,
|
||||
deployPreviewApplication,
|
||||
IS_CLOUD,
|
||||
rebuildApplication,
|
||||
rebuildCompose,
|
||||
rebuildPreviewApplication,
|
||||
@@ -10,87 +9,69 @@ import {
|
||||
updateCompose,
|
||||
updatePreviewDeployment,
|
||||
} from "@dokploy/server";
|
||||
import { type Job, Worker } from "bullmq";
|
||||
import type { DeploymentJob } from "./queue-types";
|
||||
import { redisConfig } from "./redis-connection";
|
||||
import type { InMemoryJob } from "./in-memory-queue";
|
||||
|
||||
const createDeploymentWorker = () =>
|
||||
new Worker(
|
||||
"deployments",
|
||||
async (job: Job<DeploymentJob>) => {
|
||||
try {
|
||||
if (job.data.applicationType === "application") {
|
||||
await updateApplicationStatus(job.data.applicationId, "running");
|
||||
/**
|
||||
* Processes a single deployment job. Shared by the in-memory queue worker and
|
||||
* (in cloud) the direct background execution path.
|
||||
*/
|
||||
export const processDeploymentJob = async (job: InMemoryJob) => {
|
||||
try {
|
||||
if (job.data.applicationType === "application") {
|
||||
await updateApplicationStatus(job.data.applicationId, "running");
|
||||
|
||||
if (job.data.type === "redeploy") {
|
||||
await rebuildApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
} else if (job.data.type === "deploy") {
|
||||
await deployApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
}
|
||||
} else if (job.data.applicationType === "compose") {
|
||||
await updateCompose(job.data.composeId, {
|
||||
composeStatus: "running",
|
||||
});
|
||||
if (job.data.type === "deploy") {
|
||||
await deployCompose({
|
||||
composeId: job.data.composeId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
} else if (job.data.type === "redeploy") {
|
||||
await rebuildCompose({
|
||||
composeId: job.data.composeId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
}
|
||||
} else if (job.data.applicationType === "application-preview") {
|
||||
await updatePreviewDeployment(job.data.previewDeploymentId, {
|
||||
previewStatus: "running",
|
||||
});
|
||||
|
||||
if (job.data.type === "redeploy") {
|
||||
await rebuildPreviewApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
previewDeploymentId: job.data.previewDeploymentId,
|
||||
});
|
||||
} else if (job.data.type === "deploy") {
|
||||
await deployPreviewApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
previewDeploymentId: job.data.previewDeploymentId,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("Error", error);
|
||||
if (job.data.type === "redeploy") {
|
||||
await rebuildApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
} else if (job.data.type === "deploy") {
|
||||
await deployApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
}
|
||||
},
|
||||
{
|
||||
autorun: false,
|
||||
connection: redisConfig,
|
||||
},
|
||||
);
|
||||
} else if (job.data.applicationType === "compose") {
|
||||
await updateCompose(job.data.composeId, {
|
||||
composeStatus: "running",
|
||||
});
|
||||
if (job.data.type === "deploy") {
|
||||
await deployCompose({
|
||||
composeId: job.data.composeId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
} else if (job.data.type === "redeploy") {
|
||||
await rebuildCompose({
|
||||
composeId: job.data.composeId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
});
|
||||
}
|
||||
} else if (job.data.applicationType === "application-preview") {
|
||||
await updatePreviewDeployment(job.data.previewDeploymentId, {
|
||||
previewStatus: "running",
|
||||
});
|
||||
|
||||
/** No-op worker when Redis is disabled (e.g. IS_CLOUD). Avoids BullMQ connection errors. */
|
||||
const noopWorker = {
|
||||
run: () => Promise.resolve(),
|
||||
close: () => Promise.resolve(),
|
||||
cancelJob: () => Promise.resolve(),
|
||||
cancelAllJobs: () => Promise.resolve(),
|
||||
if (job.data.type === "redeploy") {
|
||||
await rebuildPreviewApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
previewDeploymentId: job.data.previewDeploymentId,
|
||||
});
|
||||
} else if (job.data.type === "deploy") {
|
||||
await deployPreviewApplication({
|
||||
applicationId: job.data.applicationId,
|
||||
titleLog: job.data.titleLog,
|
||||
descriptionLog: job.data.descriptionLog,
|
||||
previewDeploymentId: job.data.previewDeploymentId,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("Error", error);
|
||||
}
|
||||
};
|
||||
|
||||
export const deploymentWorker = !IS_CLOUD
|
||||
? createDeploymentWorker()
|
||||
: (noopWorker as unknown as Worker<DeploymentJob>);
|
||||
|
||||
262
apps/dokploy/server/queues/in-memory-queue.ts
Normal file
262
apps/dokploy/server/queues/in-memory-queue.ts
Normal file
@@ -0,0 +1,262 @@
|
||||
import type { DeploymentJob } from "./queue-types";
|
||||
|
||||
/**
|
||||
* In-memory deployment queue for self-hosted instances.
|
||||
*
|
||||
* Replaces BullMQ/Redis for deployments. The model is per-group FIFO with a
|
||||
* configurable concurrency per partition (server):
|
||||
*
|
||||
* - Jobs are partitioned by `serverId` (the local web server uses the
|
||||
* `LOCAL_PARTITION` key). Each partition runs up to `concurrency` jobs at
|
||||
* the same time, so two different applications can build concurrently.
|
||||
* - Within a partition, jobs that belong to the same group (same application
|
||||
* or compose) never run in parallel — they are serialized FIFO. This avoids
|
||||
* two builds of the same service stepping on each other (same code dir,
|
||||
* same container name, etc).
|
||||
*
|
||||
* The concurrency is resolved lazily per partition through `resolveConcurrency`
|
||||
* so it can be gated by the enterprise license at run time (a non-licensed
|
||||
* instance always resolves to 1).
|
||||
*
|
||||
* The public surface (`add`, `getJobs`, `close`, `on`) mirrors the subset of
|
||||
* BullMQ used by the routers so it can be a drop-in replacement.
|
||||
*/
|
||||
|
||||
export const LOCAL_PARTITION = "__local__";
|
||||
|
||||
export type JobState = "waiting" | "active";
|
||||
|
||||
export interface InMemoryJob {
|
||||
id: string;
|
||||
name: string;
|
||||
data: DeploymentJob;
|
||||
timestamp: number;
|
||||
processedOn?: number;
|
||||
finishedOn?: number;
|
||||
failedReason?: string;
|
||||
getState: () => Promise<JobState>;
|
||||
remove: () => Promise<void>;
|
||||
}
|
||||
|
||||
type Processor = (job: InMemoryJob) => Promise<void>;
|
||||
|
||||
/** Resolve the partition key (serverId) a job belongs to. */
|
||||
export const getPartition = (data: DeploymentJob): string =>
|
||||
data.serverId ?? LOCAL_PARTITION;
|
||||
|
||||
/** Resolve the FIFO group a job belongs to (the service being deployed). */
|
||||
export const getGroup = (data: DeploymentJob): string => {
|
||||
if (data.applicationType === "compose") {
|
||||
return `compose:${data.composeId}`;
|
||||
}
|
||||
return `application:${data.applicationId}`;
|
||||
};
|
||||
|
||||
interface InternalJob {
|
||||
id: string;
|
||||
name: string;
|
||||
data: DeploymentJob;
|
||||
timestamp: number;
|
||||
processedOn?: number;
|
||||
finishedOn?: number;
|
||||
failedReason?: string;
|
||||
state: JobState;
|
||||
partition: string;
|
||||
group: string;
|
||||
}
|
||||
|
||||
interface Partition {
|
||||
waiting: InternalJob[];
|
||||
/** Groups currently running in this partition. */
|
||||
activeGroups: Set<string>;
|
||||
active: InternalJob[];
|
||||
}
|
||||
|
||||
export interface InMemoryQueueOptions {
|
||||
/**
|
||||
* Returns the max number of jobs that may run in parallel for a given
|
||||
* partition. Called on every scheduling tick so license/config changes are
|
||||
* picked up without restarting the queue. Must return a value >= 1.
|
||||
*/
|
||||
resolveConcurrency: (partition: string) => Promise<number> | number;
|
||||
/** Monotonic clock; injectable for tests. Defaults to Date.now. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class InMemoryQueue {
|
||||
private partitions = new Map<string, Partition>();
|
||||
private processor: Processor | null = null;
|
||||
private running = false;
|
||||
private seq = 0;
|
||||
private readonly resolveConcurrency: InMemoryQueueOptions["resolveConcurrency"];
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(options: InMemoryQueueOptions) {
|
||||
this.resolveConcurrency = options.resolveConcurrency;
|
||||
this.now = options.now ?? (() => Date.now());
|
||||
}
|
||||
|
||||
private getPartitionState(key: string): Partition {
|
||||
let partition = this.partitions.get(key);
|
||||
if (!partition) {
|
||||
partition = { waiting: [], activeGroups: new Set(), active: [] };
|
||||
this.partitions.set(key, partition);
|
||||
}
|
||||
return partition;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the worker that processes each job. Registering a processor also
|
||||
* starts the queue: in dev (tsx/Next) the module that calls `run()` and the
|
||||
* module that calls `add()` can resolve to different instances, so we must
|
||||
* not depend on a separate `run()` call to flip `running` on.
|
||||
*/
|
||||
process(processor: Processor) {
|
||||
this.processor = processor;
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
run() {
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
async add(data: DeploymentJob): Promise<{ id: string }> {
|
||||
const id = `job-${++this.seq}`;
|
||||
const partitionKey = getPartition(data);
|
||||
const job: InternalJob = {
|
||||
id,
|
||||
name: "deployments",
|
||||
data,
|
||||
timestamp: this.now(),
|
||||
state: "waiting",
|
||||
partition: partitionKey,
|
||||
group: getGroup(data),
|
||||
};
|
||||
this.getPartitionState(partitionKey).waiting.push(job);
|
||||
this.schedule();
|
||||
return { id };
|
||||
}
|
||||
|
||||
private toPublic(job: InternalJob): InMemoryJob {
|
||||
return {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
timestamp: job.timestamp,
|
||||
processedOn: job.processedOn,
|
||||
finishedOn: job.finishedOn,
|
||||
getState: () => Promise.resolve(job.state),
|
||||
remove: () => this.remove(job.id),
|
||||
};
|
||||
}
|
||||
|
||||
/** Snapshot of jobs in the requested states (defaults to waiting + active). */
|
||||
getJobs(states?: JobState[]): Promise<InMemoryJob[]> {
|
||||
const wantWaiting = !states || states.includes("waiting");
|
||||
const wantActive = !states || states.includes("active");
|
||||
const jobs: InMemoryJob[] = [];
|
||||
for (const partition of this.partitions.values()) {
|
||||
if (wantWaiting) {
|
||||
jobs.push(...partition.waiting.map((job) => this.toPublic(job)));
|
||||
}
|
||||
if (wantActive) {
|
||||
jobs.push(...partition.active.map((job) => this.toPublic(job)));
|
||||
}
|
||||
}
|
||||
return Promise.resolve(jobs);
|
||||
}
|
||||
|
||||
/** Remove a single waiting job by id. Active jobs cannot be removed. */
|
||||
remove(id: string): Promise<void> {
|
||||
for (const partition of this.partitions.values()) {
|
||||
const before = partition.waiting.length;
|
||||
partition.waiting = partition.waiting.filter((job) => job.id !== id);
|
||||
if (partition.waiting.length !== before) break;
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/** Remove waiting jobs matching a predicate. Active jobs are not affected. */
|
||||
removeWaiting(predicate: (data: DeploymentJob) => boolean): number {
|
||||
let removed = 0;
|
||||
for (const partition of this.partitions.values()) {
|
||||
partition.waiting = partition.waiting.filter((job) => {
|
||||
const match = predicate(job.data);
|
||||
if (match) removed++;
|
||||
return !match;
|
||||
});
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** Drop every waiting job across all partitions. */
|
||||
clearWaiting(): number {
|
||||
let removed = 0;
|
||||
for (const partition of this.partitions.values()) {
|
||||
removed += partition.waiting.length;
|
||||
partition.waiting = [];
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
on() {
|
||||
// No-op: kept for BullMQ API compatibility (error events, etc).
|
||||
}
|
||||
|
||||
close() {
|
||||
this.running = false;
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
private schedule() {
|
||||
if (!this.running || !this.processor) return;
|
||||
for (const key of this.partitions.keys()) {
|
||||
void this.drainPartition(key);
|
||||
}
|
||||
}
|
||||
|
||||
private async drainPartition(key: string) {
|
||||
const partition = this.partitions.get(key);
|
||||
if (!partition || !this.processor) return;
|
||||
|
||||
const concurrency = Math.max(1, await this.resolveConcurrency(key));
|
||||
|
||||
while (partition.active.length < concurrency) {
|
||||
// First waiting job whose group is not already running.
|
||||
const index = partition.waiting.findIndex(
|
||||
(job) => !partition.activeGroups.has(job.group),
|
||||
);
|
||||
if (index === -1) break;
|
||||
|
||||
const job = partition.waiting.splice(index, 1)[0];
|
||||
if (!job) break;
|
||||
job.state = "active";
|
||||
job.processedOn = this.now();
|
||||
partition.activeGroups.add(job.group);
|
||||
partition.active.push(job);
|
||||
|
||||
void this.runJob(job);
|
||||
}
|
||||
}
|
||||
|
||||
private async runJob(job: InternalJob) {
|
||||
try {
|
||||
await this.processor?.(this.toPublic(job));
|
||||
} catch (error) {
|
||||
job.failedReason = error instanceof Error ? error.message : String(error);
|
||||
console.error("In-memory deployment job failed", error);
|
||||
} finally {
|
||||
job.finishedOn = this.now();
|
||||
const partition = this.partitions.get(job.partition);
|
||||
if (partition) {
|
||||
partition.active = partition.active.filter((j) => j.id !== job.id);
|
||||
partition.activeGroups.delete(job.group);
|
||||
}
|
||||
// A slot (and possibly the group) freed up — try to schedule more.
|
||||
void this.drainPartition(job.partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,32 +3,89 @@ import {
|
||||
execAsync,
|
||||
execAsyncRemote,
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import type { Job } from "bullmq";
|
||||
import { Queue } from "bullmq";
|
||||
import { deploymentWorker } from "./deployments-queue";
|
||||
import { redisConfig } from "./redis-connection";
|
||||
import { resolveBuildsConcurrency } from "./concurrency";
|
||||
import { processDeploymentJob } from "./deployments-queue";
|
||||
import { type InMemoryJob, InMemoryQueue } from "./in-memory-queue";
|
||||
import type { DeploymentJob } from "./queue-types";
|
||||
|
||||
/** No-op queue when Redis is disabled (e.g. IS_CLOUD). Avoids BullMQ connection errors. */
|
||||
const createNoopQueue = () => ({
|
||||
getJobs: () => Promise.resolve([] as Job[]),
|
||||
add: () =>
|
||||
Promise.resolve({ id: "noop", remove: () => Promise.resolve() } as Job),
|
||||
/**
|
||||
* Deployment queue.
|
||||
*
|
||||
* Self-hosted uses an in-memory, per-group FIFO queue with configurable
|
||||
* concurrency per server (enterprise-gated). Cloud does not use the queue at
|
||||
* all — deployments run directly in the background — so we expose a no-op.
|
||||
*/
|
||||
|
||||
interface DeploymentQueue {
|
||||
add: (
|
||||
name: string,
|
||||
data: DeploymentJob,
|
||||
opts?: Record<string, unknown>,
|
||||
) => Promise<{ id: string }>;
|
||||
getJobs: (states?: Array<"waiting" | "active">) => Promise<InMemoryJob[]>;
|
||||
close: () => Promise<void>;
|
||||
on: (...args: unknown[]) => void;
|
||||
run: () => Promise<void>;
|
||||
removeWaiting: (predicate: (data: DeploymentJob) => boolean) => number;
|
||||
clearWaiting: () => number;
|
||||
}
|
||||
|
||||
const createNoopQueue = (): DeploymentQueue => ({
|
||||
add: () => Promise.resolve({ id: "noop" }),
|
||||
getJobs: () => Promise.resolve([]),
|
||||
close: () => Promise.resolve(),
|
||||
on: () => {},
|
||||
run: () => Promise.resolve(),
|
||||
removeWaiting: () => 0,
|
||||
clearWaiting: () => 0,
|
||||
});
|
||||
|
||||
const myQueue = !IS_CLOUD
|
||||
? new Queue("deployments", { connection: redisConfig })
|
||||
: (createNoopQueue() as unknown as Queue);
|
||||
const createInMemoryQueue = (): DeploymentQueue => {
|
||||
const queue = new InMemoryQueue({
|
||||
resolveConcurrency: resolveBuildsConcurrency,
|
||||
});
|
||||
queue.process(processDeploymentJob);
|
||||
|
||||
return {
|
||||
add: (_name, data) => queue.add(data),
|
||||
getJobs: (states) => queue.getJobs(states),
|
||||
close: () => queue.close(),
|
||||
on: () => {},
|
||||
run: () => queue.run(),
|
||||
removeWaiting: (predicate) => queue.removeWaiting(predicate),
|
||||
clearWaiting: () => queue.clearWaiting(),
|
||||
};
|
||||
};
|
||||
|
||||
// Use a global singleton so the deployment queue is shared across every module
|
||||
// instance. In dev (tsx/Next) the same file can be evaluated more than once
|
||||
// (relative import in server.ts vs `@/` alias in the routers); without this the
|
||||
// worker and the `add()` calls would land on different queue instances.
|
||||
const globalForQueue = globalThis as unknown as {
|
||||
__dokployDeploymentQueue?: DeploymentQueue;
|
||||
};
|
||||
|
||||
if (!globalForQueue.__dokployDeploymentQueue) {
|
||||
globalForQueue.__dokployDeploymentQueue = !IS_CLOUD
|
||||
? createInMemoryQueue()
|
||||
: createNoopQueue();
|
||||
}
|
||||
|
||||
const myQueue: DeploymentQueue = globalForQueue.__dokployDeploymentQueue;
|
||||
|
||||
/** Start processing jobs. Called once on server startup (self-hosted). */
|
||||
export const startDeploymentWorker = () => myQueue.run();
|
||||
|
||||
export const getJobsByApplicationId = async (applicationId: string) => {
|
||||
const jobs = await myQueue.getJobs();
|
||||
return jobs.filter((job) => job?.data?.applicationId === applicationId);
|
||||
return jobs.filter(
|
||||
(job) => (job.data as any)?.applicationId === applicationId,
|
||||
);
|
||||
};
|
||||
|
||||
export const getJobsByComposeId = async (composeId: string) => {
|
||||
const jobs = await myQueue.getJobs();
|
||||
return jobs.filter((job) => job?.data?.composeId === composeId);
|
||||
return jobs.filter((job) => (job.data as any)?.composeId === composeId);
|
||||
};
|
||||
|
||||
if (!IS_CLOUD) {
|
||||
@@ -36,44 +93,33 @@ if (!IS_CLOUD) {
|
||||
myQueue.close();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
myQueue.on("error", (error) => {
|
||||
if ((error as any).code === "ECONNREFUSED") {
|
||||
console.error(
|
||||
"Make sure you have installed Redis and it is running.",
|
||||
error,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export const cleanQueuesByApplication = async (applicationId: string) => {
|
||||
const jobs = await myQueue.getJobs(["waiting", "delayed"]);
|
||||
const removed = myQueue.removeWaiting(
|
||||
(data) => (data as any)?.applicationId === applicationId,
|
||||
);
|
||||
if (removed > 0) {
|
||||
console.log(
|
||||
`Removed ${removed} waiting job(s) for application ${applicationId}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
for (const job of jobs) {
|
||||
if (job?.data?.applicationId === applicationId) {
|
||||
await job.remove();
|
||||
console.log(`Removed job ${job.id} for application ${applicationId}`);
|
||||
}
|
||||
export const cleanQueuesByCompose = async (composeId: string) => {
|
||||
const removed = myQueue.removeWaiting(
|
||||
(data) => (data as any)?.composeId === composeId,
|
||||
);
|
||||
if (removed > 0) {
|
||||
console.log(`Removed ${removed} waiting job(s) for compose ${composeId}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const cleanAllDeploymentQueue = async () => {
|
||||
deploymentWorker.cancelAllJobs("User requested cancellation");
|
||||
myQueue.clearWaiting();
|
||||
return true;
|
||||
};
|
||||
|
||||
export const cleanQueuesByCompose = async (composeId: string) => {
|
||||
const jobs = await myQueue.getJobs(["waiting", "delayed"]);
|
||||
|
||||
for (const job of jobs) {
|
||||
if (job?.data?.composeId === composeId) {
|
||||
await job.remove();
|
||||
console.log(`Removed job ${job.id} for compose ${composeId}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const killDockerBuild = async (
|
||||
type: "application" | "compose",
|
||||
serverId: string | null,
|
||||
|
||||
@@ -71,8 +71,8 @@ void app.prepare().then(async () => {
|
||||
|
||||
if (!IS_CLOUD) {
|
||||
console.log("Starting Deployment Worker");
|
||||
const { deploymentWorker } = await import("./queues/deployments-queue");
|
||||
await deploymentWorker.run();
|
||||
const { startDeploymentWorker } = await import("./queues/queueSetup");
|
||||
await startDeploymentWorker();
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Main Server Error", e);
|
||||
|
||||
Reference in New Issue
Block a user