Feat/concurrent deployments in memory queue (#4645)

* feat: add builds concurrency management for servers

- Introduced a new `BuildsConcurrency` component to manage the number of concurrent builds for both local and remote servers, gated by license validity.
- Implemented backend logic to resolve effective builds concurrency based on server settings and organization licenses.
- Added unit tests for concurrency resolution logic to ensure correct behavior under various licensing scenarios.
- Updated database schema to include `buildsConcurrency` field for servers and web server settings.
- Refactored deployment queue to support in-memory job processing with configurable concurrency limits.

This feature enhances deployment flexibility and control for enterprise users.

* refactor: enhance deployment cancellation logic and improve Railpack build isolation

- Reintroduced the `initCancelDeployments` function in the server initialization sequence to ensure deployments can be canceled effectively.
- Updated the Railpack build command to use a unique builder name for each build, preventing conflicts during concurrent deployments.
- Enhanced the cancellation logic to reset application and compose statuses to "idle" after canceling running deployments, improving system reliability.

* test: add buildsConcurrency setting to server configuration tests

- Introduced a new `buildsConcurrency` property in the server configuration tests to ensure proper handling of concurrent builds in deployment scenarios.

* feat: implement builds concurrency management and validation

- Added `assertBuildsConcurrencyAllowed` function to validate concurrency settings based on license status.
- Updated `resolveBuildsConcurrency` to reflect new concurrency limits for free and enterprise tiers.
- Enhanced `BuildsConcurrency` component to manage concurrent builds for servers, with UI adjustments for better user experience.
- Introduced a new settings page for managing concurrent builds across servers, ensuring proper handling of deployments.
- Updated database schema to support increased maximum concurrency values for servers and web server settings.

packages/server/src/db/schema/server.ts

@@ -41,6 +41,7 @@ export const server = pgTable("server", {
 		.notNull()
 		.$defaultFn(() => generateAppName("server")),
 	enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(false),
+	buildsConcurrency: integer("buildsConcurrency").notNull().default(1),
 	createdAt: text("createdAt").notNull(),
 	organizationId: text("organizationId")
 		.notNull()
@@ -182,6 +183,11 @@ export const apiUpdateServer = createSchema
 		enableDockerCleanup: z.boolean().default(true),
 	});
 
+export const apiUpdateServerBuildsConcurrency = z.object({
+	serverId: z.string().min(1),
+	buildsConcurrency: z.number().int().min(1).max(100),
+});
+
 export const apiUpdateServerMonitoring = createSchema
 	.pick({
 		serverId: true,

packages/server/src/db/schema/web-server-settings.ts

@@ -1,5 +1,12 @@
 import { relations } from "drizzle-orm";
-import { boolean, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core";
+import {
+	boolean,
+	integer,
+	jsonb,
+	pgTable,
+	text,
+	timestamp,
+} from "drizzle-orm/pg-core";
 import { createInsertSchema } from "drizzle-zod";
 import { nanoid } from "nanoid";
 import { z } from "zod";
@@ -98,6 +105,8 @@ export const webServerSettings = pgTable("webServerSettings", {
 		}),
 	// Deployment Configuration (self-hosted only)
 	remoteServersOnly: boolean("remoteServersOnly").notNull().default(false),
+	// Concurrent builds on the local web server (enterprise-gated to > 1)
+	buildsConcurrency: integer("buildsConcurrency").notNull().default(1),
 	// Auth Configuration (self-hosted only)
 	enforceSSO: boolean("enforceSSO").notNull().default(false),
 	// Cache Cleanup Configuration
@@ -161,6 +170,11 @@ export const apiUpdateWebServerSettings = createSchema.partial().extend({
 	cleanupCacheOnCompose: z.boolean().optional(),
 	remoteServersOnly: z.boolean().optional(),
 	enforceSSO: z.boolean().optional(),
+	buildsConcurrency: z.number().int().min(1).max(100).optional(),
+});
+
+export const apiUpdateWebServerBuildsConcurrency = z.object({
+	buildsConcurrency: z.number().int().min(1).max(100),
 });
 
 export const apiAssignDomain = z

packages/server/src/utils/builders/railpack.ts

@@ -44,10 +44,15 @@ export const getRailpackCommand = (application: ApplicationNested) => {
 	const secretsHash = calculateSecretsHash(envVariables);
 
 	const cacheKey = cleanCache ? nanoid(10) : undefined;
-	// Build command
+	// Build command.
+	// Use a unique builder name per build so concurrent deployments don't race
+	// on a shared "builder-containerd" instance (create/use/rm collisions).
+	const builderName = `railpack-${appName}-${nanoid(6)}`;
 	const buildArgs = [
 		"buildx",
 		"build",
+		"--builder",
+		builderName,
 		...(cacheKey
 			? [
 					"--build-arg",
@@ -84,17 +89,16 @@ export const getRailpackCommand = (application: ApplicationNested) => {
 
 	const bashCommand = `
 
-# Ensure we have a builder with containerd
+# Ensure we have a builder with containerd (isolated per build)
 
 export RAILPACK_VERSION=${application.railpackVersion}
 bash -c "$(curl -fsSL https://railpack.com/install.sh)"
-docker buildx create --use --name builder-containerd --driver docker-container || true
-docker buildx use builder-containerd
+docker buildx create --name ${builderName} --driver docker-container || true
 
 echo "Preparing Railpack build plan..." ;
-railpack ${prepareArgs.join(" ")} || { 
+railpack ${prepareArgs.join(" ")} || {
 	echo "❌ Railpack prepare failed" ;
-	docker buildx rm builder-containerd || true
+	docker buildx rm ${builderName} || true
 	exit 1;
 }
 echo "✅ Railpack prepare completed." ;
@@ -102,13 +106,13 @@ echo "✅ Railpack prepare completed." ;
 echo "Building with Railpack frontend..." ;
 # Export environment variables for secrets
 ${exportEnvs.join("\n")}
-docker ${buildArgs.join(" ")} || { 
+docker ${buildArgs.join(" ")} || {
 	echo "❌ Railpack build failed" ;
-	docker buildx rm builder-containerd || true
+	docker buildx rm ${builderName} || true
 	exit 1;
 }
 echo "✅ Railpack build completed." ;
-docker buildx rm builder-containerd
+docker buildx rm ${builderName} || true
 `;
 
 	return bashCommand;

packages/server/src/utils/startup/cancel-deployments.ts

@@ -1,5 +1,5 @@
-import { deployments } from "@dokploy/server/db/schema";
-import { eq } from "drizzle-orm";
+import { applications, compose, deployments } from "@dokploy/server/db/schema";
+import { eq, inArray } from "drizzle-orm";
 import { db } from "../../db/index";
 
 export const initCancelDeployments = async () => {
@@ -14,6 +14,36 @@ export const initCancelDeployments = async () => {
 			.where(eq(deployments.status, "running"))
 			.returning();
 
+		// Reset the related services so they don't stay stuck in "running".
+		const applicationIds = [
+			...new Set(
+				result
+					.map((deployment) => deployment.applicationId)
+					.filter((id): id is string => !!id),
+			),
+		];
+		const composeIds = [
+			...new Set(
+				result
+					.map((deployment) => deployment.composeId)
+					.filter((id): id is string => !!id),
+			),
+		];
+
+		if (applicationIds.length > 0) {
+			await db
+				.update(applications)
+				.set({ applicationStatus: "idle" })
+				.where(inArray(applications.applicationId, applicationIds));
+		}
+
+		if (composeIds.length > 0) {
+			await db
+				.update(compose)
+				.set({ composeStatus: "idle" })
+				.where(inArray(compose.composeId, composeIds));
+		}
+
 		console.log(`Cancelled ${result.length} deployments`);
 	} catch (error) {
 		console.error(error);