Merge pull request #3064 from fir4tozden/fix/docker-cleanup-clears-away-all-unused-residue

fix: docker cleanup clears away all unused residue

apps/dokploy/drizzle/0126_nifty_monster_badoon.sql

@@ -0,0 +1 @@
+ALTER TABLE "user" ALTER COLUMN "enableDockerCleanup" SET DEFAULT true;

apps/dokploy/drizzle/meta/_journal.json

@@ -883,6 +883,13 @@
       "when": 1764573207555,
       "tag": "0125_neat_the_phantom",
       "breakpoints": true
+    },
+    {
+      "idx": 126,
+      "version": "7",
+      "when": 1765065295708,
+      "tag": "0126_nifty_monster_badoon",
+      "breakpoints": true
     }
   ]
 }

apps/dokploy/server/api/routers/settings.ts

@@ -1,12 +1,13 @@
 import {
 	canAccessToTraefikFiles,
 	checkGPUStatus,
+	cleanupContainers,
+	cleanupBuilders,
+	cleanupSystem,
+	cleanupImages,
+	cleanupVolumes,
+	cleanupAll,
 	checkPortInUse,
-	cleanStoppedContainers,
-	cleanUpDockerBuilder,
-	cleanUpSystemPrune,
-	cleanUpUnusedImages,
-	cleanUpUnusedVolumes,
 	DEFAULT_UPDATE_DATA,
 	execAsync,
 	findServerById,
@@ -161,41 +162,38 @@ export const settingsRouter = createTRPCRouter({
 	cleanUnusedImages: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanUpUnusedImages(input?.serverId);
+			await cleanupImages(input?.serverId);
 			return true;
 		}),
 	cleanUnusedVolumes: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanUpUnusedVolumes(input?.serverId);
+			await cleanupVolumes(input?.serverId);
 			return true;
 		}),
 	cleanStoppedContainers: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanStoppedContainers(input?.serverId);
+			await cleanupContainers(input?.serverId);
 			return true;
 		}),
 	cleanDockerBuilder: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanUpDockerBuilder(input?.serverId);
+			await cleanupBuilders(input?.serverId);
 		}),
 	cleanDockerPrune: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanUpSystemPrune(input?.serverId);
-			await cleanUpDockerBuilder(input?.serverId);
+			await cleanupSystem(input?.serverId);
+			await cleanupBuilders(input?.serverId);
 
 			return true;
 		}),
 	cleanAll: adminProcedure
 		.input(apiServerSchema)
 		.mutation(async ({ input }) => {
-			await cleanUpUnusedImages(input?.serverId);
-			await cleanStoppedContainers(input?.serverId);
-			await cleanUpDockerBuilder(input?.serverId);
-			await cleanUpSystemPrune(input?.serverId);
+			await cleanupAll(input?.serverId);
 
 			return true;
 		}),
@@ -293,9 +291,9 @@ export const settingsRouter = createTRPCRouter({
 							console.log(
 								`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
 							);
-							await cleanUpUnusedImages(server.serverId);
-							await cleanUpDockerBuilder(server.serverId);
-							await cleanUpSystemPrune(server.serverId);
+
+							await cleanupAll(server.serverId);
+
 							await sendDockerCleanupNotifications(server.organizationId);
 						});
 					}
@@ -321,9 +319,9 @@ export const settingsRouter = createTRPCRouter({
 						console.log(
 							`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
 						);
-						await cleanUpUnusedImages();
-						await cleanUpDockerBuilder();
-						await cleanUpSystemPrune();
+
+						await cleanupAll();
+
 						await sendDockerCleanupNotifications(
 							ctx.session.activeOrganizationId,
 						);

apps/schedules/src/utils.ts

@@ -1,7 +1,5 @@
 import {
-	cleanUpDockerBuilder,
-	cleanUpSystemPrune,
-	cleanUpUnusedImages,
+	cleanupAll,
 	findBackupById,
 	findScheduleById,
 	findServerById,
@@ -91,9 +89,7 @@ export const runJobs = async (job: QueueJob) => {
 				logger.info("Server is inactive");
 				return;
 			}
-			await cleanUpUnusedImages(serverId);
-			await cleanUpDockerBuilder(serverId);
-			await cleanUpSystemPrune(serverId);
+			await cleanupAll(serverId);
 		} else if (job.type === "schedule") {
 			const { scheduleId } = job;
 			const schedule = await findScheduleById(scheduleId);

packages/server/src/db/schema/user.ts

@@ -56,7 +56,7 @@ export const user = pgTable("user", {
 	host: text("host"),
 	letsEncryptEmail: text("letsEncryptEmail"),
 	sshPrivateKey: text("sshPrivateKey"),
-	enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(false),
+	enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(true),
 	logCleanupCron: text("logCleanupCron").default("0 0 * * *"),
 	role: text("role").notNull().default("user"),
 	// Metrics

packages/server/src/services/settings.ts

@@ -1,6 +1,7 @@
 import { readdirSync } from "node:fs";
 import { join } from "node:path";
 import { docker } from "@dokploy/server/constants";
+import { cleanupAll } from "@dokploy/server/utils/docker/utils";
 import {
 	execAsync,
 	execAsyncRemote,
@@ -215,38 +216,6 @@ echo "$json_output"
 	return result;
 };
 
-export const cleanupFullDocker = async (serverId?: string | null) => {
-	const cleanupImages = "docker image prune --force";
-	const cleanupVolumes = "docker volume prune --force";
-	const cleanupContainers = "docker container prune --force";
-	const cleanupSystem = "docker system prune  --force --volumes";
-	const cleanupBuilder = "docker builder prune  --force";
-
-	try {
-		if (serverId) {
-			await execAsyncRemote(
-				serverId,
-				`
-	${cleanupImages}
-	${cleanupVolumes}
-	${cleanupContainers}
-	${cleanupSystem}
-	${cleanupBuilder}
-			`,
-			);
-		}
-		await execAsync(`
-			${cleanupImages}
-			${cleanupVolumes}
-			${cleanupContainers}
-			${cleanupSystem}
-			${cleanupBuilder}
-					`);
-	} catch (error) {
-		console.log(error);
-	}
-};
-
 export const getDockerResourceType = async (
 	resourceName: string,
 	serverId?: string,

packages/server/src/utils/backups/index.ts

@@ -6,11 +6,7 @@ import { eq } from "drizzle-orm";
 import { scheduleJob } from "node-schedule";
 import { db } from "../../db/index";
 import { startLogCleanup } from "../access-log/handler";
-import {
-	cleanUpDockerBuilder,
-	cleanUpSystemPrune,
-	cleanUpUnusedImages,
-} from "../docker/utils";
+import { cleanupAll } from "../docker/utils";
 import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
 import { getS3Credentials, scheduleBackup } from "./utils";
@@ -34,9 +30,9 @@ export const initCronJobs = async () => {
 			console.log(
 				`Docker Cleanup ${new Date().toLocaleString()}]  Running docker cleanup`,
 			);
-			await cleanUpUnusedImages();
-			await cleanUpDockerBuilder();
-			await cleanUpSystemPrune();
+
+			await cleanupAll();
+
 			await sendDockerCleanupNotifications(admin.user.id);
 		});
 	}
@@ -50,9 +46,9 @@ export const initCronJobs = async () => {
 				console.log(
 					`SERVER-BACKUP[${new Date().toLocaleString()}] Running Cleanup ${name}`,
 				);
-				await cleanUpUnusedImages(serverId);
-				await cleanUpDockerBuilder(serverId);
-				await cleanUpSystemPrune(serverId);
+
+				await cleanupAll(serverId);
+
 				await sendDockerCleanupNotifications(
 					admin.user.id,
 					`Docker cleanup for Server ${name} (${serverId})`,

packages/server/src/utils/docker/utils.ts

@@ -144,81 +144,117 @@ export const getContainerByName = (name: string): Promise<ContainerInfo> => {
 		});
 	});
 };
-export const cleanUpUnusedImages = async (serverId?: string) => {
-	try {
-		const command = "docker image prune --force";
-		if (serverId) {
-			await execAsyncRemote(serverId, command);
-		} else {
-			await execAsync(command);
-		}
-	} catch (error) {
-		console.error(error);
-		throw error;
-	}
-};
 
-export const cleanStoppedContainers = async (serverId?: string) => {
+/**
+ * Docker commands passed through this method are held during Docker's build or pull process.
+ *
+ * https://github.com/Dokploy/dokploy/pull/3064
+ * https://github.com/fir4tozden
+ */
+export const dockerSafeExec = (exec: string) => `CHECK_INTERVAL=10
+
+echo "Preparing for execution..."
+
+while true; do
+    PROCESSES=$(ps aux | grep -E "docker build|docker pull" | grep -v grep)
+
+    if [ -z "$PROCESSES" ]; then
+        echo "Docker is idle. Starting execution..."
+        break
+    else
+        echo "Docker is busy. Will check again in $CHECK_INTERVAL seconds..."
+        sleep $CHECK_INTERVAL
+    fi
+done
+
+${exec}
+
+echo "Execution completed."`;
+
+export const cleanupContainers = async (serverId?: string) => {
 	try {
 		const command = "docker container prune --force";
+
 		if (serverId) {
-			await execAsyncRemote(serverId, command);
+			await execAsyncRemote(serverId, dockerSafeExec(command));
 		} else {
-			await execAsync(command);
+			await execAsync(dockerSafeExec(command));
 		}
 	} catch (error) {
 		console.error(error);
+
 		throw error;
 	}
 };
 
-export const cleanUpUnusedVolumes = async (serverId?: string) => {
+export const cleanupImages = async (serverId?: string) => {
 	try {
-		const command = "docker volume prune --force";
+		const command = "docker image prune --all --force";
+
 		if (serverId) {
-			await execAsyncRemote(serverId, command);
+			await execAsyncRemote(serverId, dockerSafeExec(command));
+		} else await execAsync(dockerSafeExec(command));
+	} catch (error) {
+		console.error(error);
+
+		throw error;
+	}
+};
+
+export const cleanupVolumes = async (serverId?: string) => {
+	try {
+		const command = "docker volume prune --all --force";
+
+		if (serverId) {
+			await execAsyncRemote(serverId, dockerSafeExec(command));
 		} else {
-			await execAsync(command);
+			await execAsync(dockerSafeExec(command));
 		}
 	} catch (error) {
 		console.error(error);
+
 		throw error;
 	}
 };
 
-export const cleanUpInactiveContainers = async () => {
+export const cleanupBuilders = async (serverId?: string) => {
 	try {
-		const containers = await docker.listContainers({ all: true });
-		const inactiveContainers = containers.filter(
-			(container) => container.State !== "running",
-		);
+		const command = "docker builder prune --all --force";
 
-		for (const container of inactiveContainers) {
-			await docker.getContainer(container.Id).remove({ force: true });
-			console.log(`Cleaning up inactive container: ${container.Id}`);
+		if (serverId) {
+			await execAsyncRemote(serverId, dockerSafeExec(command));
+		} else {
+			await execAsync(dockerSafeExec(command));
 		}
 	} catch (error) {
-		console.error("Error cleaning up inactive containers:", error);
+		console.error(error);
+
 		throw error;
 	}
 };
 
-export const cleanUpDockerBuilder = async (serverId?: string) => {
-	const command = "docker builder prune --all --force";
-	if (serverId) {
-		await execAsyncRemote(serverId, command);
-	} else {
-		await execAsync(command);
+export const cleanupSystem = async (serverId?: string) => {
+	try {
+		const command = "docker system prune --all --volumes --force";
+
+		if (serverId) {
+			await execAsyncRemote(serverId, dockerSafeExec(command));
+		} else {
+			await execAsync(dockerSafeExec(command));
+		}
+	} catch (error) {
+		console.error(error);
+
+		throw error;
 	}
 };
 
-export const cleanUpSystemPrune = async (serverId?: string) => {
-	const command = "docker system prune --force --volumes";
-	if (serverId) {
-		await execAsyncRemote(serverId, command);
-	} else {
-		await execAsync(command);
-	}
+export const cleanupAll = async (serverId?: string) => {
+	await cleanupContainers(serverId);
+	await cleanupImages(serverId);
+	await cleanupVolumes(serverId);
+	await cleanupBuilders(serverId);
+	await cleanupSystem(serverId);
 };
 
 export const startService = async (appName: string) => {