feat(libsql): add support for libsql database backups and restores

- Updated backup and restore functionalities to include support for the 'libsql' database type.
- Enhanced the backup process with new methods for running and restoring libsql backups.
- Modified existing components and schemas to accommodate libsql, including updates to the database type enumerations and backup schemas.
- Removed obsolete bottomless replication features from the libsql schema.
- Updated related UI components to reflect changes in backup handling for libsql.

packages/server/src/db/schema/backups.ts

@@ -15,6 +15,7 @@ import { generateAppName } from ".";
 import { compose } from "./compose";
 import { deployments } from "./deployment";
 import { destinations } from "./destination";
+import { libsql } from "./libsql";
 import { mariadb } from "./mariadb";
 import { mongo } from "./mongo";
 import { mysql } from "./mysql";
@@ -26,6 +27,7 @@ export const databaseType = pgEnum("databaseType", [
 	"mysql",
 	"mongo",
 	"web-server",
+	"libsql",
 ]);
 
 export const backupType = pgEnum("backupType", ["database", "compose"]);
@@ -74,6 +76,12 @@ export const backups = pgTable("backup", {
 	mongoId: text("mongoId").references((): AnyPgColumn => mongo.mongoId, {
 		onDelete: "cascade",
 	}),
+	libsqlId: text("libsqlId").references(
+		(): AnyPgColumn => libsql.libsqlId,
+		{
+			onDelete: "cascade",
+		},
+	),
 	userId: text("userId").references(() => user.id),
 	// Only for compose backups
 	metadata: jsonb("metadata").$type<
@@ -118,6 +126,10 @@ export const backupsRelations = relations(backups, ({ one, many }) => ({
 		fields: [backups.mongoId],
 		references: [mongo.mongoId],
 	}),
+	libsql: one(libsql, {
+		fields: [backups.libsqlId],
+		references: [libsql.libsqlId],
+	}),
 	user: one(user, {
 		fields: [backups.userId],
 		references: [user.id],
@@ -137,11 +149,12 @@ const createSchema = createInsertSchema(backups, {
 	database: z.string().min(1),
 	schedule: z.string(),
 	keepLatestCount: z.number().optional(),
-	databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo", "web-server"]),
+	databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo", "web-server", "libsql"]),
 	postgresId: z.string().optional(),
 	mariadbId: z.string().optional(),
 	mysqlId: z.string().optional(),
 	mongoId: z.string().optional(),
+	libsqlId: z.string().optional(),
 	userId: z.string().optional(),
 	metadata: z.any().optional(),
 });
@@ -157,6 +170,7 @@ export const apiCreateBackup = createSchema.pick({
 	mysqlId: true,
 	postgresId: true,
 	mongoId: true,
+	libsqlId: true,
 	databaseType: true,
 	userId: true,
 	backupType: true,
@@ -192,7 +206,7 @@ export const apiUpdateBackup = createSchema
 
 export const apiRestoreBackup = z.object({
 	databaseId: z.string(),
-	databaseType: z.enum(["postgres", "mysql", "mariadb", "mongo", "web-server"]),
+	databaseType: z.enum(["postgres", "mysql", "mariadb", "mongo", "web-server", "libsql"]),
 	backupType: z.enum(["database", "compose"]),
 	databaseName: z.string().min(1),
 	backupFile: z.string().min(1),

packages/server/src/db/schema/libsql.ts

@@ -3,7 +3,7 @@ import { boolean, integer, json, pgTable, text } from "drizzle-orm/pg-core";
 import { createInsertSchema } from "drizzle-zod";
 import { nanoid } from "nanoid";
 import { z } from "zod";
-import { destinations } from "./destination";
+import { backups } from "./backups";
 import { environments } from "./environment";
 import { mounts } from "./mount";
 import { server } from "./server";
@@ -43,14 +43,6 @@ export const libsql = pgTable("libsql", {
 	sqldNode: sqldNode("sqldNode").notNull().default("primary"),
 	sqldPrimaryUrl: text("sqldPrimaryUrl"),
 	enableNamespaces: boolean("enableNamespaces").notNull().default(false),
-	enableBottomlessReplication: boolean("enableBottomlessReplication")
-		.notNull()
-		.default(false),
-	bottomlessReplicationDestinationId: text(
-		"bottomlessReplicationDestinationId",
-	).references(() => destinations.destinationId, {
-		onDelete: "set null",
-	}),
 	dockerImage: text("dockerImage").notNull(),
 	command: text("command"),
 	env: text("env"),
@@ -92,16 +84,12 @@ export const libsqlRelations = relations(libsql, ({ one, many }) => ({
 		fields: [libsql.environmentId],
 		references: [environments.environmentId],
 	}),
-	//backups: many(backups),
+	backups: many(backups),
 	mounts: many(mounts),
 	server: one(server, {
 		fields: [libsql.serverId],
 		references: [server.serverId],
 	}),
-	bottomlessReplicationDestination: one(destinations, {
-		fields: [libsql.bottomlessReplicationDestinationId],
-		references: [destinations.destinationId],
-	}),
 }));
 
 const createSchema = createInsertSchema(libsql, {
@@ -119,9 +107,7 @@ const createSchema = createInsertSchema(libsql, {
 	sqldNode: z.enum(sqldNode.enumValues),
 	sqldPrimaryUrl: z.string().nullable(),
 	enableNamespaces: z.boolean().default(false),
-	enableBottomlessReplication: z.boolean().default(false),
-	bottomlessReplicationDestinationId: z.string().nullable(),
-	dockerImage: z.string().default("ghcr.io/tursodatabase/libsql-server:latest"),
+	dockerImage: z.string().default("ghcr.io/tursodatabase/libsql-server:v0.24.32"),
 	command: z.string().optional(),
 	env: z.string().optional(),
 	memoryReservation: z.string().optional(),

packages/server/src/emails/emails/database-backup.tsx

@@ -14,7 +14,7 @@ import {
 export type TemplateProps = {
 	projectName: string;
 	applicationName: string;
-	databaseType: "postgres" | "mysql" | "mongodb" | "mariadb";
+	databaseType: "postgres" | "mysql" | "mongodb" | "mariadb" | "libsql";
 	type: "error" | "success";
 	errorMessage?: string;
 	date: string;

packages/server/src/index.ts

@@ -68,6 +68,7 @@ export * from "./utils/access-log/types";
 export * from "./utils/access-log/utils";
 export * from "./utils/backups/compose";
 export * from "./utils/backups/index";
+export * from "./utils/backups/libsql";
 export * from "./utils/backups/mariadb";
 export * from "./utils/backups/mongo";
 export * from "./utils/backups/mysql";

packages/server/src/services/backup.ts

@@ -33,6 +33,7 @@ export const findBackupById = async (backupId: string) => {
 			mysql: true,
 			mariadb: true,
 			mongo: true,
+			libsql: true,
 			destination: true,
 			compose: true,
 		},
@@ -72,7 +73,7 @@ export const removeBackupById = async (backupId: string) => {
 
 export const findBackupsByDbId = async (
 	id: string,
-	type: "postgres" | "mysql" | "mariadb" | "mongo",
+	type: "postgres" | "mysql" | "mariadb" | "mongo" | "libsql",
 ) => {
 	const result = await db.query.backups.findMany({
 		where: eq(backups[`${type}Id`], id),
@@ -81,6 +82,7 @@ export const findBackupsByDbId = async (
 			mysql: true,
 			mariadb: true,
 			mongo: true,
+			libsql: true,
 			destination: true,
 		},
 	});

packages/server/src/services/libsql.ts

@@ -1,6 +1,7 @@
 import { db } from "@dokploy/server/db";
 import {
 	type apiCreateLibsql,
+	backups,
 	buildAppName,
 	libsql,
 } from "@dokploy/server/db/schema";
@@ -9,12 +10,13 @@ import { buildLibsql } from "@dokploy/server/utils/databases/libsql";
 import { pullImage } from "@dokploy/server/utils/docker/utils";
 import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
 import { TRPCError } from "@trpc/server";
-import { eq } from "drizzle-orm";
+import { eq, getTableColumns } from "drizzle-orm";
+import type { z } from "zod";
 import { validUniqueServerAppName } from "./project";
 
 export type Libsql = typeof libsql.$inferSelect;
 
-export const createLibsql = async (input: typeof apiCreateLibsql._type) => {
+export const createLibsql = async (input: z.infer<typeof apiCreateLibsql>) => {
 	const appName = buildAppName("libsql", input.appName);
 
 	const valid = await validUniqueServerAppName(input.appName);
@@ -59,13 +61,12 @@ export const findLibsqlById = async (libsqlId: string) => {
 			},
 			mounts: true,
 			server: true,
-			bottomlessReplicationDestination: true,
-			// backups: {
-			// 	with: {
-			// 		destination: true,
-			// 		deployments: true,
-			// 	},
-			// },
+			backups: {
+				with: {
+					destination: true,
+					deployments: true,
+				},
+			},
 		},
 	});
 	if (!result) {
@@ -102,24 +103,24 @@ export const removeLibsqlById = async (libsqlId: string) => {
 	return result[0];
 };
 
-// export const findLibsqlByBackupId = async (backupId: string) => {
-// 	const result = await db
-// 		.select({
-// 			...getTableColumns(libsql),
-// 		})
-// 		.from(libsql)
-// 		.innerJoin(backups, eq(libsql.libsqlId, backups.libsqlId))
-// 		.where(eq(backups.backupId, backupId))
-// 		.limit(1);
-//
-// 	if (!result || !result[0]) {
-// 		throw new TRPCError({
-// 			code: "NOT_FOUND",
-// 			message: "Libsql not found",
-// 		});
-// 	}
-// 	return result[0];
-// };
+export const findLibsqlByBackupId = async (backupId: string) => {
+	const result = await db
+		.select({
+			...getTableColumns(libsql),
+		})
+		.from(libsql)
+		.innerJoin(backups, eq(libsql.libsqlId, backups.libsqlId))
+		.where(eq(backups.backupId, backupId))
+		.limit(1);
+
+	if (!result || !result[0]) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Libsql not found",
+		});
+	}
+	return result[0];
+};
 
 export const deployLibsql = async (
 	libsqlId: string,

packages/server/src/utils/backups/index.ts

@@ -75,6 +75,7 @@ export const initCronJobs = async () => {
 			mariadb: true,
 			mysql: true,
 			mongo: true,
+			libsql: true,
 			user: true,
 			compose: true,
 		},
@@ -116,7 +117,8 @@ const getServiceAppName = (backup: BackupSchedule): string => {
 		backup.postgres?.appName ||
 		backup.mysql?.appName ||
 		backup.mariadb?.appName ||
-		backup.mongo?.appName;
+		backup.mongo?.appName ||
+		backup.libsql?.appName;
 	return serviceAppName || backup.appName;
 };
 

packages/server/src/utils/backups/libsql.ts

@@ -0,0 +1,75 @@
+import type { BackupSchedule } from "@dokploy/server/services/backup";
+import {
+	createDeploymentBackup,
+	updateDeploymentStatus,
+} from "@dokploy/server/services/deployment";
+import { findEnvironmentById } from "@dokploy/server/services/environment";
+import type { Libsql } from "@dokploy/server/services/libsql";
+import { findProjectById } from "@dokploy/server/services/project";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getBackupCommand, getS3Credentials, normalizeS3Path } from "./utils";
+
+export const runLibsqlBackup = async (
+	libsql: Libsql,
+	backup: BackupSchedule,
+) => {
+	const { name, environmentId, appName } = libsql;
+	const environment = await findEnvironmentById(environmentId);
+	const project = await findProjectById(environment.projectId);
+
+	const deployment = await createDeploymentBackup({
+		backupId: backup.backupId,
+		title: "Initializing Backup",
+		description: "Initializing Backup",
+	});
+	const { prefix } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+
+		const backupCommand = getBackupCommand(
+			backup,
+			rcloneCommand,
+			deployment.logPath,
+		);
+		if (libsql.serverId) {
+			await execAsyncRemote(libsql.serverId, backupCommand);
+		} else {
+			await execAsync(backupCommand, {
+				shell: "/bin/bash",
+			});
+		}
+
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "libsql",
+			type: "success",
+			organizationId: project.organizationId,
+			databaseName: backup.database,
+		});
+
+		await updateDeploymentStatus(deployment.deploymentId, "done");
+	} catch (error) {
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "libsql",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+			organizationId: project.organizationId,
+			databaseName: backup.database,
+		});
+
+		await updateDeploymentStatus(deployment.deploymentId, "error");
+
+		throw error;
+	}
+};

packages/server/src/utils/backups/utils.ts

@@ -4,6 +4,7 @@ import type { Destination } from "@dokploy/server/services/destination";
 import { scheduledJobs, scheduleJob } from "node-schedule";
 import { keepLatestNBackups } from ".";
 import { runComposeBackup } from "./compose";
+import { runLibsqlBackup } from "./libsql";
 import { runMariadbBackup } from "./mariadb";
 import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
@@ -19,6 +20,7 @@ export const scheduleBackup = (backup: BackupSchedule) => {
 		mysql,
 		mongo,
 		mariadb,
+		libsql,
 		compose,
 	} = backup;
 	scheduleJob(backupId, schedule, async () => {
@@ -35,6 +37,9 @@ export const scheduleBackup = (backup: BackupSchedule) => {
 			} else if (databaseType === "mariadb" && mariadb) {
 				await runMariadbBackup(mariadb, backup);
 				await keepLatestNBackups(backup, mariadb.serverId);
+			} else if (databaseType === "libsql" && libsql) {
+				await runLibsqlBackup(libsql, backup);
+				await keepLatestNBackups(backup, libsql.serverId);
 			} else if (databaseType === "web-server") {
 				await runWebServerBackup(backup);
 				await keepLatestNBackups(backup);
@@ -107,6 +112,10 @@ export const getMongoBackupCommand = (
 	return `docker exec -i $CONTAINER_ID bash -c "set -o pipefail; mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --archive --authenticationDatabase admin --gzip"`;
 };
 
+export const getLibsqlBackupCommand = (database: string) => {
+	return `docker exec -i $CONTAINER_ID sh -c "tar cf - -C /var/lib/sqld ${database} | gzip"`;
+};
+
 export const getServiceContainerCommand = (appName: string) => {
 	return `docker ps -q --filter "status=running" --filter "label=com.docker.swarm.service.name=${appName}" | head -n 1`;
 };
@@ -123,12 +132,12 @@ export const getComposeContainerCommand = (
 };
 
 const getContainerSearchCommand = (backup: BackupSchedule) => {
-	const { backupType, postgres, mysql, mariadb, mongo, compose, serviceName } =
+	const { backupType, postgres, mysql, mariadb, mongo, libsql, compose, serviceName } =
 		backup;
 
 	if (backupType === "database") {
 		const appName =
-			postgres?.appName || mysql?.appName || mariadb?.appName || mongo?.appName;
+			postgres?.appName || mysql?.appName || mariadb?.appName || mongo?.appName || libsql?.appName;
 		return getServiceContainerCommand(appName || "");
 	}
 	if (backupType === "compose") {
@@ -209,6 +218,12 @@ export const generateBackupCommand = (backup: BackupSchedule) => {
 			}
 			break;
 		}
+		case "libsql": {
+			if (backupType === "database") {
+				return getLibsqlBackupCommand(backup.database);
+			}
+			break;
+		}
 		default:
 			throw new Error(`Database type not supported: ${databaseType}`);
 	}

packages/server/src/utils/databases/libsql.ts

@@ -15,7 +15,6 @@ export type LibsqlNested = InferResultType<
 	{
 		mounts: true;
 		environment: { with: { project: true } };
-		bottomlessReplicationDestination: true;
 	}
 >;
 export const buildLibsql = async (libsql: LibsqlNested) => {
@@ -36,8 +35,6 @@ export const buildLibsql = async (libsql: LibsqlNested) => {
 		command,
 		mounts,
 		enableNamespaces,
-		enableBottomlessReplication,
-		bottomlessReplicationDestination,
 	} = libsql;
 
 	const basicAuth = Buffer.from(
@@ -45,20 +42,10 @@ export const buildLibsql = async (libsql: LibsqlNested) => {
 		"utf-8",
 	).toString("base64");
 
-	let defaultLibsqlEnv = `SQLD_NODE="${sqldNode}"\nSQLD_HTTP_AUTH="basic:${basicAuth}"${
+	const defaultLibsqlEnv = `SQLD_NODE="${sqldNode}"\nSQLD_HTTP_AUTH="basic:${basicAuth}"${
 		env ? `\n${env}` : ""
 	}${sqldNode === "replica" ? `\nSQLD_PRIMARY_URL="${sqldPrimaryUrl}"` : ""}`;
 
-	// Add bottomless replication environment variables if destination is configured
-	if (enableBottomlessReplication && bottomlessReplicationDestination) {
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_DATABASE_ID="${appName}"`;
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_BUCKET="${bottomlessReplicationDestination.bucket}"`;
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_ENDPOINT="${bottomlessReplicationDestination.endpoint}"`;
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY="${bottomlessReplicationDestination.secretAccessKey}"`;
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_AWS_ACCESS_KEY_ID="${bottomlessReplicationDestination.accessKey}"`;
-		defaultLibsqlEnv += `\nLIBSQL_BOTTOMLESS_AWS_DEFAULT_REGION="${bottomlessReplicationDestination.region}"`;
-	}
-
 	const {
 		HealthCheck,
 		RestartPolicy,
@@ -92,16 +79,13 @@ export const buildLibsql = async (libsql: LibsqlNested) => {
 	if (enableNamespaces) {
 		finalCommand += " --enable-namespaces";
 	}
-	if (enableBottomlessReplication) {
-		finalCommand += " --enable-bottomless-replication";
-	}
 
 	const settings: CreateServiceOptions = {
 		Name: appName,
 		TaskTemplate: {
 			ContainerSpec: {
 				HealthCheck,
-				Image: "ghcr.io/tursodatabase/libsql-server:latest",
+				Image: "ghcr.io/tursodatabase/libsql-server:v0.24.32",
 				Env: envVariables,
 				Mounts: [...volumesMount, ...bindsMount, ...filesMount],
 				...(finalCommand

packages/server/src/utils/notifications/database-backup.ts

@@ -29,7 +29,7 @@ export const sendDatabaseBackupNotifications = async ({
 }: {
 	projectName: string;
 	applicationName: string;
-	databaseType: "postgres" | "mysql" | "mongodb" | "mariadb";
+	databaseType: "postgres" | "mysql" | "mongodb" | "mariadb" | "libsql";
 	type: "error" | "success";
 	organizationId: string;
 	errorMessage?: string;

packages/server/src/utils/restore/compose.ts

@@ -62,7 +62,7 @@ export const restoreComposeBackup = async (
 		const restoreCommand = getRestoreCommand({
 			appName: appName,
 			serviceName: backupInput.metadata?.serviceName,
-			type: backupInput.databaseType,
+			type: backupInput.databaseType as "postgres" | "mariadb" | "mysql" | "mongo",
 			credentials: {
 				database: backupInput.databaseName,
 				...credentials,

packages/server/src/utils/restore/index.ts

@@ -1,4 +1,5 @@
 export { restoreComposeBackup } from "./compose";
+export { restoreLibsqlBackup } from "./libsql";
 export { restoreMariadbBackup } from "./mariadb";
 export { restoreMongoBackup } from "./mongo";
 export { restoreMySqlBackup } from "./mysql";

packages/server/src/utils/restore/libsql.ts

@@ -0,0 +1,51 @@
+import type { apiRestoreBackup } from "@dokploy/server/db/schema";
+import type { Destination } from "@dokploy/server/services/destination";
+import type { Libsql } from "@dokploy/server/services/libsql";
+import type { z } from "zod";
+import { getS3Credentials, getServiceContainerCommand } from "../backups/utils";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+
+export const restoreLibsqlBackup = async (
+	libsql: Libsql,
+	destination: Destination,
+	backupInput: z.infer<typeof apiRestoreBackup>,
+	emit: (log: string) => void,
+) => {
+	try {
+		const { appName, serverId } = libsql;
+
+		const rcloneFlags = getS3Credentials(destination);
+		const bucketPath = `:s3:${destination.bucket}`;
+
+		const backupPath = `${bucketPath}/${backupInput.backupFile}`;
+
+		const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} "${backupPath}"`;
+
+		emit("Starting restore...");
+		emit(`Backup path: ${backupPath}`);
+
+		const containerSearch = getServiceContainerCommand(appName);
+		const restoreCommand = `docker exec -i $CONTAINER_ID sh -c "tar xzf - -C /var/lib/sqld"`;
+
+		const command = `CONTAINER_ID=$(${containerSearch}) && ${rcloneCommand} | ${restoreCommand}`;
+
+		emit(`Executing command: ${command}`);
+
+		if (serverId) {
+			await execAsyncRemote(serverId, command);
+		} else {
+			await execAsync(command);
+		}
+
+		emit("Restore completed successfully!");
+	} catch (error) {
+		emit(
+			`Error: ${
+				error instanceof Error
+					? error.message
+					: "Error restoring libsql backup"
+			}`,
+		);
+		throw error;
+	}
+};