diff --git a/apps/dokploy/__test__/volume-backups/volume-restore.real.test.ts b/apps/dokploy/__test__/volume-backups/volume-restore.real.test.ts index 1cdf73c42..13962040c 100644 --- a/apps/dokploy/__test__/volume-backups/volume-restore.real.test.ts +++ b/apps/dokploy/__test__/volume-backups/volume-restore.real.test.ts @@ -85,37 +85,37 @@ describe( ); }); - afterEach(async () => { - console.log(`\n๐Ÿงน Cleanup: ${currentVolumeName}`); - await cleanupDocker(currentVolumeName); - await cleanupFiles(currentAppName); + // afterEach(async () => { + // console.log(`\n๐Ÿงน Cleanup: ${currentVolumeName}`); + // await cleanupDocker(currentVolumeName); + // await cleanupFiles(currentAppName); - // Clean all test volumes - try { - const { stdout } = await execAsync( - `docker volume ls -q --filter "name=test-vol-" || true`, - ); - if (stdout.trim()) { - for (const vol of stdout.trim().split("\n")) { - await execAsync(`docker volume rm ${vol} 2>/dev/null || true`); - } - } - } catch { - // Ignore - } + // // Clean all test volumes + // try { + // const { stdout } = await execAsync( + // `docker volume ls -q --filter "name=test-vol-" || true`, + // ); + // if (stdout.trim()) { + // for (const vol of stdout.trim().split("\n")) { + // await execAsync(`docker volume rm ${vol} 2>/dev/null || true`); + // } + // } + // } catch { + // // Ignore + // } - // Clean all test backup directories - try { - const { VOLUME_BACKUPS_PATH } = paths(false); - await execAsync( - `find "${VOLUME_BACKUPS_PATH}" -maxdepth 1 -type d -name "test-*" -exec rm -rf {} + 2>/dev/null || true`, - ); - } catch { - // Ignore - } + // // Clean all test backup directories + // try { + // const { VOLUME_BACKUPS_PATH } = paths(false); + // await execAsync( + // `find "${VOLUME_BACKUPS_PATH}" -maxdepth 1 -type d -name "test-*" -exec rm -rf {} + 2>/dev/null || true`, + // ); + // } catch { + // // Ignore + // } - console.log("โœ… Cleanup done\n"); - }); + // console.log("โœ… Cleanup done\n"); + // }); it( "should restore volume using real restoreVolume ", @@ -402,6 +402,376 @@ describe( }, REAL_TEST_TIMEOUT, ); + + it( + "should restore 15k files using real restoreVolume - Issue #3301", + async () => { + console.log( + `\n๐Ÿš€ Test 15k files restore with real code: ${currentVolumeName}`, + ); + + // Step 1: Create volume with 15,000 files + await execAsync(`docker volume create ${currentVolumeName}`); + console.log("โœ… Volume created"); + + const startTime = Date.now(); + console.log("๐Ÿ“ Creating 15,000 files..."); + await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu bash -c ' + mkdir -p /data/dir1 /data/dir2 /data/dir3 /data/dir4 /data/dir5 + + # Create 3000 files in each directory + for i in $(seq 1 3000); do + echo "file content $i" > /data/dir1/file-$i.txt + done + + for i in $(seq 1 3000); do + echo "file content $i" > /data/dir2/file-$i.txt + done + + for i in $(seq 1 3000); do + echo "file content $i" > /data/dir3/file-$i.txt + done + + for i in $(seq 1 3000); do + echo "file content $i" > /data/dir4/file-$i.txt + done + + for i in $(seq 1 3000); do + echo "file content $i" > /data/dir5/file-$i.txt + done + + echo "marker-15000" > /data/marker.txt + echo "Total files created: $(find /data -type f | wc -l)" + du -sh /data + ' + `); + const createTime = ((Date.now() - startTime) / 1000).toFixed(2); + console.log(`โœ… Created 15,000 files in ${createTime}s`); + + // Step 2: Create backup + const { VOLUME_BACKUPS_PATH } = paths(false); + const volumeBackupPath = path.join( + VOLUME_BACKUPS_PATH, + currentVolumeName, + ); + await execAsync(`mkdir -p "${volumeBackupPath}"`); + + const backupFileName = `${currentVolumeName}-15k.tar`; + + const backupStartTime = Date.now(); + console.log("๐Ÿ“ฆ Creating backup of 15k files..."); + await execAsync(` + docker run --rm -v ${currentVolumeName}:/volume_data -v "${volumeBackupPath}":/backup ubuntu bash -c " + cd /volume_data && tar cf /backup/${backupFileName} . + " + `); + const backupTime = ((Date.now() - backupStartTime) / 1000).toFixed(2); + console.log(`โœ… Backup created in ${backupTime}s`); + + // Step 3: Remove original volume + await execAsync(`docker volume rm ${currentVolumeName}`); + console.log("โœ… Removed original volume"); + + // Step 4: Use REAL restoreVolume function + const mockDestination = createMockDestination(); + const mockApplication = createMockApplication(currentAppName); + + vi.mocked(destinationService.findDestinationById).mockResolvedValue( + mockDestination as any, + ); + vi.mocked(applicationService.findApplicationById).mockResolvedValue( + mockApplication as any, + ); + + const fullCommand = await restoreVolume( + mockApplication.applicationId, + mockDestination.destinationId, + currentVolumeName, + backupFileName, + "", + "application", + ); + + console.log("๐Ÿ“ฅ Executing REAL Dokploy restore for 15k files..."); + + // Execute the REAL command + const restoreStartTime = Date.now(); + const commandWithoutS3 = fullCommand.replace( + /rclone copyto[^\n]+/g, + 'echo "Skipping S3 download - file already present for test"', + ); + + try { + await execAsync(commandWithoutS3); + const restoreTime = ((Date.now() - restoreStartTime) / 1000).toFixed( + 2, + ); + console.log(`โœ… 15k files restored in ${restoreTime}s`); + } catch (error: any) { + console.error("Restore command failed:", error.message); + throw error; + } + + // Step 5: Verify data integrity + console.log("๐Ÿ” Verifying 15k restored files..."); + + // Check marker file + const { stdout: marker } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/marker.txt + `); + expect(marker.trim()).toBe("marker-15000"); + + // Count restored files + const { stdout: fileCount } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu bash -c "find /data -type f | wc -l" + `); + const totalFiles = Number(fileCount.trim()); + expect(totalFiles).toBeGreaterThanOrEqual(15000); + console.log(`โœ… Verified ${totalFiles} files restored`); + + // Verify random files from different directories + const { stdout: file1 } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/dir1/file-1500.txt + `); + expect(file1).toContain("file content"); + + const { stdout: file2 } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/dir3/file-2000.txt + `); + expect(file2).toContain("file content"); + + const { stdout: file3 } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/dir5/file-2500.txt + `); + expect(file3).toContain("file content"); + + // Verify directory structure + const { stdout: dirs } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu ls /data + `); + expect(dirs).toContain("dir1"); + expect(dirs).toContain("dir2"); + expect(dirs).toContain("dir3"); + expect(dirs).toContain("dir4"); + expect(dirs).toContain("dir5"); + + // Verify volume exists + const { stdout: volumeCheck } = await execAsync( + `docker volume ls --filter name=${currentVolumeName} --format "{{.Name}}"`, + ); + expect(volumeCheck.trim()).toBe(currentVolumeName); + + console.log("\n๐Ÿ“Š Performance Summary:"); + console.log(` - Creating 15k files: ${createTime}s`); + console.log(` - Backup: ${backupTime}s`); + console.log( + "โœ… 15k files restore test PASSED - Real Dokploy code handles many files correctly", + ); + }, + REAL_TEST_TIMEOUT, + ); + + it( + "should restore 10k files + 500MB folder - Combined stress test", + async () => { + console.log( + `\n๐Ÿš€ Test 10k files + 500MB restore: ${currentVolumeName}`, + ); + + // Step 1: Create volume with 10k files + 500MB + await execAsync(`docker volume create ${currentVolumeName}`); + console.log("โœ… Volume created"); + + const startTime = Date.now(); + console.log("๐Ÿ“ Creating 10,000 files + 500MB data..."); + await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu bash -c ' + # Create directory structure + mkdir -p /data/small-files/batch1 /data/small-files/batch2 + mkdir -p /data/large-files + + # Create 5000 files in each batch (10k total) + echo "Creating 10k small files..." + for i in $(seq 1 5000); do + echo "content-$i" > /data/small-files/batch1/file-$i.txt + done + + for i in $(seq 1 5000); do + echo "content-$i" > /data/small-files/batch2/file-$i.txt + done + + # Create ~500MB of large files + echo "Creating 500MB of large files..." + dd if=/dev/zero of=/data/large-files/large-1.dat bs=1M count=125 2>/dev/null + dd if=/dev/zero of=/data/large-files/large-2.dat bs=1M count=125 2>/dev/null + dd if=/dev/zero of=/data/large-files/large-3.dat bs=1M count=125 2>/dev/null + dd if=/dev/zero of=/data/large-files/large-4.dat bs=1M count=125 2>/dev/null + + # Create marker files + echo "marker-combined-test" > /data/marker.txt + echo "10k-files-500mb" > /data/test-type.txt + + echo "Summary:" + echo "Total files: $(find /data -type f | wc -l)" + echo "Total size: $(du -sh /data | cut -f1)" + ' + `); + const createTime = ((Date.now() - startTime) / 1000).toFixed(2); + console.log(`โœ… Created 10k files + 500MB in ${createTime}s`); + + // Step 2: Create backup + const { VOLUME_BACKUPS_PATH } = paths(false); + const volumeBackupPath = path.join( + VOLUME_BACKUPS_PATH, + currentVolumeName, + ); + await execAsync(`mkdir -p "${volumeBackupPath}"`); + + const backupFileName = `${currentVolumeName}-combined.tar`; + + const backupStartTime = Date.now(); + console.log("๐Ÿ“ฆ Creating backup of 10k files + 500MB..."); + await execAsync(` + docker run --rm -v ${currentVolumeName}:/volume_data -v "${volumeBackupPath}":/backup ubuntu bash -c ' + cd /volume_data && tar cf /backup/${backupFileName} . + ' + `); + const backupTime = ((Date.now() - backupStartTime) / 1000).toFixed(2); + console.log(`โœ… Backup created in ${backupTime}s`); + + // Verify backup size + const backupFilePath = path.join(volumeBackupPath, backupFileName); + const { stdout: backupSize } = await execAsync( + `stat -f%z "${backupFilePath}" 2>/dev/null || stat -c%s "${backupFilePath}"`, + ); + const sizeInMB = Number(backupSize.trim()) / (1024 * 1024); + console.log(`Backup size: ${sizeInMB.toFixed(2)}MB`); + // Step 3: Remove original volume + await execAsync(`docker volume rm ${currentVolumeName}`); + console.log("โœ… Removed original volume"); + + // Step 4: Use REAL restoreVolume function + const mockDestination = createMockDestination(); + const mockApplication = createMockApplication(currentAppName); + + vi.mocked(destinationService.findDestinationById).mockResolvedValue( + mockDestination as any, + ); + vi.mocked(applicationService.findApplicationById).mockResolvedValue( + mockApplication as any, + ); + + const fullCommand = await restoreVolume( + mockApplication.applicationId, + mockDestination.destinationId, + currentVolumeName, + backupFileName, + "", + "application", + ); + + console.log("๐Ÿ“ฅ Executing REAL Dokploy restore for combined test..."); + + // Execute the REAL command + const restoreStartTime = Date.now(); + const commandWithoutS3 = fullCommand.replace( + /rclone copyto[^\n]+/g, + 'echo "Skipping S3 download - file already present for test"', + ); + + try { + await execAsync(commandWithoutS3); + const restoreTime = ((Date.now() - restoreStartTime) / 1000).toFixed( + 2, + ); + console.log(`โœ… Combined restore executed in ${restoreTime}s`); + } catch (error: any) { + console.error("Restore command failed:", error.message); + throw error; + } + + // Step 5: Verify data integrity + console.log("๐Ÿ” Verifying restored data..."); + + // Check marker files + const { stdout: marker } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/marker.txt + `); + expect(marker.trim()).toBe("marker-combined-test"); + + const { stdout: testType } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/test-type.txt + `); + expect(testType.trim()).toBe("10k-files-500mb"); + + // Count restored files + const { stdout: fileCount } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu bash -c "find /data -type f | wc -l" + `); + const totalFiles = Number(fileCount.trim()); + expect(totalFiles).toBeGreaterThanOrEqual(10000); + console.log(`โœ… Verified ${totalFiles} files restored`); + + // Verify random small files + const { stdout: smallFile1 } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/small-files/batch1/file-2500.txt + `); + expect(smallFile1).toContain("content-2500"); + + const { stdout: smallFile2 } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu cat /data/small-files/batch2/file-3000.txt + `); + expect(smallFile2).toContain("content-3000"); + + // Verify large files exist + const { stdout: largeFiles } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu ls -lh /data/large-files/ + `); + expect(largeFiles).toContain("large-1.dat"); + expect(largeFiles).toContain("large-2.dat"); + expect(largeFiles).toContain("large-3.dat"); + expect(largeFiles).toContain("large-4.dat"); + + // Verify large file size + const { stdout: large1Size } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu bash -c "stat -c%s /data/large-files/large-1.dat 2>/dev/null || stat -f%z /data/large-files/large-1.dat" + `); + const fileSizeInMB = Number(large1Size.trim()) / (1024 * 1024); + expect(fileSizeInMB).toBeGreaterThan(100); // Should be ~125MB + console.log( + `โœ… Large file size verified: ${fileSizeInMB.toFixed(2)}MB`, + ); + + // Verify directory structure + const { stdout: dirs } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu ls /data + `); + expect(dirs).toContain("small-files"); + expect(dirs).toContain("large-files"); + + // Verify total volume size + const { stdout: totalSize } = await execAsync(` + docker run --rm -v ${currentVolumeName}:/data ubuntu du -sh /data + `); + console.log(`Restored volume size: ${totalSize.trim()}`); + + // Verify volume exists + const { stdout: volumeCheck } = await execAsync( + `docker volume ls --filter name=${currentVolumeName} --format "{{.Name}}"`, + ); + expect(volumeCheck.trim()).toBe(currentVolumeName); + + console.log("\n๐Ÿ“Š Performance Summary:"); + console.log(` - Creating 10k files + 500MB: ${createTime}s`); + console.log(` - Backup: ${backupTime}s`); + console.log(` - Backup size: ${sizeInMB.toFixed(2)}MB`); + console.log( + "โœ… Combined stress test PASSED - Real Dokploy code handles 10k files + 500MB correctly", + ); + }, + REAL_TEST_TIMEOUT, + ); }, REAL_TEST_TIMEOUT, );