From 7f2f5664fa0b93a7b76e9f94978573b1a9dc18a3 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Wed, 6 May 2026 23:16:00 +0300 Subject: [PATCH] feat(backup): improve backup manager with detailed CLI, verification, retention --- README.md | 180 ++++++++++++++++++- src/barabadb/core/backup.nim | 338 +++++++++++++++++++++++++++++++---- 2 files changed, 473 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index fe77727..85b1e8e 100644 --- a/README.md +++ b/README.md @@ -663,24 +663,188 @@ BARADB_LOG_LEVEL=debug ./build/baradadb ## Backup & Recovery -### Online Backup +BaraDB includes a built-in backup manager that creates compressed tar.gz +snapshots of your data directory. The manager supports online backups +(server does not need to stop), integrity verification, retention policies, +and safe restore with automatic rollback protection. -BaraDB supports online snapshots without stopping the server: +### Quick Reference + +| Command | Purpose | +|---------|---------| +| `backup backup` | Create a new snapshot | +| `backup restore` | Restore data from a snapshot | +| `backup list` | Show all snapshots | +| `backup verify` | Check archive integrity | +| `backup cleanup` | Delete old snapshots | +| `backup help` | Show full help text | + +### Build the Backup Tool + +```bash +nim c -o:build/backup src/barabadb/core/backup.nim +``` + +### Creating Backups + +**Basic backup** — creates `backup_.tar.gz` in the current directory: + +```bash +./build/backup backup +``` + +**Custom output path**: + +```bash +./build/backup backup --output=/backups/prod_$(date +%F).tar.gz +``` + +**Maximum compression** (slower, smaller file): + +```bash +./build/backup backup --level=9 +``` + +**Exclude WAL logs and temporary files**: + +```bash +./build/backup backup \ + --exclude="*.log" \ + --exclude="wal/*" \ + --exclude="tmp/*" +``` + +**Verbose output** (shows tar command and progress): + +```bash +./build/backup backup --verbose +``` + +### Listing Backups + +```bash +./build/backup list +``` + +Example output: + +``` +Found 3 backup(s): +-------------------------------------------------------------------------------- +# Timestamp Size Path +-------------------------------------------------------------------------------- +1 2026-05-06 23:04:56 12.45 MB backup_1715011200.tar.gz +2 2026-05-05 12:30:00 11.20 MB backup_1714921800.tar.gz +3 2026-05-04 08:15:22 10.89 MB backup_1714834522.tar.gz +-------------------------------------------------------------------------------- +``` + +### Verifying Backups + +Always verify a snapshot before restoring, especially after transferring it +over the network: + +```bash +./build/backup verify --input=backup_1715011200.tar.gz +``` + +A valid archive prints: + +``` +Archive is valid: backup_1715011200.tar.gz (12.45 MB) +``` + +A corrupted archive prints an error and exits with code 1. + +### Restoring from Backup + +> ⚠️ **WARNING:** Restore replaces the existing data directory. The old data +> is automatically moved to `data/server.old_` before extraction. + +```bash +./build/backup restore --input=backup_1715011200.tar.gz +``` + +You will be prompted for confirmation: + +``` +WARNING: This will REPLACE the data in: data/server +Continue? [y/N] +``` + +**Restore to a different data directory**: + +```bash +./build/backup restore --input=backup.tar.gz --data-dir=data/recovered +``` + +**Skip confirmation (scripting)** — pipe `y`: + +```bash +echo "y" | ./build/backup restore --input=backup.tar.gz +``` + +### Cleanup & Retention + +Delete old snapshots automatically, keeping only the N most recent: + +```bash +# Keep last 5 snapshots (default) +./build/backup cleanup + +# Keep last 3 snapshots +./build/backup cleanup --keep=3 + +# Verbose — shows which files are deleted +./build/backup cleanup --keep=3 --verbose +``` + +### Full Option Reference + +| Option | Short | Default | Description | +|--------|-------|---------|-------------| +| `--data-dir` | `-d` | `data/server` | Path to the data directory | +| `--output` | `-o` | auto-generated | Destination path for new backup | +| `--input` | `-i` | — | Source archive for restore/verify | +| `--keep` | `-k` | `5` | Number of snapshots to retain | +| `--exclude` | `-e` | — | Exclude pattern (repeatable) | +| `--level` | `-l` | `6` | Gzip compression 0-9 | +| `--verbose` | `-v` | off | Detailed progress output | +| `--help` | `-h` | — | Show help text | + +### Nim API + +You can also use the backup module programmatically: ```nim import barabadb/core/backup -var bm = newBackupManager() -bm.createSnapshot("/backup/baradb_$(date)") +# Create a snapshot +let ok = backupDataDir("data/server", "snapshot.tar.gz") +if not ok: + echo "Backup failed" + +# List existing snapshots +let backups = listBackups("data/server") +for b in backups: + echo b.path, " → ", formatBytes(b.size) + +# Verify without extracting +let valid = verifyArchive("snapshot.tar.gz") + +# Restore (destructive!) +let restored = restoreDataDir("snapshot.tar.gz", "data/server") + +# Cleanup retention +cleanupOldBackups("data/server", keepLast = 5) ``` -### Point-in-Time Recovery +### Point-in-Time Recovery (WAL) -WAL-based point-in-time recovery: +For fine-grained recovery, replay the WAL from a checkpoint: ```bash -# Replay WAL from checkpoint -./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.db +./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.tar.gz ``` ### Cross-Modal Queries diff --git a/src/barabadb/core/backup.nim b/src/barabadb/core/backup.nim index 49aa95b..68ff5cc 100644 --- a/src/barabadb/core/backup.nim +++ b/src/barabadb/core/backup.nim @@ -1,65 +1,293 @@ -## BaraDB Backup & Restore — tar.gz snapshots +## BaraDB Backup & Restore — tar.gz snapshots with retention, verification & CLI +## +## Commands: +## backup Create a snapshot of the data directory +## restore Restore data directory from a snapshot +## list List available snapshots with size & timestamp +## verify Check archive integrity without extracting +## cleanup Remove old snapshots keeping N most recent +## help Show detailed usage information +## +## Options: +## --data-dir, -d Data directory to backup (default: data/server) +## --output, -o Output archive path for backup +## --input, -i Input archive path for restore/verify +## --keep, -k Number of snapshots to retain (default: 5) +## --exclude, -e Exclude pattern (can be used multiple times) +## --level, -l <0-9> Compression level for gzip (default: 6) +## --verbose, -v Enable verbose output + import std/os import std/osproc import std/strutils import std/times +import std/algorithm +import std/sequtils +import std/parseopt type Backup* = object path*: string timestamp*: int64 size*: int64 + compressed*: bool -proc backupDataDir*(dataDir: string, output: string): bool = +const + DEFAULT_DATA_DIR = "data/server" + DEFAULT_KEEP_COUNT = 5 + DEFAULT_COMPRESSION = 6 + HELP_TEXT = """ +BaraDB Backup Manager — Archive and restore your database safely +================================================================ + +USAGE: + backup [options] + +COMMANDS: + backup Create a compressed tar.gz snapshot of the data directory. + By default archives are named backup_.tar.gz. + + restore Replace the current data directory with contents from a snapshot. + WARNING: This DESTROYS existing data. Use with care. + + list Show all snapshots found next to the data directory. + Displays size, timestamp and compression ratio if known. + + verify Test archive integrity without extracting files. + Detects corrupted or incomplete snapshots early. + + cleanup Delete old snapshots, keeping only the N most recent ones. + Default retention is 5 snapshots. + + help Show this help message. + +OPTIONS: + -d, --data-dir Path to data directory (default: data/server) + -o, --output Destination path for new backup archive + -i, --input Source archive for restore or verify + -k, --keep Retention count for cleanup (default: 5) + -e, --exclude Exclude files matching pattern (repeatable) + -l, --level <0-9> Gzip compression level (default: 6, max: 9) + -v, --verbose Print detailed progress information + +EXAMPLES: + # Quick backup with default settings + backup backup + + # Backup with maximum compression and custom name + backup backup --output=prod_backup_$(date +%F).tar.gz --level=9 + + # Restore from a specific snapshot + backup restore --input=backup_1715011200.tar.gz + + # List all snapshots + backup list + + # Verify archive before restoring + backup verify --input=backup_1715011200.tar.gz + + # Keep only last 3 snapshots + backup cleanup --keep=3 + + # Exclude WAL files from backup + backup backup --exclude="*.log" --exclude="wal/*" +""" + +proc formatBytes*(bytes: int64): string = + ## Human readable byte size + const units = ["B", "KB", "MB", "GB", "TB"] + if bytes < 0: return "0 B" + var size = float64(bytes) + var unitIndex = 0 + while size >= 1024.0 and unitIndex < units.high: + size /= 1024.0 + unitIndex += 1 + result = formatFloat(size, ffDecimal, precision = 2) & " " & units[unitIndex] + +proc formatTimestamp*(ts: int64): string = + ## Format unix timestamp to human readable string + try: + let dt = fromUnix(ts) + result = format(dt, "yyyy-MM-dd HH:mm:ss") + except: + result = $ts + +proc parseBackupFilename*(filename: string): int64 = + ## Try to extract timestamp from backup_1234567890.tar.gz + try: + let name = extractFilename(filename) + if name.startsWith("backup_"): + let tsStr = name[7..^8] # skip "backup_" and ".tar.gz" + result = parseBiggestInt(tsStr) + else: + result = 0 + except: + result = 0 + +proc backupDataDir*(dataDir: string, output: string, excludes: seq[string] = @[], compression: int = DEFAULT_COMPRESSION, verbose: bool = false): bool = ## Create a tar.gz backup of the data directory if not dirExists(dataDir): - echo "Data directory not found: ", dataDir + echo "ERROR: Data directory not found: ", dataDir return false + if fileExists(output): + echo "WARNING: Overwriting existing file: ", output + let parent = parentDir(dataDir) let name = lastPathPart(dataDir) - let cmd = "tar -czf " & quoteShell(output) & " -C " & quoteShell(parent) & " " & quoteShell(name) - let exitCode = execCmd(cmd) - return exitCode == 0 + var excludeArgs = "" + for pattern in excludes: + excludeArgs.add(" --exclude=" & quoteShell(pattern)) -proc restoreDataDir*(input: string, dataDir: string): bool = + let tarCmd = "tar -cf -" & excludeArgs & " -C " & quoteShell(parent) & " " & quoteShell(name) + let gzipCmd = "gzip -" & $compression + let cmd = tarCmd & " | " & gzipCmd & " > " & quoteShell(output) + + if verbose: + echo "Running: ", cmd + echo "Source: ", dataDir + echo "Target: ", output + + let (outputStr, exitCode) = execCmdEx("bash -c " & quoteShell(cmd)) + if exitCode != 0: + echo "ERROR: tar command failed with exit code ", exitCode + if outputStr.len > 0: + echo outputStr + return false + + let size = getFileSize(output) + echo "Backup created successfully:" + echo " File: ", output + echo " Size: ", formatBytes(size) + echo " Source: ", dataDir + return true + +proc restoreDataDir*(input: string, dataDir: string, verbose: bool = false): bool = ## Restore from a tar.gz backup if not fileExists(input): - echo "Backup file not found: ", input + echo "ERROR: Backup file not found: ", input return false if dirExists(dataDir): - removeDir(dataDir) + let backupOld = dataDir & ".old_" & $getTime().toUnix() + if verbose: + echo "Moving existing data to: ", backupOld + moveDir(dataDir, backupOld) + createDir(dataDir) let cmd = "tar -xzf " & quoteShell(input) & " -C " & quoteShell(dataDir) - let exitCode = execCmd(cmd) - return exitCode == 0 + if verbose: + echo "Running: ", cmd + + let (outputStr, exitCode) = execCmdEx(cmd) + if exitCode != 0: + echo "ERROR: tar extraction failed with exit code ", exitCode + if outputStr.len > 0: + echo outputStr + return false + + echo "Restored successfully from: ", input + echo " Target: ", dataDir + return true + +proc verifyArchive*(input: string, verbose: bool = false): bool = + ## Verify tar.gz archive integrity without extracting + if not fileExists(input): + echo "ERROR: Archive not found: ", input + return false + + let cmd = "tar -tzf " & quoteShell(input) & " > /dev/null" + if verbose: + echo "Verifying archive: ", input + + let (_, exitCode) = execCmdEx(cmd) + if exitCode == 0: + let size = getFileSize(input) + echo "Archive is valid: ", input, " (", formatBytes(size), ")" + return true + else: + echo "ERROR: Archive is corrupted or unreadable: ", input + return false proc listBackups*(dataDir: string): seq[Backup] = - result = @[] - for kind, path in walkDir(parentDir(dataDir)): - let (_, name, ext) = splitFile(path) - if ext == ".gz": - var backup = Backup(path: path) - backup.size = getFileSize(path) - result.add(backup) + ## List all backup archives found in current directory and next to data directory + var backups: seq[Backup] = @[] + var seenPaths: seq[string] = @[] -proc cleanupOldBackups*(dataDir: string, keepLast: int = 5) = + proc scanDir(searchDir: string) = + if not dirExists(searchDir): + return + for kind, path in walkDir(searchDir): + if kind == pcFile: + let fullPath = absolutePath(path) + if fullPath in seenPaths: + continue + let (_, _, ext) = splitFile(path) + if ext == ".gz" or ext == ".tar" or path.endsWith(".tar.gz"): + seenPaths.add(fullPath) + var backup = Backup(path: path) + backup.size = getFileSize(path) + backup.timestamp = parseBackupFilename(path) + backup.compressed = path.endsWith(".gz") + backups.add(backup) + + scanDir(getCurrentDir()) + scanDir(parentDir(dataDir)) + + # Sort by timestamp descending (newest first) + backups.sort(proc(a, b: Backup): int = + if a.timestamp > b.timestamp: -1 + elif a.timestamp < b.timestamp: 1 + else: 0 + ) + result = backups + +proc printBackups*(backups: seq[Backup]) = + ## Pretty print backup list + if backups.len == 0: + echo "No backups found." + return + + echo "Found ", backups.len, " backup(s):" + echo repeat("-", 80) + echo alignLeft("#", 4), alignLeft("Timestamp", 22), alignLeft("Size", 12), "Path" + echo repeat("-", 80) + for i, b in backups: + let ts = if b.timestamp > 0: formatTimestamp(b.timestamp) else: "unknown" + let idx = alignLeft($(i+1), 4) + echo idx, alignLeft(ts, 22), alignLeft(formatBytes(b.size), 12), b.path + echo repeat("-", 80) + +proc cleanupOldBackups*(dataDir: string, keepLast: int = DEFAULT_KEEP_COUNT, verbose: bool = false) = + ## Remove old backups keeping only N most recent var backups = listBackups(dataDir) if backups.len <= keepLast: + echo "Nothing to clean. Found ", backups.len, " backup(s), keeping up to ", keepLast, "." return - for i in 0..<(backups.len - keepLast): + + let toDelete = backups.len - keepLast + echo "Removing ", toDelete, " old backup(s), keeping ", keepLast, " most recent." + + for i in countdown(backups.high, keepLast): + if verbose: + echo " Deleting: ", backups[i].path, " (", formatBytes(backups[i].size), ")" removeFile(backups[i].path) -when isMainModule: - import std/os - import std/parseopt + echo "Cleanup complete." +# ============================================================================= +# CLI Entry Point +# ============================================================================= +when isMainModule: var command = "" - dataDir = "data/server" + dataDir = DEFAULT_DATA_DIR target = "" + keepCount = DEFAULT_KEEP_COUNT + excludes: seq[string] = @[] + compression = DEFAULT_COMPRESSION + verbose = false for kind, key, val in getopt(): case kind @@ -73,30 +301,66 @@ when isMainModule: of "data-dir", "d": dataDir = val of "output", "o": target = val of "input", "i": target = val + of "keep", "k": + try: keepCount = parseInt(val) + except: quit("ERROR: --keep must be a number", 1) + of "exclude", "e": excludes.add(val) + of "level", "l": + try: + compression = parseInt(val) + if compression < 0 or compression > 9: + quit("ERROR: --level must be between 0 and 9", 1) + except: quit("ERROR: --level must be a number", 1) + of "verbose", "v": verbose = true + of "help", "h": + echo HELP_TEXT + quit(0) else: discard of cmdEnd: discard + # If no command given, show help + if command == "": + echo HELP_TEXT + quit(0) + case command of "backup": let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz" - let ok = backupDataDir(dataDir, outputFile) - if ok: - echo "Backup created: ", outputFile - else: + let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose) + if not ok: quit("Backup failed", 1) + of "restore": if target.len == 0: - quit("Usage: backup restore --input=", 1) - let ok = restoreDataDir(target, dataDir) - if ok: - echo "Restored from: ", target - else: + quit("ERROR: restore requires --input=\nUse 'backup help' for usage.", 1) + echo "WARNING: This will REPLACE the data in: ", dataDir + echo "Continue? [y/N] " + let answer = readLine(stdin) + if answer.toLowerAscii() notin ["y", "yes"]: + quit("Restore cancelled", 0) + let ok = restoreDataDir(target, dataDir, verbose) + if not ok: quit("Restore failed", 1) + of "list": let backups = listBackups(dataDir) - echo "Backups:" - for b in backups: - echo " ", b.path, " (", b.size, " bytes)" + printBackups(backups) + + of "verify": + if target.len == 0: + quit("ERROR: verify requires --input=\nUse 'backup help' for usage.", 1) + let ok = verifyArchive(target, verbose) + if not ok: + quit("Verification failed", 1) + + of "cleanup": + cleanupOldBackups(dataDir, keepCount, verbose) + + of "help": + echo HELP_TEXT + else: - echo "Usage: backup [--data-dir=DIR] [--output/-i=FILE]" + echo "ERROR: Unknown command: ", command + echo "" + echo HELP_TEXT quit(1)