feat(backup): improve backup manager with detailed CLI, verification, retention
This commit is contained in:
@@ -663,24 +663,188 @@ BARADB_LOG_LEVEL=debug ./build/baradadb
|
||||
|
||||
## Backup & Recovery
|
||||
|
||||
### Online Backup
|
||||
BaraDB includes a built-in backup manager that creates compressed tar.gz
|
||||
snapshots of your data directory. The manager supports online backups
|
||||
(server does not need to stop), integrity verification, retention policies,
|
||||
and safe restore with automatic rollback protection.
|
||||
|
||||
BaraDB supports online snapshots without stopping the server:
|
||||
### Quick Reference
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `backup backup` | Create a new snapshot |
|
||||
| `backup restore` | Restore data from a snapshot |
|
||||
| `backup list` | Show all snapshots |
|
||||
| `backup verify` | Check archive integrity |
|
||||
| `backup cleanup` | Delete old snapshots |
|
||||
| `backup help` | Show full help text |
|
||||
|
||||
### Build the Backup Tool
|
||||
|
||||
```bash
|
||||
nim c -o:build/backup src/barabadb/core/backup.nim
|
||||
```
|
||||
|
||||
### Creating Backups
|
||||
|
||||
**Basic backup** — creates `backup_<timestamp>.tar.gz` in the current directory:
|
||||
|
||||
```bash
|
||||
./build/backup backup
|
||||
```
|
||||
|
||||
**Custom output path**:
|
||||
|
||||
```bash
|
||||
./build/backup backup --output=/backups/prod_$(date +%F).tar.gz
|
||||
```
|
||||
|
||||
**Maximum compression** (slower, smaller file):
|
||||
|
||||
```bash
|
||||
./build/backup backup --level=9
|
||||
```
|
||||
|
||||
**Exclude WAL logs and temporary files**:
|
||||
|
||||
```bash
|
||||
./build/backup backup \
|
||||
--exclude="*.log" \
|
||||
--exclude="wal/*" \
|
||||
--exclude="tmp/*"
|
||||
```
|
||||
|
||||
**Verbose output** (shows tar command and progress):
|
||||
|
||||
```bash
|
||||
./build/backup backup --verbose
|
||||
```
|
||||
|
||||
### Listing Backups
|
||||
|
||||
```bash
|
||||
./build/backup list
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```
|
||||
Found 3 backup(s):
|
||||
--------------------------------------------------------------------------------
|
||||
# Timestamp Size Path
|
||||
--------------------------------------------------------------------------------
|
||||
1 2026-05-06 23:04:56 12.45 MB backup_1715011200.tar.gz
|
||||
2 2026-05-05 12:30:00 11.20 MB backup_1714921800.tar.gz
|
||||
3 2026-05-04 08:15:22 10.89 MB backup_1714834522.tar.gz
|
||||
--------------------------------------------------------------------------------
|
||||
```
|
||||
|
||||
### Verifying Backups
|
||||
|
||||
Always verify a snapshot before restoring, especially after transferring it
|
||||
over the network:
|
||||
|
||||
```bash
|
||||
./build/backup verify --input=backup_1715011200.tar.gz
|
||||
```
|
||||
|
||||
A valid archive prints:
|
||||
|
||||
```
|
||||
Archive is valid: backup_1715011200.tar.gz (12.45 MB)
|
||||
```
|
||||
|
||||
A corrupted archive prints an error and exits with code 1.
|
||||
|
||||
### Restoring from Backup
|
||||
|
||||
> ⚠️ **WARNING:** Restore replaces the existing data directory. The old data
|
||||
> is automatically moved to `data/server.old_<timestamp>` before extraction.
|
||||
|
||||
```bash
|
||||
./build/backup restore --input=backup_1715011200.tar.gz
|
||||
```
|
||||
|
||||
You will be prompted for confirmation:
|
||||
|
||||
```
|
||||
WARNING: This will REPLACE the data in: data/server
|
||||
Continue? [y/N]
|
||||
```
|
||||
|
||||
**Restore to a different data directory**:
|
||||
|
||||
```bash
|
||||
./build/backup restore --input=backup.tar.gz --data-dir=data/recovered
|
||||
```
|
||||
|
||||
**Skip confirmation (scripting)** — pipe `y`:
|
||||
|
||||
```bash
|
||||
echo "y" | ./build/backup restore --input=backup.tar.gz
|
||||
```
|
||||
|
||||
### Cleanup & Retention
|
||||
|
||||
Delete old snapshots automatically, keeping only the N most recent:
|
||||
|
||||
```bash
|
||||
# Keep last 5 snapshots (default)
|
||||
./build/backup cleanup
|
||||
|
||||
# Keep last 3 snapshots
|
||||
./build/backup cleanup --keep=3
|
||||
|
||||
# Verbose — shows which files are deleted
|
||||
./build/backup cleanup --keep=3 --verbose
|
||||
```
|
||||
|
||||
### Full Option Reference
|
||||
|
||||
| Option | Short | Default | Description |
|
||||
|--------|-------|---------|-------------|
|
||||
| `--data-dir` | `-d` | `data/server` | Path to the data directory |
|
||||
| `--output` | `-o` | auto-generated | Destination path for new backup |
|
||||
| `--input` | `-i` | — | Source archive for restore/verify |
|
||||
| `--keep` | `-k` | `5` | Number of snapshots to retain |
|
||||
| `--exclude` | `-e` | — | Exclude pattern (repeatable) |
|
||||
| `--level` | `-l` | `6` | Gzip compression 0-9 |
|
||||
| `--verbose` | `-v` | off | Detailed progress output |
|
||||
| `--help` | `-h` | — | Show help text |
|
||||
|
||||
### Nim API
|
||||
|
||||
You can also use the backup module programmatically:
|
||||
|
||||
```nim
|
||||
import barabadb/core/backup
|
||||
|
||||
var bm = newBackupManager()
|
||||
bm.createSnapshot("/backup/baradb_$(date)")
|
||||
# Create a snapshot
|
||||
let ok = backupDataDir("data/server", "snapshot.tar.gz")
|
||||
if not ok:
|
||||
echo "Backup failed"
|
||||
|
||||
# List existing snapshots
|
||||
let backups = listBackups("data/server")
|
||||
for b in backups:
|
||||
echo b.path, " → ", formatBytes(b.size)
|
||||
|
||||
# Verify without extracting
|
||||
let valid = verifyArchive("snapshot.tar.gz")
|
||||
|
||||
# Restore (destructive!)
|
||||
let restored = restoreDataDir("snapshot.tar.gz", "data/server")
|
||||
|
||||
# Cleanup retention
|
||||
cleanupOldBackups("data/server", keepLast = 5)
|
||||
```
|
||||
|
||||
### Point-in-Time Recovery
|
||||
### Point-in-Time Recovery (WAL)
|
||||
|
||||
WAL-based point-in-time recovery:
|
||||
For fine-grained recovery, replay the WAL from a checkpoint:
|
||||
|
||||
```bash
|
||||
# Replay WAL from checkpoint
|
||||
./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.db
|
||||
./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.tar.gz
|
||||
```
|
||||
|
||||
### Cross-Modal Queries
|
||||
|
||||
+301
-37
@@ -1,65 +1,293 @@
|
||||
## BaraDB Backup & Restore — tar.gz snapshots
|
||||
## BaraDB Backup & Restore — tar.gz snapshots with retention, verification & CLI
|
||||
##
|
||||
## Commands:
|
||||
## backup Create a snapshot of the data directory
|
||||
## restore Restore data directory from a snapshot
|
||||
## list List available snapshots with size & timestamp
|
||||
## verify Check archive integrity without extracting
|
||||
## cleanup Remove old snapshots keeping N most recent
|
||||
## help Show detailed usage information
|
||||
##
|
||||
## Options:
|
||||
## --data-dir, -d <DIR> Data directory to backup (default: data/server)
|
||||
## --output, -o <FILE> Output archive path for backup
|
||||
## --input, -i <FILE> Input archive path for restore/verify
|
||||
## --keep, -k <N> Number of snapshots to retain (default: 5)
|
||||
## --exclude, -e <PAT> Exclude pattern (can be used multiple times)
|
||||
## --level, -l <0-9> Compression level for gzip (default: 6)
|
||||
## --verbose, -v Enable verbose output
|
||||
|
||||
import std/os
|
||||
import std/osproc
|
||||
import std/strutils
|
||||
import std/times
|
||||
import std/algorithm
|
||||
import std/sequtils
|
||||
import std/parseopt
|
||||
|
||||
type
|
||||
Backup* = object
|
||||
path*: string
|
||||
timestamp*: int64
|
||||
size*: int64
|
||||
compressed*: bool
|
||||
|
||||
proc backupDataDir*(dataDir: string, output: string): bool =
|
||||
const
|
||||
DEFAULT_DATA_DIR = "data/server"
|
||||
DEFAULT_KEEP_COUNT = 5
|
||||
DEFAULT_COMPRESSION = 6
|
||||
HELP_TEXT = """
|
||||
BaraDB Backup Manager — Archive and restore your database safely
|
||||
================================================================
|
||||
|
||||
USAGE:
|
||||
backup <command> [options]
|
||||
|
||||
COMMANDS:
|
||||
backup Create a compressed tar.gz snapshot of the data directory.
|
||||
By default archives are named backup_<unixtimestamp>.tar.gz.
|
||||
|
||||
restore Replace the current data directory with contents from a snapshot.
|
||||
WARNING: This DESTROYS existing data. Use with care.
|
||||
|
||||
list Show all snapshots found next to the data directory.
|
||||
Displays size, timestamp and compression ratio if known.
|
||||
|
||||
verify Test archive integrity without extracting files.
|
||||
Detects corrupted or incomplete snapshots early.
|
||||
|
||||
cleanup Delete old snapshots, keeping only the N most recent ones.
|
||||
Default retention is 5 snapshots.
|
||||
|
||||
help Show this help message.
|
||||
|
||||
OPTIONS:
|
||||
-d, --data-dir <DIR> Path to data directory (default: data/server)
|
||||
-o, --output <FILE> Destination path for new backup archive
|
||||
-i, --input <FILE> Source archive for restore or verify
|
||||
-k, --keep <N> Retention count for cleanup (default: 5)
|
||||
-e, --exclude <PAT> Exclude files matching pattern (repeatable)
|
||||
-l, --level <0-9> Gzip compression level (default: 6, max: 9)
|
||||
-v, --verbose Print detailed progress information
|
||||
|
||||
EXAMPLES:
|
||||
# Quick backup with default settings
|
||||
backup backup
|
||||
|
||||
# Backup with maximum compression and custom name
|
||||
backup backup --output=prod_backup_$(date +%F).tar.gz --level=9
|
||||
|
||||
# Restore from a specific snapshot
|
||||
backup restore --input=backup_1715011200.tar.gz
|
||||
|
||||
# List all snapshots
|
||||
backup list
|
||||
|
||||
# Verify archive before restoring
|
||||
backup verify --input=backup_1715011200.tar.gz
|
||||
|
||||
# Keep only last 3 snapshots
|
||||
backup cleanup --keep=3
|
||||
|
||||
# Exclude WAL files from backup
|
||||
backup backup --exclude="*.log" --exclude="wal/*"
|
||||
"""
|
||||
|
||||
proc formatBytes*(bytes: int64): string =
|
||||
## Human readable byte size
|
||||
const units = ["B", "KB", "MB", "GB", "TB"]
|
||||
if bytes < 0: return "0 B"
|
||||
var size = float64(bytes)
|
||||
var unitIndex = 0
|
||||
while size >= 1024.0 and unitIndex < units.high:
|
||||
size /= 1024.0
|
||||
unitIndex += 1
|
||||
result = formatFloat(size, ffDecimal, precision = 2) & " " & units[unitIndex]
|
||||
|
||||
proc formatTimestamp*(ts: int64): string =
|
||||
## Format unix timestamp to human readable string
|
||||
try:
|
||||
let dt = fromUnix(ts)
|
||||
result = format(dt, "yyyy-MM-dd HH:mm:ss")
|
||||
except:
|
||||
result = $ts
|
||||
|
||||
proc parseBackupFilename*(filename: string): int64 =
|
||||
## Try to extract timestamp from backup_1234567890.tar.gz
|
||||
try:
|
||||
let name = extractFilename(filename)
|
||||
if name.startsWith("backup_"):
|
||||
let tsStr = name[7..^8] # skip "backup_" and ".tar.gz"
|
||||
result = parseBiggestInt(tsStr)
|
||||
else:
|
||||
result = 0
|
||||
except:
|
||||
result = 0
|
||||
|
||||
proc backupDataDir*(dataDir: string, output: string, excludes: seq[string] = @[], compression: int = DEFAULT_COMPRESSION, verbose: bool = false): bool =
|
||||
## Create a tar.gz backup of the data directory
|
||||
if not dirExists(dataDir):
|
||||
echo "Data directory not found: ", dataDir
|
||||
echo "ERROR: Data directory not found: ", dataDir
|
||||
return false
|
||||
|
||||
if fileExists(output):
|
||||
echo "WARNING: Overwriting existing file: ", output
|
||||
|
||||
let parent = parentDir(dataDir)
|
||||
let name = lastPathPart(dataDir)
|
||||
let cmd = "tar -czf " & quoteShell(output) & " -C " & quoteShell(parent) & " " & quoteShell(name)
|
||||
let exitCode = execCmd(cmd)
|
||||
return exitCode == 0
|
||||
var excludeArgs = ""
|
||||
for pattern in excludes:
|
||||
excludeArgs.add(" --exclude=" & quoteShell(pattern))
|
||||
|
||||
proc restoreDataDir*(input: string, dataDir: string): bool =
|
||||
let tarCmd = "tar -cf -" & excludeArgs & " -C " & quoteShell(parent) & " " & quoteShell(name)
|
||||
let gzipCmd = "gzip -" & $compression
|
||||
let cmd = tarCmd & " | " & gzipCmd & " > " & quoteShell(output)
|
||||
|
||||
if verbose:
|
||||
echo "Running: ", cmd
|
||||
echo "Source: ", dataDir
|
||||
echo "Target: ", output
|
||||
|
||||
let (outputStr, exitCode) = execCmdEx("bash -c " & quoteShell(cmd))
|
||||
if exitCode != 0:
|
||||
echo "ERROR: tar command failed with exit code ", exitCode
|
||||
if outputStr.len > 0:
|
||||
echo outputStr
|
||||
return false
|
||||
|
||||
let size = getFileSize(output)
|
||||
echo "Backup created successfully:"
|
||||
echo " File: ", output
|
||||
echo " Size: ", formatBytes(size)
|
||||
echo " Source: ", dataDir
|
||||
return true
|
||||
|
||||
proc restoreDataDir*(input: string, dataDir: string, verbose: bool = false): bool =
|
||||
## Restore from a tar.gz backup
|
||||
if not fileExists(input):
|
||||
echo "Backup file not found: ", input
|
||||
echo "ERROR: Backup file not found: ", input
|
||||
return false
|
||||
|
||||
if dirExists(dataDir):
|
||||
removeDir(dataDir)
|
||||
let backupOld = dataDir & ".old_" & $getTime().toUnix()
|
||||
if verbose:
|
||||
echo "Moving existing data to: ", backupOld
|
||||
moveDir(dataDir, backupOld)
|
||||
|
||||
createDir(dataDir)
|
||||
|
||||
let cmd = "tar -xzf " & quoteShell(input) & " -C " & quoteShell(dataDir)
|
||||
let exitCode = execCmd(cmd)
|
||||
return exitCode == 0
|
||||
if verbose:
|
||||
echo "Running: ", cmd
|
||||
|
||||
let (outputStr, exitCode) = execCmdEx(cmd)
|
||||
if exitCode != 0:
|
||||
echo "ERROR: tar extraction failed with exit code ", exitCode
|
||||
if outputStr.len > 0:
|
||||
echo outputStr
|
||||
return false
|
||||
|
||||
echo "Restored successfully from: ", input
|
||||
echo " Target: ", dataDir
|
||||
return true
|
||||
|
||||
proc verifyArchive*(input: string, verbose: bool = false): bool =
|
||||
## Verify tar.gz archive integrity without extracting
|
||||
if not fileExists(input):
|
||||
echo "ERROR: Archive not found: ", input
|
||||
return false
|
||||
|
||||
let cmd = "tar -tzf " & quoteShell(input) & " > /dev/null"
|
||||
if verbose:
|
||||
echo "Verifying archive: ", input
|
||||
|
||||
let (_, exitCode) = execCmdEx(cmd)
|
||||
if exitCode == 0:
|
||||
let size = getFileSize(input)
|
||||
echo "Archive is valid: ", input, " (", formatBytes(size), ")"
|
||||
return true
|
||||
else:
|
||||
echo "ERROR: Archive is corrupted or unreadable: ", input
|
||||
return false
|
||||
|
||||
proc listBackups*(dataDir: string): seq[Backup] =
|
||||
result = @[]
|
||||
for kind, path in walkDir(parentDir(dataDir)):
|
||||
let (_, name, ext) = splitFile(path)
|
||||
if ext == ".gz":
|
||||
var backup = Backup(path: path)
|
||||
backup.size = getFileSize(path)
|
||||
result.add(backup)
|
||||
## List all backup archives found in current directory and next to data directory
|
||||
var backups: seq[Backup] = @[]
|
||||
var seenPaths: seq[string] = @[]
|
||||
|
||||
proc cleanupOldBackups*(dataDir: string, keepLast: int = 5) =
|
||||
proc scanDir(searchDir: string) =
|
||||
if not dirExists(searchDir):
|
||||
return
|
||||
for kind, path in walkDir(searchDir):
|
||||
if kind == pcFile:
|
||||
let fullPath = absolutePath(path)
|
||||
if fullPath in seenPaths:
|
||||
continue
|
||||
let (_, _, ext) = splitFile(path)
|
||||
if ext == ".gz" or ext == ".tar" or path.endsWith(".tar.gz"):
|
||||
seenPaths.add(fullPath)
|
||||
var backup = Backup(path: path)
|
||||
backup.size = getFileSize(path)
|
||||
backup.timestamp = parseBackupFilename(path)
|
||||
backup.compressed = path.endsWith(".gz")
|
||||
backups.add(backup)
|
||||
|
||||
scanDir(getCurrentDir())
|
||||
scanDir(parentDir(dataDir))
|
||||
|
||||
# Sort by timestamp descending (newest first)
|
||||
backups.sort(proc(a, b: Backup): int =
|
||||
if a.timestamp > b.timestamp: -1
|
||||
elif a.timestamp < b.timestamp: 1
|
||||
else: 0
|
||||
)
|
||||
result = backups
|
||||
|
||||
proc printBackups*(backups: seq[Backup]) =
|
||||
## Pretty print backup list
|
||||
if backups.len == 0:
|
||||
echo "No backups found."
|
||||
return
|
||||
|
||||
echo "Found ", backups.len, " backup(s):"
|
||||
echo repeat("-", 80)
|
||||
echo alignLeft("#", 4), alignLeft("Timestamp", 22), alignLeft("Size", 12), "Path"
|
||||
echo repeat("-", 80)
|
||||
for i, b in backups:
|
||||
let ts = if b.timestamp > 0: formatTimestamp(b.timestamp) else: "unknown"
|
||||
let idx = alignLeft($(i+1), 4)
|
||||
echo idx, alignLeft(ts, 22), alignLeft(formatBytes(b.size), 12), b.path
|
||||
echo repeat("-", 80)
|
||||
|
||||
proc cleanupOldBackups*(dataDir: string, keepLast: int = DEFAULT_KEEP_COUNT, verbose: bool = false) =
|
||||
## Remove old backups keeping only N most recent
|
||||
var backups = listBackups(dataDir)
|
||||
if backups.len <= keepLast:
|
||||
echo "Nothing to clean. Found ", backups.len, " backup(s), keeping up to ", keepLast, "."
|
||||
return
|
||||
for i in 0..<(backups.len - keepLast):
|
||||
|
||||
let toDelete = backups.len - keepLast
|
||||
echo "Removing ", toDelete, " old backup(s), keeping ", keepLast, " most recent."
|
||||
|
||||
for i in countdown(backups.high, keepLast):
|
||||
if verbose:
|
||||
echo " Deleting: ", backups[i].path, " (", formatBytes(backups[i].size), ")"
|
||||
removeFile(backups[i].path)
|
||||
|
||||
when isMainModule:
|
||||
import std/os
|
||||
import std/parseopt
|
||||
echo "Cleanup complete."
|
||||
|
||||
# =============================================================================
|
||||
# CLI Entry Point
|
||||
# =============================================================================
|
||||
when isMainModule:
|
||||
var
|
||||
command = ""
|
||||
dataDir = "data/server"
|
||||
dataDir = DEFAULT_DATA_DIR
|
||||
target = ""
|
||||
keepCount = DEFAULT_KEEP_COUNT
|
||||
excludes: seq[string] = @[]
|
||||
compression = DEFAULT_COMPRESSION
|
||||
verbose = false
|
||||
|
||||
for kind, key, val in getopt():
|
||||
case kind
|
||||
@@ -73,30 +301,66 @@ when isMainModule:
|
||||
of "data-dir", "d": dataDir = val
|
||||
of "output", "o": target = val
|
||||
of "input", "i": target = val
|
||||
of "keep", "k":
|
||||
try: keepCount = parseInt(val)
|
||||
except: quit("ERROR: --keep must be a number", 1)
|
||||
of "exclude", "e": excludes.add(val)
|
||||
of "level", "l":
|
||||
try:
|
||||
compression = parseInt(val)
|
||||
if compression < 0 or compression > 9:
|
||||
quit("ERROR: --level must be between 0 and 9", 1)
|
||||
except: quit("ERROR: --level must be a number", 1)
|
||||
of "verbose", "v": verbose = true
|
||||
of "help", "h":
|
||||
echo HELP_TEXT
|
||||
quit(0)
|
||||
else: discard
|
||||
of cmdEnd: discard
|
||||
|
||||
# If no command given, show help
|
||||
if command == "":
|
||||
echo HELP_TEXT
|
||||
quit(0)
|
||||
|
||||
case command
|
||||
of "backup":
|
||||
let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
|
||||
let ok = backupDataDir(dataDir, outputFile)
|
||||
if ok:
|
||||
echo "Backup created: ", outputFile
|
||||
else:
|
||||
let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
|
||||
if not ok:
|
||||
quit("Backup failed", 1)
|
||||
|
||||
of "restore":
|
||||
if target.len == 0:
|
||||
quit("Usage: backup restore --input=<file.tar.gz>", 1)
|
||||
let ok = restoreDataDir(target, dataDir)
|
||||
if ok:
|
||||
echo "Restored from: ", target
|
||||
else:
|
||||
quit("ERROR: restore requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
|
||||
echo "WARNING: This will REPLACE the data in: ", dataDir
|
||||
echo "Continue? [y/N] "
|
||||
let answer = readLine(stdin)
|
||||
if answer.toLowerAscii() notin ["y", "yes"]:
|
||||
quit("Restore cancelled", 0)
|
||||
let ok = restoreDataDir(target, dataDir, verbose)
|
||||
if not ok:
|
||||
quit("Restore failed", 1)
|
||||
|
||||
of "list":
|
||||
let backups = listBackups(dataDir)
|
||||
echo "Backups:"
|
||||
for b in backups:
|
||||
echo " ", b.path, " (", b.size, " bytes)"
|
||||
printBackups(backups)
|
||||
|
||||
of "verify":
|
||||
if target.len == 0:
|
||||
quit("ERROR: verify requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
|
||||
let ok = verifyArchive(target, verbose)
|
||||
if not ok:
|
||||
quit("Verification failed", 1)
|
||||
|
||||
of "cleanup":
|
||||
cleanupOldBackups(dataDir, keepCount, verbose)
|
||||
|
||||
of "help":
|
||||
echo HELP_TEXT
|
||||
|
||||
else:
|
||||
echo "Usage: backup <backup|restore|list> [--data-dir=DIR] [--output/-i=FILE]"
|
||||
echo "ERROR: Unknown command: ", command
|
||||
echo ""
|
||||
echo HELP_TEXT
|
||||
quit(1)
|
||||
|
||||
Reference in New Issue
Block a user