baraba/Baradb
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backup): improve backup manager with detailed CLI, verification, retention

Browse Source
This commit is contained in:
dimgigov
2026-05-06 23:16:00 +03:00
parent c3c14d64df
commit 7f2f5664fa
2 changed files with 473 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+172 -8
View File
@@ -663,24 +663,188 @@ BARADB_LOG_LEVEL=debug ./build/baradadb
## Backup & Recovery ## Backup & Recovery
### Online Backup BaraDB includes a built-in backup manager that creates compressed tar.gz
snapshots of your data directory. The manager supports online backups
(server does not need to stop), integrity verification, retention policies,
and safe restore with automatic rollback protection.
BaraDB supports online snapshots without stopping the server: ### Quick Reference
| Command | Purpose |
|---------|---------|
| `backup backup` | Create a new snapshot |
| `backup restore` | Restore data from a snapshot |
| `backup list` | Show all snapshots |
| `backup verify` | Check archive integrity |
| `backup cleanup` | Delete old snapshots |
| `backup help` | Show full help text |
### Build the Backup Tool
```bash
nim c -o:build/backup src/barabadb/core/backup.nim
```
### Creating Backups
**Basic backup** — creates `backup_<timestamp>.tar.gz` in the current directory:
```bash
./build/backup backup
```
**Custom output path**:
```bash
./build/backup backup --output=/backups/prod_$(date +%F).tar.gz
```
**Maximum compression** (slower, smaller file):
```bash
./build/backup backup --level=9
```
**Exclude WAL logs and temporary files**:
```bash
./build/backup backup \
--exclude="*.log" \
--exclude="wal/*" \
--exclude="tmp/*"
```
**Verbose output** (shows tar command and progress):
```bash
./build/backup backup --verbose
```
### Listing Backups
```bash
./build/backup list
```
Example output:
```
Found 3 backup(s):
--------------------------------------------------------------------------------
# Timestamp Size Path
--------------------------------------------------------------------------------
1 2026-05-06 23:04:56 12.45 MB backup_1715011200.tar.gz
2 2026-05-05 12:30:00 11.20 MB backup_1714921800.tar.gz
3 2026-05-04 08:15:22 10.89 MB backup_1714834522.tar.gz
--------------------------------------------------------------------------------
```
### Verifying Backups
Always verify a snapshot before restoring, especially after transferring it
over the network:
```bash
./build/backup verify --input=backup_1715011200.tar.gz
```
A valid archive prints:
```
Archive is valid: backup_1715011200.tar.gz (12.45 MB)
```
A corrupted archive prints an error and exits with code 1.
### Restoring from Backup
> ⚠️ **WARNING:** Restore replaces the existing data directory. The old data
> is automatically moved to `data/server.old_<timestamp>` before extraction.
```bash
./build/backup restore --input=backup_1715011200.tar.gz
```
You will be prompted for confirmation:
```
WARNING: This will REPLACE the data in: data/server
Continue? [y/N]
```
**Restore to a different data directory**:
```bash
./build/backup restore --input=backup.tar.gz --data-dir=data/recovered
```
**Skip confirmation (scripting)** — pipe `y`:
```bash
echo "y" | ./build/backup restore --input=backup.tar.gz
```
### Cleanup & Retention
Delete old snapshots automatically, keeping only the N most recent:
```bash
# Keep last 5 snapshots (default)
./build/backup cleanup
# Keep last 3 snapshots
./build/backup cleanup --keep=3
# Verbose — shows which files are deleted
./build/backup cleanup --keep=3 --verbose
```
### Full Option Reference
| Option | Short | Default | Description |
|--------|-------|---------|-------------|
| `--data-dir` | `-d` | `data/server` | Path to the data directory |
| `--output` | `-o` | auto-generated | Destination path for new backup |
| `--input` | `-i` | — | Source archive for restore/verify |
| `--keep` | `-k` | `5` | Number of snapshots to retain |
| `--exclude` | `-e` | — | Exclude pattern (repeatable) |
| `--level` | `-l` | `6` | Gzip compression 0-9 |
| `--verbose` | `-v` | off | Detailed progress output |
| `--help` | `-h` | — | Show help text |
### Nim API
You can also use the backup module programmatically:
```nim ```nim
import barabadb/core/backup import barabadb/core/backup
var bm = newBackupManager() # Create a snapshot
bm.createSnapshot("/backup/baradb_$(date)") let ok = backupDataDir("data/server", "snapshot.tar.gz")
if not ok:
echo "Backup failed"
# List existing snapshots
let backups = listBackups("data/server")
for b in backups:
echo b.path, " → ", formatBytes(b.size)
# Verify without extracting
let valid = verifyArchive("snapshot.tar.gz")
# Restore (destructive!)
let restored = restoreDataDir("snapshot.tar.gz", "data/server")
# Cleanup retention
cleanupOldBackups("data/server", keepLast = 5)
``` ```
### Point-in-Time Recovery ### Point-in-Time Recovery (WAL)
WAL-based point-in-time recovery: For fine-grained recovery, replay the WAL from a checkpoint:
```bash ```bash
# Replay WAL from checkpoint ./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.tar.gz
./build/baradadb --recover --wal-dir=./wal --checkpoint=/backup/snapshot.db
``` ```
### Cross-Modal Queries ### Cross-Modal Queries
src/barabadb/core/backup.nim
+299 -35
View File
@@ -1,65 +1,293 @@
## BaraDB Backup & Restore — tar.gz snapshots ## BaraDB Backup & Restore — tar.gz snapshots with retention, verification & CLI
##
## Commands:
## backup Create a snapshot of the data directory
## restore Restore data directory from a snapshot
## list List available snapshots with size & timestamp
## verify Check archive integrity without extracting
## cleanup Remove old snapshots keeping N most recent
## help Show detailed usage information
##
## Options:
## --data-dir, -d <DIR> Data directory to backup (default: data/server)
## --output, -o <FILE> Output archive path for backup
## --input, -i <FILE> Input archive path for restore/verify
## --keep, -k <N> Number of snapshots to retain (default: 5)
## --exclude, -e <PAT> Exclude pattern (can be used multiple times)
## --level, -l <0-9> Compression level for gzip (default: 6)
## --verbose, -v Enable verbose output
import std/os import std/os
import std/osproc import std/osproc
import std/strutils import std/strutils
import std/times import std/times
import std/algorithm
import std/sequtils
import std/parseopt
type type
Backup* = object Backup* = object
path*: string path*: string
timestamp*: int64 timestamp*: int64
size*: int64 size*: int64
compressed*: bool
proc backupDataDir*(dataDir: string, output: string): bool = const
DEFAULT_DATA_DIR = "data/server"
DEFAULT_KEEP_COUNT = 5
DEFAULT_COMPRESSION = 6
HELP_TEXT = """
BaraDB Backup Manager — Archive and restore your database safely
================================================================
USAGE:
backup <command> [options]
COMMANDS:
backup Create a compressed tar.gz snapshot of the data directory.
By default archives are named backup_<unixtimestamp>.tar.gz.
restore Replace the current data directory with contents from a snapshot.
WARNING: This DESTROYS existing data. Use with care.
list Show all snapshots found next to the data directory.
Displays size, timestamp and compression ratio if known.
verify Test archive integrity without extracting files.
Detects corrupted or incomplete snapshots early.
cleanup Delete old snapshots, keeping only the N most recent ones.
Default retention is 5 snapshots.
help Show this help message.
OPTIONS:
-d, --data-dir <DIR> Path to data directory (default: data/server)
-o, --output <FILE> Destination path for new backup archive
-i, --input <FILE> Source archive for restore or verify
-k, --keep <N> Retention count for cleanup (default: 5)
-e, --exclude <PAT> Exclude files matching pattern (repeatable)
-l, --level <0-9> Gzip compression level (default: 6, max: 9)
-v, --verbose Print detailed progress information
EXAMPLES:
# Quick backup with default settings
backup backup
# Backup with maximum compression and custom name
backup backup --output=prod_backup_$(date +%F).tar.gz --level=9
# Restore from a specific snapshot
backup restore --input=backup_1715011200.tar.gz
# List all snapshots
backup list
# Verify archive before restoring
backup verify --input=backup_1715011200.tar.gz
# Keep only last 3 snapshots
backup cleanup --keep=3
# Exclude WAL files from backup
backup backup --exclude="*.log" --exclude="wal/*"
"""
proc formatBytes*(bytes: int64): string =
## Human readable byte size
const units = ["B", "KB", "MB", "GB", "TB"]
if bytes < 0: return "0 B"
var size = float64(bytes)
var unitIndex = 0
while size >= 1024.0 and unitIndex < units.high:
size /= 1024.0
unitIndex += 1
result = formatFloat(size, ffDecimal, precision = 2) & " " & units[unitIndex]
proc formatTimestamp*(ts: int64): string =
## Format unix timestamp to human readable string
try:
let dt = fromUnix(ts)
result = format(dt, "yyyy-MM-dd HH:mm:ss")
except:
result = $ts
proc parseBackupFilename*(filename: string): int64 =
## Try to extract timestamp from backup_1234567890.tar.gz
try:
let name = extractFilename(filename)
if name.startsWith("backup_"):
let tsStr = name[7..^8] # skip "backup_" and ".tar.gz"
result = parseBiggestInt(tsStr)
else:
result = 0
except:
result = 0
proc backupDataDir*(dataDir: string, output: string, excludes: seq[string] = @[], compression: int = DEFAULT_COMPRESSION, verbose: bool = false): bool =
## Create a tar.gz backup of the data directory ## Create a tar.gz backup of the data directory
if not dirExists(dataDir): if not dirExists(dataDir):
echo "Data directory not found: ", dataDir echo "ERROR: Data directory not found: ", dataDir
return false return false
if fileExists(output):
echo "WARNING: Overwriting existing file: ", output
let parent = parentDir(dataDir) let parent = parentDir(dataDir)
let name = lastPathPart(dataDir) let name = lastPathPart(dataDir)
let cmd = "tar -czf " & quoteShell(output) & " -C " & quoteShell(parent) & " " & quoteShell(name) var excludeArgs = ""
let exitCode = execCmd(cmd) for pattern in excludes:
return exitCode == 0 excludeArgs.add(" --exclude=" & quoteShell(pattern))
proc restoreDataDir*(input: string, dataDir: string): bool = let tarCmd = "tar -cf -" & excludeArgs & " -C " & quoteShell(parent) & " " & quoteShell(name)
let gzipCmd = "gzip -" & $compression
let cmd = tarCmd & " | " & gzipCmd & " > " & quoteShell(output)
if verbose:
echo "Running: ", cmd
echo "Source: ", dataDir
echo "Target: ", output
let (outputStr, exitCode) = execCmdEx("bash -c " & quoteShell(cmd))
if exitCode != 0:
echo "ERROR: tar command failed with exit code ", exitCode
if outputStr.len > 0:
echo outputStr
return false
let size = getFileSize(output)
echo "Backup created successfully:"
echo " File: ", output
echo " Size: ", formatBytes(size)
echo " Source: ", dataDir
return true
proc restoreDataDir*(input: string, dataDir: string, verbose: bool = false): bool =
## Restore from a tar.gz backup ## Restore from a tar.gz backup
if not fileExists(input): if not fileExists(input):
echo "Backup file not found: ", input echo "ERROR: Backup file not found: ", input
return false return false
if dirExists(dataDir): if dirExists(dataDir):
removeDir(dataDir) let backupOld = dataDir & ".old_" & $getTime().toUnix()
if verbose:
echo "Moving existing data to: ", backupOld
moveDir(dataDir, backupOld)
createDir(dataDir) createDir(dataDir)
let cmd = "tar -xzf " & quoteShell(input) & " -C " & quoteShell(dataDir) let cmd = "tar -xzf " & quoteShell(input) & " -C " & quoteShell(dataDir)
let exitCode = execCmd(cmd) if verbose:
return exitCode == 0 echo "Running: ", cmd
let (outputStr, exitCode) = execCmdEx(cmd)
if exitCode != 0:
echo "ERROR: tar extraction failed with exit code ", exitCode
if outputStr.len > 0:
echo outputStr
return false
echo "Restored successfully from: ", input
echo " Target: ", dataDir
return true
proc verifyArchive*(input: string, verbose: bool = false): bool =
## Verify tar.gz archive integrity without extracting
if not fileExists(input):
echo "ERROR: Archive not found: ", input
return false
let cmd = "tar -tzf " & quoteShell(input) & " > /dev/null"
if verbose:
echo "Verifying archive: ", input
let (_, exitCode) = execCmdEx(cmd)
if exitCode == 0:
let size = getFileSize(input)
echo "Archive is valid: ", input, " (", formatBytes(size), ")"
return true
else:
echo "ERROR: Archive is corrupted or unreadable: ", input
return false
proc listBackups*(dataDir: string): seq[Backup] = proc listBackups*(dataDir: string): seq[Backup] =
result = @[] ## List all backup archives found in current directory and next to data directory
for kind, path in walkDir(parentDir(dataDir)): var backups: seq[Backup] = @[]
let (_, name, ext) = splitFile(path) var seenPaths: seq[string] = @[]
if ext == ".gz":
proc scanDir(searchDir: string) =
if not dirExists(searchDir):
return
for kind, path in walkDir(searchDir):
if kind == pcFile:
let fullPath = absolutePath(path)
if fullPath in seenPaths:
continue
let (_, _, ext) = splitFile(path)
if ext == ".gz" or ext == ".tar" or path.endsWith(".tar.gz"):
seenPaths.add(fullPath)
var backup = Backup(path: path) var backup = Backup(path: path)
backup.size = getFileSize(path) backup.size = getFileSize(path)
result.add(backup) backup.timestamp = parseBackupFilename(path)
backup.compressed = path.endsWith(".gz")
backups.add(backup)
proc cleanupOldBackups*(dataDir: string, keepLast: int = 5) = scanDir(getCurrentDir())
scanDir(parentDir(dataDir))
# Sort by timestamp descending (newest first)
backups.sort(proc(a, b: Backup): int =
if a.timestamp > b.timestamp: -1
elif a.timestamp < b.timestamp: 1
else: 0
)
result = backups
proc printBackups*(backups: seq[Backup]) =
## Pretty print backup list
if backups.len == 0:
echo "No backups found."
return
echo "Found ", backups.len, " backup(s):"
echo repeat("-", 80)
echo alignLeft("#", 4), alignLeft("Timestamp", 22), alignLeft("Size", 12), "Path"
echo repeat("-", 80)
for i, b in backups:
let ts = if b.timestamp > 0: formatTimestamp(b.timestamp) else: "unknown"
let idx = alignLeft($(i+1), 4)
echo idx, alignLeft(ts, 22), alignLeft(formatBytes(b.size), 12), b.path
echo repeat("-", 80)
proc cleanupOldBackups*(dataDir: string, keepLast: int = DEFAULT_KEEP_COUNT, verbose: bool = false) =
## Remove old backups keeping only N most recent
var backups = listBackups(dataDir) var backups = listBackups(dataDir)
if backups.len <= keepLast: if backups.len <= keepLast:
echo "Nothing to clean. Found ", backups.len, " backup(s), keeping up to ", keepLast, "."
return return
for i in 0..<(backups.len - keepLast):
let toDelete = backups.len - keepLast
echo "Removing ", toDelete, " old backup(s), keeping ", keepLast, " most recent."
for i in countdown(backups.high, keepLast):
if verbose:
echo " Deleting: ", backups[i].path, " (", formatBytes(backups[i].size), ")"
removeFile(backups[i].path) removeFile(backups[i].path)
when isMainModule: echo "Cleanup complete."
import std/os
import std/parseopt
# =============================================================================
# CLI Entry Point
# =============================================================================
when isMainModule:
var var
command = "" command = ""
dataDir = "data/server" dataDir = DEFAULT_DATA_DIR
target = "" target = ""
keepCount = DEFAULT_KEEP_COUNT
excludes: seq[string] = @[]
compression = DEFAULT_COMPRESSION
verbose = false
for kind, key, val in getopt(): for kind, key, val in getopt():
case kind case kind
@@ -73,30 +301,66 @@ when isMainModule:
of "data-dir", "d": dataDir = val of "data-dir", "d": dataDir = val
of "output", "o": target = val of "output", "o": target = val
of "input", "i": target = val of "input", "i": target = val
of "keep", "k":
try: keepCount = parseInt(val)
except: quit("ERROR: --keep must be a number", 1)
of "exclude", "e": excludes.add(val)
of "level", "l":
try:
compression = parseInt(val)
if compression < 0 or compression > 9:
quit("ERROR: --level must be between 0 and 9", 1)
except: quit("ERROR: --level must be a number", 1)
of "verbose", "v": verbose = true
of "help", "h":
echo HELP_TEXT
quit(0)
else: discard else: discard
of cmdEnd: discard of cmdEnd: discard
# If no command given, show help
if command == "":
echo HELP_TEXT
quit(0)
case command case command
of "backup": of "backup":
let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz" let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
let ok = backupDataDir(dataDir, outputFile) let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
if ok: if not ok:
echo "Backup created: ", outputFile
else:
quit("Backup failed", 1) quit("Backup failed", 1)
of "restore": of "restore":
if target.len == 0: if target.len == 0:
quit("Usage: backup restore --input=<file.tar.gz>", 1) quit("ERROR: restore requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
let ok = restoreDataDir(target, dataDir) echo "WARNING: This will REPLACE the data in: ", dataDir
if ok: echo "Continue? [y/N] "
echo "Restored from: ", target let answer = readLine(stdin)
else: if answer.toLowerAscii() notin ["y", "yes"]:
quit("Restore cancelled", 0)
let ok = restoreDataDir(target, dataDir, verbose)
if not ok:
quit("Restore failed", 1) quit("Restore failed", 1)
of "list": of "list":
let backups = listBackups(dataDir) let backups = listBackups(dataDir)
echo "Backups:" printBackups(backups)
for b in backups:
echo " ", b.path, " (", b.size, " bytes)" of "verify":
if target.len == 0:
quit("ERROR: verify requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
let ok = verifyArchive(target, verbose)
if not ok:
quit("Verification failed", 1)
of "cleanup":
cleanupOldBackups(dataDir, keepCount, verbose)
of "help":
echo HELP_TEXT
else: else:
echo "Usage: backup <backup|restore|list> [--data-dir=DIR] [--output/-i=FILE]" echo "ERROR: Unknown command: ", command
echo ""
echo HELP_TEXT
quit(1) quit(1)