Стабилизация на storage слоя — 6 фази
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled

Фаза 1: SSTable Integrity
- SSTable v3 формат с CRC32 footer (data/index/bloom CRC)
- Нов модул storage/crc32.nim (zero-dep, IEEE 802.3)
- verifySSTable() — проверка на magic, version, CRC
- loadSSTable strict mode — отхвърля корумпирани файлове
- newLSMTree логва [WARN] при corrupt SSTables

Фаза 2: baradb repair
- Нов модул tools/repair.nim
- Сканира SSTables, проверява CRC, премества битите в corrupt/
- WAL replay (CrashRecovery) за възстановяване на данни
- CLI: ./baradadb repair --data-dir=... [--dry-run]

Фаза 3: MANIFEST File
- Atomic write MANIFEST (tmp + rename)
- newLSMTree зарежда от MANIFEST, fallback към walkDir
- checkStorageConsistency() — orphan/missing detection
- flushUnsafe и compaction записват MANIFEST

Фаза 4: WAL Rotation & Incremental Backup
- WAL segment rotation при 64MB (wal_archive/wal.000NNN.log)
- maybeRotate() на всеки 1000 записа + при flush
- backup incremental — архивира MANIFEST + SSTables + WAL
- CRC verification на SSTables преди архивиране

Фаза 5: Online Consistent Backup
- checkpoint() — freeze memtable + flush + WAL rotate
- ./baradadb checkpoint — offline consistent snapshot
- backup --online — checkpoint + incremental backup

Фаза 6: SSTable Version Migration
- SSTable.fileVersion поле
- listLegacySSTables() — намира v1/v2 файлове
- migrateSSTable() — пренаписва към v3 (tmp + rename)
- ./baradadb migrate [--dry-run] — offline migration

Документация:
- Обновени docs/en/backup.md, docs/bg/backup.md
- Обновени docs/en/storage.md, docs/bg/storage.md
- Добавени тестове в tests/test_all.nim
This commit is contained in:
2026-05-18 15:04:49 +03:00
parent cf2aba104f
commit 7e6a45e6b7
13 changed files with 1975 additions and 348 deletions
+121 -4
View File
@@ -26,6 +26,8 @@ import std/strutils
import std/times
import std/algorithm
import std/parseopt
import std/json
import barabadb/storage/lsm
type
Backup* = object
@@ -47,8 +49,11 @@ USAGE:
backup <command> [options]
COMMANDS:
backup Create a compressed tar.gz snapshot of the data directory.
By default archives are named backup_<unixtimestamp>.tar.gz.
backup Create a compressed tar.gz snapshot of the data directory.
By default archives are named backup_<unixtimestamp>.tar.gz.
incremental Create a consistent incremental backup including MANIFEST,
active SSTables, and all WAL segments (current + archive).
restore Replace the current data directory with contents from a snapshot.
WARNING: This DESTROYS existing data. Use with care.
@@ -387,6 +392,94 @@ proc printHistory*() =
echo entry
echo repeat("-", 80)
proc incrementalBackupDataDir*(dataDir: string, output: string, verbose: bool = false): bool =
## Create incremental backup: MANIFEST + active SSTables + WAL segments.
let manifestPath = dataDir / "MANIFEST"
if not fileExists(manifestPath):
echo "ERROR: MANIFEST not found at ", manifestPath
echo " Run a full backup first, or ensure the database has flushed data."
return false
var filesToInclude: seq[string] = @[manifestPath]
# Include SSTables from MANIFEST
try:
let manifest = parseJson(readFile(manifestPath))
for node in manifest{"sstables"}:
let sstPath = node{"path"}.getStr()
if fileExists(sstPath):
filesToInclude.add(sstPath)
else:
if verbose:
echo "WARNING: SSTable missing: ", sstPath
except CatchableError as e:
echo "ERROR: Failed to parse MANIFEST: ", e.msg
return false
# Include current WAL
let walPath = dataDir / "wal" / "wal.log"
if fileExists(walPath):
filesToInclude.add(walPath)
# Include WAL archive
let walArchiveDir = dataDir / "wal" / "wal_archive"
if dirExists(walArchiveDir):
for kind, path in walkDir(walArchiveDir):
if kind == pcFile and path.endsWith(".log"):
filesToInclude.add(path)
if filesToInclude.len == 0:
echo "ERROR: No files to backup"
return false
# Verify all SSTables before archiving
var verifyErrors = 0
for path in filesToInclude:
if path.endsWith(".sst"):
let (ok, msg) = verifySSTable(path)
if not ok:
echo "ERROR: SSTable verification failed: ", msg
inc verifyErrors
elif verbose:
echo "", extractFilename(path), " — CRC OK"
if verifyErrors > 0:
echo "ERROR: ", verifyErrors, " SSTable(s) failed verification. Backup aborted."
return false
# Write file list for tar -T
let fileListPath = output & ".files"
var f: File
if open(f, fileListPath, fmWrite):
for path in filesToInclude:
f.writeLine(path)
close(f)
else:
echo "ERROR: Cannot write file list: ", fileListPath
return false
let tarCmd = "tar -czf " & quoteShell(output) & " -T " & quoteShell(fileListPath)
if verbose:
echo "Running: ", tarCmd
echo "Including ", filesToInclude.len, " files"
for path in filesToInclude:
echo " + ", path
let (outStr, exitCode) = execCmdEx(tarCmd)
removeFile(fileListPath)
if exitCode != 0:
echo "ERROR: tar failed with exit code ", exitCode
if outStr.len > 0:
echo outStr
return false
let size = getFileSize(output)
echo "Incremental backup created successfully:"
echo " File: ", output
echo " Size: ", formatBytes(size)
echo " Files: ", filesToInclude.len
return true
# =============================================================================
# CLI Entry Point
# =============================================================================
@@ -401,6 +494,7 @@ when isMainModule:
verbose = false
dryRun = false
force = false
online = false
for kind, key, val in getopt():
case kind
@@ -426,6 +520,7 @@ when isMainModule:
except: quit("ERROR: --level must be a number", 1)
of "dry-run": dryRun = true
of "force", "f": force = true
of "online": online = true
of "verbose", "v": verbose = true
of "help", "h":
echo HELP_TEXT
@@ -441,9 +536,31 @@ when isMainModule:
case command
of "backup":
let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
if online:
echo "Creating online backup with checkpoint..."
echo " Data dir: ", dataDir
echo " Output: ", outputFile
try:
var db = newLSMTree(dataDir)
db.checkpoint()
db.close()
echo "Checkpoint complete."
except CatchableError as e:
echo "ERROR: Checkpoint failed: ", e.msg
quit(1)
let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
if not ok:
quit("Online backup failed", 1)
else:
let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
if not ok:
quit("Backup failed", 1)
of "incremental":
let outputFile = if target.len > 0: target else: "backup_inc_" & $getTime().toUnix() & ".tar.gz"
let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
if not ok:
quit("Backup failed", 1)
quit("Incremental backup failed", 1)
of "restore":
if target.len == 0: