Стабилизация на storage слоя — 6 фази

Фаза 1: SSTable Integrity
- SSTable v3 формат с CRC32 footer (data/index/bloom CRC)
- Нов модул storage/crc32.nim (zero-dep, IEEE 802.3)
- verifySSTable() — проверка на magic, version, CRC
- loadSSTable strict mode — отхвърля корумпирани файлове
- newLSMTree логва [WARN] при corrupt SSTables

Фаза 2: baradb repair
- Нов модул tools/repair.nim
- Сканира SSTables, проверява CRC, премества битите в corrupt/
- WAL replay (CrashRecovery) за възстановяване на данни
- CLI: ./baradadb repair --data-dir=... [--dry-run]

Фаза 3: MANIFEST File
- Atomic write MANIFEST (tmp + rename)
- newLSMTree зарежда от MANIFEST, fallback към walkDir
- checkStorageConsistency() — orphan/missing detection
- flushUnsafe и compaction записват MANIFEST

Фаза 4: WAL Rotation & Incremental Backup
- WAL segment rotation при 64MB (wal_archive/wal.000NNN.log)
- maybeRotate() на всеки 1000 записа + при flush
- backup incremental — архивира MANIFEST + SSTables + WAL
- CRC verification на SSTables преди архивиране

Фаза 5: Online Consistent Backup
- checkpoint() — freeze memtable + flush + WAL rotate
- ./baradadb checkpoint — offline consistent snapshot
- backup --online — checkpoint + incremental backup

Фаза 6: SSTable Version Migration
- SSTable.fileVersion поле
- listLegacySSTables() — намира v1/v2 файлове
- migrateSSTable() — пренаписва към v3 (tmp + rename)
- ./baradadb migrate [--dry-run] — offline migration

Документация:
- Обновени docs/en/backup.md, docs/bg/backup.md
- Обновени docs/en/storage.md, docs/bg/storage.md
- Добавени тестове в tests/test_all.nim

src/barabadb/core/backup.nim

@@ -26,6 +26,8 @@ import std/strutils
 import std/times
 import std/algorithm
 import std/parseopt
+import std/json
+import barabadb/storage/lsm
 
 type
   Backup* = object
@@ -47,8 +49,11 @@ USAGE:
   backup <command> [options]
 
 COMMANDS:
-  backup   Create a compressed tar.gz snapshot of the data directory.
-           By default archives are named backup_<unixtimestamp>.tar.gz.
+  backup       Create a compressed tar.gz snapshot of the data directory.
+               By default archives are named backup_<unixtimestamp>.tar.gz.
+
+  incremental  Create a consistent incremental backup including MANIFEST,
+               active SSTables, and all WAL segments (current + archive).
 
   restore  Replace the current data directory with contents from a snapshot.
            WARNING: This DESTROYS existing data. Use with care.
@@ -387,6 +392,94 @@ proc printHistory*() =
     echo entry
   echo repeat("-", 80)
 
+proc incrementalBackupDataDir*(dataDir: string, output: string, verbose: bool = false): bool =
+  ## Create incremental backup: MANIFEST + active SSTables + WAL segments.
+  let manifestPath = dataDir / "MANIFEST"
+  if not fileExists(manifestPath):
+    echo "ERROR: MANIFEST not found at ", manifestPath
+    echo "       Run a full backup first, or ensure the database has flushed data."
+    return false
+
+  var filesToInclude: seq[string] = @[manifestPath]
+
+  # Include SSTables from MANIFEST
+  try:
+    let manifest = parseJson(readFile(manifestPath))
+    for node in manifest{"sstables"}:
+      let sstPath = node{"path"}.getStr()
+      if fileExists(sstPath):
+        filesToInclude.add(sstPath)
+      else:
+        if verbose:
+          echo "WARNING: SSTable missing: ", sstPath
+  except CatchableError as e:
+    echo "ERROR: Failed to parse MANIFEST: ", e.msg
+    return false
+
+  # Include current WAL
+  let walPath = dataDir / "wal" / "wal.log"
+  if fileExists(walPath):
+    filesToInclude.add(walPath)
+
+  # Include WAL archive
+  let walArchiveDir = dataDir / "wal" / "wal_archive"
+  if dirExists(walArchiveDir):
+    for kind, path in walkDir(walArchiveDir):
+      if kind == pcFile and path.endsWith(".log"):
+        filesToInclude.add(path)
+
+  if filesToInclude.len == 0:
+    echo "ERROR: No files to backup"
+    return false
+
+  # Verify all SSTables before archiving
+  var verifyErrors = 0
+  for path in filesToInclude:
+    if path.endsWith(".sst"):
+      let (ok, msg) = verifySSTable(path)
+      if not ok:
+        echo "ERROR: SSTable verification failed: ", msg
+        inc verifyErrors
+      elif verbose:
+        echo "  ✓ ", extractFilename(path), " — CRC OK"
+  if verifyErrors > 0:
+    echo "ERROR: ", verifyErrors, " SSTable(s) failed verification. Backup aborted."
+    return false
+
+  # Write file list for tar -T
+  let fileListPath = output & ".files"
+  var f: File
+  if open(f, fileListPath, fmWrite):
+    for path in filesToInclude:
+      f.writeLine(path)
+    close(f)
+  else:
+    echo "ERROR: Cannot write file list: ", fileListPath
+    return false
+
+  let tarCmd = "tar -czf " & quoteShell(output) & " -T " & quoteShell(fileListPath)
+  if verbose:
+    echo "Running: ", tarCmd
+    echo "Including ", filesToInclude.len, " files"
+    for path in filesToInclude:
+      echo "  + ", path
+
+  let (outStr, exitCode) = execCmdEx(tarCmd)
+  removeFile(fileListPath)
+
+  if exitCode != 0:
+    echo "ERROR: tar failed with exit code ", exitCode
+    if outStr.len > 0:
+      echo outStr
+    return false
+
+  let size = getFileSize(output)
+  echo "Incremental backup created successfully:"
+  echo "  File:   ", output
+  echo "  Size:   ", formatBytes(size)
+  echo "  Files:  ", filesToInclude.len
+  return true
+
 # =============================================================================
 # CLI Entry Point
 # =============================================================================
@@ -401,6 +494,7 @@ when isMainModule:
     verbose = false
     dryRun = false
     force = false
+    online = false
 
   for kind, key, val in getopt():
     case kind
@@ -426,6 +520,7 @@ when isMainModule:
         except: quit("ERROR: --level must be a number", 1)
       of "dry-run": dryRun = true
       of "force", "f": force = true
+      of "online": online = true
       of "verbose", "v": verbose = true
       of "help", "h":
         echo HELP_TEXT
@@ -441,9 +536,31 @@ when isMainModule:
   case command
   of "backup":
     let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
-    let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
+    if online:
+      echo "Creating online backup with checkpoint..."
+      echo "  Data dir: ", dataDir
+      echo "  Output:   ", outputFile
+      try:
+        var db = newLSMTree(dataDir)
+        db.checkpoint()
+        db.close()
+        echo "Checkpoint complete."
+      except CatchableError as e:
+        echo "ERROR: Checkpoint failed: ", e.msg
+        quit(1)
+      let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
+      if not ok:
+        quit("Online backup failed", 1)
+    else:
+      let ok = backupDataDir(dataDir, outputFile, excludes, compression, verbose)
+      if not ok:
+        quit("Backup failed", 1)
+
+  of "incremental":
+    let outputFile = if target.len > 0: target else: "backup_inc_" & $getTime().toUnix() & ".tar.gz"
+    let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
     if not ok:
-      quit("Backup failed", 1)
+      quit("Incremental backup failed", 1)
 
   of "restore":
     if target.len == 0:

src/barabadb/storage/crc32.nim

@@ -0,0 +1,120 @@
+## CRC32 — IEEE 802.3 (Ethernet, ZIP, PNG)
+## Zero-dependency implementation for SSTable and WAL integrity checks.
+import std/strutils
+##
+## Polynomial: 0xEDB88320
+## Initial:    0xFFFFFFFF
+## Final XOR:  0xFFFFFFFF
+
+const CRC32_TABLE: array[256, uint32] = [
+  0x00000000'u32, 0x77073096'u32, 0xee0e612c'u32, 0x990951ba'u32,
+  0x076dc419'u32, 0x706af48f'u32, 0xe963a535'u32, 0x9e6495a3'u32,
+  0x0edb8832'u32, 0x79dcb8a4'u32, 0xe0d5e91e'u32, 0x97d2d988'u32,
+  0x09b64c2b'u32, 0x7eb17cbd'u32, 0xe7b82d07'u32, 0x90bf1d91'u32,
+  0x1db71064'u32, 0x6ab020f2'u32, 0xf3b97148'u32, 0x84be41de'u32,
+  0x1adad47d'u32, 0x6ddde4eb'u32, 0xf4d4b551'u32, 0x83d385c7'u32,
+  0x136c9856'u32, 0x646ba8c0'u32, 0xfd62f97a'u32, 0x8a65c9ec'u32,
+  0x14015c4f'u32, 0x63066cd9'u32, 0xfa0f3d63'u32, 0x8d080df5'u32,
+  0x3b6e20c8'u32, 0x4c69105e'u32, 0xd56041e4'u32, 0xa2677172'u32,
+  0x3c03e4d1'u32, 0x4b04d447'u32, 0xd20d85fd'u32, 0xa50ab56b'u32,
+  0x35b5a8fa'u32, 0x42b2986c'u32, 0xdbbbc9d6'u32, 0xacbcf940'u32,
+  0x32d86ce3'u32, 0x45df5c75'u32, 0xdcd60dcf'u32, 0xabd13d59'u32,
+  0x26d930ac'u32, 0x51de003a'u32, 0xc8d75180'u32, 0xbfd06116'u32,
+  0x21b4f4b5'u32, 0x56b3c423'u32, 0xcfba9599'u32, 0xb8bda50f'u32,
+  0x2802b89e'u32, 0x5f058808'u32, 0xc60cd9b2'u32, 0xb10be924'u32,
+  0x2f6f7c87'u32, 0x58684c11'u32, 0xc1611dab'u32, 0xb6662d3d'u32,
+  0x76dc4190'u32, 0x01db7106'u32, 0x98d220bc'u32, 0xefd5102a'u32,
+  0x71b18589'u32, 0x06b6b51f'u32, 0x9fbfe4a5'u32, 0xe8b8d433'u32,
+  0x7807c9a2'u32, 0x0f00f934'u32, 0x9609a88e'u32, 0xe10e9818'u32,
+  0x7f6a0dbb'u32, 0x086d3d2d'u32, 0x91646c97'u32, 0xe6630c01'u32,
+  0x6b6b51f4'u32, 0x1c6c6162'u32, 0x856530d8'u32, 0xf262004e'u32,
+  0x6c0695ed'u32, 0x1b01a57b'u32, 0x8208f4c1'u32, 0xf50fc457'u32,
+  0x65b0d9c6'u32, 0x12b7e950'u32, 0x8bbeb8ea'u32, 0xfcb9887c'u32,
+  0x62dd1ddf'u32, 0x15da2d49'u32, 0x8cd37cf3'u32, 0xfbd44c65'u32,
+  0x4db26158'u32, 0x3ab551ce'u32, 0xa3bc0074'u32, 0xd4bb30e2'u32,
+  0x4adfa541'u32, 0x3dd895d7'u32, 0xa4d1c46d'u32, 0xd3d6f4fb'u32,
+  0x4369e96a'u32, 0x346ed9fc'u32, 0xad678846'u32, 0xda60b8d0'u32,
+  0x44042d73'u32, 0x33031de5'u32, 0xaa0a4c5f'u32, 0xdd0d7cc9'u32,
+  0x5005713c'u32, 0x270241aa'u32, 0xbe0b1010'u32, 0xc90c2086'u32,
+  0x5768b525'u32, 0x206f85b3'u32, 0xb966d409'u32, 0xce61e49f'u32,
+  0x5edef90e'u32, 0x29d9c998'u32, 0xb0d09822'u32, 0xc7d7a8b4'u32,
+  0x59b33d17'u32, 0x2eb40d81'u32, 0xb7bd5c3b'u32, 0xc0ba6cad'u32,
+  0xedb88320'u32, 0x9abfb3b6'u32, 0x03b6e20c'u32, 0x74b1d29a'u32,
+  0xead54739'u32, 0x9dd277af'u32, 0x04db2615'u32, 0x73dc1683'u32,
+  0xe3630b12'u32, 0x94643b84'u32, 0x0d6d6a3e'u32, 0x7a6a5aa8'u32,
+  0xe40ecf0b'u32, 0x9309ff9d'u32, 0x0a00ae27'u32, 0x7d079eb1'u32,
+  0xf00f9344'u32, 0x8708a3d2'u32, 0x1e01f268'u32, 0x6906c2fe'u32,
+  0xf762575d'u32, 0x806567cb'u32, 0x196c3671'u32, 0x6e6b06e7'u32,
+  0xfed41b76'u32, 0x89d32be0'u32, 0x10da7a5a'u32, 0x67dd4acc'u32,
+  0xf9b9df6f'u32, 0x8ebeeff9'u32, 0x17b7be43'u32, 0x60b08ed5'u32,
+  0xd6d6a3e8'u32, 0xa1d1937e'u32, 0x38d8c2c4'u32, 0x4fdff252'u32,
+  0xd1bb67f1'u32, 0xa6bc5767'u32, 0x3fb506dd'u32, 0x48b2364b'u32,
+  0xd80d2bda'u32, 0xaf0a1b4c'u32, 0x36034af6'u32, 0x41047a60'u32,
+  0xdf60efc3'u32, 0xa867df55'u32, 0x316e8eef'u32, 0x4669be79'u32,
+  0xcb61b38c'u32, 0xbc66831a'u32, 0x256fd2a0'u32, 0x5268e236'u32,
+  0xcc0c7795'u32, 0xbb0b4703'u32, 0x220216b9'u32, 0x5505262f'u32,
+  0xc5ba3bbe'u32, 0xb2bd0b28'u32, 0x2bb45a92'u32, 0x5cb36a04'u32,
+  0xc2d7ffa7'u32, 0xb5d0cf31'u32, 0x2cd99e8b'u32, 0x5bdeae1d'u32,
+  0x9b64c2b0'u32, 0xec63f226'u32, 0x756aa39c'u32, 0x026d930a'u32,
+  0x9c0906a9'u32, 0xeb0e363f'u32, 0x72076785'u32, 0x05005713'u32,
+  0x95bf4a82'u32, 0xe2b87a14'u32, 0x7bb12bae'u32, 0x0cb61b38'u32,
+  0x92d28e9b'u32, 0xe5d5be0d'u32, 0x7cdcefb7'u32, 0x0bdbdf21'u32,
+  0x86d3d2d4'u32, 0xf1d4e242'u32, 0x68ddb3f8'u32, 0x1fda836e'u32,
+  0x81be16cd'u32, 0xf6b9265b'u32, 0x6fb077e1'u32, 0x18b74777'u32,
+  0x88085ae6'u32, 0xff0f6a70'u32, 0x66063bca'u32, 0x11010b5c'u32,
+  0x8f659eff'u32, 0xf862ae69'u32, 0x616bffd3'u32, 0x166ccf45'u32,
+  0xa00ae278'u32, 0xd70dd2ee'u32, 0x4e048354'u32, 0x3903b3c2'u32,
+  0xa7672661'u32, 0xd06016f7'u32, 0x4969474d'u32, 0x3e6e77db'u32,
+  0xaed16a4a'u32, 0xd9d65adc'u32, 0x40df0b66'u32, 0x37d83bf0'u32,
+  0xa9bcae53'u32, 0xdebb9ec5'u32, 0x47b2cf7f'u32, 0x30b5ffe9'u32,
+  0xbdbdf21c'u32, 0xcabac28a'u32, 0x53b39330'u32, 0x24b4a3a6'u32,
+  0xbad03605'u32, 0xcdd70693'u32, 0x54de5729'u32, 0x23d967bf'u32,
+  0xb3667a2e'u32, 0xc4614ab8'u32, 0x5d681b02'u32, 0x2a6f2b94'u32,
+  0xb40bbe37'u32, 0xc30c8ea1'u32, 0x5a05df1b'u32, 0x2d02ef8d'u32,
+]
+
+proc crc32*(data: openArray[byte]): uint32 =
+  ## Compute CRC32 of a byte sequence.
+  result = 0xFFFFFFFF'u32
+  for b in data:
+    result = CRC32_TABLE[int((result xor uint32(b)) and 0xFF)] xor (result shr 8)
+  result = result xor 0xFFFFFFFF'u32
+
+proc crc32*(data: openArray[byte], seed: uint32): uint32 =
+  ## Incremental CRC32: continue from a previous seed.
+  ## To start incremental computation, pass 0xFFFFFFFF as seed.
+  ## Final XOR must be applied manually by caller when done.
+  result = seed
+  for b in data:
+    result = CRC32_TABLE[int((result xor uint32(b)) and 0xFF)] xor (result shr 8)
+
+proc crc32*(data: pointer, size: int): uint32 =
+  ## Compute CRC32 of a raw memory buffer.
+  result = 0xFFFFFFFF'u32
+  let bytes = cast[ptr UncheckedArray[byte]](data)
+  for i in 0..<size:
+    result = CRC32_TABLE[int((result xor uint32(bytes[i])) and 0xFF)] xor (result shr 8)
+  result = result xor 0xFFFFFFFF'u32
+
+proc crc32*(s: string): uint32 =
+  ## Compute CRC32 of a string.
+  result = 0xFFFFFFFF'u32
+  for b in s:
+    result = CRC32_TABLE[int((result xor uint32(b)) and 0xFF)] xor (result shr 8)
+  result = result xor 0xFFFFFFFF'u32
+
+proc crc32ToHex*(crc: uint32): string =
+  ## Format CRC32 as zero-padded hex string.
+  result = toHex(int64(crc))
+  # toHex returns uppercase, 16 chars for int64; trim to 8
+  result = result[^8..^1]
+
+when isMainModule:
+  # Self-test with known vectors
+  assert crc32("") == 0x00000000'u32
+  assert crc32("a") == 0xe8b7be43'u32
+  assert crc32("abc") == 0x352441c2'u32
+  assert crc32("message digest") == 0x20159d7f'u32
+  assert crc32("abcdefghijklmnopqrstuvwxyz") == 0x4c2750bd'u32
+  assert crc32("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == 0x1fc2e6d2'u32
+  echo "CRC32 self-test passed"

src/barabadb/storage/lsm.nim

@@ -5,17 +5,22 @@ import std/hashes
 import std/strutils
 import std/tables
 import std/monotimes
+import std/times
 import std/streams
 import std/locks
+import std/json
 import bloom
 import wal
 import mmap
+import crc32
 
 const
   SSTableMagic* = 0x53535442'u32  # "SSTB"
-  SSTableVersion* = 2'u32
+  SSTableVersion* = 3'u32
   DefaultMemTableSize* = 4 * 1024 * 1024  # 4MB
   DefaultBloomFpRate* = 0.01
+  ManifestVersion* = 1
+  ManifestFileName* = "MANIFEST"
 
 type
   Entry* = object
@@ -38,6 +43,7 @@ type
     minKey*: string
     maxKey*: string
     entryCount*: int
+    fileVersion*: uint32
     mmapFile*: MmapFile
 
   LSMTree* = ref object
@@ -45,10 +51,11 @@ type
     memTable: MemTable
     immutableMem: MemTable
     sstables*: seq[SSTable]
-    wal: WriteAheadLog
+    wal*: WriteAheadLog
     memMaxSize: int
     currentSeq: uint64
-    nextSSTableId: int
+    nextSSTableId*: int
+    manifestSequence*: int64
     lock*: Lock
     walLock*: Lock
 
@@ -100,12 +107,14 @@ proc clear*(mt: var MemTable) =
 
 # ----------------------------------------------------------------------
 # SSTable serialization format (native endianness):
-# [Header] 28 bytes
-#   magic: uint32
-#   version: uint32
+# [Header] 36 bytes (v3)
+#   magic: uint32        (0x53535442 = "SSTB")
+#   version: uint32      (3 = current, 2 = legacy with level, 1 = legacy)
 #   entryCount: uint32
+#   level: uint32
 #   indexOffset: uint64
 #   bloomOffset: uint64
+#   footerOffset: uint64   # v3 only
 #
 # [Data Block]
 #   For each entry:
@@ -125,14 +134,23 @@ proc clear*(mt: var MemTable) =
 # [Bloom Filter Block]
 #   bloomSize: uint32
 #   bloomData: bytes[bloomSize]
+#
+# [Footer] 16 bytes (v3 only)
+#   dataCrc32: uint32    # CRC32 of Data Block
+#   indexCrc32: uint32   # CRC32 of Index Block
+#   bloomCrc32: uint32   # CRC32 of Bloom Block
+#   reserved: uint32     # must be 0
 # ----------------------------------------------------------------------
 
+const
+  SSTableFooterSize* = 16
+
 proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
   let s = newFileStream(path, fmWrite)
   if s.isNil:
     raise newException(IOError, "Cannot create SSTable file: " & path)
 
-  # Write header
+  # Write header (v3: 36 bytes)
   s.write(SSTableMagic)
   s.write(SSTableVersion)
   if entries.len > high(uint32).int:
@@ -143,6 +161,8 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
   s.write(0'u64)  # patched after data+bloom are written
   let bloomOffsetPos = s.getPosition()
   s.write(0'u64)  # patched after data+bloom are written
+  let footerOffsetPos = s.getPosition()
+  s.write(0'u64)  # patched after footer is written
 
   # Write data block
   var offsets = newSeq[(string, int64)](entries.len)
@@ -179,13 +199,39 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
   if bloomData.len > 0:
     s.writeData(addr bloomData[0], bloomData.len)
 
-  # Patch header with correct offsets before closing
-  s.setPosition(int(indexOffsetPos))
-  s.write(indexOffset)
-  s.setPosition(int(bloomOffsetPos))
-  s.write(bloomOffset)
+  let footerOffset = uint64(s.getPosition())
   s.close()
 
+  # Compute CRCs via mmap
+  let mf = openMmap(path, mmReadOnly)
+  if mf.regions.len == 0:
+    raise newException(IOError, "Cannot mmap SSTable for CRC: " & path)
+
+  let headerSize = 36
+  let dataCrc = crc32(unsafeAddr mf.regions[0].data[headerSize], int(indexOffset) - headerSize)
+  let indexCrc = crc32(unsafeAddr mf.regions[0].data[int(indexOffset)], int(bloomOffset) - int(indexOffset))
+  let bloomCrc = crc32(unsafeAddr mf.regions[0].data[int(bloomOffset)], int(footerOffset) - int(bloomOffset))
+  mf.close()
+
+  # Write footer and patch header
+  let s2 = newFileStream(path, fmReadWriteExisting)
+  if s2.isNil:
+    raise newException(IOError, "Cannot reopen SSTable for footer write: " & path)
+
+  s2.setPosition(int(footerOffset))
+  s2.write(dataCrc)
+  s2.write(indexCrc)
+  s2.write(bloomCrc)
+  s2.write(0'u32)  # reserved
+
+  s2.setPosition(int(indexOffsetPos))
+  s2.write(indexOffset)
+  s2.setPosition(int(bloomOffsetPos))
+  s2.write(bloomOffset)
+  s2.setPosition(int(footerOffsetPos))
+  s2.write(footerOffset)
+  s2.close()
+
   # Build in-memory index
   var idxTable = initTable[string, int64]()
   var minK = ""
@@ -204,9 +250,58 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
     minKey: minK,
     maxKey: maxK,
     entryCount: entries.len,
+    fileVersion: SSTableVersion,
     mmapFile: openMmap(path),
   )
 
+proc verifySSTable*(path: string): (bool, string) =
+  ## Verify SSTable integrity: magic, version, CRC footer.
+  ## Returns (ok, message).
+  let mf = openMmap(path, mmReadOnly)
+  if mf.regions.len == 0:
+    return (false, "Cannot mmap: " & path)
+  if mf.totalSize < 40:
+    return (false, "File too small: " & path)
+
+  if mf.readUint32(0) != SSTableMagic:
+    return (false, "Invalid SSTable magic: " & path)
+  let fileVersion = mf.readUint32(4)
+  if fileVersion == 1'u32:
+    return (true, "OK (legacy v1): " & path)
+  elif fileVersion == 2'u32:
+    return (true, "OK (legacy v2): " & path)
+  elif fileVersion != SSTableVersion:
+    return (false, "Unsupported SSTable version " & $fileVersion & ": " & path)
+
+  # v3 CRC verification
+  let footerOffset = int(mf.readUint64(32))
+  if footerOffset + SSTableFooterSize > mf.totalSize:
+    return (false, "Footer extends past EOF (" & $footerOffset & " + 16 > " & $mf.totalSize & "): " & path)
+
+  let indexOffset = int(mf.readUint64(16))
+  let bloomOffset = int(mf.readUint64(24))
+
+  let storedDataCrc = mf.readUint32(footerOffset)
+  let storedIndexCrc = mf.readUint32(footerOffset + 4)
+  let storedBloomCrc = mf.readUint32(footerOffset + 8)
+  let reserved = mf.readUint32(footerOffset + 12)
+  if reserved != 0:
+    return (false, "Non-zero reserved field in footer: " & path)
+
+  let headerSize = 36
+  let computedDataCrc = crc32(unsafeAddr mf.regions[0].data[headerSize], indexOffset - headerSize)
+  let computedIndexCrc = crc32(unsafeAddr mf.regions[0].data[indexOffset], bloomOffset - indexOffset)
+  let computedBloomCrc = crc32(unsafeAddr mf.regions[0].data[bloomOffset], footerOffset - bloomOffset)
+
+  if computedDataCrc != storedDataCrc:
+    return (false, "Data block CRC mismatch (expected " & crc32ToHex(storedDataCrc) & ", got " & crc32ToHex(computedDataCrc) & "): " & path)
+  if computedIndexCrc != storedIndexCrc:
+    return (false, "Index block CRC mismatch (expected " & crc32ToHex(storedIndexCrc) & ", got " & crc32ToHex(computedIndexCrc) & "): " & path)
+  if computedBloomCrc != storedBloomCrc:
+    return (false, "Bloom block CRC mismatch (expected " & crc32ToHex(storedBloomCrc) & ", got " & crc32ToHex(computedBloomCrc) & "): " & path)
+
+  return (true, "OK (v3 CRC verified): " & path)
+
 proc loadSSTable*(path: string): SSTable =
   let mf = openMmap(path)
   if mf.regions.len == 0:
@@ -215,14 +310,34 @@ proc loadSSTable*(path: string): SSTable =
   if mf.readUint32(0) != SSTableMagic:
     raise newException(ValueError, "Invalid SSTable magic")
   let fileVersion = mf.readUint32(4)
-  if fileVersion != SSTableVersion and fileVersion != 1'u32:
-    raise newException(ValueError, "Unsupported SSTable version")
+  if fileVersion != SSTableVersion and fileVersion != 2'u32 and fileVersion != 1'u32:
+    raise newException(ValueError, "Unsupported SSTable version " & $fileVersion)
 
   let entryCount = int(mf.readUint32(8))
   var level = 0
   var indexOffset = 0
   var bloomOffset = 0
-  if fileVersion == 2'u32:
+  var footerOffset = 0
+
+  if fileVersion == 3'u32:
+    level = int(mf.readUint32(12))
+    indexOffset = int(mf.readUint64(16))
+    bloomOffset = int(mf.readUint64(24))
+    footerOffset = int(mf.readUint64(32))
+    # Verify CRC before proceeding
+    if footerOffset + SSTableFooterSize <= mf.totalSize:
+      let storedDataCrc = mf.readUint32(footerOffset)
+      let storedIndexCrc = mf.readUint32(footerOffset + 4)
+      let storedBloomCrc = mf.readUint32(footerOffset + 8)
+      let headerSize = 36
+      let computedDataCrc = crc32(unsafeAddr mf.regions[0].data[headerSize], indexOffset - headerSize)
+      let computedIndexCrc = crc32(unsafeAddr mf.regions[0].data[indexOffset], bloomOffset - indexOffset)
+      let computedBloomCrc = crc32(unsafeAddr mf.regions[0].data[bloomOffset], footerOffset - bloomOffset)
+      if computedDataCrc != storedDataCrc or computedIndexCrc != storedIndexCrc or computedBloomCrc != storedBloomCrc:
+        raise newException(ValueError, "SSTable CRC check failed: " & path)
+    else:
+      raise newException(ValueError, "SSTable footer extends past EOF: " & path)
+  elif fileVersion == 2'u32:
     level = int(mf.readUint32(12))
     indexOffset = int(mf.readUint64(16))
     bloomOffset = int(mf.readUint64(24))
@@ -267,6 +382,7 @@ proc loadSSTable*(path: string): SSTable =
     minKey: minK,
     maxKey: maxK,
     entryCount: entryCount,
+    fileVersion: fileVersion,
     mmapFile: mf,
   )
 
@@ -312,6 +428,161 @@ proc readSSTableEntry*(sst: SSTable, key: string): (bool, Entry) =
 proc close*(sst: var SSTable) =
   sst.mmapFile.close()
 
+# ----------------------------------------------------------------------
+# SSTable Version Migration
+# ----------------------------------------------------------------------
+
+proc listLegacySSTables*(dir: string): seq[(string, uint32)] =
+  ## Scan sstables directory and return paths of v1/v2 SSTables.
+  result = @[]
+  let sstDir = dir / "sstables"
+  if not dirExists(sstDir):
+    return
+  for kind, path in walkDir(sstDir):
+    if kind == pcFile and path.endsWith(".sst"):
+      try:
+        let sst = loadSSTable(path)
+        if sst.fileVersion < SSTableVersion:
+          result.add((path, sst.fileVersion))
+      except:
+        discard
+
+proc migrateSSTable*(path: string): bool =
+  ## Read a legacy SSTable and rewrite it as current version (v3).
+  ## The original file is replaced atomically via temp + rename.
+  try:
+    let sst = loadSSTable(path)
+    if sst.fileVersion == SSTableVersion:
+      return true  # already current
+
+    # Read all entries
+    var entries: seq[Entry] = @[]
+    for key, offset in sst.index:
+      let (found, entry) = readSSTableEntry(sst, key)
+      if found:
+        entries.add(entry)
+
+    if entries.len == 0:
+      return false
+
+    # Sort by key for writeSSTable
+    entries.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
+
+    let tmpPath = path & ".migrate"
+    discard writeSSTable(entries, tmpPath, sst.level)
+
+    # Close mmap before replacing
+    var mutableSst = sst
+    mutableSst.close()
+
+    # Atomic replace
+    removeFile(path)
+    moveFile(tmpPath, path)
+    return true
+  except CatchableError as e:
+    echo "[ERROR] Failed to migrate ", path, ": ", e.msg
+    return false
+
+# ----------------------------------------------------------------------
+# MANIFEST — Atomic catalog of active SSTables
+# ----------------------------------------------------------------------
+
+proc writeManifest*(db: LSMTree) =
+  ## Atomically write MANIFEST file with current SSTable set.
+  let manifestPath = db.dir / ManifestFileName
+  let tmpPath = manifestPath & ".tmp"
+
+  var j = newJObject()
+  j["version"] = newJInt(ManifestVersion)
+  j["sequence"] = newJInt(db.manifestSequence)
+  j["createdAt"] = newJInt(int64(getTime().toUnix()))
+
+  var sstArr = newJArray()
+  for sst in db.sstables:
+    var obj = newJObject()
+    obj["id"] = newJInt(sst.id)
+    obj["path"] = newJString(sst.path)
+    obj["level"] = newJInt(sst.level)
+    obj["minKey"] = newJString(sst.minKey)
+    obj["maxKey"] = newJString(sst.maxKey)
+    obj["entryCount"] = newJInt(sst.entryCount)
+    sstArr.add(obj)
+  j["sstables"] = sstArr
+
+  let content = pretty(j)
+  var f: File
+  if open(f, tmpPath, fmWrite):
+    try:
+      f.write(content)
+    finally:
+      close(f)
+    try:
+      moveFile(tmpPath, manifestPath)
+    except IOError:
+      removeFile(tmpPath)
+      raise
+  else:
+    raise newException(IOError, "Cannot write MANIFEST: " & tmpPath)
+
+proc readManifest*(dir: string): (seq[SSTable], int64) =
+  ## Read MANIFEST and return (sstables, sequence). If no MANIFEST, returns empty.
+  let manifestPath = dir / ManifestFileName
+  if not fileExists(manifestPath):
+    return (@[], 0'i64)
+
+  let content = readFile(manifestPath)
+  if content.len == 0:
+    return (@[], 0'i64)
+
+  let j = parseJson(content)
+  if j{"version"}.getInt() != ManifestVersion:
+    raise newException(ValueError, "Unsupported MANIFEST version")
+
+  var sstables: seq[SSTable] = @[]
+  let seqNum = j{"sequence"}.getInt()
+  let sstArr = j{"sstables"}
+  for node in sstArr:
+    let path = node{"path"}.getStr()
+    if not fileExists(path):
+      continue  # skip missing SSTables
+    try:
+      var sst = loadSSTable(path)
+      sst.id = node{"id"}.getInt()
+      sst.level = node{"level"}.getInt()
+      sstables.add(sst)
+    except CatchableError as e:
+      echo "[WARN] MANIFEST references corrupt SSTable: ", path, " — ", e.msg
+
+  return (sstables, int64(seqNum))
+
+proc checkStorageConsistency*(db: LSMTree): seq[string] =
+  ## Return list of warnings: orphan files, missing SSTables, etc.
+  result = @[]
+  let manifestPath = db.dir / ManifestFileName
+  var manifestPaths: seq[string] = @[]
+
+  if fileExists(manifestPath):
+    try:
+      let j = parseJson(readFile(manifestPath))
+      for node in j{"sstables"}:
+        manifestPaths.add(node{"path"}.getStr())
+    except:
+      result.add("MANIFEST is corrupt or unreadable")
+      return
+
+  # Check for orphan SSTables on disk
+  let sstDir = db.dir / "sstables"
+  if dirExists(sstDir):
+    for kind, path in walkDir(sstDir):
+      if kind == pcFile and path.endsWith(".sst"):
+        if path notin manifestPaths and manifestPaths.len > 0:
+          result.add("Orphan SSTable (not in MANIFEST): " & path)
+
+  # Check for missing SSTables referenced in MANIFEST
+  for p in manifestPaths:
+    if not fileExists(p):
+      result.add("Missing SSTable (in MANIFEST but not on disk): " & p)
+
 # ----------------------------------------------------------------------
 # LSMTree API
 # ----------------------------------------------------------------------
@@ -324,20 +595,38 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
 
   var sstables: seq[SSTable] = @[]
   var nextId = 1
+  var manifestSeq: int64 = 0
 
-  # Load existing SSTables
-  for kind, path in walkDir(dir / "sstables"):
-    if kind == pcFile and path.endsWith(".sst"):
-      try:
-        var sst = loadSSTable(path)
-        let name = splitFile(path).name
-        sst.id = parseInt(name)
-        sstables.add(sst)
+  # Try loading from MANIFEST first
+  let manifestPath = dir / ManifestFileName
+  if fileExists(manifestPath):
+    try:
+      let (loaded, seqNum) = readManifest(dir)
+      sstables = loaded
+      manifestSeq = seqNum
+      for sst in sstables:
         nextId = max(nextId, sst.id + 1)
-      except:
-        discard  # skip corrupt SSTables
+      if sstables.len > 0:
+        echo "[INFO] Loaded ", sstables.len, " SSTable(s) from MANIFEST (seq=", manifestSeq, ")"
+    except CatchableError as e:
+      echo "[WARN] Failed to read MANIFEST: ", e.msg, " — falling back to directory scan"
+      sstables.setLen(0)
 
-  sstables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
+  # Fallback: directory scan if MANIFEST missing or empty
+  if sstables.len == 0:
+    for kind, path in walkDir(dir / "sstables"):
+      if kind == pcFile and path.endsWith(".sst"):
+        try:
+          var sst = loadSSTable(path)
+          let name = splitFile(path).name
+          sst.id = parseInt(name)
+          sstables.add(sst)
+          nextId = max(nextId, sst.id + 1)
+        except CatchableError as e:
+          echo "[WARN] Skipping corrupt SSTable: ", path, " — ", e.msg
+    sstables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
+    if sstables.len > 0:
+      echo "[INFO] Loaded ", sstables.len, " SSTable(s) from directory scan"
 
   new(result)
   initLock(result.lock)
@@ -350,6 +639,7 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
   result.memMaxSize = memMaxSize
   result.currentSeq = 0
   result.nextSSTableId = nextId
+  result.manifestSequence = manifestSeq
 
   # WAL crash recovery — replay unflushed entries into memTable
   let walPath = dir / "wal" / "wal.log"
@@ -506,7 +796,15 @@ proc flushUnsafe(db: LSMTree) =
   db.sstables.add(sst)
   # SSTables are kept in insertion order (newest last) so getUnsafe can search newest-first
 
+  # Update MANIFEST atomically
+  inc db.manifestSequence
+  try:
+    writeManifest(db)
+  except CatchableError as e:
+    echo "[WARN] Failed to write MANIFEST: ", e.msg
+
   db.wal.writeCommit(uint64(getMonoTime().ticks()))
+  db.wal.maybeRotate()
   db.wal.sync()
 
 proc flush*(db: LSMTree) =
@@ -514,6 +812,31 @@ proc flush*(db: LSMTree) =
   defer: release(db.lock)
   flushUnsafe(db)
 
+proc checkpoint*(db: LSMTree) =
+  ## Create a consistent checkpoint: freeze memtable, flush to SSTable,
+  ## rotate WAL, and write MANIFEST. This provides a clean boundary
+  ## for online backup without stopping the server.
+  acquire(db.lock)
+  
+  # Flush any pending immutable memtable first
+  if db.immutableMem.len > 0:
+    flushUnsafe(db)
+  
+  # Freeze current memtable so writes can continue on a new one
+  if db.memTable.len > 0:
+    db.immutableMem = db.memTable
+    db.memTable = newMemTable(db.memMaxSize)
+  
+  release(db.lock)
+  
+  # Flush the frozen memtable outside the lock (writes proceed concurrently)
+  if db.immutableMem.len > 0:
+    flushUnsafe(db)
+  
+  # Rotate WAL for a clean backup boundary
+  db.wal.maybeRotate()
+  db.wal.sync()
+
 proc close*(db: LSMTree) =
   acquire(db.lock)
   defer: release(db.lock)

src/barabadb/storage/wal.nim

@@ -1,11 +1,15 @@
 ## WAL — Write-Ahead Log for durability
+import std/algorithm
 import std/os
 import std/streams
+import std/strutils
 import std/posix
 
 const
   WALMagic* = 0x42415241'u32  # "BARA"
   WALVersion* = 1'u32
+  DefaultMaxWalSegmentSize* = 64 * 1024 * 1024  # 64MB
+  WalArchiveDir* = "wal_archive"
 
 type
   WalEntryKind* = enum
@@ -20,13 +24,87 @@ type
     key*: seq[byte]
     value*: seq[byte]
 
+  WalSegment* = object
+    sequence*: int64
+    path*: string
+    size*: int64
+
   WriteAheadLog* = object
+    dir*: string
     path: string
     stream: FileStream
     entryCount: uint64
     syncOnWrite: bool
+    maxSegmentSize: int64
+    currentSequence: int64
 
 proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry]
+proc listWalArchive*(dir: string): seq[WalSegment]
+proc maybeRotate*(wal: var WriteAheadLog)
+
+proc parseWalSequence*(filename: string): int64 =
+  ## Extract sequence from "wal.000042.log"
+  try:
+    if filename.startsWith("wal.") and filename.endsWith(".log"):
+      let numStr = filename[4..^5]
+      result = parseBiggestInt(numStr)
+    else:
+      result = 0
+  except:
+    result = 0
+
+proc listWalArchive*(dir: string): seq[WalSegment] =
+  ## Return all archived WAL segments sorted by sequence.
+  result = @[]
+  let archiveDir = dir / WalArchiveDir
+  if not dirExists(archiveDir):
+    return
+  for kind, path in walkDir(archiveDir):
+    if kind == pcFile and path.endsWith(".log"):
+      let seqNum = parseWalSequence(extractFilename(path))
+      if seqNum > 0:
+        let size = try: getFileSize(path) except: 0
+        result.add(WalSegment(sequence: seqNum, path: path, size: size))
+  result.sort(proc(a, b: WalSegment): int = cmp(a.sequence, b.sequence))
+
+proc nextWalSequence*(dir: string): int64 =
+  ## Find the next available WAL sequence number.
+  let segments = listWalArchive(dir)
+  if segments.len == 0:
+    return 1
+  return segments[^1].sequence + 1
+
+proc rotate*(wal: var WriteAheadLog) =
+  ## Close current WAL and archive it, then start a new one.
+  if wal.stream != nil:
+    wal.stream.flush()
+    wal.stream.close()
+
+  let archiveDir = wal.dir / WalArchiveDir
+  createDir(archiveDir)
+  let archivePath = archiveDir / ("wal." & align($wal.currentSequence, 6, '0') & ".log")
+
+  try:
+    moveFile(wal.path, archivePath)
+  except IOError as e:
+    raise newException(IOError, "WAL rotation failed: cannot move " & wal.path & " to " & archivePath & ": " & e.msg)
+
+  wal.currentSequence = wal.currentSequence + 1
+  wal.stream = newFileStream(wal.path, fmWrite)
+  if wal.stream == nil:
+    raise newException(IOError, "Cannot create new WAL after rotation: " & wal.path)
+  wal.stream.write(WALMagic)
+  wal.stream.write(WALVersion)
+  wal.stream.flush()
+  wal.entryCount = 0
+
+proc maybeRotate*(wal: var WriteAheadLog) =
+  ## Rotate if current WAL exceeds max segment size.
+  if wal.maxSegmentSize <= 0:
+    return
+  let currentSize = try: getFileSize(wal.path) except: 0
+  if currentSize >= wal.maxSegmentSize:
+    wal.rotate()
 
 proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
   createDir(dir)
@@ -46,7 +124,17 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
   if exists and not isEmpty:
     for e in readEntries(path):
       inc count
-  WriteAheadLog(path: path, stream: stream, entryCount: count, syncOnWrite: syncOnWrite)
+
+  let seqNum = nextWalSequence(dir)
+  WriteAheadLog(
+    dir: dir,
+    path: path,
+    stream: stream,
+    entryCount: count,
+    syncOnWrite: syncOnWrite,
+    maxSegmentSize: DefaultMaxWalSegmentSize,
+    currentSequence: seqNum,
+  )
 
 proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
   wal.stream.write(uint8(entry.kind))
@@ -60,6 +148,9 @@ proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
   if wal.syncOnWrite:
     wal.stream.flush()
   inc wal.entryCount
+  # Check rotation every 1000 entries to avoid stat on every write
+  if wal.entryCount mod 1000 == 0:
+    wal.maybeRotate()
 
 proc writePut*(wal: var WriteAheadLog, key, value: openArray[byte], timestamp: uint64) =
   wal.writeEntry(WalEntry(
@@ -95,6 +186,9 @@ proc sync*(wal: var WriteAheadLog) =
     discard posix.fsync(fd)
     discard posix.close(fd)
 
+proc setMaxSegmentSize*(wal: var WriteAheadLog, size: int64) =
+  wal.maxSegmentSize = size
+
 proc close*(wal: var WriteAheadLog) =
   wal.stream.flush()
   let fd = posix.open(cstring(wal.path), O_RDWR)

src/barabadb/tools/migrate.nim

@@ -0,0 +1,126 @@
+## BaraDB SSTable Migration Tool — rewrite legacy v1/v2 SSTables to v3
+##
+## Usage:
+##   nim c -r src/barabadb/tools/migrate.nim --data-dir=./data/server
+##
+## Or via baradadb binary:
+##   ./baradadb migrate --data-dir=./data/server
+
+import std/os
+import ../storage/lsm
+
+type
+  MigrateResult* = object
+    scanned*: int
+    migrated*: int
+    skipped*: int
+    errors*: seq[string]
+
+const
+  DEFAULT_DATA_DIR = "data/server"
+  HelpText* = """
+BaraDB SSTable Migration — Upgrade legacy SSTables to current format
+=====================================================================
+
+USAGE:
+  migrate [options]
+
+OPTIONS:
+  -d, --data-dir <DIR>   Path to data directory (default: data/server)
+  --dry-run              Show what would be migrated without making changes
+  -h, --help             Show this help message
+
+DESCRIPTION:
+  Scans all SSTables in <DIR>/sstables/ and rewrites any v1 or v2 files
+  to the current v3 format (with CRC footer). The original files are
+  replaced atomically. This is an offline operation — the server should
+  not be running during migration.
+
+EXAMPLES:
+  # Preview only
+  migrate --dry-run
+
+  # Migrate default data directory
+  migrate
+
+  # Migrate specific directory
+  migrate --data-dir=/var/lib/baradb
+"""
+
+proc runMigration*(dataDir: string, dryRun: bool = false): MigrateResult =
+  result.scanned = 0
+  result.migrated = 0
+  result.skipped = 0
+  result.errors = @[]
+
+  let legacy = listLegacySSTables(dataDir)
+  result.scanned = legacy.len
+
+  if legacy.len == 0:
+    echo "No legacy SSTables found. All files are already at version ", SSTableVersion, "."
+    return
+
+  echo "Found ", legacy.len, " legacy SSTable(s) to migrate:"
+  for (path, version) in legacy:
+    echo "  ", extractFilename(path), " (v", version, ")"
+
+  if dryRun:
+    echo "DRY-RUN: No changes made."
+    return
+
+  for (path, version) in legacy:
+    echo "Migrating: ", extractFilename(path), " (v", version, " → v", SSTableVersion, ")"
+    if migrateSSTable(path):
+      inc result.migrated
+      echo "  ✓ Done"
+    else:
+      result.errors.add("Failed: " & path)
+      echo "  ✗ Failed"
+
+  # After migration, rewrite MANIFEST so it references current versions
+  if result.migrated > 0:
+    try:
+      var db = newLSMTree(dataDir)
+      writeManifest(db)
+      db.close()
+      echo "MANIFEST updated."
+    except CatchableError as e:
+      result.errors.add("MANIFEST update failed: " & e.msg)
+
+# =============================================================================
+# CLI Entry Point
+# =============================================================================
+when isMainModule:
+  var
+    dataDir = DEFAULT_DATA_DIR
+    dryRun = false
+
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      discard
+    of cmdLongOption, cmdShortOption:
+      case key
+      of "data-dir", "d": dataDir = val
+      of "dry-run": dryRun = true
+      of "help", "h":
+        echo HelpText
+        quit(0)
+      else: discard
+    of cmdEnd: discard
+
+  if dryRun:
+    echo "DRY-RUN mode: no changes will be made."
+
+  let result = runMigration(dataDir, dryRun)
+  echo ""
+  echo "Migration complete:"
+  echo "  Scanned:  ", result.scanned
+  echo "  Migrated: ", result.migrated
+  echo "  Errors:   ", result.errors.len
+  if result.errors.len > 0:
+    for e in result.errors:
+      echo "  ! ", e
+    quit(1)
+  else:
+    quit(0)

src/barabadb/tools/repair.nim

@@ -0,0 +1,276 @@
+## BaraDB Repair Tool — Storage verification, cleanup, and WAL replay
+##
+## Usage:
+##   nim c -r src/barabadb/tools/repair.nim --data-dir=./data/server
+##
+## Or via baradadb binary (if wired):
+##   ./baradadb repair --data-dir=./data/server
+
+import std/os
+import std/strutils
+import std/times
+import std/parseopt
+import ../storage/lsm
+import ../storage/recovery
+
+type
+  SSTableRepairStatus* = enum
+    srsOk
+    srsCorrupt
+    srsRebuilt
+    srsRemoved
+
+  SSTableRepairResult* = object
+    path*: string
+    status*: SSTableRepairStatus
+    message*: string
+    version*: int
+
+  RepairReport* = object
+    dataDir*: string
+    sstablesChecked*: int
+    sstablesOk*: int
+    sstablesCorrupt*: int
+    sstablesRemoved*: int
+    walEntriesRecovered*: int
+    walRedone*: int
+    walUndone*: int
+    errors*: seq[string]
+    results*: seq[SSTableRepairResult]
+    startedAt*: string
+    completedAt*: string
+
+const
+  DEFAULT_DATA_DIR = "data/server"
+  HelpText* = """
+BaraDB Storage Repair — Verify SSTables, remove corruption, replay WAL
+BaraDB Storage Repair — Verify SSTables, remove corruption, replay WAL
+=======================================================================
+
+USAGE:
+  repair [options]
+
+OPTIONS:
+  -d, --data-dir <DIR>   Path to data directory (default: data/server)
+  -f, --force            Remove corrupt SSTables without prompting
+  -q, --quiet            Only print errors and summary
+  --dry-run              Show what would be done without making changes
+  -h, --help             Show this help message
+
+DESCRIPTION:
+  Scans all SSTable files in <DIR>/sstables/, verifies CRC checksums
+  (v3) and magic/version (v1/v2). Corrupt files are moved to
+  <DIR>/corrupt/ (or deleted with --force). After cleanup, WAL is
+  replayed to recover any unflushed committed entries.
+
+EXAMPLES:
+  # Dry run — preview only
+  repair --dry-run
+
+  # Repair with default data dir
+  repair
+
+  # Repair specific directory, auto-remove corrupt files
+  repair --data-dir=/var/lib/baradb --force
+"""
+
+proc formatBytes*(bytes: int64): string =
+  const units = ["B", "KB", "MB", "GB", "TB"]
+  if bytes < 0: return "0 B"
+  var size = float64(bytes)
+  var unitIndex = 0
+  while size >= 1024.0 and unitIndex < units.high:
+    size /= 1024.0
+    unitIndex += 1
+  result = formatFloat(size, ffDecimal, precision = 2) & " " & units[unitIndex]
+
+proc runRepair*(dataDir: string, dryRun: bool = false, quiet: bool = false): RepairReport =
+  ## Main repair routine.
+  result.dataDir = dataDir
+  result.startedAt = format(getTime(), "yyyy-MM-dd HH:mm:ss")
+  result.sstablesChecked = 0
+  result.sstablesOk = 0
+  result.sstablesCorrupt = 0
+  result.sstablesRemoved = 0
+  result.walEntriesRecovered = 0
+  result.walRedone = 0
+  result.walUndone = 0
+
+  let sstDir = dataDir / "sstables"
+  let corruptDir = dataDir / "corrupt"
+
+  if not dirExists(sstDir):
+    result.errors.add("SSTables directory not found: " & sstDir)
+    result.completedAt = format(getTime(), "yyyy-MM-dd HH:mm:ss")
+    return
+
+  if not dryRun:
+    createDir(corruptDir)
+
+  # ------------------------------------------------------------------
+  # Phase 1: Scan and verify all SSTables
+  # ------------------------------------------------------------------
+  if not quiet:
+    echo "Scanning SSTables in ", sstDir, " ..."
+
+  for kind, path in walkDir(sstDir):
+    if kind != pcFile or not path.endsWith(".sst"):
+      continue
+
+    inc result.sstablesChecked
+    let (ok, msg) = verifySSTable(path)
+    var res = SSTableRepairResult(path: path, message: msg)
+
+    if ok:
+      res.status = srsOk
+      inc result.sstablesOk
+      # Try to detect version from message
+      if msg.contains("v3"): res.version = 3
+      elif msg.contains("v2"): res.version = 2
+      elif msg.contains("v1"): res.version = 1
+    else:
+      res.status = srsCorrupt
+      inc result.sstablesCorrupt
+      result.errors.add(msg)
+
+      # Move to corrupt dir (or delete in dry-run we just report)
+      if not dryRun:
+        let fname = extractFilename(path)
+        let dest = corruptDir / fname
+        try:
+          moveFile(path, dest)
+          res.status = srsRemoved
+          inc result.sstablesRemoved
+          res.message = msg & " → moved to " & dest
+        except IOError as e:
+          res.message = msg & " → failed to move: " & e.msg
+      else:
+        res.message = msg & " → would move to " & corruptDir
+
+    result.results.add(res)
+
+  if not quiet:
+    echo "SSTable scan complete: ", result.sstablesOk, " OK, ", result.sstablesCorrupt, " corrupt"
+
+  # ------------------------------------------------------------------
+  # Phase 2: WAL replay to recover unflushed data
+  # ------------------------------------------------------------------
+  let walPath = dataDir / "wal" / "wal.log"
+  if fileExists(walPath):
+    if not quiet:
+      echo "Replaying WAL: ", walPath, " ..."
+
+    if dryRun:
+      # Just count entries without applying
+      var rec = newCrashRecovery(dataDir / "wal", dataDir)
+      let analysis = rec.analyze()
+      result.walEntriesRecovered = analysis.totalEntries
+      result.walRedone = analysis.redone
+      result.walUndone = analysis.undone
+      if not quiet:
+        echo "WAL dry-run: ", analysis.totalEntries, " entries (", analysis.redone, " redo, ", analysis.undone, " undo)"
+    else:
+      # Create a temporary LSMTree just for recovery, then flush
+      var tmpDb = newLSMTree(dataDir, DefaultMemTableSize)
+      var rec = newCrashRecovery(dataDir / "wal", dataDir)
+      let recoveryResult = rec.recover(tmpDb)
+      result.walEntriesRecovered = recoveryResult.totalEntries
+      result.walRedone = recoveryResult.redone
+      result.walUndone = recoveryResult.undone
+
+      # Flush recovered data to SSTables
+      if recoveryResult.redone > 0:
+        tmpDb.flush()
+        if not quiet:
+          echo "WAL replay complete: ", recoveryResult.redone, " entries redone, ", recoveryResult.undone, " undone"
+      tmpDb.close()
+  else:
+    if not quiet:
+      echo "No WAL found at ", walPath
+
+  result.completedAt = format(getTime(), "yyyy-MM-dd HH:mm:ss")
+
+proc printReport*(report: RepairReport, quiet: bool = false) =
+  if quiet and report.errors.len == 0 and report.sstablesCorrupt == 0:
+    echo "Repair complete. No issues found."
+    return
+
+  echo ""
+  echo "═══════════════════════════════════════════════════════════════════════"
+  echo "                    BaraDB Repair Report"
+  echo "═══════════════════════════════════════════════════════════════════════"
+  echo "Data directory:  ", report.dataDir
+  echo "Started:         ", report.startedAt
+  echo "Completed:       ", report.completedAt
+  echo ""
+  echo "SSTables:"
+  echo "  Checked:       ", report.sstablesChecked
+  echo "  OK:            ", report.sstablesOk
+  echo "  Corrupt:       ", report.sstablesCorrupt
+  echo "  Removed:       ", report.sstablesRemoved
+  echo ""
+  echo "WAL Recovery:"
+  echo "  Entries:       ", report.walEntriesRecovered
+  echo "  Redone:        ", report.walRedone
+  echo "  Undone:        ", report.walUndone
+  echo ""
+
+  if report.errors.len > 0:
+    echo "Errors (", report.errors.len, "):"
+    for e in report.errors:
+      echo "  • ", e
+    echo ""
+
+  if report.results.len > 0:
+    echo "Details:"
+    for r in report.results:
+      let icon = case r.status
+        of srsOk: "✓"
+        of srsCorrupt: "✗"
+        of srsRebuilt: "↻"
+        of srsRemoved: "→"
+      echo "  ", icon, " ", extractFilename(r.path), " — ", r.message
+    echo ""
+
+  if report.sstablesCorrupt == 0 and report.errors.len == 0:
+    echo "Result: ALL OK — storage is healthy."
+  else:
+    echo "Result: ", report.sstablesCorrupt, " corrupt SSTable(s) handled."
+  echo "═══════════════════════════════════════════════════════════════════════"
+
+# =============================================================================
+# CLI Entry Point
+# =============================================================================
+when isMainModule:
+  var
+    dataDir = DEFAULT_DATA_DIR
+    dryRun = false
+    force = false
+    quiet = false
+
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      discard  # no positional args
+    of cmdLongOption, cmdShortOption:
+      case key
+      of "data-dir", "d": dataDir = val
+      of "dry-run": dryRun = true
+      of "force", "f": force = true
+      of "quiet", "q": quiet = true
+      of "help", "h":
+        echo HelpText
+        quit(0)
+      else: discard
+    of cmdEnd: discard
+
+  if dryRun and not quiet:
+    echo "DRY-RUN mode: no changes will be made."
+
+  let report = runRepair(dataDir, dryRun, quiet)
+  printReport(report, quiet)
+
+  if report.sstablesCorrupt > 0:
+    quit(1)  # exit with error if corruption was found
+  else:
+    quit(0)

src/baradadb.nim

@@ -7,6 +7,8 @@ import std/threadpool
 import std/locks
 import std/os
 import std/strutils
+import std/tables
+import std/algorithm
 import barabadb/core/server
 import barabadb/core/httpserver
 import barabadb/core/config
@@ -18,6 +20,8 @@ import barabadb/core/raft
 import barabadb/core/gossip
 import barabadb/core/replication
 import barabadb/core/disttxn
+import barabadb/tools/repair
+import barabadb/tools/migrate
 
 type
   CompactionManager* = ref object
@@ -43,7 +47,41 @@ proc compact*(cm: CompactionManager) =
   defer: release(cm.db.lock)
   for level in 0 ..< compaction.MaxLevel:
     if cm.strategy.needsCompaction(level):
-      discard cm.strategy.compact(level)
+      let result = cm.strategy.compact(level)
+      if result.outputTables.len == 0:
+        continue
+
+      # Remove compacted input SSTables from LSMTree
+      var newSSTables: seq[SSTable] = @[]
+      var removedPaths = initTable[string, bool]()
+      for t in result.inputTables:
+        removedPaths[t.path] = true
+      for sst in cm.db.sstables:
+        if sst.path notin removedPaths:
+          newSSTables.add(sst)
+
+      # Load and add output SSTables
+      for meta in result.outputTables:
+        try:
+          var sst = loadSSTable(meta.path)
+          let name = splitFile(meta.path).name
+          # Extract numeric id from filename if possible
+          sst.id = try: parseInt(name) except: cm.db.nextSSTableId
+          sst.level = meta.level
+          newSSTables.add(sst)
+          cm.db.nextSSTableId = max(cm.db.nextSSTableId, sst.id + 1)
+        except CatchableError as e:
+          warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg)
+
+      newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
+      cm.db.sstables = newSSTables
+
+      # Update MANIFEST
+      inc cm.db.manifestSequence
+      try:
+        writeManifest(cm.db)
+      except CatchableError as e:
+        warn("Failed to write MANIFEST after compaction: " & e.msg)
 
 proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.async.} =
   while true:
@@ -82,6 +120,93 @@ proc wireReplicationDistTxn(rm: ReplicationManager, dtm: DistTxnManager) =
   discard
 
 proc main() =
+  # CLI command dispatch (before server startup)
+  if paramCount() >= 1:
+    let cmd = paramStr(1).toLowerAscii()
+    if cmd == "repair":
+      var dataDir = "data/server"
+      var dryRun = false
+      var quiet = false
+      var i = 2
+      while i <= paramCount():
+        let arg = paramStr(i)
+        if arg.startsWith("--data-dir="):
+          dataDir = arg[11..^1]
+        elif arg == "--dry-run":
+          dryRun = true
+        elif arg == "--quiet" or arg == "-q":
+          quiet = true
+        elif arg == "--help" or arg == "-h":
+          echo repair.HelpText
+          return
+        inc i
+      let rep = repair.runRepair(dataDir, dryRun, quiet)
+      repair.printReport(rep, quiet)
+      if rep.sstablesCorrupt > 0:
+        quit(1)
+      else:
+        quit(0)
+
+    elif cmd == "checkpoint":
+      var dataDir = "data/server"
+      var i = 2
+      while i <= paramCount():
+        let arg = paramStr(i)
+        if arg.startsWith("--data-dir="):
+          dataDir = arg[11..^1]
+        elif arg == "--help" or arg == "-h":
+          echo "BaraDB Checkpoint — Create a consistent storage checkpoint"
+          echo ""
+          echo "USAGE:"
+          echo "  checkpoint [options]"
+          echo ""
+          echo "OPTIONS:"
+          echo "  -d, --data-dir <DIR>  Path to data directory (default: data/server)"
+          echo "  -h, --help            Show this help message"
+          return
+        inc i
+      try:
+        var db = newLSMTree(dataDir)
+        db.checkpoint()
+        db.close()
+        var sstCount = 0
+        for kind, path in walkDir(dataDir / "sstables"):
+          if kind == pcFile: inc sstCount
+        echo "Checkpoint created successfully at: ", dataDir
+        echo "  SSTables: ", sstCount
+        echo "  MANIFEST: ", dataDir / "MANIFEST"
+        quit(0)
+      except CatchableError as e:
+        echo "ERROR: Checkpoint failed: ", e.msg
+        quit(1)
+
+    elif cmd == "migrate":
+      var dataDir = "data/server"
+      var dryRun = false
+      var i = 2
+      while i <= paramCount():
+        let arg = paramStr(i)
+        if arg.startsWith("--data-dir="):
+          dataDir = arg[11..^1]
+        elif arg == "--dry-run":
+          dryRun = true
+        elif arg == "--help" or arg == "-h":
+          echo migrate.HelpText
+          return
+        inc i
+      let result = migrate.runMigration(dataDir, dryRun)
+      echo ""
+      echo "Migration complete:"
+      echo "  Scanned:  ", result.scanned
+      echo "  Migrated: ", result.migrated
+      echo "  Errors:   ", result.errors.len
+      if result.errors.len > 0:
+        for e in result.errors:
+          echo "  ! ", e
+        quit(1)
+      else:
+        quit(0)
+
   var config = loadConfig()
   # Init structured logger from config
   let logLvl = parseEnum[LogLevel]("ll" & capitalizeAscii(config.logLevel))