Стабилизация на storage слоя — 6 фази
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
Фаза 1: SSTable Integrity - SSTable v3 формат с CRC32 footer (data/index/bloom CRC) - Нов модул storage/crc32.nim (zero-dep, IEEE 802.3) - verifySSTable() — проверка на magic, version, CRC - loadSSTable strict mode — отхвърля корумпирани файлове - newLSMTree логва [WARN] при corrupt SSTables Фаза 2: baradb repair - Нов модул tools/repair.nim - Сканира SSTables, проверява CRC, премества битите в corrupt/ - WAL replay (CrashRecovery) за възстановяване на данни - CLI: ./baradadb repair --data-dir=... [--dry-run] Фаза 3: MANIFEST File - Atomic write MANIFEST (tmp + rename) - newLSMTree зарежда от MANIFEST, fallback към walkDir - checkStorageConsistency() — orphan/missing detection - flushUnsafe и compaction записват MANIFEST Фаза 4: WAL Rotation & Incremental Backup - WAL segment rotation при 64MB (wal_archive/wal.000NNN.log) - maybeRotate() на всеки 1000 записа + при flush - backup incremental — архивира MANIFEST + SSTables + WAL - CRC verification на SSTables преди архивиране Фаза 5: Online Consistent Backup - checkpoint() — freeze memtable + flush + WAL rotate - ./baradadb checkpoint — offline consistent snapshot - backup --online — checkpoint + incremental backup Фаза 6: SSTable Version Migration - SSTable.fileVersion поле - listLegacySSTables() — намира v1/v2 файлове - migrateSSTable() — пренаписва към v3 (tmp + rename) - ./baradadb migrate [--dry-run] — offline migration Документация: - Обновени docs/en/backup.md, docs/bg/backup.md - Обновени docs/en/storage.md, docs/bg/storage.md - Добавени тестове в tests/test_all.nim
This commit is contained in:
@@ -119,6 +119,84 @@ suite "LSM-Tree Storage":
|
||||
check db.contains(key)
|
||||
db.close()
|
||||
|
||||
suite "SSTable Integrity":
|
||||
test "SSTable v3 CRC verified on write and load":
|
||||
let testDir = "/tmp/baradb_test_lsm_crc"
|
||||
removeDir(testDir)
|
||||
var db = newLSMTree(testDir, 1024)
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
db.put("key2", cast[seq[byte]]("value2"))
|
||||
db.flush()
|
||||
let sstPath = testDir / "sstables" / "1.sst"
|
||||
let (ok, msg) = verifySSTable(sstPath)
|
||||
check ok == true
|
||||
check msg.contains("v3 CRC verified")
|
||||
db.close()
|
||||
# Reopen and verify data survives
|
||||
var db2 = newLSMTree(testDir, 1024)
|
||||
let (found, val) = db2.get("key1")
|
||||
check found
|
||||
check cast[string](val) == "value1"
|
||||
db2.close()
|
||||
|
||||
test "Corrupted SSTable is rejected by verifySSTable and loadSSTable":
|
||||
let testDir = "/tmp/baradb_test_lsm_crc2"
|
||||
removeDir(testDir)
|
||||
var db = newLSMTree(testDir, 1024)
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
db.flush()
|
||||
db.close()
|
||||
let sstPath = testDir / "sstables" / "1.sst"
|
||||
var f = open(sstPath, fmReadWriteExisting)
|
||||
setFilePos(f, 40)
|
||||
var b: byte = 0
|
||||
discard readBuffer(f, addr b, 1)
|
||||
b = b xor 0xFF
|
||||
setFilePos(f, 40)
|
||||
discard writeBuffer(f, addr b, 1)
|
||||
close(f)
|
||||
let (ok, msg) = verifySSTable(sstPath)
|
||||
check ok == false
|
||||
check msg.contains("CRC mismatch")
|
||||
expect ValueError:
|
||||
discard loadSSTable(sstPath)
|
||||
|
||||
suite "MANIFEST Catalog":
|
||||
test "MANIFEST written and loaded on reopen":
|
||||
let testDir = "/tmp/baradb_test_manifest"
|
||||
removeDir(testDir)
|
||||
var db = newLSMTree(testDir, 512)
|
||||
db.put("a", cast[seq[byte]]("val_a"))
|
||||
db.put("b", cast[seq[byte]]("val_b"))
|
||||
db.flush()
|
||||
db.put("c", cast[seq[byte]]("val_c"))
|
||||
db.flush()
|
||||
let manifestPath = testDir / "MANIFEST"
|
||||
check fileExists(manifestPath)
|
||||
db.close()
|
||||
var db2 = newLSMTree(testDir, 512)
|
||||
check db2.sstableCount() == 2
|
||||
let (found, val) = db2.get("a")
|
||||
check found
|
||||
check cast[string](val) == "val_a"
|
||||
db2.close()
|
||||
|
||||
test "Orphan SSTable detected by checkStorageConsistency":
|
||||
let testDir = "/tmp/baradb_test_manifest_orphan"
|
||||
removeDir(testDir)
|
||||
var db = newLSMTree(testDir, 512)
|
||||
db.put("x", cast[seq[byte]]("val_x"))
|
||||
db.flush()
|
||||
# Create orphan file
|
||||
let orphan = testDir / "sstables" / "99.sst"
|
||||
var f = open(orphan, fmWrite)
|
||||
f.write("fake")
|
||||
close(f)
|
||||
let issues = checkStorageConsistency(db)
|
||||
check issues.len == 1
|
||||
check issues[0].contains("Orphan")
|
||||
db.close()
|
||||
|
||||
suite "BaraQL Lexer":
|
||||
test "Tokenize simple SELECT":
|
||||
let tokens = lex.tokenize("SELECT name FROM users WHERE age > 18")
|
||||
|
||||
Reference in New Issue
Block a user