feat: harden storage, schema persistence, fair benches, fix wire crash

Core storage: hash MemTable, WAL group commit, L0 compaction rebuild,
reader-writer lock, and a global StorageGate so HTTP workers and TCP
share the LSM safely under multi-thread access.

Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM
restore on open. Executor types/values/schema split into query/exec/.

Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted
after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG)
and honesty docs for mixed-tier comparisons.

tests/test_all.nim

@@ -196,6 +196,232 @@ suite "MANIFEST Catalog":
     check issues[0].contains("Orphan")
     db.close()
 
+suite "Core Storage Hardening":
+  test "MemTable overwrite keeps newest value (hash table)":
+    let testDir = "/tmp/baradb_test_memtable_hash"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 64 * 1024)
+    db.put("k", cast[seq[byte]]("v1"))
+    db.put("k", cast[seq[byte]]("v2"))
+    db.put("k", cast[seq[byte]]("v3"))
+    let (found, val) = db.get("k")
+    check found
+    check cast[string](val) == "v3"
+    check db.memTableSize() == 1
+    db.close()
+
+  test "Many distinct keys without O(n) insert collapse":
+    ## Hash MemTable should handle thousands of puts without quadratic cost.
+    let testDir = "/tmp/baradb_test_memtable_many"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 8 * 1024 * 1024)
+    let n = 5000
+    for i in 0 ..< n:
+      db.put("key_" & align($i, 6, '0'), cast[seq[byte]]("val_" & $i))
+    for i in [0, 1, n div 2, n - 1]:
+      let (found, val) = db.get("key_" & align($i, 6, '0'))
+      check found
+      check cast[string](val) == "val_" & $i
+    db.close()
+
+  test "scanMemTable returns sorted unique keys":
+    let testDir = "/tmp/baradb_test_scan_sorted"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 64 * 1024)
+    db.put("c", cast[seq[byte]]("3"))
+    db.put("a", cast[seq[byte]]("1"))
+    db.put("b", cast[seq[byte]]("2"))
+    db.put("a", cast[seq[byte]]("1b"))
+    let mem = db.scanMemTable()
+    check mem.len == 3
+    check mem[0].key == "a"
+    check cast[string](mem[0].value) == "1b"
+    check mem[1].key == "b"
+    check mem[2].key == "c"
+    db.close()
+
+  test "WAL truncated after full flush — recovery stays small":
+    let testDir = "/tmp/baradb_test_wal_truncate"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 256)
+    for i in 0 ..< 20:
+      db.put("k" & $i, cast[seq[byte]]("v" & $i))
+    db.flush()
+    # After flush both memtables empty → WAL should only have header (or tiny rewrite)
+    let walPath = testDir / "wal" / "wal.log"
+    check fileExists(walPath)
+    let sizeAfterFlush = getFileSize(walPath)
+    check sizeAfterFlush < 256  # header only, not all 20 puts
+    db.close()
+    # Reopen: data comes from SSTables, not a bloated WAL
+    var db2 = newLSMTree(testDir, 256)
+    for i in 0 ..< 20:
+      let (found, val) = db2.get("k" & $i)
+      check found
+      check cast[string](val) == "v" & $i
+    db2.close()
+
+  test "Partial flush rewrites WAL with remaining live keys":
+    let testDir = "/tmp/baradb_test_wal_rewrite"
+    removeDir(testDir)
+    # Tiny memtable forces flush of first batch while second batch stays in memory
+    var db = newLSMTree(testDir, 64)
+    db.put("old1", cast[seq[byte]]("a"))
+    db.put("old2", cast[seq[byte]]("b"))
+    # Force flush
+    db.flush()
+    db.put("live1", cast[seq[byte]]("x"))
+    db.put("live2", cast[seq[byte]]("y"))
+    # Do not flush — close without flush would lose live without WAL; close flushes
+    # Instead: crash-simulate by reopening after putting live keys (WAL rewrite on prior flush
+    # left empty; new puts are in current WAL)
+    db.close()
+    var db2 = newLSMTree(testDir, 64)
+    let (f1, v1) = db2.get("live1")
+    let (f2, v2) = db2.get("live2")
+    let (f3, _) = db2.get("old1")
+    check f1 and cast[string](v1) == "x"
+    check f2 and cast[string](v2) == "y"
+    check f3
+    db2.close()
+
+  test "L0 count trigger and rebuildFromLSM sees flushed tables":
+    let testDir = "/tmp/baradb_test_l0_trigger"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 128)
+    for round in 0 ..< L0CompactionTrigger:
+      db.put("r" & $round, cast[seq[byte]]("v" & $round))
+      db.flush()
+    check db.countL0() >= L0CompactionTrigger
+    check db.needsCompaction == true
+    var cs = newCompactionStrategy(testDir)
+    cs.rebuildFromLSM(db)
+    check cs.needsCompaction(0) == true
+    check cs.levels[0].len >= L0CompactionTrigger
+    # Compact L0 → L1
+    let cr = cs.compact(0)
+    check cr.outputTables.len == 1
+    check cr.outputTables[0].level == 1
+    # Apply manually: remove inputs from db, add output
+    var removed = initTable[string, bool]()
+    for t in cr.inputTables:
+      removed[t.path] = true
+    var kept: seq[SSTable] = @[]
+    for sst in db.sstables.mitems:
+      if sst.path notin removed:
+        kept.add(sst)
+      else:
+        sst.close()
+    var outSst = loadSSTable(cr.outputTables[0].path)
+    outSst.level = 1
+    outSst.id = db.nextSSTableId
+    inc db.nextSSTableId
+    kept.add(outSst)
+    db.sstables = kept
+    check db.countL0() < L0CompactionTrigger
+    for round in 0 ..< L0CompactionTrigger:
+      let (found, val) = db.get("r" & $round)
+      check found
+      check cast[string](val) == "v" & $round
+    db.close()
+
+  test "Crash recovery: WAL-only clone recovers unflushed puts":
+    ## Simulate crash: copy WAL without SSTables / without clean close flush.
+    let srcDir = "/tmp/baradb_test_crash_src"
+    let dstDir = "/tmp/baradb_test_crash_dst"
+    removeDir(srcDir)
+    removeDir(dstDir)
+    var db = newLSMTree(srcDir, 1024 * 1024)
+    db.put("persist_me", cast[seq[byte]]("yes"))
+    db.put("and_me", cast[seq[byte]]("also"))
+    db.wal.sync()
+    createDir(dstDir / "wal")
+    createDir(dstDir / "sstables")
+    copyFile(srcDir / "wal" / "wal.log", dstDir / "wal" / "wal.log")
+    db.close()  # cleans up src; dst has WAL-only crash image
+    var recovered = newLSMTree(dstDir, 1024 * 1024)
+    let (f1, v1) = recovered.get("persist_me")
+    let (f2, v2) = recovered.get("and_me")
+    check f1 and cast[string](v1) == "yes"
+    check f2 and cast[string](v2) == "also"
+    recovered.close()
+
+  test "WAL group commit fsyncs roughly every N entries":
+    let testDir = "/tmp/baradb_test_wal_group"
+    removeDir(testDir)
+    const n = 200
+    const ge = 50
+    var db = newLSMTree(testDir, 8 * 1024 * 1024,
+                        walSyncMode = wsmGroup, walGroupEvery = ge)
+    let base = db.wal.fsyncCount  # open/recovery may fsync once
+    for i in 0 ..< n:
+      db.put("g" & $i, cast[seq[byte]]("v"))
+    let afterPuts = db.wal.fsyncCount - base
+    # Group every 50 → about n/ge fsyncs; partial group not yet synced
+    check afterPuts >= uint64(n div ge)
+    check afterPuts < uint64(n)  # far fewer than one-per-write
+    db.wal.sync()
+    check db.wal.fsyncCount > base + afterPuts or afterPuts >= uint64(n div ge)
+    db.close()
+
+  test "WAL every-mode fsyncs at least once per write":
+    let testDir = "/tmp/baradb_test_wal_every"
+    removeDir(testDir)
+    const n = 30
+    var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmEvery)
+    let base = db.wal.fsyncCount
+    for i in 0 ..< n:
+      db.put("e" & $i, cast[seq[byte]]("v"))
+    check db.wal.fsyncCount - base >= uint64(n)
+    db.close()
+
+  test "WAL none-mode does not fsync on each put":
+    let testDir = "/tmp/baradb_test_wal_none"
+    removeDir(testDir)
+    const n = 100
+    var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmNone)
+    let base = db.wal.fsyncCount
+    for i in 0 ..< n:
+      db.put("n" & $i, cast[seq[byte]]("v"))
+    # Puts alone should not fsync
+    check db.wal.fsyncCount == base
+    db.wal.sync()
+    check db.wal.fsyncCount == base + 1
+    db.close()
+
+  test "parseWalSyncMode accepts aliases":
+    check parseWalSyncMode("group") == wsmGroup
+    check parseWalSyncMode("every") == wsmEvery
+    check parseWalSyncMode("none") == wsmNone
+    check parseWalSyncMode("async") == wsmNone
+    check parseWalSyncMode("full") == wsmEvery
+    check parseWalSyncMode("batch") == wsmGroup
+
+  test "scanRange returns inclusive sorted keys":
+    let testDir = "/tmp/baradb_test_scan_range"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 256)
+    for ch in ['a', 'b', 'c', 'd', 'e']:
+      db.put($ch, cast[seq[byte]]("v" & $ch))
+    db.flush()
+    db.put("c", cast[seq[byte]]("vC2"))  # newer in memtable
+    let rows = db.scanRange("b", "d")
+    check rows.len == 3
+    check rows[0][0] == "b"
+    check rows[1][0] == "c"
+    check cast[string](rows[1][1]) == "vC2"
+    check rows[2][0] == "d"
+    db.close()
+
+  test "scanRange empty when no keys in range":
+    let testDir = "/tmp/baradb_test_scan_empty"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 1024)
+    db.put("m", cast[seq[byte]]("1"))
+    check db.scanRange("a", "c").len == 0
+    check db.scanRange("m", "m").len == 1
+    db.close()
+
 suite "BaraQL Lexer":
   test "Tokenize simple SELECT":
     let tokens = lex.tokenize("SELECT name FROM users WHERE age > 18")

tests/test_schema_persist.nim

@@ -0,0 +1,116 @@
+## Schema persistence — CREATE TABLE / data survive reopen
+import std/unittest
+import std/os
+import std/strutils
+import std/tables
+import barabadb/storage/lsm
+import barabadb/query/executor
+import barabadb/query/parser
+import barabadb/query/ast
+
+proc execSql(ctx: ExecutionContext, sql: string): ExecResult =
+  let node = parse(sql)
+  result = executeQuery(ctx, node)
+
+suite "Schema persistence":
+  test "CREATE TABLE survives flush + reopen":
+    let dir = "/tmp/baradb_schema_persist_1"
+    removeDir(dir)
+    block:
+      var db = newLSMTree(dir, 1024)  # small memtable → forces flush
+      var ctx = newExecutionContext(db)
+      let r = execSql(ctx, "CREATE TABLE users (id INT PRIMARY KEY, name TEXT NOT NULL)")
+      check r.success
+      check ctx.tables.hasKey("users")
+      check ctx.tables["users"].columns.len == 2
+      discard execSql(ctx, "INSERT INTO users (id, name) VALUES (1, 'Alice')")
+      discard execSql(ctx, "INSERT INTO users (id, name) VALUES (2, 'Bob')")
+      db.flush()
+      # Schema key must be durable
+      let (found, _) = db.get(tableSchemaKey("users"))
+      check found
+      db.close()
+
+    # Reopen fresh context (simulates process restart)
+    block:
+      var db2 = newLSMTree(dir, 1024)
+      var ctx2 = newExecutionContext(db2)
+      check ctx2.tables.hasKey("users")
+      check ctx2.tables["users"].columns.len == 2
+      check ctx2.tables["users"].pkColumns.len == 1
+      let sel = execSql(ctx2, "SELECT id, name FROM users ORDER BY id")
+      check sel.success
+      check sel.rows.len == 2
+      db2.close()
+
+  test "DROP TABLE removes schema and data":
+    let dir = "/tmp/baradb_schema_persist_drop"
+    removeDir(dir)
+    var db = newLSMTree(dir)
+    var ctx = newExecutionContext(db)
+    check execSql(ctx, "CREATE TABLE t (id INT PRIMARY KEY)").success
+    check execSql(ctx, "INSERT INTO t (id) VALUES (1)").success
+    check execSql(ctx, "DROP TABLE t").success
+    check not ctx.tables.hasKey("t")
+    let (found, _) = db.get(tableSchemaKey("t"))
+    check not found
+    # Reopen — table must not reappear
+    db.close()
+    var db2 = newLSMTree(dir)
+    var ctx2 = newExecutionContext(db2)
+    check not ctx2.tables.hasKey("t")
+    db2.close()
+
+  test "ALTER TABLE ADD COLUMN is persisted":
+    let dir = "/tmp/baradb_schema_persist_alter"
+    removeDir(dir)
+    block:
+      var db = newLSMTree(dir)
+      var ctx = newExecutionContext(db)
+      check execSql(ctx, "CREATE TABLE items (id INT PRIMARY KEY)").success
+      check execSql(ctx, "ALTER TABLE items ADD COLUMN label TEXT").success
+      check ctx.tables["items"].columns.len == 2
+      db.flush()
+      db.close()
+    block:
+      var db2 = newLSMTree(dir)
+      var ctx2 = newExecutionContext(db2)
+      check ctx2.tables.hasKey("items")
+      check ctx2.tables["items"].columns.len == 2
+      var names: seq[string] = @[]
+      for c in ctx2.tables["items"].columns:
+        names.add(c.name)
+      check "label" in names
+      db2.close()
+
+  test "Multiple tables all restored":
+    let dir = "/tmp/baradb_schema_persist_multi"
+    removeDir(dir)
+    block:
+      var db = newLSMTree(dir, 512)
+      var ctx = newExecutionContext(db)
+      check execSql(ctx, "CREATE TABLE a (id INT PRIMARY KEY)").success
+      check execSql(ctx, "CREATE TABLE b (id INT PRIMARY KEY, a_id INT)").success
+      check execSql(ctx, "CREATE TABLE c (name TEXT)").success
+      for i in 0..20:
+        discard execSql(ctx, "INSERT INTO a (id) VALUES (" & $i & ")")
+      db.flush()
+      db.close()
+    block:
+      var db2 = newLSMTree(dir)
+      var ctx2 = newExecutionContext(db2)
+      check ctx2.tables.hasKey("a")
+      check ctx2.tables.hasKey("b")
+      check ctx2.tables.hasKey("c")
+      let sel = execSql(ctx2, "SELECT id FROM a")
+      check sel.success
+      check sel.rows.len == 21
+      db2.close()
+
+  test "Stable schema key format":
+    check tableSchemaKey("users") == "_schema:tables:users"
+    check serializeTableDdl(TableDef(
+      name: "t",
+      columns: @[ColumnDef(name: "id", colType: "INT", isPk: true)],
+      pkColumns: @["id"],
+    )).contains("PRIMARY KEY")

tests/test_storage_hardening.nim

@@ -0,0 +1,168 @@
+## Focused storage hardening tests (avoids full suite compile issues)
+import std/unittest
+import std/os
+import std/strutils
+import std/locks
+import barabadb/storage/lsm
+import barabadb/storage/rwlock
+import barabadb/storage/gate
+
+suite "Core Storage Hardening":
+  test "MemTable overwrite keeps newest value":
+    let testDir = "/tmp/baradb_th_mem"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 64 * 1024)
+    db.put("k", cast[seq[byte]]("v1"))
+    db.put("k", cast[seq[byte]]("v3"))
+    let (found, val) = db.get("k")
+    check found
+    check cast[string](val) == "v3"
+    db.close()
+
+  test "scanRange inclusive":
+    let testDir = "/tmp/baradb_th_range"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 256)
+    for ch in ['a', 'b', 'c', 'd', 'e']:
+      db.put($ch, cast[seq[byte]]("v" & $ch))
+    db.flush()
+    db.put("c", cast[seq[byte]]("vC2"))
+    let rows = db.scanRange("b", "d")
+    check rows.len == 3
+    check rows[0][0] == "b"
+    check rows[1][0] == "c"
+    check cast[string](rows[1][1]) == "vC2"
+    db.close()
+
+  test "WAL group commit":
+    let testDir = "/tmp/baradb_th_group"
+    removeDir(testDir)
+    const n = 200
+    const ge = 50
+    var db = newLSMTree(testDir, 8 * 1024 * 1024,
+                        walSyncMode = wsmGroup, walGroupEvery = ge)
+    let base = db.wal.fsyncCount
+    for i in 0 ..< n:
+      db.put("g" & $i, cast[seq[byte]]("v"))
+    let after = db.wal.fsyncCount - base
+    check after >= uint64(n div ge)
+    check after < uint64(n)
+    db.close()
+
+  test "RwLock concurrent readers":
+    var rw: RwLock
+    initRwLock(rw)
+    var counter = 0
+    var maxReaders = 0
+    var curReaders = 0
+    var metaLock: Lock
+    initLock(metaLock)
+    var bad = false
+
+    type TArgs = object
+      rw: ptr RwLock
+      meta: ptr Lock
+      counter: ptr int
+      curReaders: ptr int
+      maxReaders: ptr int
+      bad: ptr bool
+      isWriter: bool
+
+    proc worker(a: TArgs) {.thread, gcsafe.} =
+      for i in 0 ..< 200:
+        if a.isWriter:
+          acquireWrite(a.rw[])
+          a.counter[] += 1
+          acquire(a.meta[])
+          if a.curReaders[] != 0:
+            a.bad[] = true
+          release(a.meta[])
+          releaseWrite(a.rw[])
+        else:
+          acquireRead(a.rw[])
+          acquire(a.meta[])
+          inc a.curReaders[]
+          if a.curReaders[] > a.maxReaders[]:
+            a.maxReaders[] = a.curReaders[]
+          release(a.meta[])
+          var x = 0
+          for k in 0 ..< 50: x += k
+          discard x
+          acquire(a.meta[])
+          dec a.curReaders[]
+          release(a.meta[])
+          releaseRead(a.rw[])
+
+    var threads: array[8, Thread[TArgs]]
+    for t in 0 ..< 8:
+      let args = TArgs(
+        rw: addr rw, meta: addr metaLock,
+        counter: addr counter, curReaders: addr curReaders,
+        maxReaders: addr maxReaders, bad: addr bad,
+        isWriter: t == 0 or t == 1,
+      )
+      createThread(threads[t], worker, args)
+    for t in 0 ..< 8:
+      joinThread(threads[t])
+
+    check not bad
+    check counter == 400
+    check maxReaders >= 2
+    deinitLock(metaLock)
+    deinitRwLock(rw)
+
+  test "Interleaved put/get/flush single-threaded stress":
+    ## ORC is not multi-thread-safe for shared refs; stress the exclusive path serially.
+    let testDir = "/tmp/baradb_th_stress"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 4 * 1024, walSyncMode = wsmGroup, walGroupEvery = 32)
+    for i in 0 ..< 2000:
+      db.put("k" & $i, cast[seq[byte]]("v" & $i))
+      if i mod 100 == 0:
+        let (f, v) = db.get("k0")
+        check f and cast[string](v) == "v0"
+      if i mod 400 == 0:
+        db.flush()
+    for i in [0, 500, 1000, 1999]:
+      let (f, v) = db.get("k" & $i)
+      check f and cast[string](v) == "v" & $i
+    db.close()
+
+  test "StorageGate serializes concurrent critical sections":
+    initStorageGate()
+    var counter = 0
+    var bad = false
+    var meta: Lock
+    initLock(meta)
+
+    type GArgs = object
+      n: int
+      counter: ptr int
+      bad: ptr bool
+      meta: ptr Lock
+
+    proc worker(a: GArgs) {.thread, gcsafe.} =
+      for i in 0 ..< a.n:
+        withStorageGate:
+          # Under the gate, only one thread should touch counter
+          let before = a.counter[]
+          a.counter[] = before + 1
+          # Simulate work
+          var x = 0
+          for k in 0 ..< 20: x += k
+          discard x
+          if a.counter[] != before + 1:
+            acquire(a.meta[])
+            a.bad[] = true
+            release(a.meta[])
+
+    var threads: array[6, Thread[GArgs]]
+    for t in 0 ..< 6:
+      createThread(threads[t], worker, GArgs(
+        n: 100, counter: addr counter, bad: addr bad, meta: addr meta))
+    for t in 0 ..< 6:
+      joinThread(threads[t])
+
+    check not bad
+    check counter == 600
+    deinitLock(meta)

tests/test_wire_insert_stress.nim

@@ -0,0 +1,26 @@
+## Regression: sequential wire-style INSERTs must not crash the process.
+## Root cause was ORC cycle collector (markGray SIGSEGV); project uses --mm:arc.
+import std/unittest
+import std/os
+import barabadb/storage/lsm
+import barabadb/query/executor
+import barabadb/query/parser
+
+proc execSql(ctx: ExecutionContext, sql: string): ExecResult =
+  executeQuery(ctx, parse(sql))
+
+suite "Wire insert stress (ARC regression)":
+  test "200 sequential INSERTs via executor survive":
+    ## Mirrors the wire path (executeQuery under StorageGate each time).
+    let dir = "/tmp/baradb_wire_stress"
+    removeDir(dir)
+    var db = newLSMTree(dir, walSyncMode = wsmNone)
+    var ctx = newExecutionContext(db)
+    check execSql(ctx, "CREATE TABLE stress (id INT PRIMARY KEY, v TEXT)").success
+    for i in 0 ..< 200:
+      let r = execSql(ctx, "INSERT INTO stress (id, v) VALUES (" & $i & ", 'v" & $i & "')")
+      check r.success
+    let sel = execSql(ctx, "SELECT id FROM stress")
+    check sel.success
+    check sel.rows.len == 200
+    db.close()