feat: harden storage, schema persistence, fair benches, fix wire crash

Core storage: hash MemTable, WAL group commit, L0 compaction rebuild, reader-writer lock, and a global StorageGate so HTTP workers and TCP share the LSM safely under multi-thread access. Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM restore on open. Executor types/values/schema split into query/exec/. Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG) and honesty docs for mixed-tier comparisons.
2026-07-18 16:55:50 +03:00
parent aa4ab11210
commit 8db5cfe7e1
31 changed files with 3131 additions and 797 deletions
@@ -22,6 +22,11 @@ type
    logFormat*: string
    memtableSizeMb*: int
    cacheSizeMb*: int
+    ## WAL durability: "none" | "group" (default) | "every"
+    walSyncMode*: string
+    ## Group commit batch size (entries between fsyncs when mode=group)
+    walGroupEvery*: int
+    ## Time-based group fsync interval in ms (0 = off); also used as legacy name
    walSyncIntervalMs*: int
    compactionIntervalMs*: int
    bloomBitsPerKey*: int
@@ -58,6 +63,8 @@ proc defaultConfig*(): BaraConfig =
    logFormat: "json",
    memtableSizeMb: 64,
    cacheSizeMb: 256,
+    walSyncMode: "group",
+    walGroupEvery: 64,
    walSyncIntervalMs: 0,
    compactionIntervalMs: 60_000,
    bloomBitsPerKey: 10,
@@ -93,6 +100,8 @@ proc loadConfigFromJson*(path: string, cfg: var BaraConfig) =
      if s.hasKey("data_dir"): cfg.dataDir = s["data_dir"].getStr()
      if s.hasKey("memtable_size_mb"): cfg.memtableSizeMb = s["memtable_size_mb"].getInt()
      if s.hasKey("cache_size_mb"): cfg.cacheSizeMb = s["cache_size_mb"].getInt()
+      if s.hasKey("wal_sync_mode"): cfg.walSyncMode = s["wal_sync_mode"].getStr()
+      if s.hasKey("wal_group_every"): cfg.walGroupEvery = s["wal_group_every"].getInt()
      if s.hasKey("wal_sync_interval_ms"): cfg.walSyncIntervalMs = s["wal_sync_interval_ms"].getInt()
      if s.hasKey("compaction_interval_ms"): cfg.compactionIntervalMs = s["compaction_interval_ms"].getInt()
      if s.hasKey("bloom_bits_per_key"): cfg.bloomBitsPerKey = s["bloom_bits_per_key"].getInt()
@@ -153,6 +162,8 @@ proc loadConfigFromEnv*(cfg: var BaraConfig) =
  cfg.logFormat = getEnv("BARADB_LOG_FORMAT", cfg.logFormat)
  cfg.memtableSizeMb = parseEnvInt(getEnv("BARADB_MEMTABLE_SIZE_MB", ""), cfg.memtableSizeMb)
  cfg.cacheSizeMb = parseEnvInt(getEnv("BARADB_CACHE_SIZE_MB", ""), cfg.cacheSizeMb)
+  cfg.walSyncMode = getEnv("BARADB_WAL_SYNC_MODE", cfg.walSyncMode)
+  cfg.walGroupEvery = parseEnvInt(getEnv("BARADB_WAL_GROUP_EVERY", ""), cfg.walGroupEvery)
  cfg.walSyncIntervalMs = parseEnvInt(getEnv("BARADB_WAL_SYNC_INTERVAL_MS", ""), cfg.walSyncIntervalMs)
  cfg.compactionIntervalMs = parseEnvInt(getEnv("BARADB_COMPACTION_INTERVAL_MS", ""), cfg.compactionIntervalMs)
  cfg.bloomBitsPerKey = parseEnvInt(getEnv("BARADB_BLOOM_BITS_PER_KEY", ""), cfg.bloomBitsPerKey)
@@ -15,6 +15,7 @@ import ../query/parser
 import ../query/executor
 import ../core/types
 import ../storage/lsm
+import ../storage/gate
 import ../core/mvcc
 import ../protocol/wire
 import ../core/websocket
@@ -196,17 +197,7 @@ proc queryHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Empty query"}, 400)
        return

-      var reqCtx = getRequestDatabaseContext(server, request)
-      reqCtx.currentUser = userId
-      reqCtx.currentRole = role
-      let tokens = tokenize(queryStr)
-      let astNode = parse(tokens)
-
-      if astNode.stmts.len == 0:
-        ctx.json(%*{"rows": [], "affectedRows": 0, "columns": []})
-        return
-
-      # Extract optional params from JSON body
+      # Extract optional params from JSON body (no storage access yet)
      var params: seq[WireValue] = @[]
      if "params" in body and body["params"].kind == JArray:
        for p in body["params"]:
@@ -218,31 +209,50 @@ proc queryHandler(server: HttpServer): RequestHandler =
          of JString: params.add(WireValue(kind: fkString, strVal: p.getStr()))
          else: params.add(WireValue(kind: fkString, strVal: $p))

-      let res = executor.executeQuery(reqCtx, astNode, params)
+      # StorageGate: serialize against TCP + other Hunos workers (ORC safety)
+      var success: bool
+      var jsonRows = newJArray()
+      var jsonCols = newJArray()
+      var affected = 0
+      var msg = ""
+      var errMsg = ""
+      withStorageGate:
+        var reqCtx = getRequestDatabaseContext(server, request)
+        reqCtx.currentUser = userId
+        reqCtx.currentRole = role
+        let tokens = tokenize(queryStr)
+        let astNode = parse(tokens)
+        if astNode.stmts.len == 0:
+          success = true
+        else:
+          let res = executor.executeQuery(reqCtx, astNode, params)
+          success = res.success
+          if res.success:
+            affected = res.affectedRows
+            msg = res.message
+            for row in res.rows:
+              var jsonRow = newJObject()
+              for col in res.columns:
+                if col in row and row[col].kind != vkNull:
+                  jsonRow[col] = %valueToString(row[col])
+                else:
+                  jsonRow[col] = newJNull()
+              jsonRows.add(jsonRow)
+            for c in res.columns:
+              jsonCols.add(%c)
+          else:
+            errMsg = res.message

-      if res.success:
-        var jsonRows = newJArray()
-        for row in res.rows:
-          var jsonRow = newJObject()
-          for col in res.columns:
-            let key = col
-            if key in row and row[key].kind != vkNull:
-              jsonRow[key] = %valueToString(row[key])
-            else:
-              jsonRow[key] = newJNull()
-          jsonRows.add(jsonRow)
-        var jsonCols = newJArray()
-        for c in res.columns:
-          jsonCols.add(%c)
+      if success:
        ctx.json(%*{
          "rows": jsonRows,
-          "affectedRows": res.affectedRows,
+          "affectedRows": affected,
          "columns": jsonCols,
-          "message": if res.message.len > 0: %res.message else: newJNull()
+          "message": if msg.len > 0: %msg else: newJNull()
        })
      else:
        server.metrics.queryErrors += 1
-        ctx.json(%*{"error": res.message}, 400)
+        ctx.json(%*{"error": errMsg}, 400)

 proc healthHandler(): RequestHandler =
  return proc(request: Request) {.gcsafe.} =
@@ -376,15 +386,16 @@ proc tablesHandler(server: HttpServer): RequestHandler =
      let ctx = newContext(request)
      if not server.checkAuth(request, ctx):
        return
-      let reqCtx = getRequestDatabaseContext(server, request)
      var tables = newJArray()
-      for name, tbl in reqCtx.tables:
-        var cols = newJArray()
-        for col in tbl.columns:
-          cols.add(%*{"name": col.name, "type": col.colType,
-            "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique})
-        tables.add(%*{"name": name, "columns": cols,
-          "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
+      withStorageGate:
+        let reqCtx = getRequestDatabaseContext(server, request)
+        for name, tbl in reqCtx.tables:
+          var cols = newJArray()
+          for col in tbl.columns:
+            cols.add(%*{"name": col.name, "type": col.colType,
+              "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique})
+          tables.add(%*{"name": name, "columns": cols,
+            "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
      ctx.json(%*{"tables": tables})

 proc databasesHandler(server: HttpServer): RequestHandler =
@@ -393,24 +404,25 @@ proc databasesHandler(server: HttpServer): RequestHandler =
      let ctx = newContext(request)
      if not server.checkAuth(request, ctx):
        return
-      let dbs = server.registry.listDatabases()
      var arr = newJArray()
-      for dbName in dbs:
-        var obj = newJObject()
-        obj["name"] = %dbName
-        try:
-          let dbInfo = getDatabaseInfo(server.registry, dbName)
-          if dbInfo != nil and dbInfo.ctx != nil:
-            let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx))
-            obj["tables"] = %dbCtx.tables.len
-            obj["connections"] = %getConnectionCount(server.registry, dbName)
-          else:
+      withStorageGate:
+        let dbs = server.registry.listDatabases()
+        for dbName in dbs:
+          var obj = newJObject()
+          obj["name"] = %dbName
+          try:
+            let dbInfo = getDatabaseInfo(server.registry, dbName)
+            if dbInfo != nil and dbInfo.ctx != nil:
+              let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx))
+              obj["tables"] = %dbCtx.tables.len
+              obj["connections"] = %getConnectionCount(server.registry, dbName)
+            else:
+              obj["tables"] = %0
+              obj["connections"] = %0
+          except CatchableError:
            obj["tables"] = %0
            obj["connections"] = %0
-        except CatchableError:
-          obj["tables"] = %0
-          obj["connections"] = %0
-        arr.add(obj)
+          arr.add(obj)
      ctx.json(%*{"databases": arr})

 proc createDatabaseHandler(server: HttpServer): RequestHandler =
@@ -428,7 +440,8 @@ proc createDatabaseHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Empty database name"}, 400)
        return
      try:
-        discard getOrCreateDatabase(server.registry, dbName)
+        withStorageGate:
+          discard getOrCreateDatabase(server.registry, dbName)
        ctx.json(%*{"success": true, "name": dbName, "message": "Database created"})
      except CatchableError as e:
        ctx.json(%*{"error": e.msg}, 400)
@@ -444,7 +457,9 @@ proc dropDatabaseHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Missing database name"}, 400)
        return
      try:
-        let ok = dropDatabase(server.registry, dbName)
+        var ok = false
+        withStorageGate:
+          ok = dropDatabase(server.registry, dbName)
        if ok:
          ctx.json(%*{"success": true, "name": dbName, "message": "Database dropped"})
        else:
@@ -470,13 +485,15 @@ proc backupHandler(server: HttpServer): RequestHandler =
      let compression = if body != nil and "level" in body: body["level"].getInt() else: 6
      try:
        var ok = false
-        if allDatabases:
-          ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
-        elif dbName.len > 0:
-          let dbDir = dataRoot / dbName
-          ok = backupDataDir(dbDir, outputFile, @[], compression, false)
-        else:
-          ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
+        # Gate held so live writers/compactors don't mutate files mid-backup
+        withStorageGate:
+          if allDatabases:
+            ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
+          elif dbName.len > 0:
+            let dbDir = dataRoot / dbName
+            ok = backupDataDir(dbDir, outputFile, @[], compression, false)
+          else:
+            ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
        if ok:
          ctx.json(%*{"success": true, "output": outputFile, "message": "Backup created"})
        else:
@@ -541,18 +558,20 @@ proc restoreHandler(server: HttpServer): RequestHandler =
        let meta = readBackupMeta(inputFile)
        let isMultiDb = meta != nil and meta{"databases"} != nil
        var ok = false
-        if isMultiDb or allDatabases:
-          ok = restoreAllDatabases(inputFile, dataRoot, false, false)
-        elif dbName.len > 0:
-          let dbDir = dataRoot / dbName
-          ok = restoreDataDir(inputFile, dbDir, false, false)
-        else:
-          ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+        withStorageGate:
+          if isMultiDb or allDatabases:
+            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+          elif dbName.len > 0:
+            let dbDir = dataRoot / dbName
+            ok = restoreDataDir(inputFile, dbDir, false, false)
+          else:
+            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+          if ok:
+            # Reload under same gate after files are restored
+            server.registry.loadExistingDatabases()

        logRestore(inputFile, dataRoot, ok)
        if ok:
-          # Reload databases after restore
-          server.registry.loadExistingDatabases()
          ctx.json(%*{"success": true, "message": "Restore completed"})
        else:
          ctx.json(%*{"error": "Restore failed"}, 500)
@@ -890,10 +909,14 @@ proc run*(server: HttpServer, port: int = 9470) =
  asyncCheck server.ws.run(port + 1)
  hunosServer.serve(Port(port))

-proc stop*(server: HttpServer) =
+proc stop*(server: HttpServer, closeStorage: bool = false) =
+  ## Stop HTTP listeners. By default does **not** close the shared registry —
+  ## when HTTP is spawned alongside TCP they share one registry owned by main.
  server.running = false
  server.ws.stop()
-  if server.registry != nil:
-    server.registry.closeAll()
-  else:
-    server.db.close()
+  if closeStorage:
+    withStorageGate:
+      if server.registry != nil:
+        server.registry.closeAll()
+      elif server.db != nil:
+        server.db.close()
@@ -29,6 +29,17 @@ type

 const reservedDbNames* = ["system", "information_schema", "pg_catalog"]

+proc openLsmForRegistry(reg: DatabaseRegistry, dbDir: string): LSMTree =
+  ## Open LSM with WAL durability settings from registry config.
+  let memBytes = max(1, reg.config.memtableSizeMb) * 1024 * 1024
+  newLSMTree(
+    dbDir,
+    memMaxSize = memBytes,
+    walSyncMode = parseWalSyncMode(reg.config.walSyncMode),
+    walGroupEvery = reg.config.walGroupEvery,
+    walGroupIntervalMs = reg.config.walSyncIntervalMs,
+  )
+
 proc isValidDbName*(name: string): bool =
  if name.len == 0: return false
  if '/' in name or '\\' in name: return false
@@ -63,7 +74,7 @@ proc loadExistingDatabases*(reg: DatabaseRegistry) =
      if dbName.len > 0 and isValidDbName(dbName):
        let dbDir = reg.dataRoot / dbName
        info("Loading database '" & dbName & "' from " & dbDir)
-        let db = newLSMTree(dbDir)
+        let db = openLsmForRegistry(reg, dbDir)
        let ctx = reg.ctxFactory(db, reg)
        acquire(reg.lock)
        reg.databases[dbName] = DatabaseInfo(
@@ -89,7 +100,7 @@ proc ensureDefaultDatabase*(reg: DatabaseRegistry) =
  if not exists:
    let dbDir = reg.dataRoot / defaultDbName
    info("Creating default database at " & dbDir)
-    let db = newLSMTree(dbDir)
+    let db = openLsmForRegistry(reg, dbDir)
    let ctx = reg.ctxFactory(db, reg)
    acquire(reg.lock)
    reg.databases[defaultDbName] = DatabaseInfo(
@@ -113,7 +124,7 @@ proc getOrCreateDatabase*(reg: DatabaseRegistry, name: string): DatabaseInfo =
  # Create new database
  let dbDir = reg.dataRoot / name
  info("Creating database '" & name & "' at " & dbDir)
-  let db = newLSMTree(dbDir)
+  let db = openLsmForRegistry(reg, dbDir)
  let ctx = reg.ctxFactory(db, reg)
  let info = DatabaseInfo(name: name, db: db, ctx: ctx, activeConnections: 0)
  reg.databases[name] = info
@@ -21,6 +21,7 @@ import ../query/parser
 import ../query/ast
 import ../query/executor
 import ../storage/lsm
+import ../storage/gate
 import ../core/mvcc
 import ../core/disttxn
 import ../core/replication
@@ -206,61 +207,64 @@ proc valueToWire(val: string, colType: string): WireValue =

 proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq[WireValue] = @[],
                   replication: ReplicationManager = nil): (bool, QueryResult, string) =
-  try:
-    let tokens = tokenize(query)
-    let astNode = parse(tokens)
+  ## All storage access is under the global StorageGate so HTTP worker threads
+  ## and the TCP event loop never touch ORC-managed LSM/executor state concurrently.
+  withStorageGate:
+    try:
+      let tokens = tokenize(query)
+      let astNode = parse(tokens)

-    if astNode.stmts.len == 0:
-      return (true, QueryResult(), "")
+      if astNode.stmts.len == 0:
+        return (true, QueryResult(), "")

-    let res = executor.executeQuery(ctx, astNode, params)
-    if res.success:
-      # Ship written key-value pairs to replicas
-      if replication != nil and res.keyValuePairs.len > 0:
-        for (key, value) in res.keyValuePairs:
-          var data = newSeq[byte](key.len + 1 + value.len)
-          for i, c in key: data[i] = byte(c)
-          data[key.len] = byte(0)
-          for i, c in value: data[key.len + 1 + i] = c
-          discard replication.writeLsn(data)
-      var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len)
-      qr.columns = res.columns
+      let res = executor.executeQuery(ctx, astNode, params)
+      if res.success:
+        # Ship written key-value pairs to replicas
+        if replication != nil and res.keyValuePairs.len > 0:
+          for (key, value) in res.keyValuePairs:
+            var data = newSeq[byte](key.len + 1 + value.len)
+            for i, c in key: data[i] = byte(c)
+            data[key.len] = byte(0)
+            for i, c in value: data[key.len + 1 + i] = c
+            discard replication.writeLsn(data)
+        var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len)
+        qr.columns = res.columns

-      var colTypes: seq[string] = @[]
-      var tableName = ""
-      if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil:
-        tableName = astNode.stmts[0].selFrom.fromTable
-      elif astNode.stmts[0].kind == nkInsert:
-        tableName = astNode.stmts[0].insTarget
-      elif astNode.stmts[0].kind == nkUpdate:
-        tableName = astNode.stmts[0].updTarget
+        var colTypes: seq[string] = @[]
+        var tableName = ""
+        if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil:
+          tableName = astNode.stmts[0].selFrom.fromTable
+        elif astNode.stmts[0].kind == nkInsert:
+          tableName = astNode.stmts[0].insTarget
+        elif astNode.stmts[0].kind == nkUpdate:
+          tableName = astNode.stmts[0].updTarget

-      if tableName.len > 0 and tableName in ctx.tables:
-        let tbl = ctx.tables[tableName]
-        for col in res.columns:
-          var found = ""
-          for c in tbl.columns:
-            if c.name.toLower() == col.toLower():
-              found = c.colType
-              break
-          colTypes.add(found)
+        if tableName.len > 0 and tableName in ctx.tables:
+          let tbl = ctx.tables[tableName]
+          for col in res.columns:
+            var found = ""
+            for c in tbl.columns:
+              if c.name.toLower() == col.toLower():
+                found = c.colType
+                break
+            colTypes.add(found)
+        else:
+          colTypes = newSeq[string](res.columns.len)
+
+        qr.columnTypes = colTypes.mapIt(typeToFieldKind(it))
+        qr.rows = @[]
+        for row in res.rows:
+          var wireRow: seq[WireValue] = @[]
+          for i, col in res.columns:
+            let val = if col in row: valueToString(row[col]) else: "\\N"
+            let cType = if i < colTypes.len: colTypes[i] else: ""
+            wireRow.add(valueToWire(val, cType))
+          qr.rows.add(wireRow)
+        return (true, qr, res.message)
      else:
-        colTypes = newSeq[string](res.columns.len)
-
-      qr.columnTypes = colTypes.mapIt(typeToFieldKind(it))
-      qr.rows = @[]
-      for row in res.rows:
-        var wireRow: seq[WireValue] = @[]
-        for i, col in res.columns:
-          let val = if col in row: valueToString(row[col]) else: "\\N"
-          let cType = if i < colTypes.len: colTypes[i] else: ""
-          wireRow.add(valueToWire(val, cType))
-        qr.rows.add(wireRow)
-      return (true, qr, res.message)
-    else:
-      return (false, QueryResult(), res.message)
-  except Exception as e:
-    return (false, QueryResult(), e.msg)
+        return (false, QueryResult(), res.message)
+    except Exception as e:
+      return (false, QueryResult(), e.msg)

 # ----------------------------------------------------------------------
 # Response Serialization
@@ -0,0 +1,29 @@
+# Executor package (`query/exec/`)
+
+The original `executor.nim` was a ~5.8k-line god object. Shared pieces live here;
+`../executor.nim` remains the main execution engine and **re-exports** this package
+so existing `import barabadb/query/executor` keeps working.
+
+## Modules
+
+| Module | Responsibility |
+|--------|----------------|
+| `types.nim` | `ExecutionContext`, `TableDef`, `Row`, `ExecResult`, … |
+| `values.nim` | Null/string conversion, row payload parse/escape, SQL escapes |
+| `schema.nim` | Durable catalog (`_schema:tables:*`), restore, index rebuild |
+
+## Import rules
+
+- **No cycles:** `types` → nothing in `exec/`; `values` → `types`; `schema` → `types` + `values`.
+- `executor.nim` imports all three and `export`s them.
+- Prefer adding new shared helpers under `exec/` instead of growing `executor.nim`.
+
+## Sensible next extractions (not done yet)
+
+1. `dml.nim` — `execScan` / `execInsert` / `execUpdate` / `execDelete` (needs eval/triggers hooks)
+2. `rls.nim` — row-level security + privileges
+3. `lower.nim` — AST → IR (`lowerExpr` / `lowerSelect`)
+4. `plan_exec.nim` — IR plan walker / window functions
+5. `hybrid.nim` — hybrid vector+FTS search helpers
+
+Keep statement dispatch (`executeQueryImpl`) in `executor.nim` until those land.
@@ -0,0 +1,231 @@
+## Schema catalog persistence — CREATE/DROP/ALTER survive restart
+import std/strutils
+import std/tables
+import std/sequtils
+import ../ast
+import ../lexer as qlex
+import ../parser as qpar
+import ../../storage/lsm
+import ../../storage/btree
+import types
+import values
+
+const
+  SchemaTablePrefix* = "_schema:tables:"
+  SchemaViewPrefix* = "_schema:views:"
+  SchemaTriggerPrefix* = "_schema:triggers:"
+  SchemaUserPrefix* = "_schema:users:"
+  SchemaPolicyPrefix* = "_schema:policies:"
+  ## Legacy CREATE TABLE keys (pre-fix) used a migrations: counter suffix
+  SchemaLegacyCreatePrefix* = "_schema:migrations:"
+
+proc tableSchemaKey*(tableName: string): string =
+  SchemaTablePrefix & tableName
+
+proc litToString(node: Node): string =
+  ## Evaluate simple literal defaults for schema materialization (no full expr engine).
+  if node == nil: return ""
+  case node.kind
+  of nkStringLit: return node.strVal
+  of nkIntLit: return $node.intVal
+  of nkFloatLit: return $node.floatVal
+  of nkBoolLit: return $node.boolVal
+  of nkNullLit: return "\\N"
+  else: return ""
+
+proc serializeTableDdl*(tbl: TableDef): string =
+  ## Stable DDL for a table definition (survives restart via LSM).
+  var colDefs: seq[string] = @[]
+  let multiPk = tbl.pkColumns.len > 1
+  for col in tbl.columns:
+    var parts: seq[string] = @[col.name, col.colType]
+    if col.isPk and not multiPk:
+      parts.add("PRIMARY KEY")
+    if col.autoIncrement:
+      parts.add("AUTO_INCREMENT")
+    if col.isNotNull:
+      parts.add("NOT NULL")
+    if col.isUnique and not col.isPk:
+      parts.add("UNIQUE")
+    if col.defaultVal.len > 0:
+      parts.add("DEFAULT '" & sqlEscapeString(col.defaultVal) & "'")
+    if col.fkTable.len > 0:
+      parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")")
+      if col.fkOnDelete.len > 0:
+        parts.add("ON DELETE " & col.fkOnDelete)
+      if col.fkOnUpdate.len > 0:
+        parts.add("ON UPDATE " & col.fkOnUpdate)
+    colDefs.add(parts.join(" "))
+  if multiPk:
+    colDefs.add("PRIMARY KEY (" & tbl.pkColumns.join(", ") & ")")
+  result = "CREATE TABLE " & tbl.name & " (" & colDefs.join(", ") & ")"
+
+proc persistTableSchema*(ctx: ExecutionContext, tbl: TableDef) =
+  ## Write table DDL under a stable key so restore finds it after flush/restart.
+  let ddl = serializeTableDdl(tbl)
+  ctx.db.put(tableSchemaKey(tbl.name), cast[seq[byte]](ddl))
+
+proc dropTableSchema*(ctx: ExecutionContext, tableName: string) =
+  ctx.db.delete(tableSchemaKey(tableName))
+
+proc applyCreateTableStmt*(ctx: ExecutionContext, stmt: Node) =
+  ## Materialize CREATE TABLE AST into ctx.tables + empty secondary indexes.
+  var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[],
+                     foreignKeys: @[], checks: @[], triggers: @[])
+  for col in stmt.crtColumns:
+    if col.kind == nkColumnDef:
+      var colDef = ColumnDef(name: col.cdName, colType: col.cdType)
+      colDef.autoIncrement = col.cdAutoIncrement
+      for cst in col.cdConstraints:
+        if cst.kind == nkConstraintDef:
+          case cst.cstType
+          of "pkey":
+            colDef.isPk = true
+            if col.cdName notin tbl.pkColumns:
+              tbl.pkColumns.add(col.cdName)
+            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
+          of "notnull": colDef.isNotNull = true
+          of "unique":
+            colDef.isUnique = true
+            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
+          of "default":
+            if cst.cstDefault != nil:
+              colDef.defaultVal = litToString(cst.cstDefault)
+          of "fkey":
+            colDef.fkTable = cst.cstRefTable
+            colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: ""
+            colDef.fkOnDelete = cst.cstOnDelete
+            colDef.fkOnUpdate = cst.cstOnUpdate
+          else: discard
+      tbl.columns.add(colDef)
+  # Table-level constraints
+  for cstNode in stmt.crtConstraints:
+    if cstNode.kind == nkConstraintDef:
+      if cstNode.cstType == "pkey":
+        for c in cstNode.cstColumns:
+          if c notin tbl.pkColumns:
+            tbl.pkColumns.add(c)
+          for i, col in tbl.columns:
+            if col.name == c:
+              tbl.columns[i].isPk = true
+              let idxName = stmt.crtName & "." & c
+              if idxName notin ctx.btrees:
+                ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
+      elif cstNode.cstType == "fkey":
+        tbl.foreignKeys.add(ForeignKeyDef(
+          refTable: cstNode.cstRefTable,
+          refColumn: if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: "",
+          onDelete: cstNode.cstOnDelete,
+          onUpdate: cstNode.cstOnUpdate))
+        if cstNode.cstColumns.len > 0:
+          for i, c in tbl.columns:
+            if c.name in cstNode.cstColumns:
+              tbl.columns[i].fkTable = cstNode.cstRefTable
+              tbl.columns[i].fkColumn = if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: ""
+              tbl.columns[i].fkOnDelete = cstNode.cstOnDelete
+              tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate
+      elif cstNode.cstType == "check":
+        tbl.checks.add(CheckDef(name: "check_" & $tbl.checks.len, checkNode: cstNode.cstCheck))
+  ctx.tables[stmt.crtName] = tbl
+
+proc rebuildSecondaryIndexes*(ctx: ExecutionContext) =
+  ## Rebuild in-memory B-Tree indexes from durable row data after schema restore.
+  for tableName, tbl in ctx.tables.pairs:
+    for col in tbl.columns:
+      if col.isPk or col.isUnique:
+        let idxName = tableName & "." & col.name
+        if idxName notin ctx.btrees:
+          ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
+    let prefix = tableName & "."
+    for (key, value) in ctx.db.scanAll():
+      if not key.startsWith(prefix): continue
+      if key.startsWith("_schema:"): continue
+      let valStr = cast[string](value)
+      let rest = key[prefix.len..^1]
+      var colVals = initTable[string, string]()
+      let eqPos = rest.find('=')
+      if eqPos >= 0 and ':' notin rest:
+        colVals[rest[0..<eqPos]] = rest[eqPos+1..^1]
+      else:
+        for part in rest.split(':'):
+          let p = part.find('=')
+          if p >= 0:
+            colVals[part[0..<p]] = part[p+1..^1]
+      for k, v in parseRowData(valStr):
+        colVals[k] = v
+      for colName in ctx.btrees.keys.toSeq():
+        if not colName.startsWith(prefix): continue
+        let colsPart = colName[tableName.len + 1..^1]
+        let idxCols = colsPart.split(".")
+        var parts: seq[string] = @[]
+        for c in idxCols:
+          parts.add(colVals.getOrDefault(c, ""))
+        let idxVal = parts.join("|")
+        if idxVal.len > 0 and not isNull(idxVal):
+          ctx.btrees[colName].insert(idxVal, IndexEntry(lsmKey: key, rowValue: valStr))
+
+proc restoreSchema*(ctx: ExecutionContext) =
+  ## Load durable schema from LSM (memtable + SSTables). Stable keys only.
+  var tableDdls: seq[string] = @[]
+  var otherDdls: seq[string] = @[]
+
+  for (key, value) in ctx.db.scanAll():
+    if not key.startsWith("_schema:"): continue
+    let ddl = cast[string](value)
+    if ddl.len == 0: continue
+    if key.startsWith(SchemaTablePrefix):
+      tableDdls.add(ddl)
+    elif key.startsWith(SchemaLegacyCreatePrefix) and ddl.toUpperAscii().startsWith("CREATE TABLE"):
+      tableDdls.add(ddl)
+    elif key.startsWith(SchemaViewPrefix) or key.startsWith(SchemaTriggerPrefix) or
+         key.startsWith(SchemaUserPrefix) or key.startsWith(SchemaPolicyPrefix):
+      otherDdls.add(ddl)
+    elif ddl.toUpperAscii().startsWith("CREATE VIEW") or
+         ddl.toUpperAscii().startsWith("CREATE TRIGGER") or
+         ddl.toUpperAscii().startsWith("CREATE USER") or
+         ddl.toUpperAscii().startsWith("CREATE POLICY"):
+      otherDdls.add(ddl)
+
+  for ddl in tableDdls:
+    try:
+      let tokens = qlex.tokenize(ddl)
+      let astNode = qpar.parse(tokens)
+      if astNode.stmts.len > 0 and astNode.stmts[0].kind == nkCreateTable:
+        applyCreateTableStmt(ctx, astNode.stmts[0])
+        if astNode.stmts[0].crtName in ctx.tables:
+          persistTableSchema(ctx, ctx.tables[astNode.stmts[0].crtName])
+    except CatchableError:
+      continue
+
+  for ddl in otherDdls:
+    var astNode: Node
+    try:
+      let tokens = qlex.tokenize(ddl)
+      astNode = qpar.parse(tokens)
+    except CatchableError:
+      continue
+    if astNode.stmts.len == 0: continue
+    let stmt = astNode.stmts[0]
+    case stmt.kind
+    of nkCreateView:
+      ctx.views[stmt.cvName] = stmt.cvQuery
+    of nkCreateTrigger:
+      if stmt.trigTable in ctx.tables:
+        ctx.tables[stmt.trigTable].triggers.add(TriggerDef(
+          name: stmt.trigName,
+          timing: stmt.trigTiming,
+          event: stmt.trigEvent,
+          action: stmt.trigAction,
+        ))
+    of nkCreateUser:
+      ctx.users[stmt.cuName] = UserDef(name: stmt.cuName,
+          passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[])
+    of nkCreatePolicy:
+      var pols = ctx.policies.getOrDefault(stmt.cpTable)
+      pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable,
+                         command: stmt.cpCommand, usingExpr: stmt.cpUsing,
+                         withCheckExpr: stmt.cpWithCheck))
+      ctx.policies[stmt.cpTable] = pols
+    else: discard
+
+  rebuildSecondaryIndexes(ctx)
@@ -0,0 +1,143 @@
+## Executor types — shared by all exec/* modules and executor.nim
+import std/tables
+import std/locks
+import ../ast
+import ../ir
+import ../../core/types
+import ../../storage/lsm
+import ../../storage/btree
+import ../../core/mvcc
+import ../../fts/engine as fts
+import ../../vector/engine as vengine
+import ../../graph/engine as gengine
+import ../../ai/embed as embedmod
+import ../../ai/llm as llmmod
+import ../../core/registry
+
+type
+  IndexEntry* = ref object
+    lsmKey*: string
+    rowValue*: string
+
+  ChangeKind* = enum
+    ckInsert, ckUpdate, ckDelete
+
+  ChangeEvent* = object
+    table*: string
+    kind*: ChangeKind
+    key*: string
+    data*: string
+
+  UserDef* = object
+    name*: string
+    passwordHash*: string
+    isSuperuser*: bool
+    roles*: seq[string]
+
+  PrivilegeDef* = object
+    tableName*: string
+    command*: string  # SELECT, INSERT, UPDATE, DELETE, ALL
+
+  PolicyDef* = object
+    name*: string
+    tableName*: string
+    command*: string   # ALL, SELECT, INSERT, UPDATE, DELETE
+    usingExpr*: Node   # parsed USING expression
+    withCheckExpr*: Node  # parsed WITH CHECK expression
+
+  SharedLock* = ref object
+    lock*: Lock
+
+  ForeignKeyDef* = object
+    refTable*: string
+    refColumn*: string
+    onDelete*: string  # CASCADE, SET NULL, RESTRICT
+    onUpdate*: string  # CASCADE, SET NULL, RESTRICT
+
+  CheckDef* = object
+    name*: string
+    expr*: string  # stored expression string
+    checkNode*: Node  # AST for runtime evaluation
+
+  TriggerDef* = object
+    name*: string
+    timing*: string   # BEFORE, AFTER
+    event*: string    # INSERT, UPDATE, DELETE
+    action*: Node     # SQL statement AST
+
+  ColumnDef* = object
+    name*: string
+    colType*: string
+    isPk*: bool
+    isNotNull*: bool
+    isUnique*: bool
+    defaultVal*: string
+    fkTable*: string
+    fkColumn*: string
+    fkOnDelete*: string
+    fkOnUpdate*: string
+    autoIncrement*: bool
+
+  TableDef* = object
+    name*: string
+    columns*: seq[ColumnDef]
+    pkColumns*: seq[string]
+    foreignKeys*: seq[ForeignKeyDef]
+    checks*: seq[CheckDef]
+    triggers*: seq[TriggerDef]
+
+  Row* = Table[string, Value]
+
+  ExecutionContext* = ref object
+    db*: LSMTree
+    tables*: Table[string, TableDef]
+    btrees*: Table[string, BTreeIndex[string, IndexEntry]]
+    views*: Table[string, Node]  # view name -> SELECT AST
+    cteTables*: Table[string, seq[Row]]  # CTE name -> rows
+    ftsIndexes*: Table[string, fts.InvertedIndex]  # table.col -> FTS index
+    vectorIndexes*: Table[string, vengine.HNSWIndex]  # table.col -> HNSW index
+    graphs*: Table[string, gengine.Graph]  # graph name -> Graph object
+    embedder*: embedmod.Embedder  # optional embedding service client
+    llmClient*: llmmod.LLMClient  # optional LLM client for NL->SQL
+    txnManager*: TxnManager
+    pendingTxn*: Transaction
+    onChange*: proc(ev: ChangeEvent) {.closure.}
+    users*: Table[string, UserDef]
+    policies*: Table[string, seq[PolicyDef]]  # table name -> policies
+    currentUser*: string
+    currentRole*: string
+    sessionVars*: Table[string, string]
+    autoIncCounters*: Table[string, int64]
+    sequences*: Table[string, int64]
+    sharedLock*: SharedLock  # shared across cloned contexts
+    outerRow*: Table[string, string]  # outer query row for correlated subqueries
+    subqueryPlan*: IRPlan  # current subquery plan being evaluated
+    currentDatabase*: string  # name of the currently selected database
+    registry*: DatabaseRegistry  # nil for single-DB mode
+
+  MigrationRecord* = object
+    name*: string
+    checksum*: string
+    appliedAt*: int64
+    appliedBy*: string
+    durationMs*: int
+    rolledBack*: bool
+
+  ExecResult* = object
+    success*: bool
+    columns*: seq[string]
+    rows*: seq[Row]
+    affectedRows*: int
+    message*: string
+    keyValuePairs*: seq[(string, seq[byte])]
+
+proc `==`*(a, b: IndexEntry): bool =
+  a.lsmKey == b.lsmKey and a.rowValue == b.rowValue
+
+proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "",
+               kvPairs: seq[(string, seq[byte])] = @[]): ExecResult =
+  ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg,
+             keyValuePairs: kvPairs)
+
+proc errResult*(msg: string): ExecResult =
+  ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg)
@@ -0,0 +1,119 @@
+## Value / row serialization helpers used across the executor
+import std/strutils
+import std/tables
+import std/json
+import ../../core/types
+import types
+
+proc isNull*(value: string): bool =
+  value == "\\N" or value.toLower() == "null"
+
+proc valueToString*(v: Value): string =
+  case v.kind
+  of vkNull: return "\\N"
+  of vkString: return v.strVal
+  of vkInt64: return $v.int64Val
+  of vkFloat64: return $v.float64Val
+  of vkBool: return $v.boolVal
+  else: return ""
+
+proc `%`*(v: Value): JsonNode =
+  case v.kind
+  of vkNull: return newJNull()
+  of vkString: return %v.strVal
+  of vkInt64: return %v.int64Val
+  of vkFloat64: return %v.float64Val
+  of vkBool: return %v.boolVal
+  else: return newJNull()
+
+proc toString*(v: Value): string = valueToString(v)
+
+proc `[]=`*(t: var Row, key: string, val: string) =
+  t[key] = Value(kind: vkString, strVal: val)
+
+proc escapeRowVal*(v: string): string =
+  v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
+
+proc unescapeRowVal*(v: string): string =
+  result = ""
+  var i = 0
+  while i < v.len:
+    if v[i] == '\\' and i + 1 < v.len:
+      case v[i+1]
+      of '\\', ',', '=':
+        result &= v[i+1]
+        i += 2
+        continue
+      else: discard
+    result &= v[i]
+    inc i
+
+proc parseRowData*(valStr: string): Table[string, string] =
+  ## Parse "col1=val1,col2=val2" into a table
+  result = initTable[string, string]()
+  var i = 0
+  var part = ""
+  while i < valStr.len:
+    if valStr[i] == '\\' and i + 1 < valStr.len:
+      part &= valStr[i]
+      part &= valStr[i+1]
+      i += 2
+      continue
+    if valStr[i] == ',':
+      let eqPos = part.find('=')
+      if eqPos >= 0:
+        let k = part[0..<eqPos].strip()
+        let v = unescapeRowVal(part[eqPos+1..^1].strip())
+        result[k] = v
+      part = ""
+    else:
+      part &= valStr[i]
+    inc i
+  if part.len > 0:
+    let eqPos = part.find('=')
+    if eqPos >= 0:
+      let k = part[0..<eqPos].strip()
+      let v = unescapeRowVal(part[eqPos+1..^1].strip())
+      result[k] = v
+
+proc parseRowDataToValueRow*(valStr: string): Row =
+  result = initTable[string, Value]()
+  for k, v in parseRowData(valStr):
+    result[k] = v
+
+proc sqlEscapeIdent*(ident: string): string =
+  ## Escape SQL identifiers by doubling double-quotes.
+  result = ident.replace("\"", "\"\"")
+
+proc sqlEscapeString*(s: string): string =
+  ## Escape SQL string literals by doubling single-quotes.
+  result = s.replace("'", "''")
+
+proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
+  ## Build a multi-row INSERT statement for bulk import.
+  result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
+  for i, col in columns:
+    if i > 0: result &= ", "
+    result &= "\"" & sqlEscapeIdent(col) & "\""
+  result &= ") VALUES "
+  for ri, row in rows:
+    if ri > 0: result &= ", "
+    result &= "("
+    for ci, val in row:
+      if ci > 0: result &= ", "
+      if val.len == 0 or val == "\\N":
+        result &= "NULL"
+      else:
+        result &= "'" & sqlEscapeString(val) & "'"
+    result &= ")"
+
+proc getValue*(values: seq[string], fields: seq[string], colName: string): string =
+  for i, f in fields:
+    if f.toLower() == colName.toLower():
+      if i < values.len: return values[i]
+      return "\\N"
+  return "\\N"
+
+proc getTableDef*(ctx: ExecutionContext, tableName: string): TableDef =
+  if tableName in ctx.tables: return ctx.tables[tableName]
+  return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[])
@@ -1,4 +1,7 @@
 ## BaraQL Executor — AST lowering, IR compilation, and execution
+##
+## Shared types/helpers live under `exec/` (re-exported below for API stability).
+## See `exec/README.md` for module map and further extraction plan.
 import std/os
 import std/strutils
 import std/tables
@@ -53,140 +56,18 @@ import ../ai/embed as embedmod
 import ../ai/llm as llmmod
 import ../graph/cypher as cyphermod

-type
-  IndexEntry* = ref object
-    lsmKey*: string
-    rowValue*: string
-
-  ChangeKind* = enum
-    ckInsert, ckUpdate, ckDelete
-
-  ChangeEvent* = object
-    table*: string
-    kind*: ChangeKind
-    key*: string
-    data*: string
-
-  UserDef* = object
-    name*: string
-    passwordHash*: string
-    isSuperuser*: bool
-    roles*: seq[string]
-
-  PrivilegeDef* = object
-    tableName*: string
-    command*: string  # SELECT, INSERT, UPDATE, DELETE, ALL
-
-  PolicyDef* = object
-    name*: string
-    tableName*: string
-    command*: string   # ALL, SELECT, INSERT, UPDATE, DELETE
-    usingExpr*: Node   # parsed USING expression
-    withCheckExpr*: Node  # parsed WITH CHECK expression
-
-  SharedLock* = ref object
-    lock*: Lock
-
-  ExecutionContext* = ref object
-    db*: LSMTree
-    tables*: Table[string, TableDef]
-    btrees*: Table[string, BTreeIndex[string, IndexEntry]]
-    views*: Table[string, Node]  # view name -> SELECT AST
-    cteTables*: Table[string, seq[Row]]  # CTE name -> rows
-    ftsIndexes*: Table[string, fts.InvertedIndex]  # table.col -> FTS index
-    vectorIndexes*: Table[string, vengine.HNSWIndex]  # table.col -> HNSW index
-    graphs*: Table[string, gengine.Graph]  # graph name -> Graph object
-    embedder*: embedmod.Embedder  # optional embedding service client
-    llmClient*: llmmod.LLMClient  # optional LLM client for NL->SQL
-    txnManager*: TxnManager
-    pendingTxn*: Transaction
-    onChange*: proc(ev: ChangeEvent) {.closure.}
-    users*: Table[string, UserDef]
-    policies*: Table[string, seq[PolicyDef]]  # table name -> policies
-    currentUser*: string
-    currentRole*: string
-    sessionVars*: Table[string, string]
-    autoIncCounters*: Table[string, int64]
-    sequences*: Table[string, int64]
-    sharedLock*: SharedLock  # shared across cloned contexts — protects tables, views, btrees, ftsIndexes, users, policies, autoIncCounters, sequences
-    outerRow*: Table[string, string]  # outer query row for correlated subqueries
-    subqueryPlan*: IRPlan  # current subquery plan being evaluated (for correlation in execScan)
-    currentDatabase*: string  # name of the currently selected database
-    registry*: DatabaseRegistry  # reference to the database registry (nil for single-DB mode)
-
-  MigrationRecord* = object
-    name*: string
-    checksum*: string
-    appliedAt*: int64
-    appliedBy*: string
-    durationMs*: int
-    rolledBack*: bool
-
-  ForeignKeyDef* = object
-    refTable*: string
-    refColumn*: string
-    onDelete*: string  # CASCADE, SET NULL, RESTRICT
-    onUpdate*: string  # CASCADE, SET NULL, RESTRICT
-
-  CheckDef* = object
-    name*: string
-    expr*: string  # stored expression string
-    checkNode*: Node  # AST for runtime evaluation
-
-  TriggerDef* = object
-    name*: string
-    timing*: string   # BEFORE, AFTER
-    event*: string    # INSERT, UPDATE, DELETE
-    action*: Node     # SQL statement AST
-
-  TableDef* = object
-    name*: string
-    columns*: seq[ColumnDef]
-    pkColumns*: seq[string]
-    foreignKeys*: seq[ForeignKeyDef]
-    checks*: seq[CheckDef]
-    triggers*: seq[TriggerDef]
-
-  ColumnDef* = object
-    name*: string
-    colType*: string
-    isPk*: bool
-    isNotNull*: bool
-    isUnique*: bool
-    defaultVal*: string
-    fkTable*: string
-    fkColumn*: string
-    fkOnDelete*: string
-    fkOnUpdate*: string
-    autoIncrement*: bool
-
-  Row* = Table[string, Value]
-
-  ExecResult* = object
-    success*: bool
-    columns*: seq[string]
-    rows*: seq[Row]
-    affectedRows*: int
-    message*: string
-    keyValuePairs*: seq[(string, seq[byte])]
-
-proc `==`*(a, b: IndexEntry): bool =
-  a.lsmKey == b.lsmKey and a.rowValue == b.rowValue
-
-proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "",
-               kvPairs: seq[(string, seq[byte])] = @[]): ExecResult =
-  ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg,
-             keyValuePairs: kvPairs)
-
-proc errResult*(msg: string): ExecResult =
-  ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg)
+import exec/types
+import exec/values
+import exec/schema
+export types
+export values
+export schema

 # ----------------------------------------------------------------------
 # Context management
 # ----------------------------------------------------------------------

 proc evalNodeToString(node: Node): string
-proc restoreSchema(ctx: ExecutionContext)

 proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): ExecutionContext =
  result = ExecutionContext(db: db, tables: initTable[string, TableDef](),
@@ -213,32 +94,6 @@ proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): Execut
 # AST to SQL serializer (for VIEW DDL persistence)
 # ----------------------------------------------------------------------

-proc sqlEscapeIdent*(ident: string): string =
-  ## Escape SQL identifiers by doubling double-quotes.
-  result = ident.replace("\"", "\"\"")
-
-proc sqlEscapeString*(s: string): string =
-  ## Escape SQL string literals by doubling single-quotes.
-  result = s.replace("'", "''")
-
-proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
-  ## Build a multi-row INSERT statement for bulk import.
-  result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
-  for i, col in columns:
-    if i > 0: result &= ", "
-    result &= "\"" & sqlEscapeIdent(col) & "\""
-  result &= ") VALUES "
-  for ri, row in rows:
-    if ri > 0: result &= ", "
-    result &= "("
-    for ci, val in row:
-      if ci > 0: result &= ", "
-      if val.len == 0 or val == "\\N":
-        result &= "NULL"
-      else:
-        result &= "'" & sqlEscapeString(val) & "'"
-    result &= ")"
-
 proc exprToSql(node: Node): string =
  if node == nil:
    return ""
@@ -343,76 +198,6 @@ proc selectToSql(node: Node): string =
  if node.selOffset != nil and node.selOffset.offsetExpr.kind == nkIntLit:
    result.add(" OFFSET " & $node.selOffset.offsetExpr.intVal)

-# ----------------------------------------------------------------------
-# Schema restore
-# ----------------------------------------------------------------------
-
-proc restoreSchema(ctx: ExecutionContext) =
-  for entry in ctx.db.scanMemTable():
-    if entry.deleted: continue
-    if not entry.key.startsWith("_schema:"): continue
-    let ddl = cast[string](entry.value)
-    if ddl.len == 0: continue
-    var astNode: Node
-    try:
-      let tokens = qlex.tokenize(ddl)
-      astNode = qpar.parse(tokens)
-    except:
-      # Skip corrupted schema entries during startup
-      continue
-    if astNode.stmts.len > 0:
-      let stmt = astNode.stmts[0]
-      case stmt.kind
-      of nkCreateTable:
-        var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[],
-                           foreignKeys: @[], checks: @[], triggers: @[])
-        for col in stmt.crtColumns:
-          if col.kind == nkColumnDef:
-            var colDef = ColumnDef(name: col.cdName, colType: col.cdType)
-            colDef.autoIncrement = col.cdAutoIncrement
-            for cst in col.cdConstraints:
-              if cst.kind == nkConstraintDef:
-                case cst.cstType
-                of "pkey":
-                  colDef.isPk = true
-                  tbl.pkColumns.add(col.cdName)
-                  ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
-                of "notnull": colDef.isNotNull = true
-                of "unique":
-                  colDef.isUnique = true
-                  ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
-                of "default":
-                  if cst.cstDefault != nil:
-                    colDef.defaultVal = evalNodeToString(cst.cstDefault)
-                of "fkey":
-                  colDef.fkTable = cst.cstRefTable
-                  colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: ""
-                  colDef.fkOnDelete = cst.cstOnDelete
-                  colDef.fkOnUpdate = cst.cstOnUpdate
-                else: discard
-            tbl.columns.add(colDef)
-        ctx.tables[stmt.crtName] = tbl
-      of nkCreateView:
-        ctx.views[stmt.cvName] = stmt.cvQuery
-      of nkCreateTrigger:
-        if stmt.trigTable in ctx.tables:
-          ctx.tables[stmt.trigTable].triggers.add(TriggerDef(
-            name: stmt.trigName,
-            timing: stmt.trigTiming,
-            event: stmt.trigEvent,
-            action: stmt.trigAction,
-          ))
-      of nkCreateUser:
-        ctx.users[stmt.cuName] = UserDef(name: stmt.cuName,
-            passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[])
-      of nkCreatePolicy:
-        var pols = ctx.policies.getOrDefault(stmt.cpTable)
-        pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable,
-                           command: stmt.cpCommand, usingExpr: stmt.cpUsing,
-                           withCheckExpr: stmt.cpWithCheck))
-        ctx.policies[stmt.cpTable] = pols
-      else: discard
-
 proc cloneForConnection*(ctx: ExecutionContext): ExecutionContext =
  var svCopy = initTable[string, string]()
  for k, v in ctx.sessionVars:
@@ -512,97 +297,6 @@ proc getMigrationBody(ctx: ExecutionContext, name: string): (bool, string, strin
    else:
      return (true, ddl, "")
  return (false, "", "")
-
-# ----------------------------------------------------------------------
-# Helpers
-# ----------------------------------------------------------------------
-
-proc getTableDef(ctx: ExecutionContext, tableName: string): TableDef =
-  if tableName in ctx.tables: return ctx.tables[tableName]
-  return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[])
-
-proc getValue(values: seq[string], fields: seq[string], colName: string): string =
-  for i, f in fields:
-    if f.toLower() == colName.toLower() and i < values.len:
-      return values[i]
-  return "\\N"
-
-proc isNull*(value: string): bool =
-  value == "\\N" or value.toLower() == "null"
-
-proc valueToString*(v: Value): string =
-  case v.kind
-  of vkNull: return "\\N"
-  of vkString: return v.strVal
-  of vkInt64: return $v.int64Val
-  of vkFloat64: return $v.float64Val
-  of vkBool: return $v.boolVal
-  else: return ""
-
-proc `%`*(v: Value): JsonNode =
-  case v.kind
-  of vkNull: return newJNull()
-  of vkString: return %v.strVal
-  of vkInt64: return %v.int64Val
-  of vkFloat64: return %v.float64Val
-  of vkBool: return %v.boolVal
-  else: return newJNull()
-
-proc toString*(v: Value): string = valueToString(v)
-
-proc `[]=`*(t: var Row, key: string, val: string) =
-  t[key] = Value(kind: vkString, strVal: val)
-
-proc escapeRowVal(v: string): string =
-  v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
-
-proc unescapeRowVal(v: string): string =
-  result = ""
-  var i = 0
-  while i < v.len:
-    if v[i] == '\\' and i + 1 < v.len:
-      case v[i+1]
-      of '\\', ',', '=':
-        result &= v[i+1]
-        i += 2
-        continue
-      else: discard
-    result &= v[i]
-    inc i
-
-proc parseRowData(valStr: string): Table[string, string] =
-  ## Parse "col1=val1,col2=val2" into a table
-  result = initTable[string, string]()
-  var i = 0
-  var part = ""
-  while i < valStr.len:
-    if valStr[i] == '\\' and i + 1 < valStr.len:
-      part &= valStr[i]
-      part &= valStr[i+1]
-      i += 2
-      continue
-    if valStr[i] == ',':
-      let eqPos = part.find('=')
-      if eqPos >= 0:
-        let k = part[0..<eqPos].strip()
-        let v = unescapeRowVal(part[eqPos+1..^1].strip())
-        result[k] = v
-      part = ""
-    else:
-      part &= valStr[i]
-    inc i
-  if part.len > 0:
-    let eqPos = part.find('=')
-    if eqPos >= 0:
-      let k = part[0..<eqPos].strip()
-      let v = unescapeRowVal(part[eqPos+1..^1].strip())
-      result[k] = v
-
-proc parseRowDataToValueRow(valStr: string): Row =
-  result = initTable[string, Value]()
-  for k, v in parseRowData(valStr):
-    result[k] = v
-
 proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row]

 proc extractJoinEquality*(expr: IRExpr): (string, string) =
@@ -4988,30 +4682,35 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
                tbl.columns[i].fkOnDelete = cstNode.cstOnDelete
                tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate
    ctx.tables[stmt.crtName] = tbl
-
-    # Persist schema
-    var colDefs: seq[string] = @[]
-    for col in tbl.columns:
-      var parts = @[col.name, col.colType]
-      if col.isPk: parts.add("PRIMARY KEY")
-      if col.autoIncrement: parts.add("AUTO_INCREMENT")
-      if col.isNotNull: parts.add("NOT NULL")
-      if col.isUnique: parts.add("UNIQUE")
-      if col.defaultVal.len > 0: parts.add("DEFAULT '" & col.defaultVal & "'")
-      if col.fkTable.len > 0:
-        parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")")
-      colDefs.add(parts.join(" "))
-    let schemaKey = "_schema:migrations:" & $ctx.tables.len
-    ctx.db.put(schemaKey, cast[seq[byte]]("CREATE TABLE " & stmt.crtName & " (" & colDefs.join(", ") & ")"))
-
+    persistTableSchema(ctx, tbl)
    return okResult()

  of nkDropTable:
-    ctx.tables.del(stmt.drtName)
+    let dropName = stmt.drtName
+    ctx.tables.del(dropName)
    var toDelete: seq[string] = @[]
    for idxName in ctx.btrees.keys.toSeq():
-      if idxName.startsWith(stmt.drtName & "."): toDelete.add(idxName)
+      if idxName.startsWith(dropName & "."): toDelete.add(idxName)
    for idxName in toDelete: ctx.btrees.del(idxName)
+    # Remove durable schema entry
+    dropTableSchema(ctx, dropName)
+    # Remove row data for this table
+    var dataKeys: seq[string] = @[]
+    let prefix = dropName & "."
+    for (key, _) in ctx.db.scanAll():
+      if key.startsWith(prefix):
+        dataKeys.add(key)
+    for key in dataKeys:
+      ctx.db.delete(key)
+    # Drop orphan legacy schema keys that mentioned this table
+    var legacyKeys: seq[string] = @[]
+    for (key, value) in ctx.db.scanAll():
+      if key.startsWith(SchemaLegacyCreatePrefix):
+        let ddl = cast[string](value)
+        if ddl.contains("CREATE TABLE " & dropName) or ddl.contains("CREATE TABLE \"" & dropName):
+          legacyKeys.add(key)
+    for key in legacyKeys:
+      ctx.db.delete(key)
    return okResult()

  of nkCreateGraph:
@@ -5113,6 +4812,7 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
          var colDef = ColumnDef(name: op.cdName, colType: op.cdType)
          tbl.columns.add(colDef)
      ctx.tables[stmt.altName] = tbl
+      persistTableSchema(ctx, tbl)
      return okResult(msg="ALTER TABLE " & stmt.altName & " executed")
    return errResult("Table '" & stmt.altName & "' does not exist")

@@ -8,6 +8,7 @@ import ../storage/lsm
 const
  MaxLevel* = 7
  LevelMultiplier* = 10  # each level is 10x the previous
+  ## L0 uses file-count trigger (overlapping ranges); lower levels use size.

 type
  SSTableMeta* = object
@@ -29,20 +30,44 @@ type
    levels*: seq[seq[SSTableMeta]]
    dataDir*: string
    maxSizePerLevel*: seq[int]
+    l0FileLimit*: int

-proc newCompactionStrategy*(dataDir: string): CompactionStrategy =
+proc newCompactionStrategy*(dataDir: string, l0FileLimit: int = L0CompactionTrigger): CompactionStrategy =
  result = CompactionStrategy(
    levels: newSeq[seq[SSTableMeta]](MaxLevel),
    dataDir: dataDir,
    maxSizePerLevel: newSeq[int](MaxLevel),
+    l0FileLimit: l0FileLimit,
  )
  for i in 0..<MaxLevel:
    result.levels[i] = @[]
    result.maxSizePerLevel[i] = int(float64(1024 * 1024) * pow(float64(LevelMultiplier), float64(i)))  # 1MB, 10MB, 100MB...

+proc clear*(cs: CompactionStrategy) =
+  ## Drop all registered tables (used before rebuild-from-LSM).
+  for i in 0..<MaxLevel:
+    cs.levels[i].setLen(0)
+
 proc addTable*(cs: CompactionStrategy, meta: SSTableMeta) =
-  if meta.level < MaxLevel:
-    cs.levels[meta.level].add(meta)
+  let lvl = clamp(meta.level, 0, MaxLevel - 1)
+  cs.levels[lvl].add(meta)
+
+proc rebuildFromLSM*(cs: CompactionStrategy, db: LSMTree) =
+  ## Rebuild level layout from the live LSMTree catalog — single source of truth.
+  ## Avoids drift when flushes add SSTables the strategy never saw.
+  cs.clear()
+  cs.dataDir = db.dir
+  for sst in db.sstables:
+    let size = try: int(getFileSize(sst.path)) except: sst.entryCount * 64
+    cs.addTable(SSTableMeta(
+      path: sst.path,
+      level: sst.level,
+      minKey: sst.minKey,
+      maxKey: sst.maxKey,
+      entryCount: sst.entryCount,
+      sizeBytes: size,
+      createdAt: sst.id,  # stable ordering by id / creation sequence
+    ))

 proc totalSize*(cs: CompactionStrategy, level: int): int =
  result = 0
@@ -52,6 +77,9 @@ proc totalSize*(cs: CompactionStrategy, level: int): int =
 proc needsCompaction*(cs: CompactionStrategy, level: int): bool =
  if level >= MaxLevel - 1:
    return false
+  if level == 0:
+    # L0 files can overlap — count-based trigger (RocksDB-style)
+    return cs.levels[0].len >= cs.l0FileLimit
  return cs.totalSize(level) > cs.maxSizePerLevel[level]

 proc pickTablesForCompaction*(cs: CompactionStrategy, level: int): seq[SSTableMeta] =
@@ -0,0 +1,41 @@
+## Global storage gate — exclusive multi-thread entry to LSM / executor.
+##
+## Why: Hunos HTTP runs handlers on a worker-thread pool (`spawn` + internal
+## workers). The TCP server runs on the main async loop. Both share the same
+## `LSMTree` / `ExecutionContext` refs. Nim's default ORC memory manager is not
+## safe for concurrent refcount ops on the same objects from multiple OS threads.
+##
+## Holding this gate for the full duration of a query/compaction/DDL ensures
+## only one thread mutates or reads GC-managed storage state at a time.
+##
+## Ordering: always acquire StorageGate **before** any per-DB `LSMTree.lock`.
+## Call `initStorageGate()` once from main before accepting connections.
+import std/locks
+
+var
+  gGate: Lock
+  gInited*: bool
+
+proc initStorageGate*() =
+  ## Idempotent when called from a single thread at startup.
+  if not gInited:
+    initLock(gGate)
+    gInited = true
+
+proc acquireStorageGate*() {.inline.} =
+  ## Prefer calling initStorageGate() once at process start (main).
+  ## Lazy-init is allowed for unit tests (single-threaded).
+  if not gInited:
+    initStorageGate()
+  acquire(gGate)
+
+proc releaseStorageGate*() {.inline.} =
+  release(gGate)
+
+template withStorageGate*(body: untyped) =
+  ## Exclusive ownership of the storage engine for `body`.
+  acquireStorageGate()
+  try:
+    body
+  finally:
+    releaseStorageGate()
@@ -13,6 +13,11 @@ import bloom
 import wal
 import mmap
 import crc32
+import rwlock
+
+# Re-export WAL durability knobs for callers of newLSMTree
+export wal
+export rwlock

 const
  SSTableMagic* = 0x53535442'u32  # "SSTB"
@@ -21,6 +26,8 @@ const
  DefaultBloomFpRate* = 0.01
  ManifestVersion* = 1
  ManifestFileName* = "MANIFEST"
+  ## Trigger L0 compaction when this many L0 SSTables exist.
+  L0CompactionTrigger* = 4

 type
  Entry* = object
@@ -29,9 +36,10 @@ type
    timestamp*: uint64
    deleted*: bool

+  ## Hash-table MemTable: O(1) put/get. Sorted only when flushing to SSTable.
  MemTable* = object
-    entries: seq[Entry]
-    size: int
+    map: Table[string, Entry]
+    size: int      ## approximate byte size of live entries
    maxSize: int

  SSTable* = object
@@ -56,55 +64,64 @@ type
    currentSeq: uint64
    nextSSTableId*: int
    manifestSequence*: int64
-    lock*: Lock
+    ## Reader-writer lock: concurrent gets; exclusive put/flush/compact.
+    ## `acquire(db.lock)` is exclusive (write) for backward compatibility.
+    lock*: RwLock
    walLock*: Lock
+    ## Set by flush when L0 file count hits L0CompactionTrigger (hint for compactors).
+    needsCompaction*: bool
+    ## When true, flushUnsafe skips WAL rewrite (recovery still holds the WAL file open).
+    recovering: bool

 proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable =
-  MemTable(entries: @[], size: 0, maxSize: maxSize)
+  MemTable(map: initTable[string, Entry](), size: 0, maxSize: maxSize)

-proc len*(mt: MemTable): int = mt.entries.len
+proc len*(mt: MemTable): int = mt.map.len
+
+proc byteSize*(mt: MemTable): int = mt.size

 proc put*(mt: var MemTable, key: string, value: seq[byte], timestamp: uint64, deleted: bool = false): bool =
+  ## O(1) average-case insert/update. Returns false if the new key would exceed maxSize.
  let entrySize = key.len + value.len + 16
  if entrySize > mt.maxSize:
    return false
  let entry = Entry(key: key, value: value, timestamp: timestamp, deleted: deleted)
-  let pos = mt.entries.lowerBound(entry, proc(a, b: Entry): int = cmp(a.key, b.key))
-  if pos < mt.entries.len and mt.entries[pos].key == key:
-    let oldSize = mt.entries[pos].key.len + mt.entries[pos].value.len + 16
-    mt.entries[pos] = entry
+  if key in mt.map:
+    let old = mt.map[key]
+    # Only accept equal-or-newer timestamps (WAL recovery may replay older values)
+    if timestamp < old.timestamp:
+      return true
+    let oldSize = old.key.len + old.value.len + 16
+    mt.map[key] = entry
    mt.size += entrySize - oldSize
  else:
-    if mt.size + entrySize > mt.maxSize and mt.entries.len > 0:
+    if mt.size + entrySize > mt.maxSize and mt.map.len > 0:
      return false
-    mt.entries.insert(entry, pos)
+    mt.map[key] = entry
    mt.size += entrySize
  return true

 proc get*(mt: MemTable, key: string): (bool, Entry) =
-  if mt.entries.len == 0:
-    return (false, Entry())
-  var lo = 0
-  var hi = mt.entries.len - 1
-  while lo <= hi:
-    let mid = (lo + hi) div 2
-    let c = cmp(mt.entries[mid].key, key)
-    if c == 0:
-      return (true, mt.entries[mid])
-    elif c < 0:
-      lo = mid + 1
-    else:
-      hi = mid - 1
+  if key in mt.map:
+    return (true, mt.map[key])
  return (false, Entry())

+proc sortedEntries*(mt: MemTable): seq[Entry] =
+  ## Materialize entries sorted by key — used for SSTable flush and ordered scans.
+  result = newSeqOfCap[Entry](mt.map.len)
+  for _, entry in mt.map:
+    result.add(entry)
+  result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
+
 proc scan*(mt: MemTable, startKey, endKey: string): seq[Entry] =
  result = @[]
-  for entry in mt.entries:
-    if entry.key >= startKey and entry.key <= endKey:
+  for key, entry in mt.map:
+    if key >= startKey and key <= endKey:
      result.add(entry)
+  result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))

 proc clear*(mt: var MemTable) =
-  mt.entries.setLen(0)
+  mt.map.clear()
  mt.size = 0

 # ----------------------------------------------------------------------
@@ -600,8 +617,15 @@ proc checkStorageConsistency*(db: LSMTree): seq[string] =
 # ----------------------------------------------------------------------

 proc flushUnsafe(db: LSMTree) {.gcsafe.}
+proc countL0*(db: LSMTree): int

-proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
+proc newLSMTree*(
+    dir: string,
+    memMaxSize: int = DefaultMemTableSize,
+    walSyncMode: WalSyncMode = wsmGroup,
+    walGroupEvery: int = DefaultWalGroupEvery,
+    walGroupIntervalMs: int = 0,
+): LSMTree =
  createDir(dir)
  createDir(dir / "sstables")

@@ -641,21 +665,29 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
      echo "[INFO] Loaded ", sstables.len, " SSTable(s) from directory scan"

  new(result)
-  initLock(result.lock)
+  initRwLock(result.lock)
  initLock(result.walLock)
  result.dir = dir
  result.memTable = newMemTable(memMaxSize)
  result.immutableMem = newMemTable(0)
  result.sstables = sstables
-  result.wal = newWriteAheadLog(dir / "wal")
+  result.wal = newWriteAheadLog(
+    dir / "wal",
+    syncMode = walSyncMode,
+    groupEvery = walGroupEvery,
+    groupIntervalMs = walGroupIntervalMs,
+  )
  result.memMaxSize = memMaxSize
  result.currentSeq = 0
  result.nextSSTableId = nextId
  result.manifestSequence = manifestSeq
+  result.recovering = false
+  result.needsCompaction = result.countL0() >= L0CompactionTrigger

  # WAL crash recovery — replay unflushed entries into memTable
  let walPath = dir / "wal" / "wal.log"
  if fileExists(walPath):
+    result.recovering = true
    var stream: FileStream = nil
    try:
      stream = newFileStream(walPath, fmRead)
@@ -697,11 +729,30 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
    finally:
      if stream != nil:
        stream.close()
+      result.recovering = false
+      # After recovery, shrink WAL to live unflushed state only
+      acquire(result.walLock)
+      try:
+        var liveKeys: seq[string] = @[]
+        var liveVals: seq[seq[byte]] = @[]
+        var liveTs: seq[uint64] = @[]
+        var liveDel: seq[bool] = @[]
+        for e in result.immutableMem.sortedEntries():
+          liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted)
+        for e in result.memTable.sortedEntries():
+          liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted)
+        if liveKeys.len == 0:
+          result.wal.truncate()
+        else:
+          result.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel)
+      finally:
+        release(result.walLock)

 proc put*(db: LSMTree, key: string, value: seq[byte]) =
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
-  defer: release(db.lock)
+  acquireWrite(db.lock)
+  defer: releaseWrite(db.lock)
+  # WAL then memtable under the same exclusive lock → crash recovery sees a total order
  acquire(db.walLock)
  db.wal.writePut(cast[seq[byte]](key), value, ts)
  release(db.walLock)
@@ -716,8 +767,8 @@ proc put*(db: LSMTree, key: string, value: seq[byte]) =

 proc delete*(db: LSMTree, key: string) =
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
-  defer: release(db.lock)
+  acquireWrite(db.lock)
+  defer: releaseWrite(db.lock)
  acquire(db.walLock)
  db.wal.writeDelete(cast[seq[byte]](key), ts)
  release(db.walLock)
@@ -732,8 +783,8 @@ proc delete*(db: LSMTree, key: string) =
 proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = false) =
  ## Direct LSM insert without WAL logging — used by recovery.
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
-  defer: release(db.lock)
+  acquireWrite(db.lock)
+  defer: releaseWrite(db.lock)
  if not db.memTable.put(key, value, ts, deleted):
    if db.immutableMem.len > 0:
      db.flushUnsafe()
@@ -745,18 +796,26 @@ proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = fals
 proc deleteUnsafe*(db: LSMTree, key: string) =
  putUnsafe(db, key, @[], deleted = true)

+proc copyBytes(s: seq[byte]): seq[byte] =
+  ## Deep copy so callers on other threads never share ORC-managed seq buffers.
+  result = newSeq[byte](s.len)
+  if s.len > 0:
+    copyMem(addr result[0], unsafeAddr s[0], s.len)
+
 proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
+  ## Caller must hold at least a read lock.
+  ## Returned values are deep-copied for multi-thread ORC safety (HTTP + TCP share LSM).
  let (found, entry) = db.memTable.get(key)
  if found:
    if entry.deleted:
      return (false, @[])
-    return (true, entry.value)
+    return (true, copyBytes(entry.value))

  let (found2, entry2) = db.immutableMem.get(key)
  if found2:
    if entry2.deleted:
      return (false, @[])
-    return (true, entry2.value)
+    return (true, copyBytes(entry2.value))

  # Search SSTables from newest to oldest
  for i in countdown(db.sstables.high, db.sstables.low):
@@ -769,21 +828,40 @@ proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
    if found3:
      if entry3.deleted:
        return (false, @[])
-      return (true, entry3.value)
+      return (true, copyBytes(entry3.value))

  return (false, @[])

 proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
-  acquire(db.lock)
-  defer: release(db.lock)
+  ## Thread-safe lookup.
+  ## Default: exclusive lock — required for Nim ORC when TCP + HTTP threads share the DB.
+  ## Compile with `-d:baraConcurrentReads` for shared read locks (needs multi-thread-safe MM
+  ## such as a future atomicArc build; unsafe with default ORC across OS threads).
+  when defined(baraConcurrentReads):
+    acquireRead(db.lock)
+    defer: releaseRead(db.lock)
+  else:
+    acquireWrite(db.lock)
+    defer: releaseWrite(db.lock)
  return getUnsafe(db, key)

 proc contains*(db: LSMTree, key: string): bool =
-  acquire(db.lock)
-  defer: release(db.lock)
+  when defined(baraConcurrentReads):
+    acquireRead(db.lock)
+    defer: releaseRead(db.lock)
+  else:
+    acquireWrite(db.lock)
+    defer: releaseWrite(db.lock)
  let (found, _) = getUnsafe(db, key)
  return found

+proc countL0*(db: LSMTree): int =
+  ## Number of level-0 SSTables (newest, uncompacted).
+  result = 0
+  for sst in db.sstables:
+    if sst.level == 0:
+      inc result
+
 proc flushUnsafe(db: LSMTree) =
  if db.immutableMem.len == 0 and db.memTable.len == 0:
    return
@@ -802,7 +880,8 @@ proc flushUnsafe(db: LSMTree) =
  let path = db.dir / "sstables" / ($db.nextSSTableId & ".sst")
  inc db.nextSSTableId

-  var sst = writeSSTable(toFlush.entries, path, level = 0)
+  # Sort once at flush time (O(n log n)) — put/get stay O(1)
+  var sst = writeSSTable(toFlush.sortedEntries(), path, level = 0)
  sst.id = db.nextSSTableId - 1
  db.sstables.add(sst)
  # SSTables are kept in insertion order (newest last) so getUnsafe can search newest-first
@@ -814,22 +893,43 @@ proc flushUnsafe(db: LSMTree) =
  except CatchableError as e:
    echo "[WARN] Failed to write MANIFEST: ", e.msg

-  acquire(db.walLock)
-  db.wal.writeCommit(uint64(getMonoTime().ticks()))
-  db.wal.maybeRotate()
-  db.wal.sync()
-  release(db.walLock)
+  # Rewrite WAL to contain only still-unflushed memtable entries.
+  # Skip during recovery — the WAL file is still open for reading.
+  if not db.recovering:
+    acquire(db.walLock)
+    var liveKeys: seq[string] = @[]
+    var liveVals: seq[seq[byte]] = @[]
+    var liveTs: seq[uint64] = @[]
+    var liveDel: seq[bool] = @[]
+    for e in db.immutableMem.sortedEntries():
+      liveKeys.add(e.key)
+      liveVals.add(e.value)
+      liveTs.add(e.timestamp)
+      liveDel.add(e.deleted)
+    for e in db.memTable.sortedEntries():
+      liveKeys.add(e.key)
+      liveVals.add(e.value)
+      liveTs.add(e.timestamp)
+      liveDel.add(e.deleted)
+    if liveKeys.len == 0:
+      db.wal.truncate()
+    else:
+      db.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel)
+    release(db.walLock)
+
+  if db.countL0() >= L0CompactionTrigger:
+    db.needsCompaction = true

 proc flush*(db: LSMTree) =
-  acquire(db.lock)
-  defer: release(db.lock)
+  acquireWrite(db.lock)
+  defer: releaseWrite(db.lock)
  flushUnsafe(db)

 proc checkpoint*(db: LSMTree) =
  ## Create a consistent checkpoint: freeze memtable, flush to SSTable,
  ## rotate WAL, and write MANIFEST. This provides a clean boundary
  ## for online backup without stopping the server.
-  acquire(db.lock)
+  acquireWrite(db.lock)

  # Flush any pending immutable memtable first
  if db.immutableMem.len > 0:
@@ -850,10 +950,10 @@ proc checkpoint*(db: LSMTree) =
  db.wal.sync()
  release(db.walLock)

-  release(db.lock)
+  releaseWrite(db.lock)

 proc close*(db: LSMTree) =
-  acquire(db.lock)
+  acquireWrite(db.lock)
  try:
    # Flush both memtables to avoid data loss
    while db.immutableMem.len > 0:
@@ -863,61 +963,109 @@ proc close*(db: LSMTree) =
      sst.close()
    db.wal.close()
  finally:
-    release(db.lock)
+    releaseWrite(db.lock)
+
+template withDataLock(db: LSMTree, body: untyped) =
+  ## Shared or exclusive depending on baraConcurrentReads (see get*).
+  when defined(baraConcurrentReads):
+    acquireRead(db.lock)
+    try:
+      body
+    finally:
+      releaseRead(db.lock)
+  else:
+    acquireWrite(db.lock)
+    try:
+      body
+    finally:
+      releaseWrite(db.lock)

 proc memTableSize*(db: LSMTree): int =
-  acquire(db.lock)
-  defer: release(db.lock)
-  return db.memTable.len
+  withDataLock(db):
+    return db.memTable.len

 proc sstableCount*(db: LSMTree): int =
-  acquire(db.lock)
-  defer: release(db.lock)
-  return db.sstables.len
+  withDataLock(db):
+    return db.sstables.len

 proc dir*(db: LSMTree): string =
-  acquire(db.lock)
-  defer: release(db.lock)
-  return db.dir
+  withDataLock(db):
+    return db.dir

 proc scanMemTable*(db: LSMTree): seq[Entry] =
-  acquire(db.lock)
-  defer: release(db.lock)
-  ## Return all entries from memory (memTable + immutableMem)
-  result = @[]
-  for e in db.memTable.entries:
-    result.add(e)
-  for e in db.immutableMem.entries:
-    result.add(e)
+  ## Return all entries from memory (memTable + immutableMem), sorted by key.
+  ## Immutable wins over active memtable only when timestamps are newer (same key rare).
+  withDataLock(db):
+    var merged = initTable[string, Entry]()
+    for e in db.immutableMem.sortedEntries():
+      merged[e.key] = e
+    for e in db.memTable.sortedEntries():
+      if e.key notin merged or e.timestamp >= merged[e.key].timestamp:
+        merged[e.key] = e
+    result = newSeqOfCap[Entry](merged.len)
+    for _, e in merged:
+      result.add(e)
+    result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
+
+proc scanRange*(db: LSMTree, startKey, endKey: string): seq[(string, seq[byte])] =
+  ## Inclusive key range scan over memtables + SSTables (newest wins).
+  withDataLock(db):
+    var best = initTable[string, Entry]()
+
+    for e in db.memTable.scan(startKey, endKey):
+      best[e.key] = e
+    for e in db.immutableMem.scan(startKey, endKey):
+      if e.key notin best or e.timestamp > best[e.key].timestamp:
+        best[e.key] = e
+
+    for i in countdown(db.sstables.high, db.sstables.low):
+      let sst = db.sstables[i]
+      if sst.maxKey < startKey or sst.minKey > endKey:
+        continue
+      for key, offset in sst.index:
+        if key < startKey or key > endKey:
+          continue
+        if key in best:
+          continue
+        let (found, entry) = readSSTableEntry(sst, key)
+        if found:
+          best[key] = entry
+
+    var keys = newSeqOfCap[string](best.len)
+    for k in best.keys:
+      keys.add(k)
+    keys.sort(cmp)
+    for k in keys:
+      let e = best[k]
+      if not e.deleted:
+        result.add((e.key, e.value))

 proc scanAll*(db: LSMTree): seq[(string, seq[byte])] =
  ## Scan all active (non-deleted) entries from memory and SSTables.
  ## Used for shard data migration.
-  acquire(db.lock)
-  defer: release(db.lock)
+  withDataLock(db):
+    var seen = initTable[string, bool]()

-  var seen = initTable[string, bool]()
+    # Scan memtable first (most recent)
+    for e in db.memTable.sortedEntries():
+      if e.key notin seen:
+        seen[e.key] = true
+        if not e.deleted:
+          result.add((e.key, e.value))

-  # Scan memtable first (most recent)
-  for e in db.memTable.entries:
-    if e.key notin seen:
-      seen[e.key] = true
-      if not e.deleted:
-        result.add((e.key, e.value))
+    # Scan immutable memtable
+    for e in db.immutableMem.sortedEntries():
+      if e.key notin seen:
+        seen[e.key] = true
+        if not e.deleted:
+          result.add((e.key, e.value))

-  # Scan immutable memtable
-  for e in db.immutableMem.entries:
-    if e.key notin seen:
-      seen[e.key] = true
-      if not e.deleted:
-        result.add((e.key, e.value))
-
-  # Scan SSTables from newest to oldest
-  for i in countdown(db.sstables.high, db.sstables.low):
-    let sst = db.sstables[i]
-    for key, offset in sst.index:
-      if key notin seen:
-        seen[key] = true
-        let (found, entry) = readSSTableEntry(sst, key)
-        if found and not entry.deleted:
-          result.add((entry.key, entry.value))
+    # Scan SSTables from newest to oldest
+    for i in countdown(db.sstables.high, db.sstables.low):
+      let sst = db.sstables[i]
+      for key, offset in sst.index:
+        if key notin seen:
+          seen[key] = true
+          let (found, entry) = readSSTableEntry(sst, key)
+          if found and not entry.deleted:
+            result.add((entry.key, entry.value))
@@ -0,0 +1,80 @@
+## Simple reader-writer lock for LSM concurrent reads.
+## Multiple readers OR one writer. Writers are exclusive.
+## `acquire` / `release` are write-side (backward compatible with Lock-style usage).
+import std/locks
+
+type
+  RwLock* = object
+    mu: Lock
+    readers: int          ## active readers
+    writer: bool          ## writer holds exclusive access
+    waitingWriters: int   ## prefer writers to avoid reader starvation of compact/flush
+    canRead: Cond
+    canWrite: Cond
+
+proc initRwLock*(rw: var RwLock) =
+  initLock(rw.mu)
+  initCond(rw.canRead)
+  initCond(rw.canWrite)
+  rw.readers = 0
+  rw.writer = false
+  rw.waitingWriters = 0
+
+proc deinitRwLock*(rw: var RwLock) =
+  deinitCond(rw.canRead)
+  deinitCond(rw.canWrite)
+  deinitLock(rw.mu)
+
+proc acquireRead*(rw: var RwLock) =
+  ## Shared read lock. Blocks while a writer is active or waiting (writer preference).
+  acquire(rw.mu)
+  while rw.writer or rw.waitingWriters > 0:
+    wait(rw.canRead, rw.mu)
+  inc rw.readers
+  release(rw.mu)
+
+proc releaseRead*(rw: var RwLock) =
+  acquire(rw.mu)
+  dec rw.readers
+  if rw.readers == 0:
+    # Wake one waiting writer
+    signal(rw.canWrite)
+  release(rw.mu)
+
+proc acquireWrite*(rw: var RwLock) =
+  ## Exclusive write lock.
+  acquire(rw.mu)
+  inc rw.waitingWriters
+  while rw.writer or rw.readers > 0:
+    wait(rw.canWrite, rw.mu)
+  dec rw.waitingWriters
+  rw.writer = true
+  release(rw.mu)
+
+proc releaseWrite*(rw: var RwLock) =
+  acquire(rw.mu)
+  rw.writer = false
+  # Prefer draining writers, else open the gate for readers
+  if rw.waitingWriters > 0:
+    signal(rw.canWrite)
+  else:
+    broadcast(rw.canRead)
+  release(rw.mu)
+
+# Lock-compatible names: default exclusive (used by compaction, put, flush)
+proc acquire*(rw: var RwLock) {.inline.} = acquireWrite(rw)
+proc release*(rw: var RwLock) {.inline.} = releaseWrite(rw)
+
+template withReadLock*(rw: var RwLock, body: untyped) =
+  acquireRead(rw)
+  try:
+    body
+  finally:
+    releaseRead(rw)
+
+template withWriteLock*(rw: var RwLock, body: untyped) =
+  acquireWrite(rw)
+  try:
+    body
+  finally:
+    releaseWrite(rw)
@@ -4,12 +4,16 @@ import std/os
 import std/streams
 import std/strutils
 import std/posix
+import std/monotimes
+import std/times

 const
  WALMagic* = 0x42415241'u32  # "BARA"
  WALVersion* = 1'u32
  DefaultMaxWalSegmentSize* = 64 * 1024 * 1024  # 64MB
  WalArchiveDir* = "wal_archive"
+  ## Default group-commit batch size (entries between fsyncs).
+  DefaultWalGroupEvery* = 64

 type
  WalEntryKind* = enum
@@ -18,6 +22,15 @@ type
    wekCheckpoint = 3
    wekCommit = 4

+  ## Durability policy for WAL writes.
+  ## - wsmNone:  flush userspace buffer only; fsync on truncate/rewrite/close/explicit sync
+  ## - wsmGroup: group commit — fsync every N entries and/or every intervalMs (default)
+  ## - wsmEvery: fsync after every entry (strict, slow)
+  WalSyncMode* = enum
+    wsmNone = "none"
+    wsmGroup = "group"
+    wsmEvery = "every"
+
  WalEntry* = object
    kind*: WalEntryKind
    timestamp*: uint64
@@ -34,14 +47,28 @@ type
    path: string
    stream: FileStream
    entryCount: uint64
-    syncOnWrite: bool
+    syncMode*: WalSyncMode
+    groupEvery*: int          ## entries between fsyncs when mode=group
+    groupIntervalMs*: int     ## time-based fsync when mode=group (0 = off)
+    unsyncedEntries: int      ## entries written since last fsync
+    lastSync: MonoTime
    maxSegmentSize: int64
    currentSequence: int64
+    ## Counters for observability / benchmarks
+    fsyncCount*: uint64
+    bytesSinceSync: int

 proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry]
 proc listWalArchive*(dir: string): seq[WalSegment]
 proc maybeRotate*(wal: var WriteAheadLog)

+proc parseWalSyncMode*(s: string): WalSyncMode =
+  case s.toLowerAscii()
+  of "none", "async", "off", "false", "0": wsmNone
+  of "every", "sync", "full", "true", "1": wsmEvery
+  of "group", "batch", "": wsmGroup
+  else: wsmGroup
+
 proc parseWalSequence*(filename: string): int64 =
  ## Extract sequence from "wal.000042.log"
  try:
@@ -74,6 +101,12 @@ proc nextWalSequence*(dir: string): int64 =
    return 1
  return segments[^1].sequence + 1

+proc fsyncPath(path: string) =
+  let fd = posix.open(cstring(path), O_RDWR)
+  if fd != -1:
+    discard posix.fsync(fd)
+    discard posix.close(fd)
+
 proc rotate*(wal: var WriteAheadLog) =
  ## Close current WAL and archive it, then start a new one.
  if wal.stream != nil:
@@ -96,7 +129,12 @@ proc rotate*(wal: var WriteAheadLog) =
  wal.stream.write(WALMagic)
  wal.stream.write(WALVersion)
  wal.stream.flush()
+  fsyncPath(wal.path)
  wal.entryCount = 0
+  wal.unsyncedEntries = 0
+  wal.bytesSinceSync = 0
+  wal.lastSync = getMonoTime()
+  inc wal.fsyncCount

 proc maybeRotate*(wal: var WriteAheadLog) =
  ## Rotate if current WAL exceeds max segment size.
@@ -106,7 +144,16 @@ proc maybeRotate*(wal: var WriteAheadLog) =
  if currentSize >= wal.maxSegmentSize:
    wal.rotate()

-proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
+proc newWriteAheadLog*(
+    dir: string,
+    syncMode: WalSyncMode = wsmGroup,
+    groupEvery: int = DefaultWalGroupEvery,
+    groupIntervalMs: int = 0,
+    syncOnWrite: bool = false,
+): WriteAheadLog =
+  ## Create a WAL.
+  ## - syncMode controls durability (see WalSyncMode).
+  ## - syncOnWrite=true is legacy and forces wsmEvery.
  createDir(dir)
  let path = dir / "wal.log"
  let exists = fileExists(path)
@@ -125,18 +172,62 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
    for e in readEntries(path):
      inc count

+  let mode = if syncOnWrite: wsmEvery else: syncMode
+  let ge = if groupEvery <= 0: DefaultWalGroupEvery else: groupEvery
  let seqNum = nextWalSequence(dir)
  WriteAheadLog(
    dir: dir,
    path: path,
    stream: stream,
    entryCount: count,
-    syncOnWrite: syncOnWrite,
+    syncMode: mode,
+    groupEvery: ge,
+    groupIntervalMs: groupIntervalMs,
+    unsyncedEntries: 0,
+    lastSync: getMonoTime(),
    maxSegmentSize: DefaultMaxWalSegmentSize,
    currentSequence: seqNum,
+    fsyncCount: 0,
+    bytesSinceSync: 0,
  )

+proc setSyncMode*(wal: var WriteAheadLog, mode: WalSyncMode) =
+  wal.syncMode = mode
+
+proc setGroupEvery*(wal: var WriteAheadLog, n: int) =
+  wal.groupEvery = if n <= 0: DefaultWalGroupEvery else: n
+
+proc setGroupIntervalMs*(wal: var WriteAheadLog, ms: int) =
+  wal.groupIntervalMs = max(0, ms)
+
+proc markSynced(wal: var WriteAheadLog) =
+  wal.unsyncedEntries = 0
+  wal.bytesSinceSync = 0
+  wal.lastSync = getMonoTime()
+  inc wal.fsyncCount
+
+proc maybeGroupSync(wal: var WriteAheadLog, entryBytes: int) =
+  ## Apply durability policy after a buffered write.
+  case wal.syncMode
+  of wsmNone:
+    discard
+  of wsmEvery:
+    fsyncPath(wal.path)
+    wal.markSynced()
+  of wsmGroup:
+    inc wal.unsyncedEntries
+    wal.bytesSinceSync += entryBytes
+    var due = wal.unsyncedEntries >= wal.groupEvery
+    if not due and wal.groupIntervalMs > 0:
+      let elapsedMs = (getMonoTime() - wal.lastSync).inMilliseconds
+      if elapsedMs >= wal.groupIntervalMs:
+        due = true
+    if due:
+      fsyncPath(wal.path)
+      wal.markSynced()
+
 proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
+  let entryBytes = 1 + 8 + 4 + entry.key.len + 4 + entry.value.len
  wal.stream.write(uint8(entry.kind))
  wal.stream.write(entry.timestamp)
  wal.stream.write(uint32(entry.key.len))
@@ -145,8 +236,9 @@ proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
  wal.stream.write(uint32(entry.value.len))
  if entry.value.len > 0:
    wal.stream.writeData(unsafeAddr entry.value[0], entry.value.len)
-  if wal.syncOnWrite:
-    wal.stream.flush()
+  # Always push to kernel page cache; durability policy decides fsync
+  wal.stream.flush()
+  wal.maybeGroupSync(entryBytes)
  inc wal.entryCount
  # Check rotation every 1000 entries to avoid stat on every write
  if wal.entryCount mod 1000 == 0:
@@ -177,28 +269,87 @@ proc writeCommit*(wal: var WriteAheadLog, timestamp: uint64) =
  ))

 proc sync*(wal: var WriteAheadLog) =
+  ## Force durability of all buffered WAL data.
  wal.stream.flush()
-  # Re-open with O_RDWR so fsync operates on a write-capable fd.
-  # Not ideal (two fds for same file) but avoids accessing private
-  # FileStream internals that vary across Nim versions.
-  let fd = posix.open(cstring(wal.path), O_RDWR)
-  if fd != -1:
-    discard posix.fsync(fd)
-    discard posix.close(fd)
+  fsyncPath(wal.path)
+  wal.markSynced()
+
+proc truncate*(wal: var WriteAheadLog) =
+  ## Reset WAL to empty (header only). Safe only when all prior entries
+  ## are durable in SSTables and nothing remains only-in-memtable.
+  if wal.stream != nil:
+    wal.stream.flush()
+    wal.stream.close()
+  wal.stream = newFileStream(wal.path, fmWrite)
+  if wal.stream == nil:
+    raise newException(IOError, "Cannot truncate WAL: " & wal.path)
+  wal.stream.write(WALMagic)
+  wal.stream.write(WALVersion)
+  wal.stream.flush()
+  fsyncPath(wal.path)
+  wal.entryCount = 0
+  wal.markSynced()
+
+proc rewriteLive*(wal: var WriteAheadLog,
+                  keys: openArray[string],
+                  values: openArray[seq[byte]],
+                  timestamps: openArray[uint64],
+                  deleted: openArray[bool]) =
+  ## Atomically replace WAL contents with a live memtable snapshot.
+  ## Used after a partial flush so unflushed keys remain recoverable.
+  doAssert keys.len == values.len and keys.len == timestamps.len and keys.len == deleted.len
+  if keys.len == 0:
+    wal.truncate()
+    return
+
+  let tmpPath = wal.path & ".rewrite"
+  let s = newFileStream(tmpPath, fmWrite)
+  if s == nil:
+    raise newException(IOError, "Cannot create WAL rewrite file: " & tmpPath)
+  s.write(WALMagic)
+  s.write(WALVersion)
+  var count: uint64 = 0
+  for i in 0 ..< keys.len:
+    let kind = if deleted[i]: wekDelete else: wekPut
+    s.write(uint8(kind))
+    s.write(timestamps[i])
+    s.write(uint32(keys[i].len))
+    if keys[i].len > 0:
+      s.write(keys[i])
+    s.write(uint32(values[i].len))
+    if values[i].len > 0:
+      s.writeData(unsafeAddr values[i][0], values[i].len)
+    inc count
+  s.flush()
+  s.close()
+  fsyncPath(tmpPath)
+
+  if wal.stream != nil:
+    wal.stream.close()
+  if fileExists(wal.path):
+    removeFile(wal.path)
+  moveFile(tmpPath, wal.path)
+  wal.stream = newFileStream(wal.path, fmAppend)
+  if wal.stream == nil:
+    raise newException(IOError, "Cannot reopen WAL after rewrite: " & wal.path)
+  wal.entryCount = count
+  wal.markSynced()

 proc setMaxSegmentSize*(wal: var WriteAheadLog, size: int64) =
  wal.maxSegmentSize = size

 proc close*(wal: var WriteAheadLog) =
  wal.stream.flush()
-  let fd = posix.open(cstring(wal.path), O_RDWR)
-  if fd != -1:
-    discard posix.fsync(fd)
-    discard posix.close(fd)
+  fsyncPath(wal.path)
+  wal.markSynced()
  wal.stream.close()

 proc entryCount*(wal: WriteAheadLog): uint64 = wal.entryCount
 proc path*(wal: WriteAheadLog): string = wal.path
+proc unsyncedEntries*(wal: WriteAheadLog): int = wal.unsyncedEntries
+
+## Legacy alias — true maps to wsmEvery
+proc syncOnWrite*(wal: WriteAheadLog): bool = wal.syncMode == wsmEvery

 proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry] =
  result = @[]
@@ -15,6 +15,7 @@ import barabadb/core/config
 import barabadb/core/logging
 import barabadb/protocol/ssl
 import barabadb/storage/lsm
+import barabadb/storage/gate
 import barabadb/storage/compaction
 import barabadb/core/raft
 import barabadb/query/executor
@@ -36,58 +37,66 @@ type

 proc newCompactionManager*(db: LSMTree): CompactionManager =
  result = CompactionManager(db: db, strategy: compaction.newCompactionStrategy(db.dir))
-  for sst in db.sstables:
-    let meta = compaction.SSTableMeta(
-      path: sst.path,
-      level: sst.level,
-      minKey: sst.minKey,
-      maxKey: sst.maxKey,
-      entryCount: sst.entryCount,
-      sizeBytes: sst.entryCount * 64,
-      createdAt: 0,
-    )
-    result.strategy.addTable(meta)
+  result.strategy.rebuildFromLSM(db)
+
+proc applyCompactionResult(db: LSMTree, result: compaction.CompactionResult) =
+  ## Apply compaction output under the caller's lock: update sstables + MANIFEST.
+  ## On Linux, compact may already have unlinked inputs; we still close our mmaps.
+  if result.outputTables.len == 0:
+    return
+
+  var newSSTables: seq[SSTable] = @[]
+  var removedPaths = initTable[string, bool]()
+  for t in result.inputTables:
+    removedPaths[t.path] = true
+  for sst in db.sstables.mitems:
+    if sst.path notin removedPaths:
+      newSSTables.add(sst)
+    else:
+      # Drop mmap after compact unlinked the path (fd remains valid until close)
+      sst.close()
+
+  for meta in result.outputTables:
+    try:
+      var sst = loadSSTable(meta.path)
+      let name = splitFile(meta.path).name
+      # Prefer numeric id from filename; otherwise allocate
+      let parsed = try: parseInt(name) except: -1
+      if parsed >= 0:
+        sst.id = parsed
+      else:
+        sst.id = db.nextSSTableId
+        inc db.nextSSTableId
+      sst.level = meta.level
+      newSSTables.add(sst)
+      db.nextSSTableId = max(db.nextSSTableId, sst.id + 1)
+    except CatchableError as e:
+      warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg)
+
+  newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
+  db.sstables = newSSTables
+  db.needsCompaction = db.countL0() >= L0CompactionTrigger
+
+  inc db.manifestSequence
+  try:
+    writeManifest(db)
+  except CatchableError as e:
+    warn("Failed to write MANIFEST after compaction: " & e.msg)

 proc compact*(cm: CompactionManager) =
-  acquire(cm.db.lock)
-  defer: release(cm.db.lock)
-  for level in 0 ..< compaction.MaxLevel:
-    if cm.strategy.needsCompaction(level):
-      let result = cm.strategy.compact(level)
-      if result.outputTables.len == 0:
-        continue
-
-      # Remove compacted input SSTables from LSMTree
-      var newSSTables: seq[SSTable] = @[]
-      var removedPaths = initTable[string, bool]()
-      for t in result.inputTables:
-        removedPaths[t.path] = true
-      for sst in cm.db.sstables:
-        if sst.path notin removedPaths:
-          newSSTables.add(sst)
-
-      # Load and add output SSTables
-      for meta in result.outputTables:
-        try:
-          var sst = loadSSTable(meta.path)
-          let name = splitFile(meta.path).name
-          # Extract numeric id from filename if possible
-          sst.id = try: parseInt(name) except: cm.db.nextSSTableId
-          sst.level = meta.level
-          newSSTables.add(sst)
-          cm.db.nextSSTableId = max(cm.db.nextSSTableId, sst.id + 1)
-        except CatchableError as e:
-          warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg)
-
-      newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
-      cm.db.sstables = newSSTables
-
-      # Update MANIFEST
-      inc cm.db.manifestSequence
-      try:
-        writeManifest(cm.db)
-      except CatchableError as e:
-        warn("Failed to write MANIFEST after compaction: " & e.msg)
+  # Gate first (cross-thread), then per-DB write lock
+  withStorageGate:
+    acquire(cm.db.lock)
+    try:
+      # Always rebuild from LSM — flushes add L0 tables the strategy never registered
+      cm.strategy.rebuildFromLSM(cm.db)
+      for level in 0 ..< compaction.MaxLevel:
+        if cm.strategy.needsCompaction(level):
+          let result = cm.strategy.compact(level)
+          applyCompactionResult(cm.db, result)
+          cm.strategy.rebuildFromLSM(cm.db)
+    finally:
+      release(cm.db.lock)

 proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.async.} =
  while true:
@@ -96,17 +105,11 @@ proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.asyn

 proc newMultiCompactionManager*(registry: DatabaseRegistry): MultiCompactionManager =
  result = MultiCompactionManager(registry: registry, strategies: initTable[string, compaction.CompactionStrategy]())
-
-  # Initialize strategies for each existing database
  for name in listDatabases(registry):
    let info = getDatabaseInfo(registry, name)
    if info != nil:
      result.strategies[name] = compaction.newCompactionStrategy(info.db.dir)
-      for sst in info.db.sstables:
-        let meta = compaction.SSTableMeta(
-          path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey,
-          entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0)
-        result.strategies[name].addTable(meta)
+      result.strategies[name].rebuildFromLSM(info.db)

 proc compactAll(mcm: MultiCompactionManager) =
  for name in listDatabases(mcm.registry):
@@ -114,51 +117,21 @@ proc compactAll(mcm: MultiCompactionManager) =
    if info == nil: continue
    let db = info.db

-    # Initialize strategy if not already
    if name notin mcm.strategies:
      mcm.strategies[name] = compaction.newCompactionStrategy(db.dir)
-      for sst in db.sstables:
-        let meta = compaction.SSTableMeta(
-          path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey,
-          entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0)
-        mcm.strategies[name].addTable(meta)

    let strategy = mcm.strategies[name]
-    acquire(db.lock)
-    try:
-      for level in 0 ..< compaction.MaxLevel:
-        if strategy.needsCompaction(level):
-          let result = strategy.compact(level)
-          if result.outputTables.len == 0: continue
-
-          var newSSTables: seq[SSTable] = @[]
-          var removedPaths = initTable[string, bool]()
-          for t in result.inputTables:
-            removedPaths[t.path] = true
-          for sst in db.sstables:
-            if sst.path notin removedPaths:
-              newSSTables.add(sst)
-
-          for meta in result.outputTables:
-            try:
-              var sst = loadSSTable(meta.path)
-              let sstName = splitFile(meta.path).name
-              sst.id = try: parseInt(sstName) except: db.nextSSTableId
-              sst.level = meta.level
-              newSSTables.add(sst)
-              db.nextSSTableId = max(db.nextSSTableId, sst.id + 1)
-            except CatchableError as e:
-              warn("Compaction output SSTable failed to load: " & meta.path & " - " & e.msg)
-
-          newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
-          db.sstables = newSSTables
-          inc db.manifestSequence
-          try:
-            writeManifest(db)
-          except CatchableError as e:
-            warn("Failed to write MANIFEST after compaction: " & e.msg)
-    finally:
-      release(db.lock)
+    withStorageGate:
+      acquire(db.lock)
+      try:
+        strategy.rebuildFromLSM(db)
+        for level in 0 ..< compaction.MaxLevel:
+          if strategy.needsCompaction(level):
+            let result = strategy.compact(level)
+            applyCompactionResult(db, result)
+            strategy.rebuildFromLSM(db)
+      finally:
+        release(db.lock)

 proc startMultiCompactionLoop*(mcm: MultiCompactionManager, intervalMs: int = 60000) {.async.} =
  while true:
@@ -303,10 +276,13 @@ proc main() =
        quit(0)

  var config = loadConfig()
+  # Global exclusive gate for multi-thread storage (HTTP workers + TCP + compact)
+  initStorageGate()
  # Init structured logger from config
  let logLvl = parseEnum[LogLevel]("ll" & capitalizeAscii(config.logLevel))
  defaultLogger = newLogger(logLvl, config.logFile)
  info("BaraDB v1.1.6 — Multimodal Database Engine")
+  info("Storage gate initialized (serializes HTTP/TCP/compaction access)")

  # Security check: warn if JWT secret is not configured
  if config.jwtSecret.len == 0:
@@ -358,12 +334,13 @@ proc main() =
    # Wire state machine to apply committed entries to the default database
    let defaultDbInfo = getDatabaseInfo(registry, "default")
    raftNode.applyCommand = proc(cmd: string, data: seq[byte]) {.gcsafe.} =
-      if cmd == "put":
-        let parts = cast[string](data).split("\x00")
-        if parts.len >= 2:
-          defaultDbInfo.db.put(parts[0], cast[seq[byte]](parts[1]))
-      elif cmd == "delete":
-        defaultDbInfo.db.delete(cast[string](data))
+      withStorageGate:
+        if cmd == "put":
+          let parts = cast[string](data).split("\x00")
+          if parts.len >= 2:
+            defaultDbInfo.db.put(parts[0], cast[seq[byte]](parts[1]))
+        elif cmd == "delete":
+          defaultDbInfo.db.delete(cast[string](data))

    # Wire RAFT ↔ DistTxn
    wireRaftDistTxn(raftNode, tcpServer)
@@ -395,11 +372,13 @@ proc main() =
  # Start TCP wire protocol server on main thread with async event loop
  waitFor runTcpServer(config)

-  # Shutdown
-  httpServer.stop()
+  # Shutdown: stop listeners first, then close storage under the gate
+  httpServer.stop(closeStorage = false)
  tcpServer.stop()
  if tcpServer.gossipProtocol != nil:
    tcpServer.gossipProtocol.stop()
+  withStorageGate:
+    registry.closeAll()

 when isMainModule:
  main()