feat: harden storage, schema persistence, fair benches, fix wire crash

Core storage: hash MemTable, WAL group commit, L0 compaction rebuild,
reader-writer lock, and a global StorageGate so HTTP workers and TCP
share the LSM safely under multi-thread access.

Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM
restore on open. Executor types/values/schema split into query/exec/.

Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted
after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG)
and honesty docs for mixed-tier comparisons.

src/barabadb/core/config.nim

@@ -22,6 +22,11 @@ type
     logFormat*: string
     memtableSizeMb*: int
     cacheSizeMb*: int
+    ## WAL durability: "none" | "group" (default) | "every"
+    walSyncMode*: string
+    ## Group commit batch size (entries between fsyncs when mode=group)
+    walGroupEvery*: int
+    ## Time-based group fsync interval in ms (0 = off); also used as legacy name
     walSyncIntervalMs*: int
     compactionIntervalMs*: int
     bloomBitsPerKey*: int
@@ -58,6 +63,8 @@ proc defaultConfig*(): BaraConfig =
     logFormat: "json",
     memtableSizeMb: 64,
     cacheSizeMb: 256,
+    walSyncMode: "group",
+    walGroupEvery: 64,
     walSyncIntervalMs: 0,
     compactionIntervalMs: 60_000,
     bloomBitsPerKey: 10,
@@ -93,6 +100,8 @@ proc loadConfigFromJson*(path: string, cfg: var BaraConfig) =
       if s.hasKey("data_dir"): cfg.dataDir = s["data_dir"].getStr()
       if s.hasKey("memtable_size_mb"): cfg.memtableSizeMb = s["memtable_size_mb"].getInt()
       if s.hasKey("cache_size_mb"): cfg.cacheSizeMb = s["cache_size_mb"].getInt()
+      if s.hasKey("wal_sync_mode"): cfg.walSyncMode = s["wal_sync_mode"].getStr()
+      if s.hasKey("wal_group_every"): cfg.walGroupEvery = s["wal_group_every"].getInt()
       if s.hasKey("wal_sync_interval_ms"): cfg.walSyncIntervalMs = s["wal_sync_interval_ms"].getInt()
       if s.hasKey("compaction_interval_ms"): cfg.compactionIntervalMs = s["compaction_interval_ms"].getInt()
       if s.hasKey("bloom_bits_per_key"): cfg.bloomBitsPerKey = s["bloom_bits_per_key"].getInt()
@@ -153,6 +162,8 @@ proc loadConfigFromEnv*(cfg: var BaraConfig) =
   cfg.logFormat = getEnv("BARADB_LOG_FORMAT", cfg.logFormat)
   cfg.memtableSizeMb = parseEnvInt(getEnv("BARADB_MEMTABLE_SIZE_MB", ""), cfg.memtableSizeMb)
   cfg.cacheSizeMb = parseEnvInt(getEnv("BARADB_CACHE_SIZE_MB", ""), cfg.cacheSizeMb)
+  cfg.walSyncMode = getEnv("BARADB_WAL_SYNC_MODE", cfg.walSyncMode)
+  cfg.walGroupEvery = parseEnvInt(getEnv("BARADB_WAL_GROUP_EVERY", ""), cfg.walGroupEvery)
   cfg.walSyncIntervalMs = parseEnvInt(getEnv("BARADB_WAL_SYNC_INTERVAL_MS", ""), cfg.walSyncIntervalMs)
   cfg.compactionIntervalMs = parseEnvInt(getEnv("BARADB_COMPACTION_INTERVAL_MS", ""), cfg.compactionIntervalMs)
   cfg.bloomBitsPerKey = parseEnvInt(getEnv("BARADB_BLOOM_BITS_PER_KEY", ""), cfg.bloomBitsPerKey)

src/barabadb/core/httpserver.nim

@@ -15,6 +15,7 @@ import ../query/parser
 import ../query/executor
 import ../core/types
 import ../storage/lsm
+import ../storage/gate
 import ../core/mvcc
 import ../protocol/wire
 import ../core/websocket
@@ -196,17 +197,7 @@ proc queryHandler(server: HttpServer): RequestHandler =
         ctx.json(%*{"error": "Empty query"}, 400)
         return
 
-      var reqCtx = getRequestDatabaseContext(server, request)
-      reqCtx.currentUser = userId
-      reqCtx.currentRole = role
-      let tokens = tokenize(queryStr)
-      let astNode = parse(tokens)
-
-      if astNode.stmts.len == 0:
-        ctx.json(%*{"rows": [], "affectedRows": 0, "columns": []})
-        return
-
-      # Extract optional params from JSON body
+      # Extract optional params from JSON body (no storage access yet)
       var params: seq[WireValue] = @[]
       if "params" in body and body["params"].kind == JArray:
         for p in body["params"]:
@@ -218,31 +209,50 @@ proc queryHandler(server: HttpServer): RequestHandler =
           of JString: params.add(WireValue(kind: fkString, strVal: p.getStr()))
           else: params.add(WireValue(kind: fkString, strVal: $p))
 
-      let res = executor.executeQuery(reqCtx, astNode, params)
+      # StorageGate: serialize against TCP + other Hunos workers (ORC safety)
+      var success: bool
+      var jsonRows = newJArray()
+      var jsonCols = newJArray()
+      var affected = 0
+      var msg = ""
+      var errMsg = ""
+      withStorageGate:
+        var reqCtx = getRequestDatabaseContext(server, request)
+        reqCtx.currentUser = userId
+        reqCtx.currentRole = role
+        let tokens = tokenize(queryStr)
+        let astNode = parse(tokens)
+        if astNode.stmts.len == 0:
+          success = true
+        else:
+          let res = executor.executeQuery(reqCtx, astNode, params)
+          success = res.success
+          if res.success:
+            affected = res.affectedRows
+            msg = res.message
+            for row in res.rows:
+              var jsonRow = newJObject()
+              for col in res.columns:
+                if col in row and row[col].kind != vkNull:
+                  jsonRow[col] = %valueToString(row[col])
+                else:
+                  jsonRow[col] = newJNull()
+              jsonRows.add(jsonRow)
+            for c in res.columns:
+              jsonCols.add(%c)
+          else:
+            errMsg = res.message
 
-      if res.success:
-        var jsonRows = newJArray()
-        for row in res.rows:
-          var jsonRow = newJObject()
-          for col in res.columns:
-            let key = col
-            if key in row and row[key].kind != vkNull:
-              jsonRow[key] = %valueToString(row[key])
-            else:
-              jsonRow[key] = newJNull()
-          jsonRows.add(jsonRow)
-        var jsonCols = newJArray()
-        for c in res.columns:
-          jsonCols.add(%c)
+      if success:
         ctx.json(%*{
           "rows": jsonRows,
-          "affectedRows": res.affectedRows,
+          "affectedRows": affected,
           "columns": jsonCols,
-          "message": if res.message.len > 0: %res.message else: newJNull()
+          "message": if msg.len > 0: %msg else: newJNull()
         })
       else:
         server.metrics.queryErrors += 1
-        ctx.json(%*{"error": res.message}, 400)
+        ctx.json(%*{"error": errMsg}, 400)
 
 proc healthHandler(): RequestHandler =
   return proc(request: Request) {.gcsafe.} =
@@ -376,15 +386,16 @@ proc tablesHandler(server: HttpServer): RequestHandler =
       let ctx = newContext(request)
       if not server.checkAuth(request, ctx):
         return
-      let reqCtx = getRequestDatabaseContext(server, request)
       var tables = newJArray()
-      for name, tbl in reqCtx.tables:
-        var cols = newJArray()
-        for col in tbl.columns:
-          cols.add(%*{"name": col.name, "type": col.colType,
-            "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique})
-        tables.add(%*{"name": name, "columns": cols,
-          "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
+      withStorageGate:
+        let reqCtx = getRequestDatabaseContext(server, request)
+        for name, tbl in reqCtx.tables:
+          var cols = newJArray()
+          for col in tbl.columns:
+            cols.add(%*{"name": col.name, "type": col.colType,
+              "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique})
+          tables.add(%*{"name": name, "columns": cols,
+            "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
       ctx.json(%*{"tables": tables})
 
 proc databasesHandler(server: HttpServer): RequestHandler =
@@ -393,24 +404,25 @@ proc databasesHandler(server: HttpServer): RequestHandler =
       let ctx = newContext(request)
       if not server.checkAuth(request, ctx):
         return
-      let dbs = server.registry.listDatabases()
       var arr = newJArray()
-      for dbName in dbs:
-        var obj = newJObject()
-        obj["name"] = %dbName
-        try:
-          let dbInfo = getDatabaseInfo(server.registry, dbName)
-          if dbInfo != nil and dbInfo.ctx != nil:
-            let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx))
-            obj["tables"] = %dbCtx.tables.len
-            obj["connections"] = %getConnectionCount(server.registry, dbName)
-          else:
+      withStorageGate:
+        let dbs = server.registry.listDatabases()
+        for dbName in dbs:
+          var obj = newJObject()
+          obj["name"] = %dbName
+          try:
+            let dbInfo = getDatabaseInfo(server.registry, dbName)
+            if dbInfo != nil and dbInfo.ctx != nil:
+              let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx))
+              obj["tables"] = %dbCtx.tables.len
+              obj["connections"] = %getConnectionCount(server.registry, dbName)
+            else:
+              obj["tables"] = %0
+              obj["connections"] = %0
+          except CatchableError:
             obj["tables"] = %0
             obj["connections"] = %0
-        except CatchableError:
-          obj["tables"] = %0
-          obj["connections"] = %0
-        arr.add(obj)
+          arr.add(obj)
       ctx.json(%*{"databases": arr})
 
 proc createDatabaseHandler(server: HttpServer): RequestHandler =
@@ -428,7 +440,8 @@ proc createDatabaseHandler(server: HttpServer): RequestHandler =
         ctx.json(%*{"error": "Empty database name"}, 400)
         return
       try:
-        discard getOrCreateDatabase(server.registry, dbName)
+        withStorageGate:
+          discard getOrCreateDatabase(server.registry, dbName)
         ctx.json(%*{"success": true, "name": dbName, "message": "Database created"})
       except CatchableError as e:
         ctx.json(%*{"error": e.msg}, 400)
@@ -444,7 +457,9 @@ proc dropDatabaseHandler(server: HttpServer): RequestHandler =
         ctx.json(%*{"error": "Missing database name"}, 400)
         return
       try:
-        let ok = dropDatabase(server.registry, dbName)
+        var ok = false
+        withStorageGate:
+          ok = dropDatabase(server.registry, dbName)
         if ok:
           ctx.json(%*{"success": true, "name": dbName, "message": "Database dropped"})
         else:
@@ -470,13 +485,15 @@ proc backupHandler(server: HttpServer): RequestHandler =
       let compression = if body != nil and "level" in body: body["level"].getInt() else: 6
       try:
         var ok = false
-        if allDatabases:
-          ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
-        elif dbName.len > 0:
-          let dbDir = dataRoot / dbName
-          ok = backupDataDir(dbDir, outputFile, @[], compression, false)
-        else:
-          ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
+        # Gate held so live writers/compactors don't mutate files mid-backup
+        withStorageGate:
+          if allDatabases:
+            ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
+          elif dbName.len > 0:
+            let dbDir = dataRoot / dbName
+            ok = backupDataDir(dbDir, outputFile, @[], compression, false)
+          else:
+            ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
         if ok:
           ctx.json(%*{"success": true, "output": outputFile, "message": "Backup created"})
         else:
@@ -541,18 +558,20 @@ proc restoreHandler(server: HttpServer): RequestHandler =
         let meta = readBackupMeta(inputFile)
         let isMultiDb = meta != nil and meta{"databases"} != nil
         var ok = false
-        if isMultiDb or allDatabases:
-          ok = restoreAllDatabases(inputFile, dataRoot, false, false)
-        elif dbName.len > 0:
-          let dbDir = dataRoot / dbName
-          ok = restoreDataDir(inputFile, dbDir, false, false)
-        else:
-          ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+        withStorageGate:
+          if isMultiDb or allDatabases:
+            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+          elif dbName.len > 0:
+            let dbDir = dataRoot / dbName
+            ok = restoreDataDir(inputFile, dbDir, false, false)
+          else:
+            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
+          if ok:
+            # Reload under same gate after files are restored
+            server.registry.loadExistingDatabases()
 
         logRestore(inputFile, dataRoot, ok)
         if ok:
-          # Reload databases after restore
-          server.registry.loadExistingDatabases()
           ctx.json(%*{"success": true, "message": "Restore completed"})
         else:
           ctx.json(%*{"error": "Restore failed"}, 500)
@@ -890,10 +909,14 @@ proc run*(server: HttpServer, port: int = 9470) =
   asyncCheck server.ws.run(port + 1)
   hunosServer.serve(Port(port))
 
-proc stop*(server: HttpServer) =
+proc stop*(server: HttpServer, closeStorage: bool = false) =
+  ## Stop HTTP listeners. By default does **not** close the shared registry —
+  ## when HTTP is spawned alongside TCP they share one registry owned by main.
   server.running = false
   server.ws.stop()
-  if server.registry != nil:
-    server.registry.closeAll()
-  else:
-    server.db.close()
+  if closeStorage:
+    withStorageGate:
+      if server.registry != nil:
+        server.registry.closeAll()
+      elif server.db != nil:
+        server.db.close()

src/barabadb/core/registry.nim

@@ -29,6 +29,17 @@ type
 
 const reservedDbNames* = ["system", "information_schema", "pg_catalog"]
 
+proc openLsmForRegistry(reg: DatabaseRegistry, dbDir: string): LSMTree =
+  ## Open LSM with WAL durability settings from registry config.
+  let memBytes = max(1, reg.config.memtableSizeMb) * 1024 * 1024
+  newLSMTree(
+    dbDir,
+    memMaxSize = memBytes,
+    walSyncMode = parseWalSyncMode(reg.config.walSyncMode),
+    walGroupEvery = reg.config.walGroupEvery,
+    walGroupIntervalMs = reg.config.walSyncIntervalMs,
+  )
+
 proc isValidDbName*(name: string): bool =
   if name.len == 0: return false
   if '/' in name or '\\' in name: return false
@@ -63,7 +74,7 @@ proc loadExistingDatabases*(reg: DatabaseRegistry) =
       if dbName.len > 0 and isValidDbName(dbName):
         let dbDir = reg.dataRoot / dbName
         info("Loading database '" & dbName & "' from " & dbDir)
-        let db = newLSMTree(dbDir)
+        let db = openLsmForRegistry(reg, dbDir)
         let ctx = reg.ctxFactory(db, reg)
         acquire(reg.lock)
         reg.databases[dbName] = DatabaseInfo(
@@ -89,7 +100,7 @@ proc ensureDefaultDatabase*(reg: DatabaseRegistry) =
   if not exists:
     let dbDir = reg.dataRoot / defaultDbName
     info("Creating default database at " & dbDir)
-    let db = newLSMTree(dbDir)
+    let db = openLsmForRegistry(reg, dbDir)
     let ctx = reg.ctxFactory(db, reg)
     acquire(reg.lock)
     reg.databases[defaultDbName] = DatabaseInfo(
@@ -113,7 +124,7 @@ proc getOrCreateDatabase*(reg: DatabaseRegistry, name: string): DatabaseInfo =
   # Create new database
   let dbDir = reg.dataRoot / name
   info("Creating database '" & name & "' at " & dbDir)
-  let db = newLSMTree(dbDir)
+  let db = openLsmForRegistry(reg, dbDir)
   let ctx = reg.ctxFactory(db, reg)
   let info = DatabaseInfo(name: name, db: db, ctx: ctx, activeConnections: 0)
   reg.databases[name] = info

src/barabadb/core/server.nim

@@ -21,6 +21,7 @@ import ../query/parser
 import ../query/ast
 import ../query/executor
 import ../storage/lsm
+import ../storage/gate
 import ../core/mvcc
 import ../core/disttxn
 import ../core/replication
@@ -206,61 +207,64 @@ proc valueToWire(val: string, colType: string): WireValue =
 
 proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq[WireValue] = @[],
                    replication: ReplicationManager = nil): (bool, QueryResult, string) =
-  try:
-    let tokens = tokenize(query)
-    let astNode = parse(tokens)
+  ## All storage access is under the global StorageGate so HTTP worker threads
+  ## and the TCP event loop never touch ORC-managed LSM/executor state concurrently.
+  withStorageGate:
+    try:
+      let tokens = tokenize(query)
+      let astNode = parse(tokens)
 
-    if astNode.stmts.len == 0:
-      return (true, QueryResult(), "")
+      if astNode.stmts.len == 0:
+        return (true, QueryResult(), "")
 
-    let res = executor.executeQuery(ctx, astNode, params)
-    if res.success:
-      # Ship written key-value pairs to replicas
-      if replication != nil and res.keyValuePairs.len > 0:
-        for (key, value) in res.keyValuePairs:
-          var data = newSeq[byte](key.len + 1 + value.len)
-          for i, c in key: data[i] = byte(c)
-          data[key.len] = byte(0)
-          for i, c in value: data[key.len + 1 + i] = c
-          discard replication.writeLsn(data)
-      var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len)
-      qr.columns = res.columns
+      let res = executor.executeQuery(ctx, astNode, params)
+      if res.success:
+        # Ship written key-value pairs to replicas
+        if replication != nil and res.keyValuePairs.len > 0:
+          for (key, value) in res.keyValuePairs:
+            var data = newSeq[byte](key.len + 1 + value.len)
+            for i, c in key: data[i] = byte(c)
+            data[key.len] = byte(0)
+            for i, c in value: data[key.len + 1 + i] = c
+            discard replication.writeLsn(data)
+        var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len)
+        qr.columns = res.columns
 
-      var colTypes: seq[string] = @[]
-      var tableName = ""
-      if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil:
-        tableName = astNode.stmts[0].selFrom.fromTable
-      elif astNode.stmts[0].kind == nkInsert:
-        tableName = astNode.stmts[0].insTarget
-      elif astNode.stmts[0].kind == nkUpdate:
-        tableName = astNode.stmts[0].updTarget
+        var colTypes: seq[string] = @[]
+        var tableName = ""
+        if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil:
+          tableName = astNode.stmts[0].selFrom.fromTable
+        elif astNode.stmts[0].kind == nkInsert:
+          tableName = astNode.stmts[0].insTarget
+        elif astNode.stmts[0].kind == nkUpdate:
+          tableName = astNode.stmts[0].updTarget
 
-      if tableName.len > 0 and tableName in ctx.tables:
-        let tbl = ctx.tables[tableName]
-        for col in res.columns:
-          var found = ""
-          for c in tbl.columns:
-            if c.name.toLower() == col.toLower():
-              found = c.colType
-              break
-          colTypes.add(found)
+        if tableName.len > 0 and tableName in ctx.tables:
+          let tbl = ctx.tables[tableName]
+          for col in res.columns:
+            var found = ""
+            for c in tbl.columns:
+              if c.name.toLower() == col.toLower():
+                found = c.colType
+                break
+            colTypes.add(found)
+        else:
+          colTypes = newSeq[string](res.columns.len)
+
+        qr.columnTypes = colTypes.mapIt(typeToFieldKind(it))
+        qr.rows = @[]
+        for row in res.rows:
+          var wireRow: seq[WireValue] = @[]
+          for i, col in res.columns:
+            let val = if col in row: valueToString(row[col]) else: "\\N"
+            let cType = if i < colTypes.len: colTypes[i] else: ""
+            wireRow.add(valueToWire(val, cType))
+          qr.rows.add(wireRow)
+        return (true, qr, res.message)
       else:
-        colTypes = newSeq[string](res.columns.len)
-
-      qr.columnTypes = colTypes.mapIt(typeToFieldKind(it))
-      qr.rows = @[]
-      for row in res.rows:
-        var wireRow: seq[WireValue] = @[]
-        for i, col in res.columns:
-          let val = if col in row: valueToString(row[col]) else: "\\N"
-          let cType = if i < colTypes.len: colTypes[i] else: ""
-          wireRow.add(valueToWire(val, cType))
-        qr.rows.add(wireRow)
-      return (true, qr, res.message)
-    else:
-      return (false, QueryResult(), res.message)
-  except Exception as e:
-    return (false, QueryResult(), e.msg)
+        return (false, QueryResult(), res.message)
+    except Exception as e:
+      return (false, QueryResult(), e.msg)
 
 # ----------------------------------------------------------------------
 # Response Serialization