feat: harden storage, schema persistence, fair benches, fix wire crash

Core storage: hash MemTable, WAL group commit, L0 compaction rebuild,
reader-writer lock, and a global StorageGate so HTTP workers and TCP
share the LSM safely under multi-thread access.

Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM
restore on open. Executor types/values/schema split into query/exec/.

Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted
after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG)
and honesty docs for mixed-tier comparisons.

src/barabadb/query/exec/README.md

@@ -0,0 +1,29 @@
+# Executor package (`query/exec/`)
+
+The original `executor.nim` was a ~5.8k-line god object. Shared pieces live here;
+`../executor.nim` remains the main execution engine and **re-exports** this package
+so existing `import barabadb/query/executor` keeps working.
+
+## Modules
+
+| Module | Responsibility |
+|--------|----------------|
+| `types.nim` | `ExecutionContext`, `TableDef`, `Row`, `ExecResult`, … |
+| `values.nim` | Null/string conversion, row payload parse/escape, SQL escapes |
+| `schema.nim` | Durable catalog (`_schema:tables:*`), restore, index rebuild |
+
+## Import rules
+
+- **No cycles:** `types` → nothing in `exec/`; `values` → `types`; `schema` → `types` + `values`.
+- `executor.nim` imports all three and `export`s them.
+- Prefer adding new shared helpers under `exec/` instead of growing `executor.nim`.
+
+## Sensible next extractions (not done yet)
+
+1. `dml.nim` — `execScan` / `execInsert` / `execUpdate` / `execDelete` (needs eval/triggers hooks)
+2. `rls.nim` — row-level security + privileges
+3. `lower.nim` — AST → IR (`lowerExpr` / `lowerSelect`)
+4. `plan_exec.nim` — IR plan walker / window functions
+5. `hybrid.nim` — hybrid vector+FTS search helpers
+
+Keep statement dispatch (`executeQueryImpl`) in `executor.nim` until those land.

src/barabadb/query/exec/schema.nim

@@ -0,0 +1,231 @@
+## Schema catalog persistence — CREATE/DROP/ALTER survive restart
+import std/strutils
+import std/tables
+import std/sequtils
+import ../ast
+import ../lexer as qlex
+import ../parser as qpar
+import ../../storage/lsm
+import ../../storage/btree
+import types
+import values
+
+const
+  SchemaTablePrefix* = "_schema:tables:"
+  SchemaViewPrefix* = "_schema:views:"
+  SchemaTriggerPrefix* = "_schema:triggers:"
+  SchemaUserPrefix* = "_schema:users:"
+  SchemaPolicyPrefix* = "_schema:policies:"
+  ## Legacy CREATE TABLE keys (pre-fix) used a migrations: counter suffix
+  SchemaLegacyCreatePrefix* = "_schema:migrations:"
+
+proc tableSchemaKey*(tableName: string): string =
+  SchemaTablePrefix & tableName
+
+proc litToString(node: Node): string =
+  ## Evaluate simple literal defaults for schema materialization (no full expr engine).
+  if node == nil: return ""
+  case node.kind
+  of nkStringLit: return node.strVal
+  of nkIntLit: return $node.intVal
+  of nkFloatLit: return $node.floatVal
+  of nkBoolLit: return $node.boolVal
+  of nkNullLit: return "\\N"
+  else: return ""
+
+proc serializeTableDdl*(tbl: TableDef): string =
+  ## Stable DDL for a table definition (survives restart via LSM).
+  var colDefs: seq[string] = @[]
+  let multiPk = tbl.pkColumns.len > 1
+  for col in tbl.columns:
+    var parts: seq[string] = @[col.name, col.colType]
+    if col.isPk and not multiPk:
+      parts.add("PRIMARY KEY")
+    if col.autoIncrement:
+      parts.add("AUTO_INCREMENT")
+    if col.isNotNull:
+      parts.add("NOT NULL")
+    if col.isUnique and not col.isPk:
+      parts.add("UNIQUE")
+    if col.defaultVal.len > 0:
+      parts.add("DEFAULT '" & sqlEscapeString(col.defaultVal) & "'")
+    if col.fkTable.len > 0:
+      parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")")
+      if col.fkOnDelete.len > 0:
+        parts.add("ON DELETE " & col.fkOnDelete)
+      if col.fkOnUpdate.len > 0:
+        parts.add("ON UPDATE " & col.fkOnUpdate)
+    colDefs.add(parts.join(" "))
+  if multiPk:
+    colDefs.add("PRIMARY KEY (" & tbl.pkColumns.join(", ") & ")")
+  result = "CREATE TABLE " & tbl.name & " (" & colDefs.join(", ") & ")"
+
+proc persistTableSchema*(ctx: ExecutionContext, tbl: TableDef) =
+  ## Write table DDL under a stable key so restore finds it after flush/restart.
+  let ddl = serializeTableDdl(tbl)
+  ctx.db.put(tableSchemaKey(tbl.name), cast[seq[byte]](ddl))
+
+proc dropTableSchema*(ctx: ExecutionContext, tableName: string) =
+  ctx.db.delete(tableSchemaKey(tableName))
+
+proc applyCreateTableStmt*(ctx: ExecutionContext, stmt: Node) =
+  ## Materialize CREATE TABLE AST into ctx.tables + empty secondary indexes.
+  var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[],
+                     foreignKeys: @[], checks: @[], triggers: @[])
+  for col in stmt.crtColumns:
+    if col.kind == nkColumnDef:
+      var colDef = ColumnDef(name: col.cdName, colType: col.cdType)
+      colDef.autoIncrement = col.cdAutoIncrement
+      for cst in col.cdConstraints:
+        if cst.kind == nkConstraintDef:
+          case cst.cstType
+          of "pkey":
+            colDef.isPk = true
+            if col.cdName notin tbl.pkColumns:
+              tbl.pkColumns.add(col.cdName)
+            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
+          of "notnull": colDef.isNotNull = true
+          of "unique":
+            colDef.isUnique = true
+            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
+          of "default":
+            if cst.cstDefault != nil:
+              colDef.defaultVal = litToString(cst.cstDefault)
+          of "fkey":
+            colDef.fkTable = cst.cstRefTable
+            colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: ""
+            colDef.fkOnDelete = cst.cstOnDelete
+            colDef.fkOnUpdate = cst.cstOnUpdate
+          else: discard
+      tbl.columns.add(colDef)
+  # Table-level constraints
+  for cstNode in stmt.crtConstraints:
+    if cstNode.kind == nkConstraintDef:
+      if cstNode.cstType == "pkey":
+        for c in cstNode.cstColumns:
+          if c notin tbl.pkColumns:
+            tbl.pkColumns.add(c)
+          for i, col in tbl.columns:
+            if col.name == c:
+              tbl.columns[i].isPk = true
+              let idxName = stmt.crtName & "." & c
+              if idxName notin ctx.btrees:
+                ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
+      elif cstNode.cstType == "fkey":
+        tbl.foreignKeys.add(ForeignKeyDef(
+          refTable: cstNode.cstRefTable,
+          refColumn: if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: "",
+          onDelete: cstNode.cstOnDelete,
+          onUpdate: cstNode.cstOnUpdate))
+        if cstNode.cstColumns.len > 0:
+          for i, c in tbl.columns:
+            if c.name in cstNode.cstColumns:
+              tbl.columns[i].fkTable = cstNode.cstRefTable
+              tbl.columns[i].fkColumn = if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: ""
+              tbl.columns[i].fkOnDelete = cstNode.cstOnDelete
+              tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate
+      elif cstNode.cstType == "check":
+        tbl.checks.add(CheckDef(name: "check_" & $tbl.checks.len, checkNode: cstNode.cstCheck))
+  ctx.tables[stmt.crtName] = tbl
+
+proc rebuildSecondaryIndexes*(ctx: ExecutionContext) =
+  ## Rebuild in-memory B-Tree indexes from durable row data after schema restore.
+  for tableName, tbl in ctx.tables.pairs:
+    for col in tbl.columns:
+      if col.isPk or col.isUnique:
+        let idxName = tableName & "." & col.name
+        if idxName notin ctx.btrees:
+          ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
+    let prefix = tableName & "."
+    for (key, value) in ctx.db.scanAll():
+      if not key.startsWith(prefix): continue
+      if key.startsWith("_schema:"): continue
+      let valStr = cast[string](value)
+      let rest = key[prefix.len..^1]
+      var colVals = initTable[string, string]()
+      let eqPos = rest.find('=')
+      if eqPos >= 0 and ':' notin rest:
+        colVals[rest[0..<eqPos]] = rest[eqPos+1..^1]
+      else:
+        for part in rest.split(':'):
+          let p = part.find('=')
+          if p >= 0:
+            colVals[part[0..<p]] = part[p+1..^1]
+      for k, v in parseRowData(valStr):
+        colVals[k] = v
+      for colName in ctx.btrees.keys.toSeq():
+        if not colName.startsWith(prefix): continue
+        let colsPart = colName[tableName.len + 1..^1]
+        let idxCols = colsPart.split(".")
+        var parts: seq[string] = @[]
+        for c in idxCols:
+          parts.add(colVals.getOrDefault(c, ""))
+        let idxVal = parts.join("|")
+        if idxVal.len > 0 and not isNull(idxVal):
+          ctx.btrees[colName].insert(idxVal, IndexEntry(lsmKey: key, rowValue: valStr))
+
+proc restoreSchema*(ctx: ExecutionContext) =
+  ## Load durable schema from LSM (memtable + SSTables). Stable keys only.
+  var tableDdls: seq[string] = @[]
+  var otherDdls: seq[string] = @[]
+
+  for (key, value) in ctx.db.scanAll():
+    if not key.startsWith("_schema:"): continue
+    let ddl = cast[string](value)
+    if ddl.len == 0: continue
+    if key.startsWith(SchemaTablePrefix):
+      tableDdls.add(ddl)
+    elif key.startsWith(SchemaLegacyCreatePrefix) and ddl.toUpperAscii().startsWith("CREATE TABLE"):
+      tableDdls.add(ddl)
+    elif key.startsWith(SchemaViewPrefix) or key.startsWith(SchemaTriggerPrefix) or
+         key.startsWith(SchemaUserPrefix) or key.startsWith(SchemaPolicyPrefix):
+      otherDdls.add(ddl)
+    elif ddl.toUpperAscii().startsWith("CREATE VIEW") or
+         ddl.toUpperAscii().startsWith("CREATE TRIGGER") or
+         ddl.toUpperAscii().startsWith("CREATE USER") or
+         ddl.toUpperAscii().startsWith("CREATE POLICY"):
+      otherDdls.add(ddl)
+
+  for ddl in tableDdls:
+    try:
+      let tokens = qlex.tokenize(ddl)
+      let astNode = qpar.parse(tokens)
+      if astNode.stmts.len > 0 and astNode.stmts[0].kind == nkCreateTable:
+        applyCreateTableStmt(ctx, astNode.stmts[0])
+        if astNode.stmts[0].crtName in ctx.tables:
+          persistTableSchema(ctx, ctx.tables[astNode.stmts[0].crtName])
+    except CatchableError:
+      continue
+
+  for ddl in otherDdls:
+    var astNode: Node
+    try:
+      let tokens = qlex.tokenize(ddl)
+      astNode = qpar.parse(tokens)
+    except CatchableError:
+      continue
+    if astNode.stmts.len == 0: continue
+    let stmt = astNode.stmts[0]
+    case stmt.kind
+    of nkCreateView:
+      ctx.views[stmt.cvName] = stmt.cvQuery
+    of nkCreateTrigger:
+      if stmt.trigTable in ctx.tables:
+        ctx.tables[stmt.trigTable].triggers.add(TriggerDef(
+          name: stmt.trigName,
+          timing: stmt.trigTiming,
+          event: stmt.trigEvent,
+          action: stmt.trigAction,
+        ))
+    of nkCreateUser:
+      ctx.users[stmt.cuName] = UserDef(name: stmt.cuName,
+          passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[])
+    of nkCreatePolicy:
+      var pols = ctx.policies.getOrDefault(stmt.cpTable)
+      pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable,
+                         command: stmt.cpCommand, usingExpr: stmt.cpUsing,
+                         withCheckExpr: stmt.cpWithCheck))
+      ctx.policies[stmt.cpTable] = pols
+    else: discard
+
+  rebuildSecondaryIndexes(ctx)

src/barabadb/query/exec/types.nim

@@ -0,0 +1,143 @@
+## Executor types — shared by all exec/* modules and executor.nim
+import std/tables
+import std/locks
+import ../ast
+import ../ir
+import ../../core/types
+import ../../storage/lsm
+import ../../storage/btree
+import ../../core/mvcc
+import ../../fts/engine as fts
+import ../../vector/engine as vengine
+import ../../graph/engine as gengine
+import ../../ai/embed as embedmod
+import ../../ai/llm as llmmod
+import ../../core/registry
+
+type
+  IndexEntry* = ref object
+    lsmKey*: string
+    rowValue*: string
+
+  ChangeKind* = enum
+    ckInsert, ckUpdate, ckDelete
+
+  ChangeEvent* = object
+    table*: string
+    kind*: ChangeKind
+    key*: string
+    data*: string
+
+  UserDef* = object
+    name*: string
+    passwordHash*: string
+    isSuperuser*: bool
+    roles*: seq[string]
+
+  PrivilegeDef* = object
+    tableName*: string
+    command*: string  # SELECT, INSERT, UPDATE, DELETE, ALL
+
+  PolicyDef* = object
+    name*: string
+    tableName*: string
+    command*: string   # ALL, SELECT, INSERT, UPDATE, DELETE
+    usingExpr*: Node   # parsed USING expression
+    withCheckExpr*: Node  # parsed WITH CHECK expression
+
+  SharedLock* = ref object
+    lock*: Lock
+
+  ForeignKeyDef* = object
+    refTable*: string
+    refColumn*: string
+    onDelete*: string  # CASCADE, SET NULL, RESTRICT
+    onUpdate*: string  # CASCADE, SET NULL, RESTRICT
+
+  CheckDef* = object
+    name*: string
+    expr*: string  # stored expression string
+    checkNode*: Node  # AST for runtime evaluation
+
+  TriggerDef* = object
+    name*: string
+    timing*: string   # BEFORE, AFTER
+    event*: string    # INSERT, UPDATE, DELETE
+    action*: Node     # SQL statement AST
+
+  ColumnDef* = object
+    name*: string
+    colType*: string
+    isPk*: bool
+    isNotNull*: bool
+    isUnique*: bool
+    defaultVal*: string
+    fkTable*: string
+    fkColumn*: string
+    fkOnDelete*: string
+    fkOnUpdate*: string
+    autoIncrement*: bool
+
+  TableDef* = object
+    name*: string
+    columns*: seq[ColumnDef]
+    pkColumns*: seq[string]
+    foreignKeys*: seq[ForeignKeyDef]
+    checks*: seq[CheckDef]
+    triggers*: seq[TriggerDef]
+
+  Row* = Table[string, Value]
+
+  ExecutionContext* = ref object
+    db*: LSMTree
+    tables*: Table[string, TableDef]
+    btrees*: Table[string, BTreeIndex[string, IndexEntry]]
+    views*: Table[string, Node]  # view name -> SELECT AST
+    cteTables*: Table[string, seq[Row]]  # CTE name -> rows
+    ftsIndexes*: Table[string, fts.InvertedIndex]  # table.col -> FTS index
+    vectorIndexes*: Table[string, vengine.HNSWIndex]  # table.col -> HNSW index
+    graphs*: Table[string, gengine.Graph]  # graph name -> Graph object
+    embedder*: embedmod.Embedder  # optional embedding service client
+    llmClient*: llmmod.LLMClient  # optional LLM client for NL->SQL
+    txnManager*: TxnManager
+    pendingTxn*: Transaction
+    onChange*: proc(ev: ChangeEvent) {.closure.}
+    users*: Table[string, UserDef]
+    policies*: Table[string, seq[PolicyDef]]  # table name -> policies
+    currentUser*: string
+    currentRole*: string
+    sessionVars*: Table[string, string]
+    autoIncCounters*: Table[string, int64]
+    sequences*: Table[string, int64]
+    sharedLock*: SharedLock  # shared across cloned contexts
+    outerRow*: Table[string, string]  # outer query row for correlated subqueries
+    subqueryPlan*: IRPlan  # current subquery plan being evaluated
+    currentDatabase*: string  # name of the currently selected database
+    registry*: DatabaseRegistry  # nil for single-DB mode
+
+  MigrationRecord* = object
+    name*: string
+    checksum*: string
+    appliedAt*: int64
+    appliedBy*: string
+    durationMs*: int
+    rolledBack*: bool
+
+  ExecResult* = object
+    success*: bool
+    columns*: seq[string]
+    rows*: seq[Row]
+    affectedRows*: int
+    message*: string
+    keyValuePairs*: seq[(string, seq[byte])]
+
+proc `==`*(a, b: IndexEntry): bool =
+  a.lsmKey == b.lsmKey and a.rowValue == b.rowValue
+
+proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "",
+               kvPairs: seq[(string, seq[byte])] = @[]): ExecResult =
+  ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg,
+             keyValuePairs: kvPairs)
+
+proc errResult*(msg: string): ExecResult =
+  ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg)

src/barabadb/query/exec/values.nim

@@ -0,0 +1,119 @@
+## Value / row serialization helpers used across the executor
+import std/strutils
+import std/tables
+import std/json
+import ../../core/types
+import types
+
+proc isNull*(value: string): bool =
+  value == "\\N" or value.toLower() == "null"
+
+proc valueToString*(v: Value): string =
+  case v.kind
+  of vkNull: return "\\N"
+  of vkString: return v.strVal
+  of vkInt64: return $v.int64Val
+  of vkFloat64: return $v.float64Val
+  of vkBool: return $v.boolVal
+  else: return ""
+
+proc `%`*(v: Value): JsonNode =
+  case v.kind
+  of vkNull: return newJNull()
+  of vkString: return %v.strVal
+  of vkInt64: return %v.int64Val
+  of vkFloat64: return %v.float64Val
+  of vkBool: return %v.boolVal
+  else: return newJNull()
+
+proc toString*(v: Value): string = valueToString(v)
+
+proc `[]=`*(t: var Row, key: string, val: string) =
+  t[key] = Value(kind: vkString, strVal: val)
+
+proc escapeRowVal*(v: string): string =
+  v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
+
+proc unescapeRowVal*(v: string): string =
+  result = ""
+  var i = 0
+  while i < v.len:
+    if v[i] == '\\' and i + 1 < v.len:
+      case v[i+1]
+      of '\\', ',', '=':
+        result &= v[i+1]
+        i += 2
+        continue
+      else: discard
+    result &= v[i]
+    inc i
+
+proc parseRowData*(valStr: string): Table[string, string] =
+  ## Parse "col1=val1,col2=val2" into a table
+  result = initTable[string, string]()
+  var i = 0
+  var part = ""
+  while i < valStr.len:
+    if valStr[i] == '\\' and i + 1 < valStr.len:
+      part &= valStr[i]
+      part &= valStr[i+1]
+      i += 2
+      continue
+    if valStr[i] == ',':
+      let eqPos = part.find('=')
+      if eqPos >= 0:
+        let k = part[0..<eqPos].strip()
+        let v = unescapeRowVal(part[eqPos+1..^1].strip())
+        result[k] = v
+      part = ""
+    else:
+      part &= valStr[i]
+    inc i
+  if part.len > 0:
+    let eqPos = part.find('=')
+    if eqPos >= 0:
+      let k = part[0..<eqPos].strip()
+      let v = unescapeRowVal(part[eqPos+1..^1].strip())
+      result[k] = v
+
+proc parseRowDataToValueRow*(valStr: string): Row =
+  result = initTable[string, Value]()
+  for k, v in parseRowData(valStr):
+    result[k] = v
+
+proc sqlEscapeIdent*(ident: string): string =
+  ## Escape SQL identifiers by doubling double-quotes.
+  result = ident.replace("\"", "\"\"")
+
+proc sqlEscapeString*(s: string): string =
+  ## Escape SQL string literals by doubling single-quotes.
+  result = s.replace("'", "''")
+
+proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
+  ## Build a multi-row INSERT statement for bulk import.
+  result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
+  for i, col in columns:
+    if i > 0: result &= ", "
+    result &= "\"" & sqlEscapeIdent(col) & "\""
+  result &= ") VALUES "
+  for ri, row in rows:
+    if ri > 0: result &= ", "
+    result &= "("
+    for ci, val in row:
+      if ci > 0: result &= ", "
+      if val.len == 0 or val == "\\N":
+        result &= "NULL"
+      else:
+        result &= "'" & sqlEscapeString(val) & "'"
+    result &= ")"
+
+proc getValue*(values: seq[string], fields: seq[string], colName: string): string =
+  for i, f in fields:
+    if f.toLower() == colName.toLower():
+      if i < values.len: return values[i]
+      return "\\N"
+  return "\\N"
+
+proc getTableDef*(ctx: ExecutionContext, tableName: string): TableDef =
+  if tableName in ctx.tables: return ctx.tables[tableName]
+  return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[])