feat: harden storage, schema persistence, fair benches, fix wire crash

Core storage: hash MemTable, WAL group commit, L0 compaction rebuild, reader-writer lock, and a global StorageGate so HTTP workers and TCP share the LSM safely under multi-thread access. Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM restore on open. Executor types/values/schema split into query/exec/. Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG) and honesty docs for mixed-tier comparisons.
2026-07-18 16:55:50 +03:00
parent aa4ab11210
commit 8db5cfe7e1
31 changed files with 3131 additions and 797 deletions
@@ -51,6 +51,18 @@ tests/nimforum_smoke_test
 benchmark_results.json
 pg_benchmark_results.json
 fair_benchmark_results.json
 benchmarks/bench_all
 benchmarks/compare
 .qwen/
 # Compiled test / module binaries
 tests/test_schema_persist
 tests/test_storage_hardening
 tests/test_wire_insert_stress
 src/barabadb/storage/lsm
 src/barabadb/storage/wal
 src/barabadb/storage/btree
 src/barabadb/storage/gate
 clients/nim/tests/test_pool
 clients/nim/tests/test_wire
@@ -4,6 +4,36 @@ All notable changes to BaraDB are documented in this file.
 ## [1.2.0] — Unreleased
 ### Core Storage Hardening
 Foundational LSM improvements for write performance, durability, and compaction correctness.
 - **Hash-table MemTable** (`storage/lsm.nim`) — O(1) put/get instead of O(n) sorted-seq insert; sort only on flush to SSTable
 - **Timestamp-aware MemTable overwrite** — WAL recovery keeps newest version per key
 - **WAL rewrite after flush** (`storage/wal.nim`) — `truncate` / `rewriteLive` so recovery is O(unflushed) not O(history); fsync on truncate/rewrite/close
 - **WAL group commit** — `WalSyncMode`: `none` | `group` (default) | `every`; fsync every N entries and/or every interval ms
 - **Config knobs** — `wal_sync_mode`, `wal_group_every`, `wal_sync_interval_ms` / env `BARADB_WAL_*`; registry opens DBs with config
 - **L0 file-count compaction trigger** — RocksDB-style `L0CompactionTrigger` (default 4) instead of size-only for overlapping L0
 - **`rebuildFromLSM` for compaction** (`storage/compaction.nim`) — strategy always syncs from live SSTable catalog (no drift after flush)
 - **Safe mmap close after compaction** — release handles for compacted inputs after unlink
 - **Recovery flag** — flush during WAL replay does not rewrite the open WAL file mid-read
 - **Fair WAL micro-bench** — `benchWalDurabilityModes` compares none/group/every on the same workload
 - **RwLock for LSM** (`storage/rwlock.nim`) — writer-preferring reader-writer lock; default API uses exclusive mode
 - **Deep-copy on get** — returned values are copied so callers never share seq buffers with the store
 - **`-d:baraConcurrentReads`** — opt-in shared read locks (unsafe with default ORC across OS threads)
 - **`scanRange(start, end)`** — inclusive multi-level range scan (memtable + SSTables)
 - **Focused tests** — `tests/test_storage_hardening.nim` (RwLock, WAL group, scanRange, stress)
 - **Note:** Nim ORC must not share GC'd `LSMTree` refs across OS threads without isolation
 - **StorageGate** (`storage/gate.nim`) — global exclusive lock serializing HTTP (Hunos workers), TCP, compaction, Raft apply
 - **HTTP stop no longer closes shared registry** — ownership stays with main; `stop(closeStorage=true)` for standalone HTTP
 - **Schema persistence** — stable keys `_schema:tables:<name>`; restore from full LSM (`scanAll`), not only memtable; DROP/ALTER update durable catalog; secondary index rebuild on open
 - **Schema tests** — `tests/test_schema_persist.nim` (create/flush/reopen, drop, alter, multi-table)
 - **Executor split** — `query/exec/{types,values,schema}.nim`; `executor.nim` re-exports for API stability; see `query/exec/README.md`
 - **Fair benchmarks** — `benchmarks/fair_bench.py` multi-tier (embedded SQLite↔LSM; client-server HTTP/wire↔PG); batch multi-row INSERT; wire protocol via Python client; `generate_report.py --fair`; `nimble bench_fair`
 - **Fix wire INSERT SIGSEGV** — ORC cycle collector crash under async TCP load; switch default MM to `--mm:arc` in `nim.cfg`; regression `tests/test_wire_insert_stress.nim`
 - **Honest bench docs** — tier warnings in `bench_all`, `pg_bench`, `compare.nim`; `benchmarks/README.md`
 - **Regression suite** — `Core Storage Hardening` tests in `tests/test_all.nim`
 ### Search Module (new)
 A unified search module combining vector similarity, full-text, and structured
@@ -19,17 +19,24 @@ task build_debug, "Build debug version":
  exec "nim c --debugger:native --linedir:on -o:build/baramcp src/baramcp.nim"
 task build_release, "Build release version":
  # mm:arc comes from nim.cfg (ORC crashes under wire INSERT load)
  exec "nim c -d:release --opt:speed -o:build/baradadb src/baradadb.nim"
  exec "nim c -d:release --opt:speed -o:build/baramcp src/baramcp.nim"
 task test, "Run all tests":
  exec "nim c -r tests/test_all.nim"
-task bench, "Run benchmarks":
+task bench, "Run embedded micro-benchmarks (in-process)":
  exec "nim c -d:release -r benchmarks/bench_all.nim"
-task bench_pg, "Run PostgreSQL comparison benchmarks":
+task bench_pg, "Run PostgreSQL client-server micro-benchmarks":
  exec "python3 benchmarks/pg_bench.py"
-task bench_report, "Generate benchmark comparison report":
+task bench_fair, "Fair multi-tier benches (SQLite embedded + optional PG/HTTP)":
  exec "python3 benchmarks/fair_bench.py"
 task bench_report, "Generate fair comparison report (needs fair_bench first)":
  exec "python3 benchmarks/generate_report.py --fair"
 task bench_report_legacy, "Legacy mixed-tier report (PG C/S vs BaraDB embedded — unfair)":
  exec "python3 benchmarks/generate_report.py"
@@ -0,0 +1,93 @@
 # Fair Benchmark Results
 Generated: **2026-07-18 13:53:46 UTC**
 ## Methodology
 - Tier `embedded`: in-process only (BaraDB LSM from nimble bench JSON; SQLite via Python sqlite3).
 - Tier `client_server`: network SQL (BaraDB HTTP /query; BaraDB binary wire TCP; PostgreSQL via psycopg2).
 - `sql_insert_row`: one INSERT statement per row (chatty).
 - `sql_insert_batch`: multi-row INSERT with batch size 50 (same SQL style across systems).
 - PostgreSQL: synchronous_commit=on|off; SQLite: PRAGMA synchronous FULL|OFF.
 - BaraDB WAL modes appear only if you ran benchmarks/bench_all.nim (WAL-* rows).
 - Never claim 'Nx faster than Postgres' using embedded BaraDB numbers.
 **Do not compare numbers across tiers.** Embedded storage is not the same
 workload as client-server SQL over the network.
 ## Tier: `embedded`
 | Bench | System | ops/s | seconds | n | notes |
 |-------|--------|------:|--------:|--:|-------|
 | kv_write | `baradb_lsm_embedded` | 41.50K | 2.410 | 100000 | benchmark_results.json |
 | kv_read | `baradb_lsm_embedded` | 3.77M | 0.026 | 100000 | benchmark_results.json |
 | wal_none | `baradb_lsm_embedded` | 232.65K | 0.215 | 50000 | benchmark_results.json |
 | wal_group64 | `baradb_lsm_embedded` | 42.25K | 1.183 | 50000 | benchmark_results.json |
 | wal_group256 | `baradb_lsm_embedded` | 107.16K | 0.467 | 50000 | benchmark_results.json |
 | wal_every | `baradb_lsm_embedded` | 825.27 | 60.586 | 50000 | benchmark_results.json |
 | kv_write | `sqlite_off` | 402.61K | 0.002 | 1000 | PRAGMA synchronous=OFF |
 | kv_read | `sqlite_off` | 195.60K | 0.005 | 1000 |  |
 | sql_insert_batch | `sqlite_off` | 559.66K | 0.002 | 1000 | multi-row INSERT batch=50, sync=OFF |
 | kv_write | `sqlite_full` | 272.62K | 0.004 | 1000 | PRAGMA synchronous=FULL |
 | kv_read | `sqlite_full` | 196.16K | 0.005 | 1000 |  |
 | sql_insert_batch | `sqlite_full` | 370.18K | 0.003 | 1000 | multi-row INSERT batch=50, sync=FULL |
 ### Same-bench ratios (`embedded`)
 **kv_read** (fastest: `baradb_lsm_embedded` @ 3.77M/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `baradb_lsm_embedded` | 1.00x |
 | `sqlite_full` | 0.05x |
 | `sqlite_off` | 0.05x |
 **kv_write** (fastest: `sqlite_off` @ 402.61K/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `sqlite_off` | 1.00x |
 | `sqlite_full` | 0.68x |
 | `baradb_lsm_embedded` | 0.10x |
 **sql_insert_batch** (fastest: `sqlite_off` @ 559.66K/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `sqlite_off` | 1.00x |
 | `sqlite_full` | 0.66x |
 ## Tier: `client_server`
 | Bench | System | ops/s | seconds | n | notes |
 |-------|--------|------:|--------:|--:|-------|
 | sql_insert_row | `baradb_http` | 979.31 | 0.511 | 500 |  |
 | sql_select_row | `baradb_http` | 235.20 | 2.126 | 500 |  |
 | sql_insert_batch | `baradb_http` | 10.19K | 0.049 | 500 | multi-row INSERT batch=50 |
 | sql_insert_row | `baradb_wire` | 4.65K | 0.108 | 500 | binary wire protocol |
 | sql_select_row | `baradb_wire` | 642.04 | 0.779 | 500 |  |
 | sql_insert_batch | `baradb_wire` | 20.47K | 0.024 | 500 | multi-row INSERT batch=50 |
 ### Same-bench ratios (`client_server`)
 **sql_insert_batch** (fastest: `baradb_wire` @ 20.47K/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `baradb_wire` | 1.00x |
 | `baradb_http` | 0.50x |
 **sql_insert_row** (fastest: `baradb_wire` @ 4.65K/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `baradb_wire` | 1.00x |
 | `baradb_http` | 0.21x |
 **sql_select_row** (fastest: `baradb_wire` @ 642.04/s)
 | System | Relative to fastest |
 |--------|--------------------:|
 | `baradb_wire` | 1.00x |
 | `baradb_http` | 0.37x |
@@ -0,0 +1,80 @@
 # BaraDB Benchmarks
 ## Tiers (read this first)
 | Tier | What is measured | Fair peers |
 |------|------------------|------------|
 | **embedded** | In-process storage API | BaraDB LSM ↔ SQLite |
 | **client_server** | Network + query protocol | BaraDB HTTP / **wire** ↔ PostgreSQL |
 **Never** quote “BaraDB is Nx faster than Postgres” using embedded LSM numbers.
 That comparison mixes tiers and is meaningless as a product claim.
 ## Quick start
 ```bash
 # 1) Embedded micro-benches (Nim, in-process)
 nimble bench
 # or: nim c -d:release -r benchmarks/bench_all.nim
 # 2) Optional: start server for client_server tier (HTTP + wire)
 ./build/baradadb
 # 3) Fair multi-tier suite (Python)
 #    - always: SQLite embedded (+ batch)
 #    - optional: BaraDB HTTP (:9912), wire TCP (:9472), PostgreSQL
 python3 benchmarks/fair_bench.py
 # 3) Markdown report
 nimble bench_report
 # or: python3 benchmarks/generate_report.py --fair
 ```
 Outputs:
 - `benchmark_results.json` — Nim embedded suite  
 - `fair_benchmark_results.json` — multi-tier fair suite  
 - `benchmarks/FAIR_COMPARISON.md` — human-readable fair report  
 - `pg_benchmark_results.json` — optional PG-only micro suite  
 ## Environment
 | Variable | Default | Meaning |
 |----------|---------|---------|
 | `FAIR_N_KV` | 20000 | embedded KV ops |
 | `FAIR_N_SQL` | 5000 | SQL loops (HTTP/wire/PG) |
 | `FAIR_BATCH` | 100 | multi-row INSERT batch size |
 | `BARADB_HTTP_HOST` | 127.0.0.1 | HTTP host |
 | `BARADB_HTTP_PORT` | 9912 | HTTP port (`TCP+440`) |
 | `BARADB_WIRE_HOST` | 127.0.0.1 | wire protocol host |
 | `BARADB_WIRE_PORT` | 9472 | wire protocol TCP port |
 | `FAIR_SKIP_HTTP=1` | — | skip BaraDB HTTP |
 | `FAIR_SKIP_WIRE=1` | — | skip BaraDB wire |
 | `FAIR_SKIP_PG=1` | — | skip PostgreSQL |
 | `PGHOST` / `PGUSER` / `PGPASSWORD` / … | — | libpq-style |
 ## Files
 | File | Role |
 |------|------|
 | `bench_all.nim` | Embedded: LSM, WAL modes, BTree, vector, FTS, graph |
 | `fair_bench.py` | Fair multi-tier runner + markdown |
 | `pg_bench.py` | PostgreSQL client-server micro suite |
 | `generate_report.py` | `--fair` report; legacy mixed report without flag |
 | `compare.nim` | **Synthetic** — do not publish |
 | `search_bench.nim` | Search-focused micro suite |
 ## Durability knobs
 - BaraDB: `wal_sync_mode` = `none` \| `group` \| `every` (see WAL-* rows from `bench_all`)  
 - SQLite: `PRAGMA synchronous = OFF` vs `FULL`  
 - PostgreSQL: `synchronous_commit = off` vs `on`  
 Match durability stories when claiming write speedups.
 ## Wire protocol note
 The Python wire client (`clients/python`) is exercised by `fair_bench.py`.
 Builds use **`--mm:arc`** (see `nim.cfg`) because Nim **ORC** cycle collection
 crashed under async wire INSERT load (`markGray` SIGSEGV). With ARC, sequential
 wire INSERTs + batch multi-row INSERT are stable.
@@ -1,38 +1,29 @@
-# BaraDB vs PostgreSQL — Real Benchmark Results
+# Legacy mixed-tier comparison
-Generated from actual execution on:
+This file used to claim large “speedups” of BaraDB over PostgreSQL by comparing:
 - **CPU:** AMD Ryzen 9 5900X
 - **PostgreSQL:** 15.18 (local)
 - **BaraDB:** git `42043f3`
-## Methodology
+- **PostgreSQL:** client-server (psycopg2, network, SQL)
 - **BaraDB:** in-process LSM (no network, no SQL)
- PostgreSQL: single-row INSERT/SELECT via psycopg2 (client-server overhead included)
+That is **not a fair product comparison**.
 - BaraDB: in-process Nim code (no network overhead)
 - Same dataset sizes for both systems
-## Results
+## Use the fair suite instead
-| Test | PostgreSQL | BaraDB | Speedup |
+```bash
-|------|-----------|--------|---------|
+nim c -d:release -r benchmarks/bench_all.nim   # embedded BaraDB
-| KV Write (100K) | 16.82K/s (5.946s) | 32.23K/s (3.103s) | 1.9x (BaraDB) |
+python3 benchmarks/fair_bench.py               # SQLite + optional PG/HTTP
-| KV Read (100K) | 15.08K/s (6.630s) | 3.95M/s (25.3ms) | 261.9x (BaraDB) |
+# report → benchmarks/FAIR_COMPARISON.md
-| BTree Insert (100K) | 17.66K/s (5.664s) | 2.52M/s (39.7ms) | 142.8x (BaraDB) |
+```
 | BTree Get (100K) | 14.50K/s (6.899s) | 2.34M/s (42.7ms) | 161.4x (BaraDB) |
 | BTree Scan (1K ranges) | 2.39K/s (419.2ms) | 11.03M/s (1.0ms) | 4623.3x (BaraDB) |
 | FTS Index (10K docs) | 17.98K/s (556.3ms) | 119.99K/s (83.3ms) | 6.7x (BaraDB) |
 | FTS Search (1K queries) | 784.12/s (1.275s) | 1.36K/s (734.0ms) | 1.7x (BaraDB) |
-## Summary
+See:
- **Total PostgreSQL time:** 27.389s
+- [`FAIR_COMPARISON.md`](FAIR_COMPARISON.md) — latest multi-tier results  
- **Total BaraDB time:** 4.029s
+- [`README.md`](README.md) — methodology and env vars  
 - **Overall speedup:** BaraDB is **6.8x faster**
-## Notes
+## If you regenerate the legacy report
- PostgreSQL includes network round-trip and SQL parsing overhead per operation.
+```bash
- BaraDB runs in-process with zero serialization/network cost.
+python3 benchmarks/generate_report.py   # without --fair
- For embedded/single-node use cases, BaraDB shows significant advantage.
+```
- BaraDB now outperforms PostgreSQL on all tested metrics including FTS search after optimizations.
+
- PostgreSQL excels at durability, replication, and complex ACID transactions.
+It will rewrite this file with an explicit **mixed tiers** warning banner.
@@ -111,9 +111,11 @@ proc formatOps(ops: int, secs: float64): string =
 proc benchLSMTree() =
  echo "=== LSM-Tree Storage ==="
  echo "  Note: in-process embedded API (no network/SQL). Not comparable to client-server DBs."
  let benchDir = getTempDir() / "baradb_bench_lsm"
  removeDir(benchDir)
-  var db = newLSMTree(benchDir)
+  # Default group-commit WAL (production default)
  var db = newLSMTree(benchDir, walSyncMode = wsmGroup, walGroupEvery = 64)
  # Write benchmark
  let n = 100_000
@@ -124,6 +126,7 @@ proc benchLSMTree() =
  let writeLabel = "LSM-Write"
  recordResult(writeLabel, n, writeTime)
  echo "  Write ", n, " keys: ", writeTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, writeTime), ")", compareResult(writeLabel, currentResults[^1].opsPerSec, previousResults)
  echo "    fsyncs: ", db.wal.fsyncCount, " (group every 64)"
  # Read benchmark
  let readStart = getMonoTime()
@@ -138,6 +141,36 @@ proc benchLSMTree() =
  db.close()
 proc benchWalDurabilityModes() =
  ## Fair comparison of WAL durability policies on the same workload.
  echo "=== WAL Durability Modes (fair micro-bench) ==="
  echo "  Same N puts, same memtable size; only sync policy differs."
  let n = 50_000
  let modes = [
    (wsmNone, "none", 0),
    (wsmGroup, "group64", 64),
    (wsmGroup, "group256", 256),
    (wsmEvery, "every", 1),
  ]
  for (mode, label, ge) in modes:
    let dir = getTempDir() / ("baradb_bench_wal_" & label)
    removeDir(dir)
    var db = newLSMTree(dir, memMaxSize = 64 * 1024 * 1024,
                        walSyncMode = mode, walGroupEvery = max(1, ge))
    let t0 = getMonoTime()
    for i in 0..<n:
      db.put("k" & $i, cast[seq[byte]]("v" & $i))
    # Ensure pending group is durable before measuring end-to-end
    db.wal.sync()
    let secs = elapsed(t0)
    let name = "WAL-" & label
    recordResult(name, n, secs)
    echo "  ", label, ": ", secs.formatFloat(ffDecimal, 3), "s (",
         formatOps(n, secs), "), fsyncs=", db.wal.fsyncCount,
         compareResult(name, currentResults[^1].opsPerSec, previousResults)
    db.close()
    removeDir(dir)
 proc benchBTree() =
  echo "=== B-Tree Index ==="
  var btree = newBTreeIndex[string, string]()
@@ -331,11 +364,17 @@ proc benchGraph() =
 proc main() =
  echo ""
  echo "╔══════════════════════════════════════════════════╗"
-  echo "║         BaraDB Performance Benchmarks            ║"
+  echo "║     BaraDB Performance Benchmarks (EMBEDDED)     ║"
  echo "╚══════════════════════════════════════════════════╝"
  echo ""
  echo "Tier: embedded / in-process (no network, no wire SQL)."
  echo "For fair multi-tier numbers (SQLite / PG / HTTP):"
  echo "  python3 benchmarks/fair_bench.py"
  echo ""
  benchLSMTree()
  echo ""
  benchWalDurabilityModes()
  echo ""
  benchBTree()
  echo ""
  benchVectorSearch()
@@ -355,6 +394,7 @@ proc main() =
  )
  saveResults(ResultsFile, report)
  echo "Results saved to ", ResultsFile
  echo "Next: python3 benchmarks/fair_bench.py"
  echo ""
 when isMainModule:
@@ -1,4 +1,11 @@
 ## Comparative Benchmarks — BaraDB vs PostgreSQL, Redis, MongoDB
 ##
 ## ⚠️  SYNTHETIC / PLACEHOLDER: several refTimeSec values are *invented*
 ## multipliers, not measured. Do not publish these as real comparisons.
 ## Use instead:
 ##   nim c -d:release -r benchmarks/bench_all.nim
 ##   python3 benchmarks/fair_bench.py
 ##   python3 benchmarks/generate_report.py --fair
 import std/times
 import std/random
 import std/strutils
@@ -0,0 +1,799 @@
 #!/usr/bin/env python3
 """
 Fair multi-tier benchmarks for BaraDB.
 Tiers (never mix across tiers in a single "speedup" claim):
  1. embedded     — in-process storage (BaraDB LSM from JSON, SQLite)
  2. client_server — network + SQL
       • BaraDB HTTP REST
       • BaraDB wire protocol (Python async client, TCP 9472)
       • PostgreSQL (psycopg2)
  Within each tier we measure both **row-at-a-time** and **batch multi-row INSERT**.
 Usage:
  # 1) optional: run BaraDB embedded micro-benches first
  nim c -d:release -r benchmarks/bench_all.nim
  # 2) start server for HTTP/wire tiers (optional)
  ./build/baradadb
  # 3) fair suite
  python3 benchmarks/fair_bench.py
  # 4) markdown report
  python3 benchmarks/generate_report.py --fair
 Env:
  BARADB_HTTP_HOST   default 127.0.0.1
  BARADB_HTTP_PORT   default 9912  (TCP 9472 + 440)
  BARADB_WIRE_HOST   default 127.0.0.1
  BARADB_WIRE_PORT   default 9472
  PGHOST / PGPORT / PGDATABASE / PGUSER / PGPASSWORD
  FAIR_N_KV          default 20000
  FAIR_N_SQL         default 5000
  FAIR_BATCH         default 100   (rows per multi-row INSERT)
  FAIR_SKIP_PG=1     skip PostgreSQL
  FAIR_SKIP_HTTP=1   skip BaraDB HTTP
  FAIR_SKIP_WIRE=1   skip BaraDB wire protocol
 """
 from __future__ import annotations
 import asyncio
 import json
 import os
 import sqlite3
 import sys
 import tempfile
 import time
 import urllib.error
 import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 ROOT = Path(__file__).resolve().parent.parent
 OUT_JSON = ROOT / "fair_benchmark_results.json"
 BARA_JSON = ROOT / "benchmark_results.json"
 CLIENTS_PY = ROOT / "clients" / "python"
 N_KV = int(os.environ.get("FAIR_N_KV", "20000"))
 N_SQL = int(os.environ.get("FAIR_N_SQL", "5000"))
 BATCH = int(os.environ.get("FAIR_BATCH", "100"))
 HTTP_HOST = os.environ.get("BARADB_HTTP_HOST", "127.0.0.1")
 HTTP_PORT = int(os.environ.get("BARADB_HTTP_PORT", "9912"))
 WIRE_HOST = os.environ.get("BARADB_WIRE_HOST", "127.0.0.1")
 WIRE_PORT = int(os.environ.get("BARADB_WIRE_PORT", "9472"))
 def now_iso() -> str:
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
 def result(name: str, system: str, tier: str, ops: int, seconds: float, **extra):
    ops_s = ops / seconds if seconds > 0 else 0.0
    r = {
        "name": name,
        "system": system,
        "tier": tier,
        "ops": ops,
        "seconds": seconds,
        "opsPerSec": ops_s,
        "timestamp": now_iso(),
    }
    r.update(extra)
    return r
 def fmt_ops(x: float) -> str:
    if x >= 1_000_000:
        return f"{x/1_000_000:.2f}M"
    if x >= 1_000:
        return f"{x/1_000:.2f}K"
    return f"{x:.2f}"
 # ─── Tier 1: Embedded ───────────────────────────────────────────────
 def load_baradb_embedded() -> list[dict]:
    """Map bench_all.nim LSM results into fair embedded tier."""
    if not BARA_JSON.exists():
        print("  [skip] benchmark_results.json missing — run: nimble bench")
        return []
    data = json.loads(BARA_JSON.read_text())
    name_map = {
        "LSM-Write": "kv_write",
        "LSM-Read": "kv_read",
        "WAL-none": "wal_none",
        "WAL-group64": "wal_group64",
        "WAL-group256": "wal_group256",
        "WAL-every": "wal_every",
    }
    out = []
    for r in data.get("results", []):
        mapped = name_map.get(r.get("name"))
        if not mapped:
            continue
        out.append(
            result(
                mapped,
                "baradb_lsm_embedded",
                "embedded",
                r.get("ops", 0),
                r.get("seconds", 0.0),
                source="benchmark_results.json",
                gitSha=data.get("gitSha", ""),
            )
        )
    return out
 def multi_values_sql(start: int, count: int) -> str:
    """Build VALUES (...),(...),... for multi-row INSERT."""
    parts = [f"({i}, 'value_{i}')" for i in range(start, start + count)]
    return ",".join(parts)
 def bench_sqlite_embedded(n: int = N_KV) -> list[dict]:
    """SQLite in-process — fair peer for BaraDB embedded LSM."""
    out = []
    fd, path = tempfile.mkstemp(suffix=".db")
    os.close(fd)
    os.unlink(path)
    # --- durability: FULL (fsync) vs OFF ---
    for mode, label in (("OFF", "sqlite_off"), ("FULL", "sqlite_full")):
        if os.path.exists(path):
            os.unlink(path)
        conn = sqlite3.connect(path)
        cur = conn.cursor()
        cur.execute(f"PRAGMA synchronous = {mode}")
        cur.execute("PRAGMA journal_mode = WAL")
        cur.execute("CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT)")
        conn.commit()
        t0 = time.perf_counter()
        for i in range(n):
            cur.execute("INSERT INTO kv(k,v) VALUES(?,?)", (f"key_{i}", f"value_{i}"))
        conn.commit()
        w = time.perf_counter() - t0
        out.append(
            result(
                "kv_write",
                label,
                "embedded",
                n,
                w,
                durable=mode == "FULL",
                note=f"PRAGMA synchronous={mode}",
            )
        )
        t0 = time.perf_counter()
        found = 0
        for i in range(n):
            cur.execute("SELECT v FROM kv WHERE k=?", (f"key_{i}",))
            if cur.fetchone():
                found += 1
        r = time.perf_counter() - t0
        out.append(
            result(
                "kv_read",
                label,
                "embedded",
                n,
                r,
                found=found,
                durable=mode == "FULL",
            )
        )
        # Batch multi-row INSERT into SQL table (embedded SQL peer for batch)
        cur.execute("DROP TABLE IF EXISTS fair_batch")
        cur.execute("CREATE TABLE fair_batch (id INTEGER PRIMARY KEY, v TEXT)")
        conn.commit()
        t0 = time.perf_counter()
        for start in range(0, n, BATCH):
            cnt = min(BATCH, n - start)
            vals = multi_values_sql(start, cnt)
            cur.execute(f"INSERT INTO fair_batch (id, v) VALUES {vals}")
        conn.commit()
        bw = time.perf_counter() - t0
        out.append(
            result(
                "sql_insert_batch",
                label,
                "embedded",
                n,
                bw,
                batch=BATCH,
                durable=mode == "FULL",
                note=f"multi-row INSERT batch={BATCH}, sync={mode}",
            )
        )
        conn.close()
    if os.path.exists(path):
        os.unlink(path)
    return out
 # ─── Tier 2: Client / server ────────────────────────────────────────
 def bara_http_query(sql: str, host: str = HTTP_HOST, port: int = HTTP_PORT) -> dict:
    body = json.dumps({"query": sql}).encode()
    req = urllib.request.Request(
        f"http://{host}:{port}/query",
        data=body,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=60) as resp:
        return json.loads(resp.read().decode())
 def bara_http_available() -> bool:
    try:
        body = json.dumps({"query": "SELECT 1"}).encode()
        # health endpoint preferred
        req = urllib.request.Request(f"http://{HTTP_HOST}:{HTTP_PORT}/health", method="GET")
        with urllib.request.urlopen(req, timeout=2) as resp:
            return resp.status == 200
    except Exception:
        try:
            bara_http_query("SELECT 1")
            return True
        except Exception:
            return False
 def bench_baradb_http(n: int = N_SQL) -> list[dict]:
    if os.environ.get("FAIR_SKIP_HTTP") == "1":
        print("  [skip] FAIR_SKIP_HTTP=1")
        return []
    if not bara_http_available():
        print(
            f"  [skip] BaraDB HTTP not reachable at {HTTP_HOST}:{HTTP_PORT} "
            f"(start: ./build/baradadb)"
        )
        return []
    out = []
    ep = f"http://{HTTP_HOST}:{HTTP_PORT}/query"
    # BaraDB parser may not support IF EXISTS — ignore DROP failures
    try:
        bara_http_query("DROP TABLE fair_bench")
    except Exception:
        pass
    try:
        bara_http_query("CREATE TABLE fair_bench (id INT PRIMARY KEY, v TEXT)")
    except Exception as e:
        print(f"  [warn] setup query failed: {e}")
    # Row-at-a-time INSERT
    t0 = time.perf_counter()
    errors = 0
    for i in range(n):
        try:
            r = bara_http_query(
                f"INSERT INTO fair_bench (id, v) VALUES ({i}, 'value_{i}')"
            )
            if isinstance(r, dict) and r.get("error"):
                errors += 1
        except Exception:
            errors += 1
    w = time.perf_counter() - t0
    out.append(
        result(
            "sql_insert_row",
            "baradb_http",
            "client_server",
            n,
            w,
            errors=errors,
            endpoint=ep,
        )
    )
    # Point SELECT
    t0 = time.perf_counter()
    found = 0
    for i in range(n):
        try:
            r = bara_http_query(f"SELECT v FROM fair_bench WHERE id = {i}")
            rows = r.get("rows") if isinstance(r, dict) else None
            if rows:
                found += 1
        except Exception:
            pass
    rd = time.perf_counter() - t0
    out.append(
        result(
            "sql_select_row",
            "baradb_http",
            "client_server",
            n,
            rd,
            found=found,
            endpoint=ep,
        )
    )
    # Multi-row batch INSERT
    try:
        try:
            bara_http_query("DROP TABLE fair_batch")
        except Exception:
            pass
        bara_http_query("CREATE TABLE fair_batch (id INT PRIMARY KEY, v TEXT)")
    except Exception as e:
        print(f"  [warn] batch setup failed: {e}")
        return out
    t0 = time.perf_counter()
    berr = 0
    for start in range(0, n, BATCH):
        cnt = min(BATCH, n - start)
        sql = f"INSERT INTO fair_batch (id, v) VALUES {multi_values_sql(start, cnt)}"
        try:
            r = bara_http_query(sql)
            if isinstance(r, dict) and r.get("error"):
                berr += 1
        except Exception:
            berr += 1
    bw = time.perf_counter() - t0
    out.append(
        result(
            "sql_insert_batch",
            "baradb_http",
            "client_server",
            n,
            bw,
            batch=BATCH,
            errors=berr,
            endpoint=ep,
            note=f"multi-row INSERT batch={BATCH}",
        )
    )
    return out
 def _import_baradb_client():
    """Load clients/python baradb package without requiring install."""
    p = str(CLIENTS_PY)
    if p not in sys.path:
        sys.path.insert(0, p)
    from baradb import Client  # type: ignore
    return Client
 def bara_wire_available() -> bool:
    if os.environ.get("FAIR_SKIP_WIRE") == "1":
        return False
    try:
        Client = _import_baradb_client()
    except Exception as e:
        print(f"  [skip] wire client import failed: {e}")
        return False
    async def _ping():
        c = Client(WIRE_HOST, WIRE_PORT, timeout=2.0)
        try:
            await c.connect()
            await c.ping()
            await c.close()
            return True
        except Exception:
            try:
                await c.close()
            except Exception:
                pass
            return False
    try:
        return asyncio.run(_ping())
    except Exception:
        return False
 def bench_baradb_wire(n: int = N_SQL) -> list[dict]:
    """BaraDB binary wire protocol (TCP) — primary high-performance client path."""
    if os.environ.get("FAIR_SKIP_WIRE") == "1":
        print("  [skip] FAIR_SKIP_WIRE=1")
        return []
    try:
        Client = _import_baradb_client()
    except Exception as e:
        print(f"  [skip] wire client not available: {e}")
        return []
    if not bara_wire_available():
        print(
            f"  [skip] BaraDB wire not reachable at {WIRE_HOST}:{WIRE_PORT} "
            f"(start: ./build/baradadb)"
        )
        return []
    async def _run() -> list[dict]:
        out: list[dict] = []
        client = Client(WIRE_HOST, WIRE_PORT, timeout=60.0)
        await client.connect()
        try:
            try:
                await client.query("DROP TABLE fair_wire")
            except Exception:
                pass
            try:
                await client.query(
                    "CREATE TABLE fair_wire (id INT PRIMARY KEY, v TEXT)"
                )
            except Exception as e:
                print(f"  [warn] wire setup: {e}")
            ep = f"tcp://{WIRE_HOST}:{WIRE_PORT}"
            # Row-at-a-time INSERT (may crash older servers under load — record partial)
            t0 = time.perf_counter()
            errors = 0
            done = 0
            crashed = False
            for i in range(n):
                try:
                    await client.query(
                        f"INSERT INTO fair_wire (id, v) VALUES ({i}, 'value_{i}')"
                    )
                    done += 1
                except (ConnectionError, OSError, Exception) as e:
                    errors += 1
                    if "reset" in str(e).lower() or "closed" in str(e).lower():
                        crashed = True
                        print(f"  [warn] wire connection lost after {done} inserts: {e}")
                        break
            w = time.perf_counter() - t0
            if done > 0:
                out.append(
                    result(
                        "sql_insert_row",
                        "baradb_wire",
                        "client_server",
                        done,
                        w,
                        errors=errors,
                        requested=n,
                        endpoint=ep,
                        note="binary wire protocol"
                        + (" (partial — server disconnect)" if crashed else ""),
                    )
                )
            if crashed:
                return out
            # Point SELECT
            t0 = time.perf_counter()
            found = 0
            for i in range(done):
                try:
                    r = await client.query(
                        f"SELECT v FROM fair_wire WHERE id = {i}"
                    )
                    if r is not None and (
                        getattr(r, "row_count", 0) > 0 or getattr(r, "rows", None)
                    ):
                        found += 1
                except (ConnectionError, OSError, Exception) as e:
                    if "reset" in str(e).lower() or "closed" in str(e).lower():
                        crashed = True
                        print(f"  [warn] wire lost during SELECT: {e}")
                        break
            rd = time.perf_counter() - t0
            out.append(
                result(
                    "sql_select_row",
                    "baradb_wire",
                    "client_server",
                    max(done, 1),
                    rd,
                    found=found,
                    endpoint=ep,
                )
            )
            if crashed:
                return out
            # Batch multi-row INSERT
            try:
                try:
                    await client.query("DROP TABLE fair_wire_batch")
                except Exception:
                    pass
                await client.query(
                    "CREATE TABLE fair_wire_batch (id INT PRIMARY KEY, v TEXT)"
                )
            except Exception as e:
                print(f"  [warn] wire batch setup: {e}")
                return out
            t0 = time.perf_counter()
            berr = 0
            bdone = 0
            for start in range(0, n, BATCH):
                cnt = min(BATCH, n - start)
                sql = (
                    "INSERT INTO fair_wire_batch (id, v) VALUES "
                    + multi_values_sql(start, cnt)
                )
                try:
                    await client.query(sql)
                    bdone += cnt
                except (ConnectionError, OSError, Exception) as e:
                    berr += 1
                    if "reset" in str(e).lower() or "closed" in str(e).lower():
                        print(f"  [warn] wire lost during batch after {bdone} rows: {e}")
                        break
            bw = time.perf_counter() - t0
            if bdone > 0:
                out.append(
                    result(
                        "sql_insert_batch",
                        "baradb_wire",
                        "client_server",
                        bdone,
                        bw,
                        batch=BATCH,
                        errors=berr,
                        requested=n,
                        endpoint=ep,
                        note=f"multi-row INSERT batch={BATCH}",
                    )
                )
        finally:
            try:
                await client.close()
            except Exception:
                pass
        return out
    try:
        return asyncio.run(_run())
    except Exception as e:
        print(f"  [skip] wire bench failed: {e}")
        return []
 def bench_postgresql(n: int = N_SQL) -> list[dict]:
    if os.environ.get("FAIR_SKIP_PG") == "1":
        print("  [skip] FAIR_SKIP_PG=1")
        return []
    try:
        import psycopg2
    except ImportError:
        print("  [skip] psycopg2 not installed")
        return []
    cfg = {
        "host": os.environ.get("PGHOST", "localhost"),
        "port": int(os.environ.get("PGPORT", "5432")),
        "dbname": os.environ.get("PGDATABASE", "postgres"),
        "user": os.environ.get("PGUSER", "postgres"),
        "password": os.environ.get("PGPASSWORD", os.environ.get("PG_PASSWORD", "")),
    }
    if not cfg["password"] and os.environ.get("PGPASSWORD") is None:
        cfg["password"] = os.environ.get("BARA_PG_PASSWORD", "pas+123")
    out = []
    try:
        conn = psycopg2.connect(**cfg)
    except Exception as e:
        print(f"  [skip] PostgreSQL connect failed: {e}")
        return []
    cur = conn.cursor()
    for sync, label in (("on", "postgresql_sync_on"), ("off", "postgresql_sync_off")):
        cur.execute(f"SET synchronous_commit = {sync}")
        cur.execute("DROP TABLE IF EXISTS fair_bench")
        cur.execute("CREATE TABLE fair_bench (id INTEGER PRIMARY KEY, v TEXT)")
        conn.commit()
        t0 = time.perf_counter()
        for i in range(n):
            cur.execute(
                "INSERT INTO fair_bench (id, v) VALUES (%s, %s)",
                (i, f"value_{i}"),
            )
        conn.commit()
        w = time.perf_counter() - t0
        out.append(
            result(
                "sql_insert_row",
                label,
                "client_server",
                n,
                w,
                durable=sync == "on",
                note=f"synchronous_commit={sync}",
            )
        )
        t0 = time.perf_counter()
        found = 0
        for i in range(n):
            cur.execute("SELECT v FROM fair_bench WHERE id = %s", (i,))
            if cur.fetchone():
                found += 1
        rd = time.perf_counter() - t0
        out.append(
            result(
                "sql_select_row",
                label,
                "client_server",
                n,
                rd,
                found=found,
                durable=sync == "on",
            )
        )
        # Batch multi-row INSERT (same durability setting)
        cur.execute("DROP TABLE IF EXISTS fair_batch")
        cur.execute("CREATE TABLE fair_batch (id INTEGER PRIMARY KEY, v TEXT)")
        conn.commit()
        t0 = time.perf_counter()
        for start in range(0, n, BATCH):
            cnt = min(BATCH, n - start)
            cur.execute(
                f"INSERT INTO fair_batch (id, v) VALUES {multi_values_sql(start, cnt)}"
            )
        conn.commit()
        bw = time.perf_counter() - t0
        out.append(
            result(
                "sql_insert_batch",
                label,
                "client_server",
                n,
                bw,
                batch=BATCH,
                durable=sync == "on",
                note=f"multi-row INSERT batch={BATCH}, sync={sync}",
            )
        )
    cur.close()
    conn.close()
    return out
 # ─── Report ──────────────────────────────────────────────────────────
 def print_tier(name: str, rows: list[dict]):
    print(f"\n=== Tier: {name} ===")
    if not rows:
        print("  (no results)")
        return
    # group by bench name
    names = []
    for r in rows:
        if r["name"] not in names:
            names.append(r["name"])
    for nm in names:
        print(f"  [{nm}]")
        for r in rows:
            if r["name"] != nm:
                continue
            print(
                f"    {r['system']:28s}  {fmt_ops(r['opsPerSec']):>10s}/s  "
                f"({r['seconds']:.3f}s, n={r['ops']})"
            )
 def write_markdown(payload: dict, path: Path):
    lines = []
    lines.append("# Fair Benchmark Results")
    lines.append("")
    lines.append(f"Generated: **{payload.get('generated', '')}**")
    lines.append("")
    lines.append("## Methodology")
    lines.append("")
    for line in payload.get("methodology", []):
        lines.append(f"- {line}")
    lines.append("")
    lines.append("**Do not compare numbers across tiers.** Embedded storage is not the same")
    lines.append("workload as client-server SQL over the network.")
    lines.append("")
    for tier in ("embedded", "client_server"):
        rows = [r for r in payload.get("results", []) if r.get("tier") == tier]
        lines.append(f"## Tier: `{tier}`")
        lines.append("")
        if not rows:
            lines.append("_No results for this tier._")
            lines.append("")
            continue
        lines.append("| Bench | System | ops/s | seconds | n | notes |")
        lines.append("|-------|--------|------:|--------:|--:|-------|")
        for r in rows:
            note = r.get("note") or r.get("source") or ""
            lines.append(
                f"| {r['name']} | `{r['system']}` | {fmt_ops(r['opsPerSec'])} | "
                f"{r['seconds']:.3f} | {r['ops']} | {note} |"
            )
        lines.append("")
        # same-bench comparison within tier
        names = sorted({r["name"] for r in rows})
        lines.append(f"### Same-bench ratios (`{tier}`)")
        lines.append("")
        for nm in names:
            group = [r for r in rows if r["name"] == nm]
            if len(group) < 2:
                continue
            best = max(group, key=lambda x: x["opsPerSec"])
            lines.append(f"**{nm}** (fastest: `{best['system']}` @ {fmt_ops(best['opsPerSec'])}/s)")
            lines.append("")
            lines.append("| System | Relative to fastest |")
            lines.append("|--------|--------------------:|")
            for r in sorted(group, key=lambda x: -x["opsPerSec"]):
                rel = r["opsPerSec"] / best["opsPerSec"] if best["opsPerSec"] else 0
                lines.append(f"| `{r['system']}` | {rel:.2f}x |")
            lines.append("")
    path.write_text("\n".join(lines) + "\n")
    print(f"\nMarkdown written to {path}")
 def main():
    print("BaraDB Fair Benchmark Suite")
    print(f"  N_KV={N_KV}  N_SQL={N_SQL}  BATCH={BATCH}")
    print(f"  HTTP={HTTP_HOST}:{HTTP_PORT}  WIRE={WIRE_HOST}:{WIRE_PORT}")
    methodology = [
        "Tier `embedded`: in-process only (BaraDB LSM from nimble bench JSON; SQLite via Python sqlite3).",
        "Tier `client_server`: network SQL (BaraDB HTTP /query; BaraDB binary wire TCP; PostgreSQL via psycopg2).",
        "`sql_insert_row`: one INSERT statement per row (chatty).",
        f"`sql_insert_batch`: multi-row INSERT with batch size {BATCH} (same SQL style across systems).",
        "PostgreSQL: synchronous_commit=on|off; SQLite: PRAGMA synchronous FULL|OFF.",
        "BaraDB WAL modes appear only if you ran benchmarks/bench_all.nim (WAL-* rows).",
        "Never claim 'Nx faster than Postgres' using embedded BaraDB numbers.",
    ]
    results: list[dict] = []
    print("\n--- Embedded tier ---")
    results.extend(load_baradb_embedded())
    print("  SQLite embedded (+ batch)…")
    results.extend(bench_sqlite_embedded())
    print("\n--- Client/server tier ---")
    print("  BaraDB HTTP…")
    results.extend(bench_baradb_http())
    print("  BaraDB wire (TCP)…")
    results.extend(bench_baradb_wire())
    print("  PostgreSQL…")
    results.extend(bench_postgresql())
    payload = {
        "generated": now_iso(),
        "methodology": methodology,
        "config": {
            "N_KV": N_KV,
            "N_SQL": N_SQL,
            "HTTP": f"{HTTP_HOST}:{HTTP_PORT}",
        },
        "results": results,
    }
    OUT_JSON.write_text(json.dumps(payload, indent=2))
    print(f"\nJSON written to {OUT_JSON}")
    print_tier("embedded", [r for r in results if r["tier"] == "embedded"])
    print_tier("client_server", [r for r in results if r["tier"] == "client_server"])
    write_markdown(payload, ROOT / "benchmarks" / "FAIR_COMPARISON.md")
    return 0
 if __name__ == "__main__":
    sys.exit(main())
@@ -1,110 +1,143 @@
 #!/usr/bin/env python3
-"""Generate a real comparison report from BaraDB and PostgreSQL benchmark results."""
+"""Generate benchmark reports.
 Modes:
  python3 benchmarks/generate_report.py           # legacy PG vs embedded (with warning)
  python3 benchmarks/generate_report.py --fair    # multi-tier fair report from fair_bench.py
 """
 from __future__ import annotations
 import argparse
 import json
 import sys
 from pathlib import Path
 ROOT = Path(__file__).resolve().parent.parent
-def format_ops(ops_per_sec):
+
 def format_ops(ops_per_sec: float) -> str:
    if ops_per_sec >= 1_000_000:
        return f"{ops_per_sec/1_000_000:.2f}M"
-    elif ops_per_sec >= 1_000:
+    if ops_per_sec >= 1_000:
        return f"{ops_per_sec/1_000:.2f}K"
    else:
    return f"{ops_per_sec:.2f}"
-def format_time(seconds):
+def format_time(seconds: float) -> str:
    if seconds < 0.001:
        return f"{seconds*1000:.3f}ms"
-    elif seconds < 1:
+    if seconds < 1:
        return f"{seconds*1000:.1f}ms"
    else:
    return f"{seconds:.3f}s"
-def main():
+def gen_fair(out: Path) -> int:
-    root = Path(__file__).parent
+    fair_path = ROOT / "fair_benchmark_results.json"
    if not fair_path.exists():
        print("Missing fair_benchmark_results.json — run: python3 benchmarks/fair_bench.py")
        return 1
    payload = json.loads(fair_path.read_text())
    # fair_bench already writes FAIR_COMPARISON.md; re-emit for consistency
    sys.path.insert(0, str(ROOT / "benchmarks"))
    from fair_bench import write_markdown  # type: ignore
-    with open(root.parent / "benchmark_results.json") as f:
+    write_markdown(payload, out)
    return 0
 def gen_legacy() -> int:
    """Legacy report: PG client-server vs BaraDB *embedded* — always labeled unfair."""
    bara_path = ROOT / "benchmark_results.json"
    pg_path = ROOT / "pg_benchmark_results.json"
    if not bara_path.exists() or not pg_path.exists():
        print("Need benchmark_results.json and pg_benchmark_results.json")
        print("  nimble bench && python3 benchmarks/pg_bench.py")
        return 1
    with open(bara_path) as f:
        bara = json.load(f)
-    with open(root.parent / "pg_benchmark_results.json") as f:
+    with open(pg_path) as f:
        pg = json.load(f)
    bara_map = {r["name"]: r for r in bara["results"]}
-    pg_map = {k: v for k, v in pg.items()}
+    # pg_bench may write list or dict
    if isinstance(pg, dict) and "results" in pg:
        pg_map = {r["name"]: r for r in pg["results"]}
    elif isinstance(pg, list):
        pg_map = {r["name"]: r for r in pg}
    else:
        pg_map = pg  # old flat dict by name
    report = []
-    report.append("# BaraDB vs PostgreSQL — Real Benchmark Results")
+    report.append("# BaraDB vs PostgreSQL — LEGACY (mixed tiers)")
    report.append("")
-    report.append("Generated from actual execution on:")
+    report.append("> ⚠️ **Unfair comparison warning**")
-    report.append(f"- **CPU:** AMD Ryzen 9 5900X")
+    report.append(">")
-    report.append(f"- **PostgreSQL:** 15.18 (local)")
+    report.append("> PostgreSQL numbers include **client-server** round-trips.")
-    report.append(f"- **BaraDB:** git `{bara['gitSha']}`")
+    report.append("> BaraDB numbers are **in-process embedded** LSM (no network, no SQL).")
    report.append("> Use `python3 benchmarks/fair_bench.py` + `--fair` for honest tiers.")
    report.append("")
-    report.append("## Methodology")
+    report.append(f"- **BaraDB git:** `{bara.get('gitSha', 'unknown')}`")
    report.append("")
-    report.append("- PostgreSQL: single-row INSERT/SELECT via psycopg2 (client-server overhead included)")
+    report.append("| Test | PostgreSQL (C/S) | BaraDB (embedded) | Ratio (not a fair speedup) |")
-    report.append("- BaraDB: in-process Nim code (no network overhead)")
+    report.append("|------|------------------|-------------------|----------------------------|")
    report.append("- Same dataset sizes for both systems")
    report.append("")
    report.append("## Results")
    report.append("")
    report.append("| Test | PostgreSQL | BaraDB | Speedup |")
    report.append("|------|-----------|--------|---------|")
    rows = [
-        ("KV Write (100K)", pg_map.get("KV Write"), bara_map.get("LSM-Write")),
+        ("KV Write", pg_map.get("KV Write"), bara_map.get("LSM-Write")),
-        ("KV Read (100K)", pg_map.get("KV Read"), bara_map.get("LSM-Read")),
+        ("KV Read", pg_map.get("KV Read"), bara_map.get("LSM-Read")),
-        ("BTree Insert (100K)", pg_map.get("BTree Insert"), bara_map.get("BTree-Insert")),
+        ("BTree Insert", pg_map.get("BTree Insert"), bara_map.get("BTree-Insert")),
-        ("BTree Get (100K)", pg_map.get("BTree Get"), bara_map.get("BTree-Get")),
+        ("BTree Get", pg_map.get("BTree Get"), bara_map.get("BTree-Get")),
-        ("BTree Scan (1K ranges)", pg_map.get("BTree Scan"), bara_map.get("BTree-Scan")),
+        ("BTree Scan", pg_map.get("BTree Scan"), bara_map.get("BTree-Scan")),
-        ("FTS Index (10K docs)", pg_map.get("FTS Index"), bara_map.get("FTS-Index")),
+        ("FTS Index", pg_map.get("FTS Index"), bara_map.get("FTS-Index")),
-        ("FTS Search (1K queries)", pg_map.get("FTS Search"), bara_map.get("FTS-Search")),
+        ("FTS Search", pg_map.get("FTS Search"), bara_map.get("FTS-Search")),
    ]
    total_pg_time = 0
    total_bara_time = 0
    for name, p, b in rows:
        if p is None or b is None:
            continue
        pg_ops = p["opsPerSec"]
        ba_ops = b["opsPerSec"]
-        ratio = ba_ops / pg_ops
+        ratio = ba_ops / pg_ops if pg_ops else 0
        winner = "BaraDB" if ratio > 1 else "PostgreSQL"
        total_pg_time += p["seconds"]
        total_bara_time += b["seconds"]
        report.append(
            f"| {name} | {format_ops(pg_ops)}/s ({format_time(p['seconds'])}) | "
            f"{format_ops(ba_ops)}/s ({format_time(b['seconds'])}) | "
-            f"{ratio:.1f}x ({winner}) |"
+            f"{ratio:.1f}x (mixed tiers) |"
        )
    report.append("")
-    report.append("## Summary")
+    report.append("## Prefer fair suite")
    report.append("")
-    report.append(f"- **Total PostgreSQL time:** {total_pg_time:.3f}s")
+    report.append("```bash")
-    report.append(f"- **Total BaraDB time:** {total_bara_time:.3f}s")
+    report.append("nim c -d:release -r benchmarks/bench_all.nim")
-    overall = total_pg_time / total_bara_time
+    report.append("python3 benchmarks/fair_bench.py")
-    report.append(f"- **Overall speedup:** BaraDB is **{overall:.1f}x faster**")
+    report.append("python3 benchmarks/generate_report.py --fair")
-    report.append("")
+    report.append("```")
    report.append("## Notes")
    report.append("")
    report.append("- PostgreSQL includes network round-trip and SQL parsing overhead per operation.")
    report.append("- BaraDB runs in-process with zero serialization/network cost.")
    report.append("- For embedded/single-node use cases, BaraDB shows significant advantage.")
    report.append("- PostgreSQL FTS Search with GIN index outperforms BaraDB on query throughput.")
    report.append("- PostgreSQL excels at durability, replication, and complex ACID transactions.")
    report.append("")
-    output = "\n".join(report)
+    out = ROOT / "benchmarks" / "REAL_COMPARISON.md"
-    print(output)
+    out.write_text("\n".join(report) + "\n")
    print(f"Wrote {out} (legacy mixed-tier; see warning banner)")
    return 0
-    with open(root / "REAL_COMPARISON.md", "w") as f:
+
-        f.write(output)
+def main():
-    print(f"\nReport saved to {root / 'REAL_COMPARISON.md'}")
+    ap = argparse.ArgumentParser()
    ap.add_argument(
        "--fair",
        action="store_true",
        help="Emit multi-tier fair report from fair_benchmark_results.json",
    )
    ap.add_argument(
        "-o",
        "--output",
        default=str(ROOT / "benchmarks" / "FAIR_COMPARISON.md"),
        help="Output path for --fair mode",
    )
    args = ap.parse_args()
    if args.fair:
        return gen_fair(Path(args.output))
    return gen_legacy()
 if __name__ == "__main__":
-    main()
+    sys.exit(main())
@@ -174,8 +174,10 @@ def format_ops(ops_per_sec):
 def print_comparison(pg_results, bara_data):
    bara = {r["name"]: r for r in bara_data["results"]}
    print("\n╔══════════════════════════════════════════════════════════════════════╗")
-    print("║           BaraDB vs PostgreSQL — Real Benchmark Results              ║")
+    print("║     PostgreSQL (client-server) vs BaraDB (EMBEDDED) — MIXED TIERS    ║")
    print("╚══════════════════════════════════════════════════════════════════════╝\n")
    print("WARNING: This mixes client-server PG with in-process BaraDB LSM.")
    print("         Prefer: python3 benchmarks/fair_bench.py\n")
    rows = [
        ("KV Write (100K)", pg_results.get("KV Write"), bara.get("LSM-Write")),
@@ -187,7 +189,7 @@ def print_comparison(pg_results, bara_data):
        ("FTS Search (1K queries)", pg_results.get("FTS Search"), bara.get("FTS-Search")),
    ]
-    print(f"{'Test':<26} {'PostgreSQL':>18} {'BaraDB':>18} {'Winner':>10}")
+    print(f"{'Test':<26} {'PostgreSQL C/S':>18} {'BaraDB embed':>18} {'Note':>14}")
    print("─" * 76)
    for name, pg, ba in rows:
@@ -195,22 +197,14 @@ def print_comparison(pg_results, bara_data):
            continue
        pg_ops = pg["opsPerSec"]
        ba_ops = ba["opsPerSec"]
-        winner = "BaraDB" if ba_ops > pg_ops else "PostgreSQL"
+        ratio = ba_ops / pg_ops if pg_ops else 0
        ratio = max(ba_ops, pg_ops) / min(ba_ops, pg_ops)
        print(
-            f"{name:<26} {format_ops(pg_ops)+'/s':>18} {format_ops(ba_ops)+'/s':>18} {winner+' ('+f'{ratio:.1f}x'+')':>10}"
+            f"{name:<26} {format_ops(pg_ops)+'/s':>18} {format_ops(ba_ops)+'/s':>18} "
            f"{'mixed '+f'{ratio:.1f}x':>14}"
        )
    print("\n" + "─" * 76)
-    # Summary
+    print("For fair tiers (SQLite↔LSM, HTTP↔PG): python3 benchmarks/fair_bench.py")
    pg_total = sum(r["seconds"] for _, r, _ in rows if r is not None)
    ba_total = sum(b["seconds"] for _, _, b in rows if b is not None)
    print(f"\nTotal time PostgreSQL: {pg_total:.3f}s")
    print(f"Total time BaraDB:     {ba_total:.3f}s")
    if ba_total < pg_total:
        print(f"BaraDB is {pg_total/ba_total:.1f}x faster overall")
    else:
        print(f"PostgreSQL is {ba_total/pg_total:.1f}x faster overall")
 def main():
@@ -247,14 +241,23 @@ def main():
    pg_results["FTS Search"] = bench_fts_search()
    print(f"      -> {format_ops(pg_results['FTS Search']['opsPerSec'])}/s ({pg_results['FTS Search']['seconds']:.3f}s)")
-    bara_data = load_baradb_results()
+    # Annotate tier for fair tooling
-    print_comparison(pg_results, bara_data)
+    for name, r in pg_results.items():
        r["tier"] = "client_server"
        r["system"] = "postgresql"
    # Save raw results
    with open("pg_benchmark_results.json", "w") as f:
        json.dump(pg_results, f, indent=2)
    print("\nPostgreSQL results saved to pg_benchmark_results.json")
    if os.path.exists("benchmark_results.json"):
        bara_data = load_baradb_results()
        print_comparison(pg_results, bara_data)
    else:
        print("\n(No benchmark_results.json — skip mixed-tier table; run nimble bench first)")
    print("\nFair multi-tier suite: python3 benchmarks/fair_bench.py")
 if __name__ == "__main__":
    main()
@@ -1,3 +1,8 @@
 -d:ssl
 --threads:on
 --path:"src"
 # ARC: ORC cycle collector crashes under async wire-protocol load
 # (markGray/trace SIGSEGV after ~20 sequential INSERTs). ARC is stable
 # for the TCP server + HTTP worker mix. Prefer breaking cycles over
 # re-enabling ORC without a reproducer.
 --mm:arc
@@ -22,6 +22,11 @@ type
    logFormat*: string
    memtableSizeMb*: int
    cacheSizeMb*: int
    ## WAL durability: "none" | "group" (default) | "every"
    walSyncMode*: string
    ## Group commit batch size (entries between fsyncs when mode=group)
    walGroupEvery*: int
    ## Time-based group fsync interval in ms (0 = off); also used as legacy name
    walSyncIntervalMs*: int
    compactionIntervalMs*: int
    bloomBitsPerKey*: int
@@ -58,6 +63,8 @@ proc defaultConfig*(): BaraConfig =
    logFormat: "json",
    memtableSizeMb: 64,
    cacheSizeMb: 256,
    walSyncMode: "group",
    walGroupEvery: 64,
    walSyncIntervalMs: 0,
    compactionIntervalMs: 60_000,
    bloomBitsPerKey: 10,
@@ -93,6 +100,8 @@ proc loadConfigFromJson*(path: string, cfg: var BaraConfig) =
      if s.hasKey("data_dir"): cfg.dataDir = s["data_dir"].getStr()
      if s.hasKey("memtable_size_mb"): cfg.memtableSizeMb = s["memtable_size_mb"].getInt()
      if s.hasKey("cache_size_mb"): cfg.cacheSizeMb = s["cache_size_mb"].getInt()
      if s.hasKey("wal_sync_mode"): cfg.walSyncMode = s["wal_sync_mode"].getStr()
      if s.hasKey("wal_group_every"): cfg.walGroupEvery = s["wal_group_every"].getInt()
      if s.hasKey("wal_sync_interval_ms"): cfg.walSyncIntervalMs = s["wal_sync_interval_ms"].getInt()
      if s.hasKey("compaction_interval_ms"): cfg.compactionIntervalMs = s["compaction_interval_ms"].getInt()
      if s.hasKey("bloom_bits_per_key"): cfg.bloomBitsPerKey = s["bloom_bits_per_key"].getInt()
@@ -153,6 +162,8 @@ proc loadConfigFromEnv*(cfg: var BaraConfig) =
  cfg.logFormat = getEnv("BARADB_LOG_FORMAT", cfg.logFormat)
  cfg.memtableSizeMb = parseEnvInt(getEnv("BARADB_MEMTABLE_SIZE_MB", ""), cfg.memtableSizeMb)
  cfg.cacheSizeMb = parseEnvInt(getEnv("BARADB_CACHE_SIZE_MB", ""), cfg.cacheSizeMb)
  cfg.walSyncMode = getEnv("BARADB_WAL_SYNC_MODE", cfg.walSyncMode)
  cfg.walGroupEvery = parseEnvInt(getEnv("BARADB_WAL_GROUP_EVERY", ""), cfg.walGroupEvery)
  cfg.walSyncIntervalMs = parseEnvInt(getEnv("BARADB_WAL_SYNC_INTERVAL_MS", ""), cfg.walSyncIntervalMs)
  cfg.compactionIntervalMs = parseEnvInt(getEnv("BARADB_COMPACTION_INTERVAL_MS", ""), cfg.compactionIntervalMs)
  cfg.bloomBitsPerKey = parseEnvInt(getEnv("BARADB_BLOOM_BITS_PER_KEY", ""), cfg.bloomBitsPerKey)
@@ -15,6 +15,7 @@ import ../query/parser
 import ../query/executor
 import ../core/types
 import ../storage/lsm
 import ../storage/gate
 import ../core/mvcc
 import ../protocol/wire
 import ../core/websocket
@@ -196,17 +197,7 @@ proc queryHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Empty query"}, 400)
        return
-      var reqCtx = getRequestDatabaseContext(server, request)
+      # Extract optional params from JSON body (no storage access yet)
      reqCtx.currentUser = userId
      reqCtx.currentRole = role
      let tokens = tokenize(queryStr)
      let astNode = parse(tokens)
      if astNode.stmts.len == 0:
        ctx.json(%*{"rows": [], "affectedRows": 0, "columns": []})
        return
      # Extract optional params from JSON body
      var params: seq[WireValue] = @[]
      if "params" in body and body["params"].kind == JArray:
        for p in body["params"]:
@@ -218,31 +209,50 @@ proc queryHandler(server: HttpServer): RequestHandler =
          of JString: params.add(WireValue(kind: fkString, strVal: p.getStr()))
          else: params.add(WireValue(kind: fkString, strVal: $p))
-      let res = executor.executeQuery(reqCtx, astNode, params)
+      # StorageGate: serialize against TCP + other Hunos workers (ORC safety)
-
+      var success: bool
      if res.success:
      var jsonRows = newJArray()
      var jsonCols = newJArray()
      var affected = 0
      var msg = ""
      var errMsg = ""
      withStorageGate:
        var reqCtx = getRequestDatabaseContext(server, request)
        reqCtx.currentUser = userId
        reqCtx.currentRole = role
        let tokens = tokenize(queryStr)
        let astNode = parse(tokens)
        if astNode.stmts.len == 0:
          success = true
        else:
          let res = executor.executeQuery(reqCtx, astNode, params)
          success = res.success
          if res.success:
            affected = res.affectedRows
            msg = res.message
            for row in res.rows:
              var jsonRow = newJObject()
              for col in res.columns:
-            let key = col
+                if col in row and row[col].kind != vkNull:
-            if key in row and row[key].kind != vkNull:
+                  jsonRow[col] = %valueToString(row[col])
              jsonRow[key] = %valueToString(row[key])
                else:
-              jsonRow[key] = newJNull()
+                  jsonRow[col] = newJNull()
              jsonRows.add(jsonRow)
        var jsonCols = newJArray()
            for c in res.columns:
              jsonCols.add(%c)
          else:
            errMsg = res.message
      if success:
        ctx.json(%*{
          "rows": jsonRows,
-          "affectedRows": res.affectedRows,
+          "affectedRows": affected,
          "columns": jsonCols,
-          "message": if res.message.len > 0: %res.message else: newJNull()
+          "message": if msg.len > 0: %msg else: newJNull()
        })
      else:
        server.metrics.queryErrors += 1
-        ctx.json(%*{"error": res.message}, 400)
+        ctx.json(%*{"error": errMsg}, 400)
 proc healthHandler(): RequestHandler =
  return proc(request: Request) {.gcsafe.} =
@@ -376,8 +386,9 @@ proc tablesHandler(server: HttpServer): RequestHandler =
      let ctx = newContext(request)
      if not server.checkAuth(request, ctx):
        return
      let reqCtx = getRequestDatabaseContext(server, request)
      var tables = newJArray()
      withStorageGate:
        let reqCtx = getRequestDatabaseContext(server, request)
        for name, tbl in reqCtx.tables:
          var cols = newJArray()
          for col in tbl.columns:
@@ -393,8 +404,9 @@ proc databasesHandler(server: HttpServer): RequestHandler =
      let ctx = newContext(request)
      if not server.checkAuth(request, ctx):
        return
      let dbs = server.registry.listDatabases()
      var arr = newJArray()
      withStorageGate:
        let dbs = server.registry.listDatabases()
        for dbName in dbs:
          var obj = newJObject()
          obj["name"] = %dbName
@@ -428,6 +440,7 @@ proc createDatabaseHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Empty database name"}, 400)
        return
      try:
        withStorageGate:
          discard getOrCreateDatabase(server.registry, dbName)
        ctx.json(%*{"success": true, "name": dbName, "message": "Database created"})
      except CatchableError as e:
@@ -444,7 +457,9 @@ proc dropDatabaseHandler(server: HttpServer): RequestHandler =
        ctx.json(%*{"error": "Missing database name"}, 400)
        return
      try:
-        let ok = dropDatabase(server.registry, dbName)
+        var ok = false
        withStorageGate:
          ok = dropDatabase(server.registry, dbName)
        if ok:
          ctx.json(%*{"success": true, "name": dbName, "message": "Database dropped"})
        else:
@@ -470,6 +485,8 @@ proc backupHandler(server: HttpServer): RequestHandler =
      let compression = if body != nil and "level" in body: body["level"].getInt() else: 6
      try:
        var ok = false
        # Gate held so live writers/compactors don't mutate files mid-backup
        withStorageGate:
          if allDatabases:
            ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
          elif dbName.len > 0:
@@ -541,6 +558,7 @@ proc restoreHandler(server: HttpServer): RequestHandler =
        let meta = readBackupMeta(inputFile)
        let isMultiDb = meta != nil and meta{"databases"} != nil
        var ok = false
        withStorageGate:
          if isMultiDb or allDatabases:
            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
          elif dbName.len > 0:
@@ -548,11 +566,12 @@ proc restoreHandler(server: HttpServer): RequestHandler =
            ok = restoreDataDir(inputFile, dbDir, false, false)
          else:
            ok = restoreAllDatabases(inputFile, dataRoot, false, false)
          if ok:
            # Reload under same gate after files are restored
            server.registry.loadExistingDatabases()
        logRestore(inputFile, dataRoot, ok)
        if ok:
          # Reload databases after restore
          server.registry.loadExistingDatabases()
          ctx.json(%*{"success": true, "message": "Restore completed"})
        else:
          ctx.json(%*{"error": "Restore failed"}, 500)
@@ -890,10 +909,14 @@ proc run*(server: HttpServer, port: int = 9470) =
  asyncCheck server.ws.run(port + 1)
  hunosServer.serve(Port(port))
-proc stop*(server: HttpServer) =
+proc stop*(server: HttpServer, closeStorage: bool = false) =
  ## Stop HTTP listeners. By default does **not** close the shared registry —
  ## when HTTP is spawned alongside TCP they share one registry owned by main.
  server.running = false
  server.ws.stop()
  if closeStorage:
    withStorageGate:
      if server.registry != nil:
        server.registry.closeAll()
-  else:
+      elif server.db != nil:
        server.db.close()
@@ -29,6 +29,17 @@ type
 const reservedDbNames* = ["system", "information_schema", "pg_catalog"]
 proc openLsmForRegistry(reg: DatabaseRegistry, dbDir: string): LSMTree =
  ## Open LSM with WAL durability settings from registry config.
  let memBytes = max(1, reg.config.memtableSizeMb) * 1024 * 1024
  newLSMTree(
    dbDir,
    memMaxSize = memBytes,
    walSyncMode = parseWalSyncMode(reg.config.walSyncMode),
    walGroupEvery = reg.config.walGroupEvery,
    walGroupIntervalMs = reg.config.walSyncIntervalMs,
  )
 proc isValidDbName*(name: string): bool =
  if name.len == 0: return false
  if '/' in name or '\\' in name: return false
@@ -63,7 +74,7 @@ proc loadExistingDatabases*(reg: DatabaseRegistry) =
      if dbName.len > 0 and isValidDbName(dbName):
        let dbDir = reg.dataRoot / dbName
        info("Loading database '" & dbName & "' from " & dbDir)
-        let db = newLSMTree(dbDir)
+        let db = openLsmForRegistry(reg, dbDir)
        let ctx = reg.ctxFactory(db, reg)
        acquire(reg.lock)
        reg.databases[dbName] = DatabaseInfo(
@@ -89,7 +100,7 @@ proc ensureDefaultDatabase*(reg: DatabaseRegistry) =
  if not exists:
    let dbDir = reg.dataRoot / defaultDbName
    info("Creating default database at " & dbDir)
-    let db = newLSMTree(dbDir)
+    let db = openLsmForRegistry(reg, dbDir)
    let ctx = reg.ctxFactory(db, reg)
    acquire(reg.lock)
    reg.databases[defaultDbName] = DatabaseInfo(
@@ -113,7 +124,7 @@ proc getOrCreateDatabase*(reg: DatabaseRegistry, name: string): DatabaseInfo =
  # Create new database
  let dbDir = reg.dataRoot / name
  info("Creating database '" & name & "' at " & dbDir)
-  let db = newLSMTree(dbDir)
+  let db = openLsmForRegistry(reg, dbDir)
  let ctx = reg.ctxFactory(db, reg)
  let info = DatabaseInfo(name: name, db: db, ctx: ctx, activeConnections: 0)
  reg.databases[name] = info
@@ -21,6 +21,7 @@ import ../query/parser
 import ../query/ast
 import ../query/executor
 import ../storage/lsm
 import ../storage/gate
 import ../core/mvcc
 import ../core/disttxn
 import ../core/replication
@@ -206,6 +207,9 @@ proc valueToWire(val: string, colType: string): WireValue =
 proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq[WireValue] = @[],
                   replication: ReplicationManager = nil): (bool, QueryResult, string) =
  ## All storage access is under the global StorageGate so HTTP worker threads
  ## and the TCP event loop never touch ORC-managed LSM/executor state concurrently.
  withStorageGate:
    try:
      let tokens = tokenize(query)
      let astNode = parse(tokens)
@@ -0,0 +1,29 @@
 # Executor package (`query/exec/`)
 The original `executor.nim` was a ~5.8k-line god object. Shared pieces live here;
 `../executor.nim` remains the main execution engine and **re-exports** this package
 so existing `import barabadb/query/executor` keeps working.
 ## Modules
 | Module | Responsibility |
 |--------|----------------|
 | `types.nim` | `ExecutionContext`, `TableDef`, `Row`, `ExecResult`, … |
 | `values.nim` | Null/string conversion, row payload parse/escape, SQL escapes |
 | `schema.nim` | Durable catalog (`_schema:tables:*`), restore, index rebuild |
 ## Import rules
 - **No cycles:** `types` → nothing in `exec/`; `values` → `types`; `schema` → `types` + `values`.
 - `executor.nim` imports all three and `export`s them.
 - Prefer adding new shared helpers under `exec/` instead of growing `executor.nim`.
 ## Sensible next extractions (not done yet)
 1. `dml.nim` — `execScan` / `execInsert` / `execUpdate` / `execDelete` (needs eval/triggers hooks)
 2. `rls.nim` — row-level security + privileges
 3. `lower.nim` — AST → IR (`lowerExpr` / `lowerSelect`)
 4. `plan_exec.nim` — IR plan walker / window functions
 5. `hybrid.nim` — hybrid vector+FTS search helpers
 Keep statement dispatch (`executeQueryImpl`) in `executor.nim` until those land.
@@ -0,0 +1,231 @@
 ## Schema catalog persistence — CREATE/DROP/ALTER survive restart
 import std/strutils
 import std/tables
 import std/sequtils
 import ../ast
 import ../lexer as qlex
 import ../parser as qpar
 import ../../storage/lsm
 import ../../storage/btree
 import types
 import values
 const
  SchemaTablePrefix* = "_schema:tables:"
  SchemaViewPrefix* = "_schema:views:"
  SchemaTriggerPrefix* = "_schema:triggers:"
  SchemaUserPrefix* = "_schema:users:"
  SchemaPolicyPrefix* = "_schema:policies:"
  ## Legacy CREATE TABLE keys (pre-fix) used a migrations: counter suffix
  SchemaLegacyCreatePrefix* = "_schema:migrations:"
 proc tableSchemaKey*(tableName: string): string =
  SchemaTablePrefix & tableName
 proc litToString(node: Node): string =
  ## Evaluate simple literal defaults for schema materialization (no full expr engine).
  if node == nil: return ""
  case node.kind
  of nkStringLit: return node.strVal
  of nkIntLit: return $node.intVal
  of nkFloatLit: return $node.floatVal
  of nkBoolLit: return $node.boolVal
  of nkNullLit: return "\\N"
  else: return ""
 proc serializeTableDdl*(tbl: TableDef): string =
  ## Stable DDL for a table definition (survives restart via LSM).
  var colDefs: seq[string] = @[]
  let multiPk = tbl.pkColumns.len > 1
  for col in tbl.columns:
    var parts: seq[string] = @[col.name, col.colType]
    if col.isPk and not multiPk:
      parts.add("PRIMARY KEY")
    if col.autoIncrement:
      parts.add("AUTO_INCREMENT")
    if col.isNotNull:
      parts.add("NOT NULL")
    if col.isUnique and not col.isPk:
      parts.add("UNIQUE")
    if col.defaultVal.len > 0:
      parts.add("DEFAULT '" & sqlEscapeString(col.defaultVal) & "'")
    if col.fkTable.len > 0:
      parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")")
      if col.fkOnDelete.len > 0:
        parts.add("ON DELETE " & col.fkOnDelete)
      if col.fkOnUpdate.len > 0:
        parts.add("ON UPDATE " & col.fkOnUpdate)
    colDefs.add(parts.join(" "))
  if multiPk:
    colDefs.add("PRIMARY KEY (" & tbl.pkColumns.join(", ") & ")")
  result = "CREATE TABLE " & tbl.name & " (" & colDefs.join(", ") & ")"
 proc persistTableSchema*(ctx: ExecutionContext, tbl: TableDef) =
  ## Write table DDL under a stable key so restore finds it after flush/restart.
  let ddl = serializeTableDdl(tbl)
  ctx.db.put(tableSchemaKey(tbl.name), cast[seq[byte]](ddl))
 proc dropTableSchema*(ctx: ExecutionContext, tableName: string) =
  ctx.db.delete(tableSchemaKey(tableName))
 proc applyCreateTableStmt*(ctx: ExecutionContext, stmt: Node) =
  ## Materialize CREATE TABLE AST into ctx.tables + empty secondary indexes.
  var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[],
                     foreignKeys: @[], checks: @[], triggers: @[])
  for col in stmt.crtColumns:
    if col.kind == nkColumnDef:
      var colDef = ColumnDef(name: col.cdName, colType: col.cdType)
      colDef.autoIncrement = col.cdAutoIncrement
      for cst in col.cdConstraints:
        if cst.kind == nkConstraintDef:
          case cst.cstType
          of "pkey":
            colDef.isPk = true
            if col.cdName notin tbl.pkColumns:
              tbl.pkColumns.add(col.cdName)
            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
          of "notnull": colDef.isNotNull = true
          of "unique":
            colDef.isUnique = true
            ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
          of "default":
            if cst.cstDefault != nil:
              colDef.defaultVal = litToString(cst.cstDefault)
          of "fkey":
            colDef.fkTable = cst.cstRefTable
            colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: ""
            colDef.fkOnDelete = cst.cstOnDelete
            colDef.fkOnUpdate = cst.cstOnUpdate
          else: discard
      tbl.columns.add(colDef)
  # Table-level constraints
  for cstNode in stmt.crtConstraints:
    if cstNode.kind == nkConstraintDef:
      if cstNode.cstType == "pkey":
        for c in cstNode.cstColumns:
          if c notin tbl.pkColumns:
            tbl.pkColumns.add(c)
          for i, col in tbl.columns:
            if col.name == c:
              tbl.columns[i].isPk = true
              let idxName = stmt.crtName & "." & c
              if idxName notin ctx.btrees:
                ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
      elif cstNode.cstType == "fkey":
        tbl.foreignKeys.add(ForeignKeyDef(
          refTable: cstNode.cstRefTable,
          refColumn: if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: "",
          onDelete: cstNode.cstOnDelete,
          onUpdate: cstNode.cstOnUpdate))
        if cstNode.cstColumns.len > 0:
          for i, c in tbl.columns:
            if c.name in cstNode.cstColumns:
              tbl.columns[i].fkTable = cstNode.cstRefTable
              tbl.columns[i].fkColumn = if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: ""
              tbl.columns[i].fkOnDelete = cstNode.cstOnDelete
              tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate
      elif cstNode.cstType == "check":
        tbl.checks.add(CheckDef(name: "check_" & $tbl.checks.len, checkNode: cstNode.cstCheck))
  ctx.tables[stmt.crtName] = tbl
 proc rebuildSecondaryIndexes*(ctx: ExecutionContext) =
  ## Rebuild in-memory B-Tree indexes from durable row data after schema restore.
  for tableName, tbl in ctx.tables.pairs:
    for col in tbl.columns:
      if col.isPk or col.isUnique:
        let idxName = tableName & "." & col.name
        if idxName notin ctx.btrees:
          ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]()
    let prefix = tableName & "."
    for (key, value) in ctx.db.scanAll():
      if not key.startsWith(prefix): continue
      if key.startsWith("_schema:"): continue
      let valStr = cast[string](value)
      let rest = key[prefix.len..^1]
      var colVals = initTable[string, string]()
      let eqPos = rest.find('=')
      if eqPos >= 0 and ':' notin rest:
        colVals[rest[0..<eqPos]] = rest[eqPos+1..^1]
      else:
        for part in rest.split(':'):
          let p = part.find('=')
          if p >= 0:
            colVals[part[0..<p]] = part[p+1..^1]
      for k, v in parseRowData(valStr):
        colVals[k] = v
      for colName in ctx.btrees.keys.toSeq():
        if not colName.startsWith(prefix): continue
        let colsPart = colName[tableName.len + 1..^1]
        let idxCols = colsPart.split(".")
        var parts: seq[string] = @[]
        for c in idxCols:
          parts.add(colVals.getOrDefault(c, ""))
        let idxVal = parts.join("|")
        if idxVal.len > 0 and not isNull(idxVal):
          ctx.btrees[colName].insert(idxVal, IndexEntry(lsmKey: key, rowValue: valStr))
 proc restoreSchema*(ctx: ExecutionContext) =
  ## Load durable schema from LSM (memtable + SSTables). Stable keys only.
  var tableDdls: seq[string] = @[]
  var otherDdls: seq[string] = @[]
  for (key, value) in ctx.db.scanAll():
    if not key.startsWith("_schema:"): continue
    let ddl = cast[string](value)
    if ddl.len == 0: continue
    if key.startsWith(SchemaTablePrefix):
      tableDdls.add(ddl)
    elif key.startsWith(SchemaLegacyCreatePrefix) and ddl.toUpperAscii().startsWith("CREATE TABLE"):
      tableDdls.add(ddl)
    elif key.startsWith(SchemaViewPrefix) or key.startsWith(SchemaTriggerPrefix) or
         key.startsWith(SchemaUserPrefix) or key.startsWith(SchemaPolicyPrefix):
      otherDdls.add(ddl)
    elif ddl.toUpperAscii().startsWith("CREATE VIEW") or
         ddl.toUpperAscii().startsWith("CREATE TRIGGER") or
         ddl.toUpperAscii().startsWith("CREATE USER") or
         ddl.toUpperAscii().startsWith("CREATE POLICY"):
      otherDdls.add(ddl)
  for ddl in tableDdls:
    try:
      let tokens = qlex.tokenize(ddl)
      let astNode = qpar.parse(tokens)
      if astNode.stmts.len > 0 and astNode.stmts[0].kind == nkCreateTable:
        applyCreateTableStmt(ctx, astNode.stmts[0])
        if astNode.stmts[0].crtName in ctx.tables:
          persistTableSchema(ctx, ctx.tables[astNode.stmts[0].crtName])
    except CatchableError:
      continue
  for ddl in otherDdls:
    var astNode: Node
    try:
      let tokens = qlex.tokenize(ddl)
      astNode = qpar.parse(tokens)
    except CatchableError:
      continue
    if astNode.stmts.len == 0: continue
    let stmt = astNode.stmts[0]
    case stmt.kind
    of nkCreateView:
      ctx.views[stmt.cvName] = stmt.cvQuery
    of nkCreateTrigger:
      if stmt.trigTable in ctx.tables:
        ctx.tables[stmt.trigTable].triggers.add(TriggerDef(
          name: stmt.trigName,
          timing: stmt.trigTiming,
          event: stmt.trigEvent,
          action: stmt.trigAction,
        ))
    of nkCreateUser:
      ctx.users[stmt.cuName] = UserDef(name: stmt.cuName,
          passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[])
    of nkCreatePolicy:
      var pols = ctx.policies.getOrDefault(stmt.cpTable)
      pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable,
                         command: stmt.cpCommand, usingExpr: stmt.cpUsing,
                         withCheckExpr: stmt.cpWithCheck))
      ctx.policies[stmt.cpTable] = pols
    else: discard
  rebuildSecondaryIndexes(ctx)
@@ -0,0 +1,143 @@
 ## Executor types — shared by all exec/* modules and executor.nim
 import std/tables
 import std/locks
 import ../ast
 import ../ir
 import ../../core/types
 import ../../storage/lsm
 import ../../storage/btree
 import ../../core/mvcc
 import ../../fts/engine as fts
 import ../../vector/engine as vengine
 import ../../graph/engine as gengine
 import ../../ai/embed as embedmod
 import ../../ai/llm as llmmod
 import ../../core/registry
 type
  IndexEntry* = ref object
    lsmKey*: string
    rowValue*: string
  ChangeKind* = enum
    ckInsert, ckUpdate, ckDelete
  ChangeEvent* = object
    table*: string
    kind*: ChangeKind
    key*: string
    data*: string
  UserDef* = object
    name*: string
    passwordHash*: string
    isSuperuser*: bool
    roles*: seq[string]
  PrivilegeDef* = object
    tableName*: string
    command*: string  # SELECT, INSERT, UPDATE, DELETE, ALL
  PolicyDef* = object
    name*: string
    tableName*: string
    command*: string   # ALL, SELECT, INSERT, UPDATE, DELETE
    usingExpr*: Node   # parsed USING expression
    withCheckExpr*: Node  # parsed WITH CHECK expression
  SharedLock* = ref object
    lock*: Lock
  ForeignKeyDef* = object
    refTable*: string
    refColumn*: string
    onDelete*: string  # CASCADE, SET NULL, RESTRICT
    onUpdate*: string  # CASCADE, SET NULL, RESTRICT
  CheckDef* = object
    name*: string
    expr*: string  # stored expression string
    checkNode*: Node  # AST for runtime evaluation
  TriggerDef* = object
    name*: string
    timing*: string   # BEFORE, AFTER
    event*: string    # INSERT, UPDATE, DELETE
    action*: Node     # SQL statement AST
  ColumnDef* = object
    name*: string
    colType*: string
    isPk*: bool
    isNotNull*: bool
    isUnique*: bool
    defaultVal*: string
    fkTable*: string
    fkColumn*: string
    fkOnDelete*: string
    fkOnUpdate*: string
    autoIncrement*: bool
  TableDef* = object
    name*: string
    columns*: seq[ColumnDef]
    pkColumns*: seq[string]
    foreignKeys*: seq[ForeignKeyDef]
    checks*: seq[CheckDef]
    triggers*: seq[TriggerDef]
  Row* = Table[string, Value]
  ExecutionContext* = ref object
    db*: LSMTree
    tables*: Table[string, TableDef]
    btrees*: Table[string, BTreeIndex[string, IndexEntry]]
    views*: Table[string, Node]  # view name -> SELECT AST
    cteTables*: Table[string, seq[Row]]  # CTE name -> rows
    ftsIndexes*: Table[string, fts.InvertedIndex]  # table.col -> FTS index
    vectorIndexes*: Table[string, vengine.HNSWIndex]  # table.col -> HNSW index
    graphs*: Table[string, gengine.Graph]  # graph name -> Graph object
    embedder*: embedmod.Embedder  # optional embedding service client
    llmClient*: llmmod.LLMClient  # optional LLM client for NL->SQL
    txnManager*: TxnManager
    pendingTxn*: Transaction
    onChange*: proc(ev: ChangeEvent) {.closure.}
    users*: Table[string, UserDef]
    policies*: Table[string, seq[PolicyDef]]  # table name -> policies
    currentUser*: string
    currentRole*: string
    sessionVars*: Table[string, string]
    autoIncCounters*: Table[string, int64]
    sequences*: Table[string, int64]
    sharedLock*: SharedLock  # shared across cloned contexts
    outerRow*: Table[string, string]  # outer query row for correlated subqueries
    subqueryPlan*: IRPlan  # current subquery plan being evaluated
    currentDatabase*: string  # name of the currently selected database
    registry*: DatabaseRegistry  # nil for single-DB mode
  MigrationRecord* = object
    name*: string
    checksum*: string
    appliedAt*: int64
    appliedBy*: string
    durationMs*: int
    rolledBack*: bool
  ExecResult* = object
    success*: bool
    columns*: seq[string]
    rows*: seq[Row]
    affectedRows*: int
    message*: string
    keyValuePairs*: seq[(string, seq[byte])]
 proc `==`*(a, b: IndexEntry): bool =
  a.lsmKey == b.lsmKey and a.rowValue == b.rowValue
 proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "",
               kvPairs: seq[(string, seq[byte])] = @[]): ExecResult =
  ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg,
             keyValuePairs: kvPairs)
 proc errResult*(msg: string): ExecResult =
  ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg)
@@ -0,0 +1,119 @@
 ## Value / row serialization helpers used across the executor
 import std/strutils
 import std/tables
 import std/json
 import ../../core/types
 import types
 proc isNull*(value: string): bool =
  value == "\\N" or value.toLower() == "null"
 proc valueToString*(v: Value): string =
  case v.kind
  of vkNull: return "\\N"
  of vkString: return v.strVal
  of vkInt64: return $v.int64Val
  of vkFloat64: return $v.float64Val
  of vkBool: return $v.boolVal
  else: return ""
 proc `%`*(v: Value): JsonNode =
  case v.kind
  of vkNull: return newJNull()
  of vkString: return %v.strVal
  of vkInt64: return %v.int64Val
  of vkFloat64: return %v.float64Val
  of vkBool: return %v.boolVal
  else: return newJNull()
 proc toString*(v: Value): string = valueToString(v)
 proc `[]=`*(t: var Row, key: string, val: string) =
  t[key] = Value(kind: vkString, strVal: val)
 proc escapeRowVal*(v: string): string =
  v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
 proc unescapeRowVal*(v: string): string =
  result = ""
  var i = 0
  while i < v.len:
    if v[i] == '\\' and i + 1 < v.len:
      case v[i+1]
      of '\\', ',', '=':
        result &= v[i+1]
        i += 2
        continue
      else: discard
    result &= v[i]
    inc i
 proc parseRowData*(valStr: string): Table[string, string] =
  ## Parse "col1=val1,col2=val2" into a table
  result = initTable[string, string]()
  var i = 0
  var part = ""
  while i < valStr.len:
    if valStr[i] == '\\' and i + 1 < valStr.len:
      part &= valStr[i]
      part &= valStr[i+1]
      i += 2
      continue
    if valStr[i] == ',':
      let eqPos = part.find('=')
      if eqPos >= 0:
        let k = part[0..<eqPos].strip()
        let v = unescapeRowVal(part[eqPos+1..^1].strip())
        result[k] = v
      part = ""
    else:
      part &= valStr[i]
    inc i
  if part.len > 0:
    let eqPos = part.find('=')
    if eqPos >= 0:
      let k = part[0..<eqPos].strip()
      let v = unescapeRowVal(part[eqPos+1..^1].strip())
      result[k] = v
 proc parseRowDataToValueRow*(valStr: string): Row =
  result = initTable[string, Value]()
  for k, v in parseRowData(valStr):
    result[k] = v
 proc sqlEscapeIdent*(ident: string): string =
  ## Escape SQL identifiers by doubling double-quotes.
  result = ident.replace("\"", "\"\"")
 proc sqlEscapeString*(s: string): string =
  ## Escape SQL string literals by doubling single-quotes.
  result = s.replace("'", "''")
 proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
  ## Build a multi-row INSERT statement for bulk import.
  result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
  for i, col in columns:
    if i > 0: result &= ", "
    result &= "\"" & sqlEscapeIdent(col) & "\""
  result &= ") VALUES "
  for ri, row in rows:
    if ri > 0: result &= ", "
    result &= "("
    for ci, val in row:
      if ci > 0: result &= ", "
      if val.len == 0 or val == "\\N":
        result &= "NULL"
      else:
        result &= "'" & sqlEscapeString(val) & "'"
    result &= ")"
 proc getValue*(values: seq[string], fields: seq[string], colName: string): string =
  for i, f in fields:
    if f.toLower() == colName.toLower():
      if i < values.len: return values[i]
      return "\\N"
  return "\\N"
 proc getTableDef*(ctx: ExecutionContext, tableName: string): TableDef =
  if tableName in ctx.tables: return ctx.tables[tableName]
  return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[])
@@ -1,4 +1,7 @@
 ## BaraQL Executor — AST lowering, IR compilation, and execution
 ##
 ## Shared types/helpers live under `exec/` (re-exported below for API stability).
 ## See `exec/README.md` for module map and further extraction plan.
 import std/os
 import std/strutils
 import std/tables
@@ -53,140 +56,18 @@ import ../ai/embed as embedmod
 import ../ai/llm as llmmod
 import ../graph/cypher as cyphermod
-type
+import exec/types
-  IndexEntry* = ref object
+import exec/values
-    lsmKey*: string
+import exec/schema
-    rowValue*: string
+export types
-
+export values
-  ChangeKind* = enum
+export schema
    ckInsert, ckUpdate, ckDelete
  ChangeEvent* = object
    table*: string
    kind*: ChangeKind
    key*: string
    data*: string
  UserDef* = object
    name*: string
    passwordHash*: string
    isSuperuser*: bool
    roles*: seq[string]
  PrivilegeDef* = object
    tableName*: string
    command*: string  # SELECT, INSERT, UPDATE, DELETE, ALL
  PolicyDef* = object
    name*: string
    tableName*: string
    command*: string   # ALL, SELECT, INSERT, UPDATE, DELETE
    usingExpr*: Node   # parsed USING expression
    withCheckExpr*: Node  # parsed WITH CHECK expression
  SharedLock* = ref object
    lock*: Lock
  ExecutionContext* = ref object
    db*: LSMTree
    tables*: Table[string, TableDef]
    btrees*: Table[string, BTreeIndex[string, IndexEntry]]
    views*: Table[string, Node]  # view name -> SELECT AST
    cteTables*: Table[string, seq[Row]]  # CTE name -> rows
    ftsIndexes*: Table[string, fts.InvertedIndex]  # table.col -> FTS index
    vectorIndexes*: Table[string, vengine.HNSWIndex]  # table.col -> HNSW index
    graphs*: Table[string, gengine.Graph]  # graph name -> Graph object
    embedder*: embedmod.Embedder  # optional embedding service client
    llmClient*: llmmod.LLMClient  # optional LLM client for NL->SQL
    txnManager*: TxnManager
    pendingTxn*: Transaction
    onChange*: proc(ev: ChangeEvent) {.closure.}
    users*: Table[string, UserDef]
    policies*: Table[string, seq[PolicyDef]]  # table name -> policies
    currentUser*: string
    currentRole*: string
    sessionVars*: Table[string, string]
    autoIncCounters*: Table[string, int64]
    sequences*: Table[string, int64]
    sharedLock*: SharedLock  # shared across cloned contexts — protects tables, views, btrees, ftsIndexes, users, policies, autoIncCounters, sequences
    outerRow*: Table[string, string]  # outer query row for correlated subqueries
    subqueryPlan*: IRPlan  # current subquery plan being evaluated (for correlation in execScan)
    currentDatabase*: string  # name of the currently selected database
    registry*: DatabaseRegistry  # reference to the database registry (nil for single-DB mode)
  MigrationRecord* = object
    name*: string
    checksum*: string
    appliedAt*: int64
    appliedBy*: string
    durationMs*: int
    rolledBack*: bool
  ForeignKeyDef* = object
    refTable*: string
    refColumn*: string
    onDelete*: string  # CASCADE, SET NULL, RESTRICT
    onUpdate*: string  # CASCADE, SET NULL, RESTRICT
  CheckDef* = object
    name*: string
    expr*: string  # stored expression string
    checkNode*: Node  # AST for runtime evaluation
  TriggerDef* = object
    name*: string
    timing*: string   # BEFORE, AFTER
    event*: string    # INSERT, UPDATE, DELETE
    action*: Node     # SQL statement AST
  TableDef* = object
    name*: string
    columns*: seq[ColumnDef]
    pkColumns*: seq[string]
    foreignKeys*: seq[ForeignKeyDef]
    checks*: seq[CheckDef]
    triggers*: seq[TriggerDef]
  ColumnDef* = object
    name*: string
    colType*: string
    isPk*: bool
    isNotNull*: bool
    isUnique*: bool
    defaultVal*: string
    fkTable*: string
    fkColumn*: string
    fkOnDelete*: string
    fkOnUpdate*: string
    autoIncrement*: bool
  Row* = Table[string, Value]
  ExecResult* = object
    success*: bool
    columns*: seq[string]
    rows*: seq[Row]
    affectedRows*: int
    message*: string
    keyValuePairs*: seq[(string, seq[byte])]
 proc `==`*(a, b: IndexEntry): bool =
  a.lsmKey == b.lsmKey and a.rowValue == b.rowValue
 proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "",
               kvPairs: seq[(string, seq[byte])] = @[]): ExecResult =
  ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg,
             keyValuePairs: kvPairs)
 proc errResult*(msg: string): ExecResult =
  ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg)
 # ----------------------------------------------------------------------
 # Context management
 # ----------------------------------------------------------------------
 proc evalNodeToString(node: Node): string
 proc restoreSchema(ctx: ExecutionContext)
 proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): ExecutionContext =
  result = ExecutionContext(db: db, tables: initTable[string, TableDef](),
@@ -213,32 +94,6 @@ proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): Execut
 # AST to SQL serializer (for VIEW DDL persistence)
 # ----------------------------------------------------------------------
 proc sqlEscapeIdent*(ident: string): string =
  ## Escape SQL identifiers by doubling double-quotes.
  result = ident.replace("\"", "\"\"")
 proc sqlEscapeString*(s: string): string =
  ## Escape SQL string literals by doubling single-quotes.
  result = s.replace("'", "''")
 proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
  ## Build a multi-row INSERT statement for bulk import.
  result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
  for i, col in columns:
    if i > 0: result &= ", "
    result &= "\"" & sqlEscapeIdent(col) & "\""
  result &= ") VALUES "
  for ri, row in rows:
    if ri > 0: result &= ", "
    result &= "("
    for ci, val in row:
      if ci > 0: result &= ", "
      if val.len == 0 or val == "\\N":
        result &= "NULL"
      else:
        result &= "'" & sqlEscapeString(val) & "'"
    result &= ")"
 proc exprToSql(node: Node): string =
  if node == nil:
    return ""
@@ -343,76 +198,6 @@ proc selectToSql(node: Node): string =
  if node.selOffset != nil and node.selOffset.offsetExpr.kind == nkIntLit:
    result.add(" OFFSET " & $node.selOffset.offsetExpr.intVal)
 # ----------------------------------------------------------------------
 # Schema restore
 # ----------------------------------------------------------------------
 proc restoreSchema(ctx: ExecutionContext) =
  for entry in ctx.db.scanMemTable():
    if entry.deleted: continue
    if not entry.key.startsWith("_schema:"): continue
    let ddl = cast[string](entry.value)
    if ddl.len == 0: continue
    var astNode: Node
    try:
      let tokens = qlex.tokenize(ddl)
      astNode = qpar.parse(tokens)
    except:
      # Skip corrupted schema entries during startup
      continue
    if astNode.stmts.len > 0:
      let stmt = astNode.stmts[0]
      case stmt.kind
      of nkCreateTable:
        var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[],
                           foreignKeys: @[], checks: @[], triggers: @[])
        for col in stmt.crtColumns:
          if col.kind == nkColumnDef:
            var colDef = ColumnDef(name: col.cdName, colType: col.cdType)
            colDef.autoIncrement = col.cdAutoIncrement
            for cst in col.cdConstraints:
              if cst.kind == nkConstraintDef:
                case cst.cstType
                of "pkey":
                  colDef.isPk = true
                  tbl.pkColumns.add(col.cdName)
                  ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
                of "notnull": colDef.isNotNull = true
                of "unique":
                  colDef.isUnique = true
                  ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]()
                of "default":
                  if cst.cstDefault != nil:
                    colDef.defaultVal = evalNodeToString(cst.cstDefault)
                of "fkey":
                  colDef.fkTable = cst.cstRefTable
                  colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: ""
                  colDef.fkOnDelete = cst.cstOnDelete
                  colDef.fkOnUpdate = cst.cstOnUpdate
                else: discard
            tbl.columns.add(colDef)
        ctx.tables[stmt.crtName] = tbl
      of nkCreateView:
        ctx.views[stmt.cvName] = stmt.cvQuery
      of nkCreateTrigger:
        if stmt.trigTable in ctx.tables:
          ctx.tables[stmt.trigTable].triggers.add(TriggerDef(
            name: stmt.trigName,
            timing: stmt.trigTiming,
            event: stmt.trigEvent,
            action: stmt.trigAction,
          ))
      of nkCreateUser:
        ctx.users[stmt.cuName] = UserDef(name: stmt.cuName,
            passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[])
      of nkCreatePolicy:
        var pols = ctx.policies.getOrDefault(stmt.cpTable)
        pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable,
                           command: stmt.cpCommand, usingExpr: stmt.cpUsing,
                           withCheckExpr: stmt.cpWithCheck))
        ctx.policies[stmt.cpTable] = pols
      else: discard
 proc cloneForConnection*(ctx: ExecutionContext): ExecutionContext =
  var svCopy = initTable[string, string]()
  for k, v in ctx.sessionVars:
@@ -512,97 +297,6 @@ proc getMigrationBody(ctx: ExecutionContext, name: string): (bool, string, strin
    else:
      return (true, ddl, "")
  return (false, "", "")
 # ----------------------------------------------------------------------
 # Helpers
 # ----------------------------------------------------------------------
 proc getTableDef(ctx: ExecutionContext, tableName: string): TableDef =
  if tableName in ctx.tables: return ctx.tables[tableName]
  return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[])
 proc getValue(values: seq[string], fields: seq[string], colName: string): string =
  for i, f in fields:
    if f.toLower() == colName.toLower() and i < values.len:
      return values[i]
  return "\\N"
 proc isNull*(value: string): bool =
  value == "\\N" or value.toLower() == "null"
 proc valueToString*(v: Value): string =
  case v.kind
  of vkNull: return "\\N"
  of vkString: return v.strVal
  of vkInt64: return $v.int64Val
  of vkFloat64: return $v.float64Val
  of vkBool: return $v.boolVal
  else: return ""
 proc `%`*(v: Value): JsonNode =
  case v.kind
  of vkNull: return newJNull()
  of vkString: return %v.strVal
  of vkInt64: return %v.int64Val
  of vkFloat64: return %v.float64Val
  of vkBool: return %v.boolVal
  else: return newJNull()
 proc toString*(v: Value): string = valueToString(v)
 proc `[]=`*(t: var Row, key: string, val: string) =
  t[key] = Value(kind: vkString, strVal: val)
 proc escapeRowVal(v: string): string =
  v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
 proc unescapeRowVal(v: string): string =
  result = ""
  var i = 0
  while i < v.len:
    if v[i] == '\\' and i + 1 < v.len:
      case v[i+1]
      of '\\', ',', '=':
        result &= v[i+1]
        i += 2
        continue
      else: discard
    result &= v[i]
    inc i
 proc parseRowData(valStr: string): Table[string, string] =
  ## Parse "col1=val1,col2=val2" into a table
  result = initTable[string, string]()
  var i = 0
  var part = ""
  while i < valStr.len:
    if valStr[i] == '\\' and i + 1 < valStr.len:
      part &= valStr[i]
      part &= valStr[i+1]
      i += 2
      continue
    if valStr[i] == ',':
      let eqPos = part.find('=')
      if eqPos >= 0:
        let k = part[0..<eqPos].strip()
        let v = unescapeRowVal(part[eqPos+1..^1].strip())
        result[k] = v
      part = ""
    else:
      part &= valStr[i]
    inc i
  if part.len > 0:
    let eqPos = part.find('=')
    if eqPos >= 0:
      let k = part[0..<eqPos].strip()
      let v = unescapeRowVal(part[eqPos+1..^1].strip())
      result[k] = v
 proc parseRowDataToValueRow(valStr: string): Row =
  result = initTable[string, Value]()
  for k, v in parseRowData(valStr):
    result[k] = v
 proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row]
 proc extractJoinEquality*(expr: IRExpr): (string, string) =
@@ -4988,30 +4682,35 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
                tbl.columns[i].fkOnDelete = cstNode.cstOnDelete
                tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate
    ctx.tables[stmt.crtName] = tbl
-
+    persistTableSchema(ctx, tbl)
    # Persist schema
    var colDefs: seq[string] = @[]
    for col in tbl.columns:
      var parts = @[col.name, col.colType]
      if col.isPk: parts.add("PRIMARY KEY")
      if col.autoIncrement: parts.add("AUTO_INCREMENT")
      if col.isNotNull: parts.add("NOT NULL")
      if col.isUnique: parts.add("UNIQUE")
      if col.defaultVal.len > 0: parts.add("DEFAULT '" & col.defaultVal & "'")
      if col.fkTable.len > 0:
        parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")")
      colDefs.add(parts.join(" "))
    let schemaKey = "_schema:migrations:" & $ctx.tables.len
    ctx.db.put(schemaKey, cast[seq[byte]]("CREATE TABLE " & stmt.crtName & " (" & colDefs.join(", ") & ")"))
    return okResult()
  of nkDropTable:
-    ctx.tables.del(stmt.drtName)
+    let dropName = stmt.drtName
    ctx.tables.del(dropName)
    var toDelete: seq[string] = @[]
    for idxName in ctx.btrees.keys.toSeq():
-      if idxName.startsWith(stmt.drtName & "."): toDelete.add(idxName)
+      if idxName.startsWith(dropName & "."): toDelete.add(idxName)
    for idxName in toDelete: ctx.btrees.del(idxName)
    # Remove durable schema entry
    dropTableSchema(ctx, dropName)
    # Remove row data for this table
    var dataKeys: seq[string] = @[]
    let prefix = dropName & "."
    for (key, _) in ctx.db.scanAll():
      if key.startsWith(prefix):
        dataKeys.add(key)
    for key in dataKeys:
      ctx.db.delete(key)
    # Drop orphan legacy schema keys that mentioned this table
    var legacyKeys: seq[string] = @[]
    for (key, value) in ctx.db.scanAll():
      if key.startsWith(SchemaLegacyCreatePrefix):
        let ddl = cast[string](value)
        if ddl.contains("CREATE TABLE " & dropName) or ddl.contains("CREATE TABLE \"" & dropName):
          legacyKeys.add(key)
    for key in legacyKeys:
      ctx.db.delete(key)
    return okResult()
  of nkCreateGraph:
@@ -5113,6 +4812,7 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
          var colDef = ColumnDef(name: op.cdName, colType: op.cdType)
          tbl.columns.add(colDef)
      ctx.tables[stmt.altName] = tbl
      persistTableSchema(ctx, tbl)
      return okResult(msg="ALTER TABLE " & stmt.altName & " executed")
    return errResult("Table '" & stmt.altName & "' does not exist")
@@ -8,6 +8,7 @@ import ../storage/lsm
 const
  MaxLevel* = 7
  LevelMultiplier* = 10  # each level is 10x the previous
  ## L0 uses file-count trigger (overlapping ranges); lower levels use size.
 type
  SSTableMeta* = object
@@ -29,20 +30,44 @@ type
    levels*: seq[seq[SSTableMeta]]
    dataDir*: string
    maxSizePerLevel*: seq[int]
    l0FileLimit*: int
-proc newCompactionStrategy*(dataDir: string): CompactionStrategy =
+proc newCompactionStrategy*(dataDir: string, l0FileLimit: int = L0CompactionTrigger): CompactionStrategy =
  result = CompactionStrategy(
    levels: newSeq[seq[SSTableMeta]](MaxLevel),
    dataDir: dataDir,
    maxSizePerLevel: newSeq[int](MaxLevel),
    l0FileLimit: l0FileLimit,
  )
  for i in 0..<MaxLevel:
    result.levels[i] = @[]
    result.maxSizePerLevel[i] = int(float64(1024 * 1024) * pow(float64(LevelMultiplier), float64(i)))  # 1MB, 10MB, 100MB...
 proc clear*(cs: CompactionStrategy) =
  ## Drop all registered tables (used before rebuild-from-LSM).
  for i in 0..<MaxLevel:
    cs.levels[i].setLen(0)
 proc addTable*(cs: CompactionStrategy, meta: SSTableMeta) =
-  if meta.level < MaxLevel:
+  let lvl = clamp(meta.level, 0, MaxLevel - 1)
-    cs.levels[meta.level].add(meta)
+  cs.levels[lvl].add(meta)
 proc rebuildFromLSM*(cs: CompactionStrategy, db: LSMTree) =
  ## Rebuild level layout from the live LSMTree catalog — single source of truth.
  ## Avoids drift when flushes add SSTables the strategy never saw.
  cs.clear()
  cs.dataDir = db.dir
  for sst in db.sstables:
    let size = try: int(getFileSize(sst.path)) except: sst.entryCount * 64
    cs.addTable(SSTableMeta(
      path: sst.path,
      level: sst.level,
      minKey: sst.minKey,
      maxKey: sst.maxKey,
      entryCount: sst.entryCount,
      sizeBytes: size,
      createdAt: sst.id,  # stable ordering by id / creation sequence
    ))
 proc totalSize*(cs: CompactionStrategy, level: int): int =
  result = 0
@@ -52,6 +77,9 @@ proc totalSize*(cs: CompactionStrategy, level: int): int =
 proc needsCompaction*(cs: CompactionStrategy, level: int): bool =
  if level >= MaxLevel - 1:
    return false
  if level == 0:
    # L0 files can overlap — count-based trigger (RocksDB-style)
    return cs.levels[0].len >= cs.l0FileLimit
  return cs.totalSize(level) > cs.maxSizePerLevel[level]
 proc pickTablesForCompaction*(cs: CompactionStrategy, level: int): seq[SSTableMeta] =
@@ -0,0 +1,41 @@
 ## Global storage gate — exclusive multi-thread entry to LSM / executor.
 ##
 ## Why: Hunos HTTP runs handlers on a worker-thread pool (`spawn` + internal
 ## workers). The TCP server runs on the main async loop. Both share the same
 ## `LSMTree` / `ExecutionContext` refs. Nim's default ORC memory manager is not
 ## safe for concurrent refcount ops on the same objects from multiple OS threads.
 ##
 ## Holding this gate for the full duration of a query/compaction/DDL ensures
 ## only one thread mutates or reads GC-managed storage state at a time.
 ##
 ## Ordering: always acquire StorageGate **before** any per-DB `LSMTree.lock`.
 ## Call `initStorageGate()` once from main before accepting connections.
 import std/locks
 var
  gGate: Lock
  gInited*: bool
 proc initStorageGate*() =
  ## Idempotent when called from a single thread at startup.
  if not gInited:
    initLock(gGate)
    gInited = true
 proc acquireStorageGate*() {.inline.} =
  ## Prefer calling initStorageGate() once at process start (main).
  ## Lazy-init is allowed for unit tests (single-threaded).
  if not gInited:
    initStorageGate()
  acquire(gGate)
 proc releaseStorageGate*() {.inline.} =
  release(gGate)
 template withStorageGate*(body: untyped) =
  ## Exclusive ownership of the storage engine for `body`.
  acquireStorageGate()
  try:
    body
  finally:
    releaseStorageGate()
@@ -13,6 +13,11 @@ import bloom
 import wal
 import mmap
 import crc32
 import rwlock
 # Re-export WAL durability knobs for callers of newLSMTree
 export wal
 export rwlock
 const
  SSTableMagic* = 0x53535442'u32  # "SSTB"
@@ -21,6 +26,8 @@ const
  DefaultBloomFpRate* = 0.01
  ManifestVersion* = 1
  ManifestFileName* = "MANIFEST"
  ## Trigger L0 compaction when this many L0 SSTables exist.
  L0CompactionTrigger* = 4
 type
  Entry* = object
@@ -29,9 +36,10 @@ type
    timestamp*: uint64
    deleted*: bool
  ## Hash-table MemTable: O(1) put/get. Sorted only when flushing to SSTable.
  MemTable* = object
-    entries: seq[Entry]
+    map: Table[string, Entry]
-    size: int
+    size: int      ## approximate byte size of live entries
    maxSize: int
  SSTable* = object
@@ -56,55 +64,64 @@ type
    currentSeq: uint64
    nextSSTableId*: int
    manifestSequence*: int64
-    lock*: Lock
+    ## Reader-writer lock: concurrent gets; exclusive put/flush/compact.
    ## `acquire(db.lock)` is exclusive (write) for backward compatibility.
    lock*: RwLock
    walLock*: Lock
    ## Set by flush when L0 file count hits L0CompactionTrigger (hint for compactors).
    needsCompaction*: bool
    ## When true, flushUnsafe skips WAL rewrite (recovery still holds the WAL file open).
    recovering: bool
 proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable =
-  MemTable(entries: @[], size: 0, maxSize: maxSize)
+  MemTable(map: initTable[string, Entry](), size: 0, maxSize: maxSize)
-proc len*(mt: MemTable): int = mt.entries.len
+proc len*(mt: MemTable): int = mt.map.len
 proc byteSize*(mt: MemTable): int = mt.size
 proc put*(mt: var MemTable, key: string, value: seq[byte], timestamp: uint64, deleted: bool = false): bool =
  ## O(1) average-case insert/update. Returns false if the new key would exceed maxSize.
  let entrySize = key.len + value.len + 16
  if entrySize > mt.maxSize:
    return false
  let entry = Entry(key: key, value: value, timestamp: timestamp, deleted: deleted)
-  let pos = mt.entries.lowerBound(entry, proc(a, b: Entry): int = cmp(a.key, b.key))
+  if key in mt.map:
-  if pos < mt.entries.len and mt.entries[pos].key == key:
+    let old = mt.map[key]
-    let oldSize = mt.entries[pos].key.len + mt.entries[pos].value.len + 16
+    # Only accept equal-or-newer timestamps (WAL recovery may replay older values)
-    mt.entries[pos] = entry
+    if timestamp < old.timestamp:
      return true
    let oldSize = old.key.len + old.value.len + 16
    mt.map[key] = entry
    mt.size += entrySize - oldSize
  else:
-    if mt.size + entrySize > mt.maxSize and mt.entries.len > 0:
+    if mt.size + entrySize > mt.maxSize and mt.map.len > 0:
      return false
-    mt.entries.insert(entry, pos)
+    mt.map[key] = entry
    mt.size += entrySize
  return true
 proc get*(mt: MemTable, key: string): (bool, Entry) =
-  if mt.entries.len == 0:
+  if key in mt.map:
-    return (false, Entry())
+    return (true, mt.map[key])
  var lo = 0
  var hi = mt.entries.len - 1
  while lo <= hi:
    let mid = (lo + hi) div 2
    let c = cmp(mt.entries[mid].key, key)
    if c == 0:
      return (true, mt.entries[mid])
    elif c < 0:
      lo = mid + 1
    else:
      hi = mid - 1
  return (false, Entry())
 proc sortedEntries*(mt: MemTable): seq[Entry] =
  ## Materialize entries sorted by key — used for SSTable flush and ordered scans.
  result = newSeqOfCap[Entry](mt.map.len)
  for _, entry in mt.map:
    result.add(entry)
  result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
 proc scan*(mt: MemTable, startKey, endKey: string): seq[Entry] =
  result = @[]
-  for entry in mt.entries:
+  for key, entry in mt.map:
-    if entry.key >= startKey and entry.key <= endKey:
+    if key >= startKey and key <= endKey:
      result.add(entry)
  result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
 proc clear*(mt: var MemTable) =
-  mt.entries.setLen(0)
+  mt.map.clear()
  mt.size = 0
 # ----------------------------------------------------------------------
@@ -600,8 +617,15 @@ proc checkStorageConsistency*(db: LSMTree): seq[string] =
 # ----------------------------------------------------------------------
 proc flushUnsafe(db: LSMTree) {.gcsafe.}
 proc countL0*(db: LSMTree): int
-proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
+proc newLSMTree*(
    dir: string,
    memMaxSize: int = DefaultMemTableSize,
    walSyncMode: WalSyncMode = wsmGroup,
    walGroupEvery: int = DefaultWalGroupEvery,
    walGroupIntervalMs: int = 0,
 ): LSMTree =
  createDir(dir)
  createDir(dir / "sstables")
@@ -641,21 +665,29 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
      echo "[INFO] Loaded ", sstables.len, " SSTable(s) from directory scan"
  new(result)
-  initLock(result.lock)
+  initRwLock(result.lock)
  initLock(result.walLock)
  result.dir = dir
  result.memTable = newMemTable(memMaxSize)
  result.immutableMem = newMemTable(0)
  result.sstables = sstables
-  result.wal = newWriteAheadLog(dir / "wal")
+  result.wal = newWriteAheadLog(
    dir / "wal",
    syncMode = walSyncMode,
    groupEvery = walGroupEvery,
    groupIntervalMs = walGroupIntervalMs,
  )
  result.memMaxSize = memMaxSize
  result.currentSeq = 0
  result.nextSSTableId = nextId
  result.manifestSequence = manifestSeq
  result.recovering = false
  result.needsCompaction = result.countL0() >= L0CompactionTrigger
  # WAL crash recovery — replay unflushed entries into memTable
  let walPath = dir / "wal" / "wal.log"
  if fileExists(walPath):
    result.recovering = true
    var stream: FileStream = nil
    try:
      stream = newFileStream(walPath, fmRead)
@@ -697,11 +729,30 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
    finally:
      if stream != nil:
        stream.close()
      result.recovering = false
      # After recovery, shrink WAL to live unflushed state only
      acquire(result.walLock)
      try:
        var liveKeys: seq[string] = @[]
        var liveVals: seq[seq[byte]] = @[]
        var liveTs: seq[uint64] = @[]
        var liveDel: seq[bool] = @[]
        for e in result.immutableMem.sortedEntries():
          liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted)
        for e in result.memTable.sortedEntries():
          liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted)
        if liveKeys.len == 0:
          result.wal.truncate()
        else:
          result.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel)
      finally:
        release(result.walLock)
 proc put*(db: LSMTree, key: string, value: seq[byte]) =
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
+  acquireWrite(db.lock)
-  defer: release(db.lock)
+  defer: releaseWrite(db.lock)
  # WAL then memtable under the same exclusive lock → crash recovery sees a total order
  acquire(db.walLock)
  db.wal.writePut(cast[seq[byte]](key), value, ts)
  release(db.walLock)
@@ -716,8 +767,8 @@ proc put*(db: LSMTree, key: string, value: seq[byte]) =
 proc delete*(db: LSMTree, key: string) =
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
+  acquireWrite(db.lock)
-  defer: release(db.lock)
+  defer: releaseWrite(db.lock)
  acquire(db.walLock)
  db.wal.writeDelete(cast[seq[byte]](key), ts)
  release(db.walLock)
@@ -732,8 +783,8 @@ proc delete*(db: LSMTree, key: string) =
 proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = false) =
  ## Direct LSM insert without WAL logging — used by recovery.
  let ts = uint64(getMonoTime().ticks())
-  acquire(db.lock)
+  acquireWrite(db.lock)
-  defer: release(db.lock)
+  defer: releaseWrite(db.lock)
  if not db.memTable.put(key, value, ts, deleted):
    if db.immutableMem.len > 0:
      db.flushUnsafe()
@@ -745,18 +796,26 @@ proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = fals
 proc deleteUnsafe*(db: LSMTree, key: string) =
  putUnsafe(db, key, @[], deleted = true)
 proc copyBytes(s: seq[byte]): seq[byte] =
  ## Deep copy so callers on other threads never share ORC-managed seq buffers.
  result = newSeq[byte](s.len)
  if s.len > 0:
    copyMem(addr result[0], unsafeAddr s[0], s.len)
 proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
  ## Caller must hold at least a read lock.
  ## Returned values are deep-copied for multi-thread ORC safety (HTTP + TCP share LSM).
  let (found, entry) = db.memTable.get(key)
  if found:
    if entry.deleted:
      return (false, @[])
-    return (true, entry.value)
+    return (true, copyBytes(entry.value))
  let (found2, entry2) = db.immutableMem.get(key)
  if found2:
    if entry2.deleted:
      return (false, @[])
-    return (true, entry2.value)
+    return (true, copyBytes(entry2.value))
  # Search SSTables from newest to oldest
  for i in countdown(db.sstables.high, db.sstables.low):
@@ -769,21 +828,40 @@ proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
    if found3:
      if entry3.deleted:
        return (false, @[])
-      return (true, entry3.value)
+      return (true, copyBytes(entry3.value))
  return (false, @[])
 proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
-  acquire(db.lock)
+  ## Thread-safe lookup.
-  defer: release(db.lock)
+  ## Default: exclusive lock — required for Nim ORC when TCP + HTTP threads share the DB.
  ## Compile with `-d:baraConcurrentReads` for shared read locks (needs multi-thread-safe MM
  ## such as a future atomicArc build; unsafe with default ORC across OS threads).
  when defined(baraConcurrentReads):
    acquireRead(db.lock)
    defer: releaseRead(db.lock)
  else:
    acquireWrite(db.lock)
    defer: releaseWrite(db.lock)
  return getUnsafe(db, key)
 proc contains*(db: LSMTree, key: string): bool =
-  acquire(db.lock)
+  when defined(baraConcurrentReads):
-  defer: release(db.lock)
+    acquireRead(db.lock)
    defer: releaseRead(db.lock)
  else:
    acquireWrite(db.lock)
    defer: releaseWrite(db.lock)
  let (found, _) = getUnsafe(db, key)
  return found
 proc countL0*(db: LSMTree): int =
  ## Number of level-0 SSTables (newest, uncompacted).
  result = 0
  for sst in db.sstables:
    if sst.level == 0:
      inc result
 proc flushUnsafe(db: LSMTree) =
  if db.immutableMem.len == 0 and db.memTable.len == 0:
    return
@@ -802,7 +880,8 @@ proc flushUnsafe(db: LSMTree) =
  let path = db.dir / "sstables" / ($db.nextSSTableId & ".sst")
  inc db.nextSSTableId
-  var sst = writeSSTable(toFlush.entries, path, level = 0)
+  # Sort once at flush time (O(n log n)) — put/get stay O(1)
  var sst = writeSSTable(toFlush.sortedEntries(), path, level = 0)
  sst.id = db.nextSSTableId - 1
  db.sstables.add(sst)
  # SSTables are kept in insertion order (newest last) so getUnsafe can search newest-first
@@ -814,22 +893,43 @@ proc flushUnsafe(db: LSMTree) =
  except CatchableError as e:
    echo "[WARN] Failed to write MANIFEST: ", e.msg
  # Rewrite WAL to contain only still-unflushed memtable entries.
  # Skip during recovery — the WAL file is still open for reading.
  if not db.recovering:
    acquire(db.walLock)
-  db.wal.writeCommit(uint64(getMonoTime().ticks()))
+    var liveKeys: seq[string] = @[]
-  db.wal.maybeRotate()
+    var liveVals: seq[seq[byte]] = @[]
-  db.wal.sync()
+    var liveTs: seq[uint64] = @[]
    var liveDel: seq[bool] = @[]
    for e in db.immutableMem.sortedEntries():
      liveKeys.add(e.key)
      liveVals.add(e.value)
      liveTs.add(e.timestamp)
      liveDel.add(e.deleted)
    for e in db.memTable.sortedEntries():
      liveKeys.add(e.key)
      liveVals.add(e.value)
      liveTs.add(e.timestamp)
      liveDel.add(e.deleted)
    if liveKeys.len == 0:
      db.wal.truncate()
    else:
      db.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel)
    release(db.walLock)
  if db.countL0() >= L0CompactionTrigger:
    db.needsCompaction = true
 proc flush*(db: LSMTree) =
-  acquire(db.lock)
+  acquireWrite(db.lock)
-  defer: release(db.lock)
+  defer: releaseWrite(db.lock)
  flushUnsafe(db)
 proc checkpoint*(db: LSMTree) =
  ## Create a consistent checkpoint: freeze memtable, flush to SSTable,
  ## rotate WAL, and write MANIFEST. This provides a clean boundary
  ## for online backup without stopping the server.
-  acquire(db.lock)
+  acquireWrite(db.lock)
  # Flush any pending immutable memtable first
  if db.immutableMem.len > 0:
@@ -850,10 +950,10 @@ proc checkpoint*(db: LSMTree) =
  db.wal.sync()
  release(db.walLock)
-  release(db.lock)
+  releaseWrite(db.lock)
 proc close*(db: LSMTree) =
-  acquire(db.lock)
+  acquireWrite(db.lock)
  try:
    # Flush both memtables to avoid data loss
    while db.immutableMem.len > 0:
@@ -863,50 +963,98 @@ proc close*(db: LSMTree) =
      sst.close()
    db.wal.close()
  finally:
-    release(db.lock)
+    releaseWrite(db.lock)
 template withDataLock(db: LSMTree, body: untyped) =
  ## Shared or exclusive depending on baraConcurrentReads (see get*).
  when defined(baraConcurrentReads):
    acquireRead(db.lock)
    try:
      body
    finally:
      releaseRead(db.lock)
  else:
    acquireWrite(db.lock)
    try:
      body
    finally:
      releaseWrite(db.lock)
 proc memTableSize*(db: LSMTree): int =
-  acquire(db.lock)
+  withDataLock(db):
  defer: release(db.lock)
    return db.memTable.len
 proc sstableCount*(db: LSMTree): int =
-  acquire(db.lock)
+  withDataLock(db):
  defer: release(db.lock)
    return db.sstables.len
 proc dir*(db: LSMTree): string =
-  acquire(db.lock)
+  withDataLock(db):
  defer: release(db.lock)
    return db.dir
 proc scanMemTable*(db: LSMTree): seq[Entry] =
-  acquire(db.lock)
+  ## Return all entries from memory (memTable + immutableMem), sorted by key.
-  defer: release(db.lock)
+  ## Immutable wins over active memtable only when timestamps are newer (same key rare).
-  ## Return all entries from memory (memTable + immutableMem)
+  withDataLock(db):
-  result = @[]
+    var merged = initTable[string, Entry]()
-  for e in db.memTable.entries:
+    for e in db.immutableMem.sortedEntries():
-    result.add(e)
+      merged[e.key] = e
-  for e in db.immutableMem.entries:
+    for e in db.memTable.sortedEntries():
      if e.key notin merged or e.timestamp >= merged[e.key].timestamp:
        merged[e.key] = e
    result = newSeqOfCap[Entry](merged.len)
    for _, e in merged:
      result.add(e)
    result.sort(proc(a, b: Entry): int = cmp(a.key, b.key))
 proc scanRange*(db: LSMTree, startKey, endKey: string): seq[(string, seq[byte])] =
  ## Inclusive key range scan over memtables + SSTables (newest wins).
  withDataLock(db):
    var best = initTable[string, Entry]()
    for e in db.memTable.scan(startKey, endKey):
      best[e.key] = e
    for e in db.immutableMem.scan(startKey, endKey):
      if e.key notin best or e.timestamp > best[e.key].timestamp:
        best[e.key] = e
    for i in countdown(db.sstables.high, db.sstables.low):
      let sst = db.sstables[i]
      if sst.maxKey < startKey or sst.minKey > endKey:
        continue
      for key, offset in sst.index:
        if key < startKey or key > endKey:
          continue
        if key in best:
          continue
        let (found, entry) = readSSTableEntry(sst, key)
        if found:
          best[key] = entry
    var keys = newSeqOfCap[string](best.len)
    for k in best.keys:
      keys.add(k)
    keys.sort(cmp)
    for k in keys:
      let e = best[k]
      if not e.deleted:
        result.add((e.key, e.value))
 proc scanAll*(db: LSMTree): seq[(string, seq[byte])] =
  ## Scan all active (non-deleted) entries from memory and SSTables.
  ## Used for shard data migration.
-  acquire(db.lock)
+  withDataLock(db):
  defer: release(db.lock)
    var seen = initTable[string, bool]()
    # Scan memtable first (most recent)
-  for e in db.memTable.entries:
+    for e in db.memTable.sortedEntries():
      if e.key notin seen:
        seen[e.key] = true
        if not e.deleted:
          result.add((e.key, e.value))
    # Scan immutable memtable
-  for e in db.immutableMem.entries:
+    for e in db.immutableMem.sortedEntries():
      if e.key notin seen:
        seen[e.key] = true
        if not e.deleted:
@@ -0,0 +1,80 @@
 ## Simple reader-writer lock for LSM concurrent reads.
 ## Multiple readers OR one writer. Writers are exclusive.
 ## `acquire` / `release` are write-side (backward compatible with Lock-style usage).
 import std/locks
 type
  RwLock* = object
    mu: Lock
    readers: int          ## active readers
    writer: bool          ## writer holds exclusive access
    waitingWriters: int   ## prefer writers to avoid reader starvation of compact/flush
    canRead: Cond
    canWrite: Cond
 proc initRwLock*(rw: var RwLock) =
  initLock(rw.mu)
  initCond(rw.canRead)
  initCond(rw.canWrite)
  rw.readers = 0
  rw.writer = false
  rw.waitingWriters = 0
 proc deinitRwLock*(rw: var RwLock) =
  deinitCond(rw.canRead)
  deinitCond(rw.canWrite)
  deinitLock(rw.mu)
 proc acquireRead*(rw: var RwLock) =
  ## Shared read lock. Blocks while a writer is active or waiting (writer preference).
  acquire(rw.mu)
  while rw.writer or rw.waitingWriters > 0:
    wait(rw.canRead, rw.mu)
  inc rw.readers
  release(rw.mu)
 proc releaseRead*(rw: var RwLock) =
  acquire(rw.mu)
  dec rw.readers
  if rw.readers == 0:
    # Wake one waiting writer
    signal(rw.canWrite)
  release(rw.mu)
 proc acquireWrite*(rw: var RwLock) =
  ## Exclusive write lock.
  acquire(rw.mu)
  inc rw.waitingWriters
  while rw.writer or rw.readers > 0:
    wait(rw.canWrite, rw.mu)
  dec rw.waitingWriters
  rw.writer = true
  release(rw.mu)
 proc releaseWrite*(rw: var RwLock) =
  acquire(rw.mu)
  rw.writer = false
  # Prefer draining writers, else open the gate for readers
  if rw.waitingWriters > 0:
    signal(rw.canWrite)
  else:
    broadcast(rw.canRead)
  release(rw.mu)
 # Lock-compatible names: default exclusive (used by compaction, put, flush)
 proc acquire*(rw: var RwLock) {.inline.} = acquireWrite(rw)
 proc release*(rw: var RwLock) {.inline.} = releaseWrite(rw)
 template withReadLock*(rw: var RwLock, body: untyped) =
  acquireRead(rw)
  try:
    body
  finally:
    releaseRead(rw)
 template withWriteLock*(rw: var RwLock, body: untyped) =
  acquireWrite(rw)
  try:
    body
  finally:
    releaseWrite(rw)
@@ -4,12 +4,16 @@ import std/os
 import std/streams
 import std/strutils
 import std/posix
 import std/monotimes
 import std/times
 const
  WALMagic* = 0x42415241'u32  # "BARA"
  WALVersion* = 1'u32
  DefaultMaxWalSegmentSize* = 64 * 1024 * 1024  # 64MB
  WalArchiveDir* = "wal_archive"
  ## Default group-commit batch size (entries between fsyncs).
  DefaultWalGroupEvery* = 64
 type
  WalEntryKind* = enum
@@ -18,6 +22,15 @@ type
    wekCheckpoint = 3
    wekCommit = 4
  ## Durability policy for WAL writes.
  ## - wsmNone:  flush userspace buffer only; fsync on truncate/rewrite/close/explicit sync
  ## - wsmGroup: group commit — fsync every N entries and/or every intervalMs (default)
  ## - wsmEvery: fsync after every entry (strict, slow)
  WalSyncMode* = enum
    wsmNone = "none"
    wsmGroup = "group"
    wsmEvery = "every"
  WalEntry* = object
    kind*: WalEntryKind
    timestamp*: uint64
@@ -34,14 +47,28 @@ type
    path: string
    stream: FileStream
    entryCount: uint64
-    syncOnWrite: bool
+    syncMode*: WalSyncMode
    groupEvery*: int          ## entries between fsyncs when mode=group
    groupIntervalMs*: int     ## time-based fsync when mode=group (0 = off)
    unsyncedEntries: int      ## entries written since last fsync
    lastSync: MonoTime
    maxSegmentSize: int64
    currentSequence: int64
    ## Counters for observability / benchmarks
    fsyncCount*: uint64
    bytesSinceSync: int
 proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry]
 proc listWalArchive*(dir: string): seq[WalSegment]
 proc maybeRotate*(wal: var WriteAheadLog)
 proc parseWalSyncMode*(s: string): WalSyncMode =
  case s.toLowerAscii()
  of "none", "async", "off", "false", "0": wsmNone
  of "every", "sync", "full", "true", "1": wsmEvery
  of "group", "batch", "": wsmGroup
  else: wsmGroup
 proc parseWalSequence*(filename: string): int64 =
  ## Extract sequence from "wal.000042.log"
  try:
@@ -74,6 +101,12 @@ proc nextWalSequence*(dir: string): int64 =
    return 1
  return segments[^1].sequence + 1
 proc fsyncPath(path: string) =
  let fd = posix.open(cstring(path), O_RDWR)
  if fd != -1:
    discard posix.fsync(fd)
    discard posix.close(fd)
 proc rotate*(wal: var WriteAheadLog) =
  ## Close current WAL and archive it, then start a new one.
  if wal.stream != nil:
@@ -96,7 +129,12 @@ proc rotate*(wal: var WriteAheadLog) =
  wal.stream.write(WALMagic)
  wal.stream.write(WALVersion)
  wal.stream.flush()
  fsyncPath(wal.path)
  wal.entryCount = 0
  wal.unsyncedEntries = 0
  wal.bytesSinceSync = 0
  wal.lastSync = getMonoTime()
  inc wal.fsyncCount
 proc maybeRotate*(wal: var WriteAheadLog) =
  ## Rotate if current WAL exceeds max segment size.
@@ -106,7 +144,16 @@ proc maybeRotate*(wal: var WriteAheadLog) =
  if currentSize >= wal.maxSegmentSize:
    wal.rotate()
-proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
+proc newWriteAheadLog*(
    dir: string,
    syncMode: WalSyncMode = wsmGroup,
    groupEvery: int = DefaultWalGroupEvery,
    groupIntervalMs: int = 0,
    syncOnWrite: bool = false,
 ): WriteAheadLog =
  ## Create a WAL.
  ## - syncMode controls durability (see WalSyncMode).
  ## - syncOnWrite=true is legacy and forces wsmEvery.
  createDir(dir)
  let path = dir / "wal.log"
  let exists = fileExists(path)
@@ -125,18 +172,62 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
    for e in readEntries(path):
      inc count
  let mode = if syncOnWrite: wsmEvery else: syncMode
  let ge = if groupEvery <= 0: DefaultWalGroupEvery else: groupEvery
  let seqNum = nextWalSequence(dir)
  WriteAheadLog(
    dir: dir,
    path: path,
    stream: stream,
    entryCount: count,
-    syncOnWrite: syncOnWrite,
+    syncMode: mode,
    groupEvery: ge,
    groupIntervalMs: groupIntervalMs,
    unsyncedEntries: 0,
    lastSync: getMonoTime(),
    maxSegmentSize: DefaultMaxWalSegmentSize,
    currentSequence: seqNum,
    fsyncCount: 0,
    bytesSinceSync: 0,
  )
 proc setSyncMode*(wal: var WriteAheadLog, mode: WalSyncMode) =
  wal.syncMode = mode
 proc setGroupEvery*(wal: var WriteAheadLog, n: int) =
  wal.groupEvery = if n <= 0: DefaultWalGroupEvery else: n
 proc setGroupIntervalMs*(wal: var WriteAheadLog, ms: int) =
  wal.groupIntervalMs = max(0, ms)
 proc markSynced(wal: var WriteAheadLog) =
  wal.unsyncedEntries = 0
  wal.bytesSinceSync = 0
  wal.lastSync = getMonoTime()
  inc wal.fsyncCount
 proc maybeGroupSync(wal: var WriteAheadLog, entryBytes: int) =
  ## Apply durability policy after a buffered write.
  case wal.syncMode
  of wsmNone:
    discard
  of wsmEvery:
    fsyncPath(wal.path)
    wal.markSynced()
  of wsmGroup:
    inc wal.unsyncedEntries
    wal.bytesSinceSync += entryBytes
    var due = wal.unsyncedEntries >= wal.groupEvery
    if not due and wal.groupIntervalMs > 0:
      let elapsedMs = (getMonoTime() - wal.lastSync).inMilliseconds
      if elapsedMs >= wal.groupIntervalMs:
        due = true
    if due:
      fsyncPath(wal.path)
      wal.markSynced()
 proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
  let entryBytes = 1 + 8 + 4 + entry.key.len + 4 + entry.value.len
  wal.stream.write(uint8(entry.kind))
  wal.stream.write(entry.timestamp)
  wal.stream.write(uint32(entry.key.len))
@@ -145,8 +236,9 @@ proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
  wal.stream.write(uint32(entry.value.len))
  if entry.value.len > 0:
    wal.stream.writeData(unsafeAddr entry.value[0], entry.value.len)
-  if wal.syncOnWrite:
+  # Always push to kernel page cache; durability policy decides fsync
  wal.stream.flush()
  wal.maybeGroupSync(entryBytes)
  inc wal.entryCount
  # Check rotation every 1000 entries to avoid stat on every write
  if wal.entryCount mod 1000 == 0:
@@ -177,28 +269,87 @@ proc writeCommit*(wal: var WriteAheadLog, timestamp: uint64) =
  ))
 proc sync*(wal: var WriteAheadLog) =
  ## Force durability of all buffered WAL data.
  wal.stream.flush()
-  # Re-open with O_RDWR so fsync operates on a write-capable fd.
+  fsyncPath(wal.path)
-  # Not ideal (two fds for same file) but avoids accessing private
+  wal.markSynced()
-  # FileStream internals that vary across Nim versions.
+
-  let fd = posix.open(cstring(wal.path), O_RDWR)
+proc truncate*(wal: var WriteAheadLog) =
-  if fd != -1:
+  ## Reset WAL to empty (header only). Safe only when all prior entries
-    discard posix.fsync(fd)
+  ## are durable in SSTables and nothing remains only-in-memtable.
-    discard posix.close(fd)
+  if wal.stream != nil:
    wal.stream.flush()
    wal.stream.close()
  wal.stream = newFileStream(wal.path, fmWrite)
  if wal.stream == nil:
    raise newException(IOError, "Cannot truncate WAL: " & wal.path)
  wal.stream.write(WALMagic)
  wal.stream.write(WALVersion)
  wal.stream.flush()
  fsyncPath(wal.path)
  wal.entryCount = 0
  wal.markSynced()
 proc rewriteLive*(wal: var WriteAheadLog,
                  keys: openArray[string],
                  values: openArray[seq[byte]],
                  timestamps: openArray[uint64],
                  deleted: openArray[bool]) =
  ## Atomically replace WAL contents with a live memtable snapshot.
  ## Used after a partial flush so unflushed keys remain recoverable.
  doAssert keys.len == values.len and keys.len == timestamps.len and keys.len == deleted.len
  if keys.len == 0:
    wal.truncate()
    return
  let tmpPath = wal.path & ".rewrite"
  let s = newFileStream(tmpPath, fmWrite)
  if s == nil:
    raise newException(IOError, "Cannot create WAL rewrite file: " & tmpPath)
  s.write(WALMagic)
  s.write(WALVersion)
  var count: uint64 = 0
  for i in 0 ..< keys.len:
    let kind = if deleted[i]: wekDelete else: wekPut
    s.write(uint8(kind))
    s.write(timestamps[i])
    s.write(uint32(keys[i].len))
    if keys[i].len > 0:
      s.write(keys[i])
    s.write(uint32(values[i].len))
    if values[i].len > 0:
      s.writeData(unsafeAddr values[i][0], values[i].len)
    inc count
  s.flush()
  s.close()
  fsyncPath(tmpPath)
  if wal.stream != nil:
    wal.stream.close()
  if fileExists(wal.path):
    removeFile(wal.path)
  moveFile(tmpPath, wal.path)
  wal.stream = newFileStream(wal.path, fmAppend)
  if wal.stream == nil:
    raise newException(IOError, "Cannot reopen WAL after rewrite: " & wal.path)
  wal.entryCount = count
  wal.markSynced()
 proc setMaxSegmentSize*(wal: var WriteAheadLog, size: int64) =
  wal.maxSegmentSize = size
 proc close*(wal: var WriteAheadLog) =
  wal.stream.flush()
-  let fd = posix.open(cstring(wal.path), O_RDWR)
+  fsyncPath(wal.path)
-  if fd != -1:
+  wal.markSynced()
    discard posix.fsync(fd)
    discard posix.close(fd)
  wal.stream.close()
 proc entryCount*(wal: WriteAheadLog): uint64 = wal.entryCount
 proc path*(wal: WriteAheadLog): string = wal.path
 proc unsyncedEntries*(wal: WriteAheadLog): int = wal.unsyncedEntries
 ## Legacy alias — true maps to wsmEvery
 proc syncOnWrite*(wal: WriteAheadLog): bool = wal.syncMode == wsmEvery
 proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry] =
  result = @[]
@@ -15,6 +15,7 @@ import barabadb/core/config
 import barabadb/core/logging
 import barabadb/protocol/ssl
 import barabadb/storage/lsm
 import barabadb/storage/gate
 import barabadb/storage/compaction
 import barabadb/core/raft
 import barabadb/query/executor
@@ -36,59 +37,67 @@ type
 proc newCompactionManager*(db: LSMTree): CompactionManager =
  result = CompactionManager(db: db, strategy: compaction.newCompactionStrategy(db.dir))
-  for sst in db.sstables:
+  result.strategy.rebuildFromLSM(db)
    let meta = compaction.SSTableMeta(
      path: sst.path,
      level: sst.level,
      minKey: sst.minKey,
      maxKey: sst.maxKey,
      entryCount: sst.entryCount,
      sizeBytes: sst.entryCount * 64,
      createdAt: 0,
    )
    result.strategy.addTable(meta)
-proc compact*(cm: CompactionManager) =
+proc applyCompactionResult(db: LSMTree, result: compaction.CompactionResult) =
-  acquire(cm.db.lock)
+  ## Apply compaction output under the caller's lock: update sstables + MANIFEST.
-  defer: release(cm.db.lock)
+  ## On Linux, compact may already have unlinked inputs; we still close our mmaps.
  for level in 0 ..< compaction.MaxLevel:
    if cm.strategy.needsCompaction(level):
      let result = cm.strategy.compact(level)
  if result.outputTables.len == 0:
-        continue
+    return
      # Remove compacted input SSTables from LSMTree
  var newSSTables: seq[SSTable] = @[]
  var removedPaths = initTable[string, bool]()
  for t in result.inputTables:
    removedPaths[t.path] = true
-      for sst in cm.db.sstables:
+  for sst in db.sstables.mitems:
    if sst.path notin removedPaths:
      newSSTables.add(sst)
    else:
      # Drop mmap after compact unlinked the path (fd remains valid until close)
      sst.close()
      # Load and add output SSTables
  for meta in result.outputTables:
    try:
      var sst = loadSSTable(meta.path)
      let name = splitFile(meta.path).name
-          # Extract numeric id from filename if possible
+      # Prefer numeric id from filename; otherwise allocate
-          sst.id = try: parseInt(name) except: cm.db.nextSSTableId
+      let parsed = try: parseInt(name) except: -1
      if parsed >= 0:
        sst.id = parsed
      else:
        sst.id = db.nextSSTableId
        inc db.nextSSTableId
      sst.level = meta.level
      newSSTables.add(sst)
-          cm.db.nextSSTableId = max(cm.db.nextSSTableId, sst.id + 1)
+      db.nextSSTableId = max(db.nextSSTableId, sst.id + 1)
    except CatchableError as e:
      warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg)
  newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
-      cm.db.sstables = newSSTables
+  db.sstables = newSSTables
  db.needsCompaction = db.countL0() >= L0CompactionTrigger
-      # Update MANIFEST
+  inc db.manifestSequence
      inc cm.db.manifestSequence
  try:
-        writeManifest(cm.db)
+    writeManifest(db)
  except CatchableError as e:
    warn("Failed to write MANIFEST after compaction: " & e.msg)
 proc compact*(cm: CompactionManager) =
  # Gate first (cross-thread), then per-DB write lock
  withStorageGate:
    acquire(cm.db.lock)
    try:
      # Always rebuild from LSM — flushes add L0 tables the strategy never registered
      cm.strategy.rebuildFromLSM(cm.db)
      for level in 0 ..< compaction.MaxLevel:
        if cm.strategy.needsCompaction(level):
          let result = cm.strategy.compact(level)
          applyCompactionResult(cm.db, result)
          cm.strategy.rebuildFromLSM(cm.db)
    finally:
      release(cm.db.lock)
 proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.async.} =
  while true:
    await sleepAsync(intervalMs)
@@ -96,17 +105,11 @@ proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.asyn
 proc newMultiCompactionManager*(registry: DatabaseRegistry): MultiCompactionManager =
  result = MultiCompactionManager(registry: registry, strategies: initTable[string, compaction.CompactionStrategy]())
  # Initialize strategies for each existing database
  for name in listDatabases(registry):
    let info = getDatabaseInfo(registry, name)
    if info != nil:
      result.strategies[name] = compaction.newCompactionStrategy(info.db.dir)
-      for sst in info.db.sstables:
+      result.strategies[name].rebuildFromLSM(info.db)
        let meta = compaction.SSTableMeta(
          path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey,
          entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0)
        result.strategies[name].addTable(meta)
 proc compactAll(mcm: MultiCompactionManager) =
  for name in listDatabases(mcm.registry):
@@ -114,49 +117,19 @@ proc compactAll(mcm: MultiCompactionManager) =
    if info == nil: continue
    let db = info.db
    # Initialize strategy if not already
    if name notin mcm.strategies:
      mcm.strategies[name] = compaction.newCompactionStrategy(db.dir)
      for sst in db.sstables:
        let meta = compaction.SSTableMeta(
          path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey,
          entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0)
        mcm.strategies[name].addTable(meta)
    let strategy = mcm.strategies[name]
    withStorageGate:
      acquire(db.lock)
      try:
        strategy.rebuildFromLSM(db)
        for level in 0 ..< compaction.MaxLevel:
          if strategy.needsCompaction(level):
            let result = strategy.compact(level)
-          if result.outputTables.len == 0: continue
+            applyCompactionResult(db, result)
-
+            strategy.rebuildFromLSM(db)
          var newSSTables: seq[SSTable] = @[]
          var removedPaths = initTable[string, bool]()
          for t in result.inputTables:
            removedPaths[t.path] = true
          for sst in db.sstables:
            if sst.path notin removedPaths:
              newSSTables.add(sst)
          for meta in result.outputTables:
            try:
              var sst = loadSSTable(meta.path)
              let sstName = splitFile(meta.path).name
              sst.id = try: parseInt(sstName) except: db.nextSSTableId
              sst.level = meta.level
              newSSTables.add(sst)
              db.nextSSTableId = max(db.nextSSTableId, sst.id + 1)
            except CatchableError as e:
              warn("Compaction output SSTable failed to load: " & meta.path & " - " & e.msg)
          newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id))
          db.sstables = newSSTables
          inc db.manifestSequence
          try:
            writeManifest(db)
          except CatchableError as e:
            warn("Failed to write MANIFEST after compaction: " & e.msg)
      finally:
        release(db.lock)
@@ -303,10 +276,13 @@ proc main() =
        quit(0)
  var config = loadConfig()
  # Global exclusive gate for multi-thread storage (HTTP workers + TCP + compact)
  initStorageGate()
  # Init structured logger from config
  let logLvl = parseEnum[LogLevel]("ll" & capitalizeAscii(config.logLevel))
  defaultLogger = newLogger(logLvl, config.logFile)
  info("BaraDB v1.1.6 — Multimodal Database Engine")
  info("Storage gate initialized (serializes HTTP/TCP/compaction access)")
  # Security check: warn if JWT secret is not configured
  if config.jwtSecret.len == 0:
@@ -358,6 +334,7 @@ proc main() =
    # Wire state machine to apply committed entries to the default database
    let defaultDbInfo = getDatabaseInfo(registry, "default")
    raftNode.applyCommand = proc(cmd: string, data: seq[byte]) {.gcsafe.} =
      withStorageGate:
        if cmd == "put":
          let parts = cast[string](data).split("\x00")
          if parts.len >= 2:
@@ -395,11 +372,13 @@ proc main() =
  # Start TCP wire protocol server on main thread with async event loop
  waitFor runTcpServer(config)
-  # Shutdown
+  # Shutdown: stop listeners first, then close storage under the gate
-  httpServer.stop()
+  httpServer.stop(closeStorage = false)
  tcpServer.stop()
  if tcpServer.gossipProtocol != nil:
    tcpServer.gossipProtocol.stop()
  withStorageGate:
    registry.closeAll()
 when isMainModule:
  main()
@@ -196,6 +196,232 @@ suite "MANIFEST Catalog":
    check issues[0].contains("Orphan")
    db.close()
 suite "Core Storage Hardening":
  test "MemTable overwrite keeps newest value (hash table)":
    let testDir = "/tmp/baradb_test_memtable_hash"
    removeDir(testDir)
    var db = newLSMTree(testDir, 64 * 1024)
    db.put("k", cast[seq[byte]]("v1"))
    db.put("k", cast[seq[byte]]("v2"))
    db.put("k", cast[seq[byte]]("v3"))
    let (found, val) = db.get("k")
    check found
    check cast[string](val) == "v3"
    check db.memTableSize() == 1
    db.close()
  test "Many distinct keys without O(n) insert collapse":
    ## Hash MemTable should handle thousands of puts without quadratic cost.
    let testDir = "/tmp/baradb_test_memtable_many"
    removeDir(testDir)
    var db = newLSMTree(testDir, 8 * 1024 * 1024)
    let n = 5000
    for i in 0 ..< n:
      db.put("key_" & align($i, 6, '0'), cast[seq[byte]]("val_" & $i))
    for i in [0, 1, n div 2, n - 1]:
      let (found, val) = db.get("key_" & align($i, 6, '0'))
      check found
      check cast[string](val) == "val_" & $i
    db.close()
  test "scanMemTable returns sorted unique keys":
    let testDir = "/tmp/baradb_test_scan_sorted"
    removeDir(testDir)
    var db = newLSMTree(testDir, 64 * 1024)
    db.put("c", cast[seq[byte]]("3"))
    db.put("a", cast[seq[byte]]("1"))
    db.put("b", cast[seq[byte]]("2"))
    db.put("a", cast[seq[byte]]("1b"))
    let mem = db.scanMemTable()
    check mem.len == 3
    check mem[0].key == "a"
    check cast[string](mem[0].value) == "1b"
    check mem[1].key == "b"
    check mem[2].key == "c"
    db.close()
  test "WAL truncated after full flush — recovery stays small":
    let testDir = "/tmp/baradb_test_wal_truncate"
    removeDir(testDir)
    var db = newLSMTree(testDir, 256)
    for i in 0 ..< 20:
      db.put("k" & $i, cast[seq[byte]]("v" & $i))
    db.flush()
    # After flush both memtables empty → WAL should only have header (or tiny rewrite)
    let walPath = testDir / "wal" / "wal.log"
    check fileExists(walPath)
    let sizeAfterFlush = getFileSize(walPath)
    check sizeAfterFlush < 256  # header only, not all 20 puts
    db.close()
    # Reopen: data comes from SSTables, not a bloated WAL
    var db2 = newLSMTree(testDir, 256)
    for i in 0 ..< 20:
      let (found, val) = db2.get("k" & $i)
      check found
      check cast[string](val) == "v" & $i
    db2.close()
  test "Partial flush rewrites WAL with remaining live keys":
    let testDir = "/tmp/baradb_test_wal_rewrite"
    removeDir(testDir)
    # Tiny memtable forces flush of first batch while second batch stays in memory
    var db = newLSMTree(testDir, 64)
    db.put("old1", cast[seq[byte]]("a"))
    db.put("old2", cast[seq[byte]]("b"))
    # Force flush
    db.flush()
    db.put("live1", cast[seq[byte]]("x"))
    db.put("live2", cast[seq[byte]]("y"))
    # Do not flush — close without flush would lose live without WAL; close flushes
    # Instead: crash-simulate by reopening after putting live keys (WAL rewrite on prior flush
    # left empty; new puts are in current WAL)
    db.close()
    var db2 = newLSMTree(testDir, 64)
    let (f1, v1) = db2.get("live1")
    let (f2, v2) = db2.get("live2")
    let (f3, _) = db2.get("old1")
    check f1 and cast[string](v1) == "x"
    check f2 and cast[string](v2) == "y"
    check f3
    db2.close()
  test "L0 count trigger and rebuildFromLSM sees flushed tables":
    let testDir = "/tmp/baradb_test_l0_trigger"
    removeDir(testDir)
    var db = newLSMTree(testDir, 128)
    for round in 0 ..< L0CompactionTrigger:
      db.put("r" & $round, cast[seq[byte]]("v" & $round))
      db.flush()
    check db.countL0() >= L0CompactionTrigger
    check db.needsCompaction == true
    var cs = newCompactionStrategy(testDir)
    cs.rebuildFromLSM(db)
    check cs.needsCompaction(0) == true
    check cs.levels[0].len >= L0CompactionTrigger
    # Compact L0 → L1
    let cr = cs.compact(0)
    check cr.outputTables.len == 1
    check cr.outputTables[0].level == 1
    # Apply manually: remove inputs from db, add output
    var removed = initTable[string, bool]()
    for t in cr.inputTables:
      removed[t.path] = true
    var kept: seq[SSTable] = @[]
    for sst in db.sstables.mitems:
      if sst.path notin removed:
        kept.add(sst)
      else:
        sst.close()
    var outSst = loadSSTable(cr.outputTables[0].path)
    outSst.level = 1
    outSst.id = db.nextSSTableId
    inc db.nextSSTableId
    kept.add(outSst)
    db.sstables = kept
    check db.countL0() < L0CompactionTrigger
    for round in 0 ..< L0CompactionTrigger:
      let (found, val) = db.get("r" & $round)
      check found
      check cast[string](val) == "v" & $round
    db.close()
  test "Crash recovery: WAL-only clone recovers unflushed puts":
    ## Simulate crash: copy WAL without SSTables / without clean close flush.
    let srcDir = "/tmp/baradb_test_crash_src"
    let dstDir = "/tmp/baradb_test_crash_dst"
    removeDir(srcDir)
    removeDir(dstDir)
    var db = newLSMTree(srcDir, 1024 * 1024)
    db.put("persist_me", cast[seq[byte]]("yes"))
    db.put("and_me", cast[seq[byte]]("also"))
    db.wal.sync()
    createDir(dstDir / "wal")
    createDir(dstDir / "sstables")
    copyFile(srcDir / "wal" / "wal.log", dstDir / "wal" / "wal.log")
    db.close()  # cleans up src; dst has WAL-only crash image
    var recovered = newLSMTree(dstDir, 1024 * 1024)
    let (f1, v1) = recovered.get("persist_me")
    let (f2, v2) = recovered.get("and_me")
    check f1 and cast[string](v1) == "yes"
    check f2 and cast[string](v2) == "also"
    recovered.close()
  test "WAL group commit fsyncs roughly every N entries":
    let testDir = "/tmp/baradb_test_wal_group"
    removeDir(testDir)
    const n = 200
    const ge = 50
    var db = newLSMTree(testDir, 8 * 1024 * 1024,
                        walSyncMode = wsmGroup, walGroupEvery = ge)
    let base = db.wal.fsyncCount  # open/recovery may fsync once
    for i in 0 ..< n:
      db.put("g" & $i, cast[seq[byte]]("v"))
    let afterPuts = db.wal.fsyncCount - base
    # Group every 50 → about n/ge fsyncs; partial group not yet synced
    check afterPuts >= uint64(n div ge)
    check afterPuts < uint64(n)  # far fewer than one-per-write
    db.wal.sync()
    check db.wal.fsyncCount > base + afterPuts or afterPuts >= uint64(n div ge)
    db.close()
  test "WAL every-mode fsyncs at least once per write":
    let testDir = "/tmp/baradb_test_wal_every"
    removeDir(testDir)
    const n = 30
    var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmEvery)
    let base = db.wal.fsyncCount
    for i in 0 ..< n:
      db.put("e" & $i, cast[seq[byte]]("v"))
    check db.wal.fsyncCount - base >= uint64(n)
    db.close()
  test "WAL none-mode does not fsync on each put":
    let testDir = "/tmp/baradb_test_wal_none"
    removeDir(testDir)
    const n = 100
    var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmNone)
    let base = db.wal.fsyncCount
    for i in 0 ..< n:
      db.put("n" & $i, cast[seq[byte]]("v"))
    # Puts alone should not fsync
    check db.wal.fsyncCount == base
    db.wal.sync()
    check db.wal.fsyncCount == base + 1
    db.close()
  test "parseWalSyncMode accepts aliases":
    check parseWalSyncMode("group") == wsmGroup
    check parseWalSyncMode("every") == wsmEvery
    check parseWalSyncMode("none") == wsmNone
    check parseWalSyncMode("async") == wsmNone
    check parseWalSyncMode("full") == wsmEvery
    check parseWalSyncMode("batch") == wsmGroup
  test "scanRange returns inclusive sorted keys":
    let testDir = "/tmp/baradb_test_scan_range"
    removeDir(testDir)
    var db = newLSMTree(testDir, 256)
    for ch in ['a', 'b', 'c', 'd', 'e']:
      db.put($ch, cast[seq[byte]]("v" & $ch))
    db.flush()
    db.put("c", cast[seq[byte]]("vC2"))  # newer in memtable
    let rows = db.scanRange("b", "d")
    check rows.len == 3
    check rows[0][0] == "b"
    check rows[1][0] == "c"
    check cast[string](rows[1][1]) == "vC2"
    check rows[2][0] == "d"
    db.close()
  test "scanRange empty when no keys in range":
    let testDir = "/tmp/baradb_test_scan_empty"
    removeDir(testDir)
    var db = newLSMTree(testDir, 1024)
    db.put("m", cast[seq[byte]]("1"))
    check db.scanRange("a", "c").len == 0
    check db.scanRange("m", "m").len == 1
    db.close()
 suite "BaraQL Lexer":
  test "Tokenize simple SELECT":
    let tokens = lex.tokenize("SELECT name FROM users WHERE age > 18")
@@ -0,0 +1,116 @@
 ## Schema persistence — CREATE TABLE / data survive reopen
 import std/unittest
 import std/os
 import std/strutils
 import std/tables
 import barabadb/storage/lsm
 import barabadb/query/executor
 import barabadb/query/parser
 import barabadb/query/ast
 proc execSql(ctx: ExecutionContext, sql: string): ExecResult =
  let node = parse(sql)
  result = executeQuery(ctx, node)
 suite "Schema persistence":
  test "CREATE TABLE survives flush + reopen":
    let dir = "/tmp/baradb_schema_persist_1"
    removeDir(dir)
    block:
      var db = newLSMTree(dir, 1024)  # small memtable → forces flush
      var ctx = newExecutionContext(db)
      let r = execSql(ctx, "CREATE TABLE users (id INT PRIMARY KEY, name TEXT NOT NULL)")
      check r.success
      check ctx.tables.hasKey("users")
      check ctx.tables["users"].columns.len == 2
      discard execSql(ctx, "INSERT INTO users (id, name) VALUES (1, 'Alice')")
      discard execSql(ctx, "INSERT INTO users (id, name) VALUES (2, 'Bob')")
      db.flush()
      # Schema key must be durable
      let (found, _) = db.get(tableSchemaKey("users"))
      check found
      db.close()
    # Reopen fresh context (simulates process restart)
    block:
      var db2 = newLSMTree(dir, 1024)
      var ctx2 = newExecutionContext(db2)
      check ctx2.tables.hasKey("users")
      check ctx2.tables["users"].columns.len == 2
      check ctx2.tables["users"].pkColumns.len == 1
      let sel = execSql(ctx2, "SELECT id, name FROM users ORDER BY id")
      check sel.success
      check sel.rows.len == 2
      db2.close()
  test "DROP TABLE removes schema and data":
    let dir = "/tmp/baradb_schema_persist_drop"
    removeDir(dir)
    var db = newLSMTree(dir)
    var ctx = newExecutionContext(db)
    check execSql(ctx, "CREATE TABLE t (id INT PRIMARY KEY)").success
    check execSql(ctx, "INSERT INTO t (id) VALUES (1)").success
    check execSql(ctx, "DROP TABLE t").success
    check not ctx.tables.hasKey("t")
    let (found, _) = db.get(tableSchemaKey("t"))
    check not found
    # Reopen — table must not reappear
    db.close()
    var db2 = newLSMTree(dir)
    var ctx2 = newExecutionContext(db2)
    check not ctx2.tables.hasKey("t")
    db2.close()
  test "ALTER TABLE ADD COLUMN is persisted":
    let dir = "/tmp/baradb_schema_persist_alter"
    removeDir(dir)
    block:
      var db = newLSMTree(dir)
      var ctx = newExecutionContext(db)
      check execSql(ctx, "CREATE TABLE items (id INT PRIMARY KEY)").success
      check execSql(ctx, "ALTER TABLE items ADD COLUMN label TEXT").success
      check ctx.tables["items"].columns.len == 2
      db.flush()
      db.close()
    block:
      var db2 = newLSMTree(dir)
      var ctx2 = newExecutionContext(db2)
      check ctx2.tables.hasKey("items")
      check ctx2.tables["items"].columns.len == 2
      var names: seq[string] = @[]
      for c in ctx2.tables["items"].columns:
        names.add(c.name)
      check "label" in names
      db2.close()
  test "Multiple tables all restored":
    let dir = "/tmp/baradb_schema_persist_multi"
    removeDir(dir)
    block:
      var db = newLSMTree(dir, 512)
      var ctx = newExecutionContext(db)
      check execSql(ctx, "CREATE TABLE a (id INT PRIMARY KEY)").success
      check execSql(ctx, "CREATE TABLE b (id INT PRIMARY KEY, a_id INT)").success
      check execSql(ctx, "CREATE TABLE c (name TEXT)").success
      for i in 0..20:
        discard execSql(ctx, "INSERT INTO a (id) VALUES (" & $i & ")")
      db.flush()
      db.close()
    block:
      var db2 = newLSMTree(dir)
      var ctx2 = newExecutionContext(db2)
      check ctx2.tables.hasKey("a")
      check ctx2.tables.hasKey("b")
      check ctx2.tables.hasKey("c")
      let sel = execSql(ctx2, "SELECT id FROM a")
      check sel.success
      check sel.rows.len == 21
      db2.close()
  test "Stable schema key format":
    check tableSchemaKey("users") == "_schema:tables:users"
    check serializeTableDdl(TableDef(
      name: "t",
      columns: @[ColumnDef(name: "id", colType: "INT", isPk: true)],
      pkColumns: @["id"],
    )).contains("PRIMARY KEY")
@@ -0,0 +1,168 @@
 ## Focused storage hardening tests (avoids full suite compile issues)
 import std/unittest
 import std/os
 import std/strutils
 import std/locks
 import barabadb/storage/lsm
 import barabadb/storage/rwlock
 import barabadb/storage/gate
 suite "Core Storage Hardening":
  test "MemTable overwrite keeps newest value":
    let testDir = "/tmp/baradb_th_mem"
    removeDir(testDir)
    var db = newLSMTree(testDir, 64 * 1024)
    db.put("k", cast[seq[byte]]("v1"))
    db.put("k", cast[seq[byte]]("v3"))
    let (found, val) = db.get("k")
    check found
    check cast[string](val) == "v3"
    db.close()
  test "scanRange inclusive":
    let testDir = "/tmp/baradb_th_range"
    removeDir(testDir)
    var db = newLSMTree(testDir, 256)
    for ch in ['a', 'b', 'c', 'd', 'e']:
      db.put($ch, cast[seq[byte]]("v" & $ch))
    db.flush()
    db.put("c", cast[seq[byte]]("vC2"))
    let rows = db.scanRange("b", "d")
    check rows.len == 3
    check rows[0][0] == "b"
    check rows[1][0] == "c"
    check cast[string](rows[1][1]) == "vC2"
    db.close()
  test "WAL group commit":
    let testDir = "/tmp/baradb_th_group"
    removeDir(testDir)
    const n = 200
    const ge = 50
    var db = newLSMTree(testDir, 8 * 1024 * 1024,
                        walSyncMode = wsmGroup, walGroupEvery = ge)
    let base = db.wal.fsyncCount
    for i in 0 ..< n:
      db.put("g" & $i, cast[seq[byte]]("v"))
    let after = db.wal.fsyncCount - base
    check after >= uint64(n div ge)
    check after < uint64(n)
    db.close()
  test "RwLock concurrent readers":
    var rw: RwLock
    initRwLock(rw)
    var counter = 0
    var maxReaders = 0
    var curReaders = 0
    var metaLock: Lock
    initLock(metaLock)
    var bad = false
    type TArgs = object
      rw: ptr RwLock
      meta: ptr Lock
      counter: ptr int
      curReaders: ptr int
      maxReaders: ptr int
      bad: ptr bool
      isWriter: bool
    proc worker(a: TArgs) {.thread, gcsafe.} =
      for i in 0 ..< 200:
        if a.isWriter:
          acquireWrite(a.rw[])
          a.counter[] += 1
          acquire(a.meta[])
          if a.curReaders[] != 0:
            a.bad[] = true
          release(a.meta[])
          releaseWrite(a.rw[])
        else:
          acquireRead(a.rw[])
          acquire(a.meta[])
          inc a.curReaders[]
          if a.curReaders[] > a.maxReaders[]:
            a.maxReaders[] = a.curReaders[]
          release(a.meta[])
          var x = 0
          for k in 0 ..< 50: x += k
          discard x
          acquire(a.meta[])
          dec a.curReaders[]
          release(a.meta[])
          releaseRead(a.rw[])
    var threads: array[8, Thread[TArgs]]
    for t in 0 ..< 8:
      let args = TArgs(
        rw: addr rw, meta: addr metaLock,
        counter: addr counter, curReaders: addr curReaders,
        maxReaders: addr maxReaders, bad: addr bad,
        isWriter: t == 0 or t == 1,
      )
      createThread(threads[t], worker, args)
    for t in 0 ..< 8:
      joinThread(threads[t])
    check not bad
    check counter == 400
    check maxReaders >= 2
    deinitLock(metaLock)
    deinitRwLock(rw)
  test "Interleaved put/get/flush single-threaded stress":
    ## ORC is not multi-thread-safe for shared refs; stress the exclusive path serially.
    let testDir = "/tmp/baradb_th_stress"
    removeDir(testDir)
    var db = newLSMTree(testDir, 4 * 1024, walSyncMode = wsmGroup, walGroupEvery = 32)
    for i in 0 ..< 2000:
      db.put("k" & $i, cast[seq[byte]]("v" & $i))
      if i mod 100 == 0:
        let (f, v) = db.get("k0")
        check f and cast[string](v) == "v0"
      if i mod 400 == 0:
        db.flush()
    for i in [0, 500, 1000, 1999]:
      let (f, v) = db.get("k" & $i)
      check f and cast[string](v) == "v" & $i
    db.close()
  test "StorageGate serializes concurrent critical sections":
    initStorageGate()
    var counter = 0
    var bad = false
    var meta: Lock
    initLock(meta)
    type GArgs = object
      n: int
      counter: ptr int
      bad: ptr bool
      meta: ptr Lock
    proc worker(a: GArgs) {.thread, gcsafe.} =
      for i in 0 ..< a.n:
        withStorageGate:
          # Under the gate, only one thread should touch counter
          let before = a.counter[]
          a.counter[] = before + 1
          # Simulate work
          var x = 0
          for k in 0 ..< 20: x += k
          discard x
          if a.counter[] != before + 1:
            acquire(a.meta[])
            a.bad[] = true
            release(a.meta[])
    var threads: array[6, Thread[GArgs]]
    for t in 0 ..< 6:
      createThread(threads[t], worker, GArgs(
        n: 100, counter: addr counter, bad: addr bad, meta: addr meta))
    for t in 0 ..< 6:
      joinThread(threads[t])
    check not bad
    check counter == 600
    deinitLock(meta)
@@ -0,0 +1,26 @@
 ## Regression: sequential wire-style INSERTs must not crash the process.
 ## Root cause was ORC cycle collector (markGray SIGSEGV); project uses --mm:arc.
 import std/unittest
 import std/os
 import barabadb/storage/lsm
 import barabadb/query/executor
 import barabadb/query/parser
 proc execSql(ctx: ExecutionContext, sql: string): ExecResult =
  executeQuery(ctx, parse(sql))
 suite "Wire insert stress (ARC regression)":
  test "200 sequential INSERTs via executor survive":
    ## Mirrors the wire path (executeQuery under StorageGate each time).
    let dir = "/tmp/baradb_wire_stress"
    removeDir(dir)
    var db = newLSMTree(dir, walSyncMode = wsmNone)
    var ctx = newExecutionContext(db)
    check execSql(ctx, "CREATE TABLE stress (id INT PRIMARY KEY, v TEXT)").success
    for i in 0 ..< 200:
      let r = execSql(ctx, "INSERT INTO stress (id, v) VALUES (" & $i & ", 'v" & $i & "')")
      check r.success
    let sel = execSql(ctx, "SELECT id FROM stress")
    check sel.success
    check sel.rows.len == 200
    db.close()