From 8db5cfe7e1d7ca26b6e3840dfa182be345aee1a6 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Sat, 18 Jul 2026 16:55:50 +0300 Subject: [PATCH] feat: harden storage, schema persistence, fair benches, fix wire crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core storage: hash MemTable, WAL group commit, L0 compaction rebuild, reader-writer lock, and a global StorageGate so HTTP workers and TCP share the LSM safely under multi-thread access. Schema: durable CREATE/ALTER/DROP under _schema:tables:* with full LSM restore on open. Executor types/values/schema split into query/exec/. Wire protocol: switch default MM to ARC — ORC cycle collector segfaulted after ~20 async INSERTs. Fair multi-tier benchmarks (SQLite/HTTP/wire/PG) and honesty docs for mixed-tier comparisons. --- .gitignore | 12 + CHANGELOG.md | 30 ++ baradadb.nimble | 13 +- benchmarks/FAIR_COMPARISON.md | 93 ++++ benchmarks/README.md | 80 +++ benchmarks/REAL_COMPARISON.md | 49 +- benchmarks/bench_all.nim | 44 +- benchmarks/compare.nim | 7 + benchmarks/fair_bench.py | 799 ++++++++++++++++++++++++++++ benchmarks/generate_report.py | 159 +++--- benchmarks/pg_bench.py | 37 +- nim.cfg | 5 + src/barabadb/core/config.nim | 11 + src/barabadb/core/httpserver.nim | 173 +++--- src/barabadb/core/registry.nim | 17 +- src/barabadb/core/server.nim | 104 ++-- src/barabadb/query/exec/README.md | 29 + src/barabadb/query/exec/schema.nim | 231 ++++++++ src/barabadb/query/exec/types.nim | 143 +++++ src/barabadb/query/exec/values.nim | 119 +++++ src/barabadb/query/executor.nim | 366 ++----------- src/barabadb/storage/compaction.nim | 34 +- src/barabadb/storage/gate.nim | 41 ++ src/barabadb/storage/lsm.nim | 342 ++++++++---- src/barabadb/storage/rwlock.nim | 80 +++ src/barabadb/storage/wal.nim | 183 ++++++- src/baradadb.nim | 191 +++---- tests/test_all.nim | 226 ++++++++ tests/test_schema_persist.nim | 116 ++++ tests/test_storage_hardening.nim | 168 ++++++ tests/test_wire_insert_stress.nim | 26 + 31 files changed, 3131 insertions(+), 797 deletions(-) create mode 100644 benchmarks/FAIR_COMPARISON.md create mode 100644 benchmarks/README.md create mode 100644 benchmarks/fair_bench.py create mode 100644 src/barabadb/query/exec/README.md create mode 100644 src/barabadb/query/exec/schema.nim create mode 100644 src/barabadb/query/exec/types.nim create mode 100644 src/barabadb/query/exec/values.nim create mode 100644 src/barabadb/storage/gate.nim create mode 100644 src/barabadb/storage/rwlock.nim create mode 100644 tests/test_schema_persist.nim create mode 100644 tests/test_storage_hardening.nim create mode 100644 tests/test_wire_insert_stress.nim diff --git a/.gitignore b/.gitignore index ae949f4..31d4f3a 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,18 @@ tests/nimforum_smoke_test benchmark_results.json pg_benchmark_results.json +fair_benchmark_results.json benchmarks/bench_all benchmarks/compare .qwen/ + +# Compiled test / module binaries +tests/test_schema_persist +tests/test_storage_hardening +tests/test_wire_insert_stress +src/barabadb/storage/lsm +src/barabadb/storage/wal +src/barabadb/storage/btree +src/barabadb/storage/gate +clients/nim/tests/test_pool +clients/nim/tests/test_wire diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ea68c5..9a0494d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,36 @@ All notable changes to BaraDB are documented in this file. ## [1.2.0] — Unreleased +### Core Storage Hardening + +Foundational LSM improvements for write performance, durability, and compaction correctness. + +- **Hash-table MemTable** (`storage/lsm.nim`) — O(1) put/get instead of O(n) sorted-seq insert; sort only on flush to SSTable +- **Timestamp-aware MemTable overwrite** — WAL recovery keeps newest version per key +- **WAL rewrite after flush** (`storage/wal.nim`) — `truncate` / `rewriteLive` so recovery is O(unflushed) not O(history); fsync on truncate/rewrite/close +- **WAL group commit** — `WalSyncMode`: `none` | `group` (default) | `every`; fsync every N entries and/or every interval ms +- **Config knobs** — `wal_sync_mode`, `wal_group_every`, `wal_sync_interval_ms` / env `BARADB_WAL_*`; registry opens DBs with config +- **L0 file-count compaction trigger** — RocksDB-style `L0CompactionTrigger` (default 4) instead of size-only for overlapping L0 +- **`rebuildFromLSM` for compaction** (`storage/compaction.nim`) — strategy always syncs from live SSTable catalog (no drift after flush) +- **Safe mmap close after compaction** — release handles for compacted inputs after unlink +- **Recovery flag** — flush during WAL replay does not rewrite the open WAL file mid-read +- **Fair WAL micro-bench** — `benchWalDurabilityModes` compares none/group/every on the same workload +- **RwLock for LSM** (`storage/rwlock.nim`) — writer-preferring reader-writer lock; default API uses exclusive mode +- **Deep-copy on get** — returned values are copied so callers never share seq buffers with the store +- **`-d:baraConcurrentReads`** — opt-in shared read locks (unsafe with default ORC across OS threads) +- **`scanRange(start, end)`** — inclusive multi-level range scan (memtable + SSTables) +- **Focused tests** — `tests/test_storage_hardening.nim` (RwLock, WAL group, scanRange, stress) +- **Note:** Nim ORC must not share GC'd `LSMTree` refs across OS threads without isolation +- **StorageGate** (`storage/gate.nim`) — global exclusive lock serializing HTTP (Hunos workers), TCP, compaction, Raft apply +- **HTTP stop no longer closes shared registry** — ownership stays with main; `stop(closeStorage=true)` for standalone HTTP +- **Schema persistence** — stable keys `_schema:tables:`; restore from full LSM (`scanAll`), not only memtable; DROP/ALTER update durable catalog; secondary index rebuild on open +- **Schema tests** — `tests/test_schema_persist.nim` (create/flush/reopen, drop, alter, multi-table) +- **Executor split** — `query/exec/{types,values,schema}.nim`; `executor.nim` re-exports for API stability; see `query/exec/README.md` +- **Fair benchmarks** — `benchmarks/fair_bench.py` multi-tier (embedded SQLite↔LSM; client-server HTTP/wire↔PG); batch multi-row INSERT; wire protocol via Python client; `generate_report.py --fair`; `nimble bench_fair` +- **Fix wire INSERT SIGSEGV** — ORC cycle collector crash under async TCP load; switch default MM to `--mm:arc` in `nim.cfg`; regression `tests/test_wire_insert_stress.nim` +- **Honest bench docs** — tier warnings in `bench_all`, `pg_bench`, `compare.nim`; `benchmarks/README.md` +- **Regression suite** — `Core Storage Hardening` tests in `tests/test_all.nim` + ### Search Module (new) A unified search module combining vector similarity, full-text, and structured diff --git a/baradadb.nimble b/baradadb.nimble index 3aabce2..d47012d 100644 --- a/baradadb.nimble +++ b/baradadb.nimble @@ -19,17 +19,24 @@ task build_debug, "Build debug version": exec "nim c --debugger:native --linedir:on -o:build/baramcp src/baramcp.nim" task build_release, "Build release version": + # mm:arc comes from nim.cfg (ORC crashes under wire INSERT load) exec "nim c -d:release --opt:speed -o:build/baradadb src/baradadb.nim" exec "nim c -d:release --opt:speed -o:build/baramcp src/baramcp.nim" task test, "Run all tests": exec "nim c -r tests/test_all.nim" -task bench, "Run benchmarks": +task bench, "Run embedded micro-benchmarks (in-process)": exec "nim c -d:release -r benchmarks/bench_all.nim" -task bench_pg, "Run PostgreSQL comparison benchmarks": +task bench_pg, "Run PostgreSQL client-server micro-benchmarks": exec "python3 benchmarks/pg_bench.py" -task bench_report, "Generate benchmark comparison report": +task bench_fair, "Fair multi-tier benches (SQLite embedded + optional PG/HTTP)": + exec "python3 benchmarks/fair_bench.py" + +task bench_report, "Generate fair comparison report (needs fair_bench first)": + exec "python3 benchmarks/generate_report.py --fair" + +task bench_report_legacy, "Legacy mixed-tier report (PG C/S vs BaraDB embedded — unfair)": exec "python3 benchmarks/generate_report.py" diff --git a/benchmarks/FAIR_COMPARISON.md b/benchmarks/FAIR_COMPARISON.md new file mode 100644 index 0000000..7a89d11 --- /dev/null +++ b/benchmarks/FAIR_COMPARISON.md @@ -0,0 +1,93 @@ +# Fair Benchmark Results + +Generated: **2026-07-18 13:53:46 UTC** + +## Methodology + +- Tier `embedded`: in-process only (BaraDB LSM from nimble bench JSON; SQLite via Python sqlite3). +- Tier `client_server`: network SQL (BaraDB HTTP /query; BaraDB binary wire TCP; PostgreSQL via psycopg2). +- `sql_insert_row`: one INSERT statement per row (chatty). +- `sql_insert_batch`: multi-row INSERT with batch size 50 (same SQL style across systems). +- PostgreSQL: synchronous_commit=on|off; SQLite: PRAGMA synchronous FULL|OFF. +- BaraDB WAL modes appear only if you ran benchmarks/bench_all.nim (WAL-* rows). +- Never claim 'Nx faster than Postgres' using embedded BaraDB numbers. + +**Do not compare numbers across tiers.** Embedded storage is not the same +workload as client-server SQL over the network. + +## Tier: `embedded` + +| Bench | System | ops/s | seconds | n | notes | +|-------|--------|------:|--------:|--:|-------| +| kv_write | `baradb_lsm_embedded` | 41.50K | 2.410 | 100000 | benchmark_results.json | +| kv_read | `baradb_lsm_embedded` | 3.77M | 0.026 | 100000 | benchmark_results.json | +| wal_none | `baradb_lsm_embedded` | 232.65K | 0.215 | 50000 | benchmark_results.json | +| wal_group64 | `baradb_lsm_embedded` | 42.25K | 1.183 | 50000 | benchmark_results.json | +| wal_group256 | `baradb_lsm_embedded` | 107.16K | 0.467 | 50000 | benchmark_results.json | +| wal_every | `baradb_lsm_embedded` | 825.27 | 60.586 | 50000 | benchmark_results.json | +| kv_write | `sqlite_off` | 402.61K | 0.002 | 1000 | PRAGMA synchronous=OFF | +| kv_read | `sqlite_off` | 195.60K | 0.005 | 1000 | | +| sql_insert_batch | `sqlite_off` | 559.66K | 0.002 | 1000 | multi-row INSERT batch=50, sync=OFF | +| kv_write | `sqlite_full` | 272.62K | 0.004 | 1000 | PRAGMA synchronous=FULL | +| kv_read | `sqlite_full` | 196.16K | 0.005 | 1000 | | +| sql_insert_batch | `sqlite_full` | 370.18K | 0.003 | 1000 | multi-row INSERT batch=50, sync=FULL | + +### Same-bench ratios (`embedded`) + +**kv_read** (fastest: `baradb_lsm_embedded` @ 3.77M/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `baradb_lsm_embedded` | 1.00x | +| `sqlite_full` | 0.05x | +| `sqlite_off` | 0.05x | + +**kv_write** (fastest: `sqlite_off` @ 402.61K/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `sqlite_off` | 1.00x | +| `sqlite_full` | 0.68x | +| `baradb_lsm_embedded` | 0.10x | + +**sql_insert_batch** (fastest: `sqlite_off` @ 559.66K/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `sqlite_off` | 1.00x | +| `sqlite_full` | 0.66x | + +## Tier: `client_server` + +| Bench | System | ops/s | seconds | n | notes | +|-------|--------|------:|--------:|--:|-------| +| sql_insert_row | `baradb_http` | 979.31 | 0.511 | 500 | | +| sql_select_row | `baradb_http` | 235.20 | 2.126 | 500 | | +| sql_insert_batch | `baradb_http` | 10.19K | 0.049 | 500 | multi-row INSERT batch=50 | +| sql_insert_row | `baradb_wire` | 4.65K | 0.108 | 500 | binary wire protocol | +| sql_select_row | `baradb_wire` | 642.04 | 0.779 | 500 | | +| sql_insert_batch | `baradb_wire` | 20.47K | 0.024 | 500 | multi-row INSERT batch=50 | + +### Same-bench ratios (`client_server`) + +**sql_insert_batch** (fastest: `baradb_wire` @ 20.47K/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `baradb_wire` | 1.00x | +| `baradb_http` | 0.50x | + +**sql_insert_row** (fastest: `baradb_wire` @ 4.65K/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `baradb_wire` | 1.00x | +| `baradb_http` | 0.21x | + +**sql_select_row** (fastest: `baradb_wire` @ 642.04/s) + +| System | Relative to fastest | +|--------|--------------------:| +| `baradb_wire` | 1.00x | +| `baradb_http` | 0.37x | + diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..ddae3e1 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,80 @@ +# BaraDB Benchmarks + +## Tiers (read this first) + +| Tier | What is measured | Fair peers | +|------|------------------|------------| +| **embedded** | In-process storage API | BaraDB LSM ↔ SQLite | +| **client_server** | Network + query protocol | BaraDB HTTP / **wire** ↔ PostgreSQL | + +**Never** quote “BaraDB is Nx faster than Postgres” using embedded LSM numbers. +That comparison mixes tiers and is meaningless as a product claim. + +## Quick start + +```bash +# 1) Embedded micro-benches (Nim, in-process) +nimble bench +# or: nim c -d:release -r benchmarks/bench_all.nim + +# 2) Optional: start server for client_server tier (HTTP + wire) +./build/baradadb + +# 3) Fair multi-tier suite (Python) +# - always: SQLite embedded (+ batch) +# - optional: BaraDB HTTP (:9912), wire TCP (:9472), PostgreSQL +python3 benchmarks/fair_bench.py + +# 3) Markdown report +nimble bench_report +# or: python3 benchmarks/generate_report.py --fair +``` + +Outputs: + +- `benchmark_results.json` — Nim embedded suite +- `fair_benchmark_results.json` — multi-tier fair suite +- `benchmarks/FAIR_COMPARISON.md` — human-readable fair report +- `pg_benchmark_results.json` — optional PG-only micro suite + +## Environment + +| Variable | Default | Meaning | +|----------|---------|---------| +| `FAIR_N_KV` | 20000 | embedded KV ops | +| `FAIR_N_SQL` | 5000 | SQL loops (HTTP/wire/PG) | +| `FAIR_BATCH` | 100 | multi-row INSERT batch size | +| `BARADB_HTTP_HOST` | 127.0.0.1 | HTTP host | +| `BARADB_HTTP_PORT` | 9912 | HTTP port (`TCP+440`) | +| `BARADB_WIRE_HOST` | 127.0.0.1 | wire protocol host | +| `BARADB_WIRE_PORT` | 9472 | wire protocol TCP port | +| `FAIR_SKIP_HTTP=1` | — | skip BaraDB HTTP | +| `FAIR_SKIP_WIRE=1` | — | skip BaraDB wire | +| `FAIR_SKIP_PG=1` | — | skip PostgreSQL | +| `PGHOST` / `PGUSER` / `PGPASSWORD` / … | — | libpq-style | + +## Files + +| File | Role | +|------|------| +| `bench_all.nim` | Embedded: LSM, WAL modes, BTree, vector, FTS, graph | +| `fair_bench.py` | Fair multi-tier runner + markdown | +| `pg_bench.py` | PostgreSQL client-server micro suite | +| `generate_report.py` | `--fair` report; legacy mixed report without flag | +| `compare.nim` | **Synthetic** — do not publish | +| `search_bench.nim` | Search-focused micro suite | + +## Durability knobs + +- BaraDB: `wal_sync_mode` = `none` \| `group` \| `every` (see WAL-* rows from `bench_all`) +- SQLite: `PRAGMA synchronous = OFF` vs `FULL` +- PostgreSQL: `synchronous_commit = off` vs `on` + +Match durability stories when claiming write speedups. + +## Wire protocol note + +The Python wire client (`clients/python`) is exercised by `fair_bench.py`. +Builds use **`--mm:arc`** (see `nim.cfg`) because Nim **ORC** cycle collection +crashed under async wire INSERT load (`markGray` SIGSEGV). With ARC, sequential +wire INSERTs + batch multi-row INSERT are stable. diff --git a/benchmarks/REAL_COMPARISON.md b/benchmarks/REAL_COMPARISON.md index a0cca9d..d3a5a5d 100644 --- a/benchmarks/REAL_COMPARISON.md +++ b/benchmarks/REAL_COMPARISON.md @@ -1,38 +1,29 @@ -# BaraDB vs PostgreSQL — Real Benchmark Results +# Legacy mixed-tier comparison -Generated from actual execution on: -- **CPU:** AMD Ryzen 9 5900X -- **PostgreSQL:** 15.18 (local) -- **BaraDB:** git `42043f3` +This file used to claim large “speedups” of BaraDB over PostgreSQL by comparing: -## Methodology +- **PostgreSQL:** client-server (psycopg2, network, SQL) +- **BaraDB:** in-process LSM (no network, no SQL) -- PostgreSQL: single-row INSERT/SELECT via psycopg2 (client-server overhead included) -- BaraDB: in-process Nim code (no network overhead) -- Same dataset sizes for both systems +That is **not a fair product comparison**. -## Results +## Use the fair suite instead -| Test | PostgreSQL | BaraDB | Speedup | -|------|-----------|--------|---------| -| KV Write (100K) | 16.82K/s (5.946s) | 32.23K/s (3.103s) | 1.9x (BaraDB) | -| KV Read (100K) | 15.08K/s (6.630s) | 3.95M/s (25.3ms) | 261.9x (BaraDB) | -| BTree Insert (100K) | 17.66K/s (5.664s) | 2.52M/s (39.7ms) | 142.8x (BaraDB) | -| BTree Get (100K) | 14.50K/s (6.899s) | 2.34M/s (42.7ms) | 161.4x (BaraDB) | -| BTree Scan (1K ranges) | 2.39K/s (419.2ms) | 11.03M/s (1.0ms) | 4623.3x (BaraDB) | -| FTS Index (10K docs) | 17.98K/s (556.3ms) | 119.99K/s (83.3ms) | 6.7x (BaraDB) | -| FTS Search (1K queries) | 784.12/s (1.275s) | 1.36K/s (734.0ms) | 1.7x (BaraDB) | +```bash +nim c -d:release -r benchmarks/bench_all.nim # embedded BaraDB +python3 benchmarks/fair_bench.py # SQLite + optional PG/HTTP +# report → benchmarks/FAIR_COMPARISON.md +``` -## Summary +See: -- **Total PostgreSQL time:** 27.389s -- **Total BaraDB time:** 4.029s -- **Overall speedup:** BaraDB is **6.8x faster** +- [`FAIR_COMPARISON.md`](FAIR_COMPARISON.md) — latest multi-tier results +- [`README.md`](README.md) — methodology and env vars -## Notes +## If you regenerate the legacy report -- PostgreSQL includes network round-trip and SQL parsing overhead per operation. -- BaraDB runs in-process with zero serialization/network cost. -- For embedded/single-node use cases, BaraDB shows significant advantage. -- BaraDB now outperforms PostgreSQL on all tested metrics including FTS search after optimizations. -- PostgreSQL excels at durability, replication, and complex ACID transactions. +```bash +python3 benchmarks/generate_report.py # without --fair +``` + +It will rewrite this file with an explicit **mixed tiers** warning banner. diff --git a/benchmarks/bench_all.nim b/benchmarks/bench_all.nim index f0a1a7c..50a0319 100644 --- a/benchmarks/bench_all.nim +++ b/benchmarks/bench_all.nim @@ -111,9 +111,11 @@ proc formatOps(ops: int, secs: float64): string = proc benchLSMTree() = echo "=== LSM-Tree Storage ===" + echo " Note: in-process embedded API (no network/SQL). Not comparable to client-server DBs." let benchDir = getTempDir() / "baradb_bench_lsm" removeDir(benchDir) - var db = newLSMTree(benchDir) + # Default group-commit WAL (production default) + var db = newLSMTree(benchDir, walSyncMode = wsmGroup, walGroupEvery = 64) # Write benchmark let n = 100_000 @@ -124,6 +126,7 @@ proc benchLSMTree() = let writeLabel = "LSM-Write" recordResult(writeLabel, n, writeTime) echo " Write ", n, " keys: ", writeTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, writeTime), ")", compareResult(writeLabel, currentResults[^1].opsPerSec, previousResults) + echo " fsyncs: ", db.wal.fsyncCount, " (group every 64)" # Read benchmark let readStart = getMonoTime() @@ -138,6 +141,36 @@ proc benchLSMTree() = db.close() +proc benchWalDurabilityModes() = + ## Fair comparison of WAL durability policies on the same workload. + echo "=== WAL Durability Modes (fair micro-bench) ===" + echo " Same N puts, same memtable size; only sync policy differs." + let n = 50_000 + let modes = [ + (wsmNone, "none", 0), + (wsmGroup, "group64", 64), + (wsmGroup, "group256", 256), + (wsmEvery, "every", 1), + ] + for (mode, label, ge) in modes: + let dir = getTempDir() / ("baradb_bench_wal_" & label) + removeDir(dir) + var db = newLSMTree(dir, memMaxSize = 64 * 1024 * 1024, + walSyncMode = mode, walGroupEvery = max(1, ge)) + let t0 = getMonoTime() + for i in 0.. str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def result(name: str, system: str, tier: str, ops: int, seconds: float, **extra): + ops_s = ops / seconds if seconds > 0 else 0.0 + r = { + "name": name, + "system": system, + "tier": tier, + "ops": ops, + "seconds": seconds, + "opsPerSec": ops_s, + "timestamp": now_iso(), + } + r.update(extra) + return r + + +def fmt_ops(x: float) -> str: + if x >= 1_000_000: + return f"{x/1_000_000:.2f}M" + if x >= 1_000: + return f"{x/1_000:.2f}K" + return f"{x:.2f}" + + +# ─── Tier 1: Embedded ─────────────────────────────────────────────── + + +def load_baradb_embedded() -> list[dict]: + """Map bench_all.nim LSM results into fair embedded tier.""" + if not BARA_JSON.exists(): + print(" [skip] benchmark_results.json missing — run: nimble bench") + return [] + data = json.loads(BARA_JSON.read_text()) + name_map = { + "LSM-Write": "kv_write", + "LSM-Read": "kv_read", + "WAL-none": "wal_none", + "WAL-group64": "wal_group64", + "WAL-group256": "wal_group256", + "WAL-every": "wal_every", + } + out = [] + for r in data.get("results", []): + mapped = name_map.get(r.get("name")) + if not mapped: + continue + out.append( + result( + mapped, + "baradb_lsm_embedded", + "embedded", + r.get("ops", 0), + r.get("seconds", 0.0), + source="benchmark_results.json", + gitSha=data.get("gitSha", ""), + ) + ) + return out + + +def multi_values_sql(start: int, count: int) -> str: + """Build VALUES (...),(...),... for multi-row INSERT.""" + parts = [f"({i}, 'value_{i}')" for i in range(start, start + count)] + return ",".join(parts) + + +def bench_sqlite_embedded(n: int = N_KV) -> list[dict]: + """SQLite in-process — fair peer for BaraDB embedded LSM.""" + out = [] + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + os.unlink(path) + + # --- durability: FULL (fsync) vs OFF --- + for mode, label in (("OFF", "sqlite_off"), ("FULL", "sqlite_full")): + if os.path.exists(path): + os.unlink(path) + conn = sqlite3.connect(path) + cur = conn.cursor() + cur.execute(f"PRAGMA synchronous = {mode}") + cur.execute("PRAGMA journal_mode = WAL") + cur.execute("CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT)") + conn.commit() + + t0 = time.perf_counter() + for i in range(n): + cur.execute("INSERT INTO kv(k,v) VALUES(?,?)", (f"key_{i}", f"value_{i}")) + conn.commit() + w = time.perf_counter() - t0 + out.append( + result( + "kv_write", + label, + "embedded", + n, + w, + durable=mode == "FULL", + note=f"PRAGMA synchronous={mode}", + ) + ) + + t0 = time.perf_counter() + found = 0 + for i in range(n): + cur.execute("SELECT v FROM kv WHERE k=?", (f"key_{i}",)) + if cur.fetchone(): + found += 1 + r = time.perf_counter() - t0 + out.append( + result( + "kv_read", + label, + "embedded", + n, + r, + found=found, + durable=mode == "FULL", + ) + ) + + # Batch multi-row INSERT into SQL table (embedded SQL peer for batch) + cur.execute("DROP TABLE IF EXISTS fair_batch") + cur.execute("CREATE TABLE fair_batch (id INTEGER PRIMARY KEY, v TEXT)") + conn.commit() + t0 = time.perf_counter() + for start in range(0, n, BATCH): + cnt = min(BATCH, n - start) + vals = multi_values_sql(start, cnt) + cur.execute(f"INSERT INTO fair_batch (id, v) VALUES {vals}") + conn.commit() + bw = time.perf_counter() - t0 + out.append( + result( + "sql_insert_batch", + label, + "embedded", + n, + bw, + batch=BATCH, + durable=mode == "FULL", + note=f"multi-row INSERT batch={BATCH}, sync={mode}", + ) + ) + conn.close() + + if os.path.exists(path): + os.unlink(path) + return out + + +# ─── Tier 2: Client / server ──────────────────────────────────────── + + +def bara_http_query(sql: str, host: str = HTTP_HOST, port: int = HTTP_PORT) -> dict: + body = json.dumps({"query": sql}).encode() + req = urllib.request.Request( + f"http://{host}:{port}/query", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=60) as resp: + return json.loads(resp.read().decode()) + + +def bara_http_available() -> bool: + try: + body = json.dumps({"query": "SELECT 1"}).encode() + # health endpoint preferred + req = urllib.request.Request(f"http://{HTTP_HOST}:{HTTP_PORT}/health", method="GET") + with urllib.request.urlopen(req, timeout=2) as resp: + return resp.status == 200 + except Exception: + try: + bara_http_query("SELECT 1") + return True + except Exception: + return False + + +def bench_baradb_http(n: int = N_SQL) -> list[dict]: + if os.environ.get("FAIR_SKIP_HTTP") == "1": + print(" [skip] FAIR_SKIP_HTTP=1") + return [] + if not bara_http_available(): + print( + f" [skip] BaraDB HTTP not reachable at {HTTP_HOST}:{HTTP_PORT} " + f"(start: ./build/baradadb)" + ) + return [] + + out = [] + ep = f"http://{HTTP_HOST}:{HTTP_PORT}/query" + # BaraDB parser may not support IF EXISTS — ignore DROP failures + try: + bara_http_query("DROP TABLE fair_bench") + except Exception: + pass + try: + bara_http_query("CREATE TABLE fair_bench (id INT PRIMARY KEY, v TEXT)") + except Exception as e: + print(f" [warn] setup query failed: {e}") + + # Row-at-a-time INSERT + t0 = time.perf_counter() + errors = 0 + for i in range(n): + try: + r = bara_http_query( + f"INSERT INTO fair_bench (id, v) VALUES ({i}, 'value_{i}')" + ) + if isinstance(r, dict) and r.get("error"): + errors += 1 + except Exception: + errors += 1 + w = time.perf_counter() - t0 + out.append( + result( + "sql_insert_row", + "baradb_http", + "client_server", + n, + w, + errors=errors, + endpoint=ep, + ) + ) + + # Point SELECT + t0 = time.perf_counter() + found = 0 + for i in range(n): + try: + r = bara_http_query(f"SELECT v FROM fair_bench WHERE id = {i}") + rows = r.get("rows") if isinstance(r, dict) else None + if rows: + found += 1 + except Exception: + pass + rd = time.perf_counter() - t0 + out.append( + result( + "sql_select_row", + "baradb_http", + "client_server", + n, + rd, + found=found, + endpoint=ep, + ) + ) + + # Multi-row batch INSERT + try: + try: + bara_http_query("DROP TABLE fair_batch") + except Exception: + pass + bara_http_query("CREATE TABLE fair_batch (id INT PRIMARY KEY, v TEXT)") + except Exception as e: + print(f" [warn] batch setup failed: {e}") + return out + + t0 = time.perf_counter() + berr = 0 + for start in range(0, n, BATCH): + cnt = min(BATCH, n - start) + sql = f"INSERT INTO fair_batch (id, v) VALUES {multi_values_sql(start, cnt)}" + try: + r = bara_http_query(sql) + if isinstance(r, dict) and r.get("error"): + berr += 1 + except Exception: + berr += 1 + bw = time.perf_counter() - t0 + out.append( + result( + "sql_insert_batch", + "baradb_http", + "client_server", + n, + bw, + batch=BATCH, + errors=berr, + endpoint=ep, + note=f"multi-row INSERT batch={BATCH}", + ) + ) + return out + + +def _import_baradb_client(): + """Load clients/python baradb package without requiring install.""" + p = str(CLIENTS_PY) + if p not in sys.path: + sys.path.insert(0, p) + from baradb import Client # type: ignore + + return Client + + +def bara_wire_available() -> bool: + if os.environ.get("FAIR_SKIP_WIRE") == "1": + return False + try: + Client = _import_baradb_client() + except Exception as e: + print(f" [skip] wire client import failed: {e}") + return False + + async def _ping(): + c = Client(WIRE_HOST, WIRE_PORT, timeout=2.0) + try: + await c.connect() + await c.ping() + await c.close() + return True + except Exception: + try: + await c.close() + except Exception: + pass + return False + + try: + return asyncio.run(_ping()) + except Exception: + return False + + +def bench_baradb_wire(n: int = N_SQL) -> list[dict]: + """BaraDB binary wire protocol (TCP) — primary high-performance client path.""" + if os.environ.get("FAIR_SKIP_WIRE") == "1": + print(" [skip] FAIR_SKIP_WIRE=1") + return [] + try: + Client = _import_baradb_client() + except Exception as e: + print(f" [skip] wire client not available: {e}") + return [] + + if not bara_wire_available(): + print( + f" [skip] BaraDB wire not reachable at {WIRE_HOST}:{WIRE_PORT} " + f"(start: ./build/baradadb)" + ) + return [] + + async def _run() -> list[dict]: + out: list[dict] = [] + client = Client(WIRE_HOST, WIRE_PORT, timeout=60.0) + await client.connect() + try: + try: + await client.query("DROP TABLE fair_wire") + except Exception: + pass + try: + await client.query( + "CREATE TABLE fair_wire (id INT PRIMARY KEY, v TEXT)" + ) + except Exception as e: + print(f" [warn] wire setup: {e}") + + ep = f"tcp://{WIRE_HOST}:{WIRE_PORT}" + # Row-at-a-time INSERT (may crash older servers under load — record partial) + t0 = time.perf_counter() + errors = 0 + done = 0 + crashed = False + for i in range(n): + try: + await client.query( + f"INSERT INTO fair_wire (id, v) VALUES ({i}, 'value_{i}')" + ) + done += 1 + except (ConnectionError, OSError, Exception) as e: + errors += 1 + if "reset" in str(e).lower() or "closed" in str(e).lower(): + crashed = True + print(f" [warn] wire connection lost after {done} inserts: {e}") + break + w = time.perf_counter() - t0 + if done > 0: + out.append( + result( + "sql_insert_row", + "baradb_wire", + "client_server", + done, + w, + errors=errors, + requested=n, + endpoint=ep, + note="binary wire protocol" + + (" (partial — server disconnect)" if crashed else ""), + ) + ) + + if crashed: + return out + + # Point SELECT + t0 = time.perf_counter() + found = 0 + for i in range(done): + try: + r = await client.query( + f"SELECT v FROM fair_wire WHERE id = {i}" + ) + if r is not None and ( + getattr(r, "row_count", 0) > 0 or getattr(r, "rows", None) + ): + found += 1 + except (ConnectionError, OSError, Exception) as e: + if "reset" in str(e).lower() or "closed" in str(e).lower(): + crashed = True + print(f" [warn] wire lost during SELECT: {e}") + break + rd = time.perf_counter() - t0 + out.append( + result( + "sql_select_row", + "baradb_wire", + "client_server", + max(done, 1), + rd, + found=found, + endpoint=ep, + ) + ) + if crashed: + return out + + # Batch multi-row INSERT + try: + try: + await client.query("DROP TABLE fair_wire_batch") + except Exception: + pass + await client.query( + "CREATE TABLE fair_wire_batch (id INT PRIMARY KEY, v TEXT)" + ) + except Exception as e: + print(f" [warn] wire batch setup: {e}") + return out + + t0 = time.perf_counter() + berr = 0 + bdone = 0 + for start in range(0, n, BATCH): + cnt = min(BATCH, n - start) + sql = ( + "INSERT INTO fair_wire_batch (id, v) VALUES " + + multi_values_sql(start, cnt) + ) + try: + await client.query(sql) + bdone += cnt + except (ConnectionError, OSError, Exception) as e: + berr += 1 + if "reset" in str(e).lower() or "closed" in str(e).lower(): + print(f" [warn] wire lost during batch after {bdone} rows: {e}") + break + bw = time.perf_counter() - t0 + if bdone > 0: + out.append( + result( + "sql_insert_batch", + "baradb_wire", + "client_server", + bdone, + bw, + batch=BATCH, + errors=berr, + requested=n, + endpoint=ep, + note=f"multi-row INSERT batch={BATCH}", + ) + ) + finally: + try: + await client.close() + except Exception: + pass + return out + + try: + return asyncio.run(_run()) + except Exception as e: + print(f" [skip] wire bench failed: {e}") + return [] + + +def bench_postgresql(n: int = N_SQL) -> list[dict]: + if os.environ.get("FAIR_SKIP_PG") == "1": + print(" [skip] FAIR_SKIP_PG=1") + return [] + try: + import psycopg2 + except ImportError: + print(" [skip] psycopg2 not installed") + return [] + + cfg = { + "host": os.environ.get("PGHOST", "localhost"), + "port": int(os.environ.get("PGPORT", "5432")), + "dbname": os.environ.get("PGDATABASE", "postgres"), + "user": os.environ.get("PGUSER", "postgres"), + "password": os.environ.get("PGPASSWORD", os.environ.get("PG_PASSWORD", "")), + } + if not cfg["password"] and os.environ.get("PGPASSWORD") is None: + cfg["password"] = os.environ.get("BARA_PG_PASSWORD", "pas+123") + + out = [] + try: + conn = psycopg2.connect(**cfg) + except Exception as e: + print(f" [skip] PostgreSQL connect failed: {e}") + return [] + + cur = conn.cursor() + for sync, label in (("on", "postgresql_sync_on"), ("off", "postgresql_sync_off")): + cur.execute(f"SET synchronous_commit = {sync}") + cur.execute("DROP TABLE IF EXISTS fair_bench") + cur.execute("CREATE TABLE fair_bench (id INTEGER PRIMARY KEY, v TEXT)") + conn.commit() + + t0 = time.perf_counter() + for i in range(n): + cur.execute( + "INSERT INTO fair_bench (id, v) VALUES (%s, %s)", + (i, f"value_{i}"), + ) + conn.commit() + w = time.perf_counter() - t0 + out.append( + result( + "sql_insert_row", + label, + "client_server", + n, + w, + durable=sync == "on", + note=f"synchronous_commit={sync}", + ) + ) + + t0 = time.perf_counter() + found = 0 + for i in range(n): + cur.execute("SELECT v FROM fair_bench WHERE id = %s", (i,)) + if cur.fetchone(): + found += 1 + rd = time.perf_counter() - t0 + out.append( + result( + "sql_select_row", + label, + "client_server", + n, + rd, + found=found, + durable=sync == "on", + ) + ) + + # Batch multi-row INSERT (same durability setting) + cur.execute("DROP TABLE IF EXISTS fair_batch") + cur.execute("CREATE TABLE fair_batch (id INTEGER PRIMARY KEY, v TEXT)") + conn.commit() + t0 = time.perf_counter() + for start in range(0, n, BATCH): + cnt = min(BATCH, n - start) + cur.execute( + f"INSERT INTO fair_batch (id, v) VALUES {multi_values_sql(start, cnt)}" + ) + conn.commit() + bw = time.perf_counter() - t0 + out.append( + result( + "sql_insert_batch", + label, + "client_server", + n, + bw, + batch=BATCH, + durable=sync == "on", + note=f"multi-row INSERT batch={BATCH}, sync={sync}", + ) + ) + + cur.close() + conn.close() + return out + + +# ─── Report ────────────────────────────────────────────────────────── + + +def print_tier(name: str, rows: list[dict]): + print(f"\n=== Tier: {name} ===") + if not rows: + print(" (no results)") + return + # group by bench name + names = [] + for r in rows: + if r["name"] not in names: + names.append(r["name"]) + for nm in names: + print(f" [{nm}]") + for r in rows: + if r["name"] != nm: + continue + print( + f" {r['system']:28s} {fmt_ops(r['opsPerSec']):>10s}/s " + f"({r['seconds']:.3f}s, n={r['ops']})" + ) + + +def write_markdown(payload: dict, path: Path): + lines = [] + lines.append("# Fair Benchmark Results") + lines.append("") + lines.append(f"Generated: **{payload.get('generated', '')}**") + lines.append("") + lines.append("## Methodology") + lines.append("") + for line in payload.get("methodology", []): + lines.append(f"- {line}") + lines.append("") + lines.append("**Do not compare numbers across tiers.** Embedded storage is not the same") + lines.append("workload as client-server SQL over the network.") + lines.append("") + + for tier in ("embedded", "client_server"): + rows = [r for r in payload.get("results", []) if r.get("tier") == tier] + lines.append(f"## Tier: `{tier}`") + lines.append("") + if not rows: + lines.append("_No results for this tier._") + lines.append("") + continue + lines.append("| Bench | System | ops/s | seconds | n | notes |") + lines.append("|-------|--------|------:|--------:|--:|-------|") + for r in rows: + note = r.get("note") or r.get("source") or "" + lines.append( + f"| {r['name']} | `{r['system']}` | {fmt_ops(r['opsPerSec'])} | " + f"{r['seconds']:.3f} | {r['ops']} | {note} |" + ) + lines.append("") + + # same-bench comparison within tier + names = sorted({r["name"] for r in rows}) + lines.append(f"### Same-bench ratios (`{tier}`)") + lines.append("") + for nm in names: + group = [r for r in rows if r["name"] == nm] + if len(group) < 2: + continue + best = max(group, key=lambda x: x["opsPerSec"]) + lines.append(f"**{nm}** (fastest: `{best['system']}` @ {fmt_ops(best['opsPerSec'])}/s)") + lines.append("") + lines.append("| System | Relative to fastest |") + lines.append("|--------|--------------------:|") + for r in sorted(group, key=lambda x: -x["opsPerSec"]): + rel = r["opsPerSec"] / best["opsPerSec"] if best["opsPerSec"] else 0 + lines.append(f"| `{r['system']}` | {rel:.2f}x |") + lines.append("") + + path.write_text("\n".join(lines) + "\n") + print(f"\nMarkdown written to {path}") + + +def main(): + print("BaraDB Fair Benchmark Suite") + print(f" N_KV={N_KV} N_SQL={N_SQL} BATCH={BATCH}") + print(f" HTTP={HTTP_HOST}:{HTTP_PORT} WIRE={WIRE_HOST}:{WIRE_PORT}") + + methodology = [ + "Tier `embedded`: in-process only (BaraDB LSM from nimble bench JSON; SQLite via Python sqlite3).", + "Tier `client_server`: network SQL (BaraDB HTTP /query; BaraDB binary wire TCP; PostgreSQL via psycopg2).", + "`sql_insert_row`: one INSERT statement per row (chatty).", + f"`sql_insert_batch`: multi-row INSERT with batch size {BATCH} (same SQL style across systems).", + "PostgreSQL: synchronous_commit=on|off; SQLite: PRAGMA synchronous FULL|OFF.", + "BaraDB WAL modes appear only if you ran benchmarks/bench_all.nim (WAL-* rows).", + "Never claim 'Nx faster than Postgres' using embedded BaraDB numbers.", + ] + + results: list[dict] = [] + + print("\n--- Embedded tier ---") + results.extend(load_baradb_embedded()) + print(" SQLite embedded (+ batch)…") + results.extend(bench_sqlite_embedded()) + + print("\n--- Client/server tier ---") + print(" BaraDB HTTP…") + results.extend(bench_baradb_http()) + print(" BaraDB wire (TCP)…") + results.extend(bench_baradb_wire()) + print(" PostgreSQL…") + results.extend(bench_postgresql()) + + payload = { + "generated": now_iso(), + "methodology": methodology, + "config": { + "N_KV": N_KV, + "N_SQL": N_SQL, + "HTTP": f"{HTTP_HOST}:{HTTP_PORT}", + }, + "results": results, + } + OUT_JSON.write_text(json.dumps(payload, indent=2)) + print(f"\nJSON written to {OUT_JSON}") + + print_tier("embedded", [r for r in results if r["tier"] == "embedded"]) + print_tier("client_server", [r for r in results if r["tier"] == "client_server"]) + + write_markdown(payload, ROOT / "benchmarks" / "FAIR_COMPARISON.md") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/generate_report.py b/benchmarks/generate_report.py index c8ee5ad..4a7be7b 100644 --- a/benchmarks/generate_report.py +++ b/benchmarks/generate_report.py @@ -1,110 +1,143 @@ #!/usr/bin/env python3 -"""Generate a real comparison report from BaraDB and PostgreSQL benchmark results.""" +"""Generate benchmark reports. + +Modes: + python3 benchmarks/generate_report.py # legacy PG vs embedded (with warning) + python3 benchmarks/generate_report.py --fair # multi-tier fair report from fair_bench.py +""" +from __future__ import annotations + +import argparse import json +import sys from pathlib import Path +ROOT = Path(__file__).resolve().parent.parent -def format_ops(ops_per_sec): + +def format_ops(ops_per_sec: float) -> str: if ops_per_sec >= 1_000_000: return f"{ops_per_sec/1_000_000:.2f}M" - elif ops_per_sec >= 1_000: + if ops_per_sec >= 1_000: return f"{ops_per_sec/1_000:.2f}K" - else: - return f"{ops_per_sec:.2f}" + return f"{ops_per_sec:.2f}" -def format_time(seconds): +def format_time(seconds: float) -> str: if seconds < 0.001: return f"{seconds*1000:.3f}ms" - elif seconds < 1: + if seconds < 1: return f"{seconds*1000:.1f}ms" - else: - return f"{seconds:.3f}s" + return f"{seconds:.3f}s" -def main(): - root = Path(__file__).parent +def gen_fair(out: Path) -> int: + fair_path = ROOT / "fair_benchmark_results.json" + if not fair_path.exists(): + print("Missing fair_benchmark_results.json — run: python3 benchmarks/fair_bench.py") + return 1 + payload = json.loads(fair_path.read_text()) + # fair_bench already writes FAIR_COMPARISON.md; re-emit for consistency + sys.path.insert(0, str(ROOT / "benchmarks")) + from fair_bench import write_markdown # type: ignore - with open(root.parent / "benchmark_results.json") as f: + write_markdown(payload, out) + return 0 + + +def gen_legacy() -> int: + """Legacy report: PG client-server vs BaraDB *embedded* — always labeled unfair.""" + bara_path = ROOT / "benchmark_results.json" + pg_path = ROOT / "pg_benchmark_results.json" + if not bara_path.exists() or not pg_path.exists(): + print("Need benchmark_results.json and pg_benchmark_results.json") + print(" nimble bench && python3 benchmarks/pg_bench.py") + return 1 + + with open(bara_path) as f: bara = json.load(f) - with open(root.parent / "pg_benchmark_results.json") as f: + with open(pg_path) as f: pg = json.load(f) bara_map = {r["name"]: r for r in bara["results"]} - pg_map = {k: v for k, v in pg.items()} + # pg_bench may write list or dict + if isinstance(pg, dict) and "results" in pg: + pg_map = {r["name"]: r for r in pg["results"]} + elif isinstance(pg, list): + pg_map = {r["name"]: r for r in pg} + else: + pg_map = pg # old flat dict by name report = [] - report.append("# BaraDB vs PostgreSQL — Real Benchmark Results") + report.append("# BaraDB vs PostgreSQL — LEGACY (mixed tiers)") report.append("") - report.append("Generated from actual execution on:") - report.append(f"- **CPU:** AMD Ryzen 9 5900X") - report.append(f"- **PostgreSQL:** 15.18 (local)") - report.append(f"- **BaraDB:** git `{bara['gitSha']}`") + report.append("> ⚠️ **Unfair comparison warning**") + report.append(">") + report.append("> PostgreSQL numbers include **client-server** round-trips.") + report.append("> BaraDB numbers are **in-process embedded** LSM (no network, no SQL).") + report.append("> Use `python3 benchmarks/fair_bench.py` + `--fair` for honest tiers.") report.append("") - report.append("## Methodology") + report.append(f"- **BaraDB git:** `{bara.get('gitSha', 'unknown')}`") report.append("") - report.append("- PostgreSQL: single-row INSERT/SELECT via psycopg2 (client-server overhead included)") - report.append("- BaraDB: in-process Nim code (no network overhead)") - report.append("- Same dataset sizes for both systems") - report.append("") - report.append("## Results") - report.append("") - report.append("| Test | PostgreSQL | BaraDB | Speedup |") - report.append("|------|-----------|--------|---------|") + report.append("| Test | PostgreSQL (C/S) | BaraDB (embedded) | Ratio (not a fair speedup) |") + report.append("|------|------------------|-------------------|----------------------------|") rows = [ - ("KV Write (100K)", pg_map.get("KV Write"), bara_map.get("LSM-Write")), - ("KV Read (100K)", pg_map.get("KV Read"), bara_map.get("LSM-Read")), - ("BTree Insert (100K)", pg_map.get("BTree Insert"), bara_map.get("BTree-Insert")), - ("BTree Get (100K)", pg_map.get("BTree Get"), bara_map.get("BTree-Get")), - ("BTree Scan (1K ranges)", pg_map.get("BTree Scan"), bara_map.get("BTree-Scan")), - ("FTS Index (10K docs)", pg_map.get("FTS Index"), bara_map.get("FTS-Index")), - ("FTS Search (1K queries)", pg_map.get("FTS Search"), bara_map.get("FTS-Search")), + ("KV Write", pg_map.get("KV Write"), bara_map.get("LSM-Write")), + ("KV Read", pg_map.get("KV Read"), bara_map.get("LSM-Read")), + ("BTree Insert", pg_map.get("BTree Insert"), bara_map.get("BTree-Insert")), + ("BTree Get", pg_map.get("BTree Get"), bara_map.get("BTree-Get")), + ("BTree Scan", pg_map.get("BTree Scan"), bara_map.get("BTree-Scan")), + ("FTS Index", pg_map.get("FTS Index"), bara_map.get("FTS-Index")), + ("FTS Search", pg_map.get("FTS Search"), bara_map.get("FTS-Search")), ] - total_pg_time = 0 - total_bara_time = 0 - for name, p, b in rows: if p is None or b is None: continue pg_ops = p["opsPerSec"] ba_ops = b["opsPerSec"] - ratio = ba_ops / pg_ops - winner = "BaraDB" if ratio > 1 else "PostgreSQL" - total_pg_time += p["seconds"] - total_bara_time += b["seconds"] - + ratio = ba_ops / pg_ops if pg_ops else 0 report.append( f"| {name} | {format_ops(pg_ops)}/s ({format_time(p['seconds'])}) | " f"{format_ops(ba_ops)}/s ({format_time(b['seconds'])}) | " - f"{ratio:.1f}x ({winner}) |" + f"{ratio:.1f}x (mixed tiers) |" ) report.append("") - report.append("## Summary") + report.append("## Prefer fair suite") report.append("") - report.append(f"- **Total PostgreSQL time:** {total_pg_time:.3f}s") - report.append(f"- **Total BaraDB time:** {total_bara_time:.3f}s") - overall = total_pg_time / total_bara_time - report.append(f"- **Overall speedup:** BaraDB is **{overall:.1f}x faster**") - report.append("") - report.append("## Notes") - report.append("") - report.append("- PostgreSQL includes network round-trip and SQL parsing overhead per operation.") - report.append("- BaraDB runs in-process with zero serialization/network cost.") - report.append("- For embedded/single-node use cases, BaraDB shows significant advantage.") - report.append("- PostgreSQL FTS Search with GIN index outperforms BaraDB on query throughput.") - report.append("- PostgreSQL excels at durability, replication, and complex ACID transactions.") + report.append("```bash") + report.append("nim c -d:release -r benchmarks/bench_all.nim") + report.append("python3 benchmarks/fair_bench.py") + report.append("python3 benchmarks/generate_report.py --fair") + report.append("```") report.append("") - output = "\n".join(report) - print(output) + out = ROOT / "benchmarks" / "REAL_COMPARISON.md" + out.write_text("\n".join(report) + "\n") + print(f"Wrote {out} (legacy mixed-tier; see warning banner)") + return 0 - with open(root / "REAL_COMPARISON.md", "w") as f: - f.write(output) - print(f"\nReport saved to {root / 'REAL_COMPARISON.md'}") + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--fair", + action="store_true", + help="Emit multi-tier fair report from fair_benchmark_results.json", + ) + ap.add_argument( + "-o", + "--output", + default=str(ROOT / "benchmarks" / "FAIR_COMPARISON.md"), + help="Output path for --fair mode", + ) + args = ap.parse_args() + if args.fair: + return gen_fair(Path(args.output)) + return gen_legacy() if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/benchmarks/pg_bench.py b/benchmarks/pg_bench.py index 1a3b80a..684cfc8 100644 --- a/benchmarks/pg_bench.py +++ b/benchmarks/pg_bench.py @@ -174,8 +174,10 @@ def format_ops(ops_per_sec): def print_comparison(pg_results, bara_data): bara = {r["name"]: r for r in bara_data["results"]} print("\n╔══════════════════════════════════════════════════════════════════════╗") - print("║ BaraDB vs PostgreSQL — Real Benchmark Results ║") + print("║ PostgreSQL (client-server) vs BaraDB (EMBEDDED) — MIXED TIERS ║") print("╚══════════════════════════════════════════════════════════════════════╝\n") + print("WARNING: This mixes client-server PG with in-process BaraDB LSM.") + print(" Prefer: python3 benchmarks/fair_bench.py\n") rows = [ ("KV Write (100K)", pg_results.get("KV Write"), bara.get("LSM-Write")), @@ -187,7 +189,7 @@ def print_comparison(pg_results, bara_data): ("FTS Search (1K queries)", pg_results.get("FTS Search"), bara.get("FTS-Search")), ] - print(f"{'Test':<26} {'PostgreSQL':>18} {'BaraDB':>18} {'Winner':>10}") + print(f"{'Test':<26} {'PostgreSQL C/S':>18} {'BaraDB embed':>18} {'Note':>14}") print("─" * 76) for name, pg, ba in rows: @@ -195,22 +197,14 @@ def print_comparison(pg_results, bara_data): continue pg_ops = pg["opsPerSec"] ba_ops = ba["opsPerSec"] - winner = "BaraDB" if ba_ops > pg_ops else "PostgreSQL" - ratio = max(ba_ops, pg_ops) / min(ba_ops, pg_ops) + ratio = ba_ops / pg_ops if pg_ops else 0 print( - f"{name:<26} {format_ops(pg_ops)+'/s':>18} {format_ops(ba_ops)+'/s':>18} {winner+' ('+f'{ratio:.1f}x'+')':>10}" + f"{name:<26} {format_ops(pg_ops)+'/s':>18} {format_ops(ba_ops)+'/s':>18} " + f"{'mixed '+f'{ratio:.1f}x':>14}" ) print("\n" + "─" * 76) - # Summary - pg_total = sum(r["seconds"] for _, r, _ in rows if r is not None) - ba_total = sum(b["seconds"] for _, _, b in rows if b is not None) - print(f"\nTotal time PostgreSQL: {pg_total:.3f}s") - print(f"Total time BaraDB: {ba_total:.3f}s") - if ba_total < pg_total: - print(f"BaraDB is {pg_total/ba_total:.1f}x faster overall") - else: - print(f"PostgreSQL is {ba_total/pg_total:.1f}x faster overall") + print("For fair tiers (SQLite↔LSM, HTTP↔PG): python3 benchmarks/fair_bench.py") def main(): @@ -247,14 +241,23 @@ def main(): pg_results["FTS Search"] = bench_fts_search() print(f" -> {format_ops(pg_results['FTS Search']['opsPerSec'])}/s ({pg_results['FTS Search']['seconds']:.3f}s)") - bara_data = load_baradb_results() - print_comparison(pg_results, bara_data) + # Annotate tier for fair tooling + for name, r in pg_results.items(): + r["tier"] = "client_server" + r["system"] = "postgresql" - # Save raw results with open("pg_benchmark_results.json", "w") as f: json.dump(pg_results, f, indent=2) print("\nPostgreSQL results saved to pg_benchmark_results.json") + if os.path.exists("benchmark_results.json"): + bara_data = load_baradb_results() + print_comparison(pg_results, bara_data) + else: + print("\n(No benchmark_results.json — skip mixed-tier table; run nimble bench first)") + + print("\nFair multi-tier suite: python3 benchmarks/fair_bench.py") + if __name__ == "__main__": main() diff --git a/nim.cfg b/nim.cfg index 7a9fb50..f10d906 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,3 +1,8 @@ -d:ssl --threads:on --path:"src" +# ARC: ORC cycle collector crashes under async wire-protocol load +# (markGray/trace SIGSEGV after ~20 sequential INSERTs). ARC is stable +# for the TCP server + HTTP worker mix. Prefer breaking cycles over +# re-enabling ORC without a reproducer. +--mm:arc diff --git a/src/barabadb/core/config.nim b/src/barabadb/core/config.nim index 9c67d7c..e3bbd46 100644 --- a/src/barabadb/core/config.nim +++ b/src/barabadb/core/config.nim @@ -22,6 +22,11 @@ type logFormat*: string memtableSizeMb*: int cacheSizeMb*: int + ## WAL durability: "none" | "group" (default) | "every" + walSyncMode*: string + ## Group commit batch size (entries between fsyncs when mode=group) + walGroupEvery*: int + ## Time-based group fsync interval in ms (0 = off); also used as legacy name walSyncIntervalMs*: int compactionIntervalMs*: int bloomBitsPerKey*: int @@ -58,6 +63,8 @@ proc defaultConfig*(): BaraConfig = logFormat: "json", memtableSizeMb: 64, cacheSizeMb: 256, + walSyncMode: "group", + walGroupEvery: 64, walSyncIntervalMs: 0, compactionIntervalMs: 60_000, bloomBitsPerKey: 10, @@ -93,6 +100,8 @@ proc loadConfigFromJson*(path: string, cfg: var BaraConfig) = if s.hasKey("data_dir"): cfg.dataDir = s["data_dir"].getStr() if s.hasKey("memtable_size_mb"): cfg.memtableSizeMb = s["memtable_size_mb"].getInt() if s.hasKey("cache_size_mb"): cfg.cacheSizeMb = s["cache_size_mb"].getInt() + if s.hasKey("wal_sync_mode"): cfg.walSyncMode = s["wal_sync_mode"].getStr() + if s.hasKey("wal_group_every"): cfg.walGroupEvery = s["wal_group_every"].getInt() if s.hasKey("wal_sync_interval_ms"): cfg.walSyncIntervalMs = s["wal_sync_interval_ms"].getInt() if s.hasKey("compaction_interval_ms"): cfg.compactionIntervalMs = s["compaction_interval_ms"].getInt() if s.hasKey("bloom_bits_per_key"): cfg.bloomBitsPerKey = s["bloom_bits_per_key"].getInt() @@ -153,6 +162,8 @@ proc loadConfigFromEnv*(cfg: var BaraConfig) = cfg.logFormat = getEnv("BARADB_LOG_FORMAT", cfg.logFormat) cfg.memtableSizeMb = parseEnvInt(getEnv("BARADB_MEMTABLE_SIZE_MB", ""), cfg.memtableSizeMb) cfg.cacheSizeMb = parseEnvInt(getEnv("BARADB_CACHE_SIZE_MB", ""), cfg.cacheSizeMb) + cfg.walSyncMode = getEnv("BARADB_WAL_SYNC_MODE", cfg.walSyncMode) + cfg.walGroupEvery = parseEnvInt(getEnv("BARADB_WAL_GROUP_EVERY", ""), cfg.walGroupEvery) cfg.walSyncIntervalMs = parseEnvInt(getEnv("BARADB_WAL_SYNC_INTERVAL_MS", ""), cfg.walSyncIntervalMs) cfg.compactionIntervalMs = parseEnvInt(getEnv("BARADB_COMPACTION_INTERVAL_MS", ""), cfg.compactionIntervalMs) cfg.bloomBitsPerKey = parseEnvInt(getEnv("BARADB_BLOOM_BITS_PER_KEY", ""), cfg.bloomBitsPerKey) diff --git a/src/barabadb/core/httpserver.nim b/src/barabadb/core/httpserver.nim index 833b383..7658a53 100644 --- a/src/barabadb/core/httpserver.nim +++ b/src/barabadb/core/httpserver.nim @@ -15,6 +15,7 @@ import ../query/parser import ../query/executor import ../core/types import ../storage/lsm +import ../storage/gate import ../core/mvcc import ../protocol/wire import ../core/websocket @@ -196,17 +197,7 @@ proc queryHandler(server: HttpServer): RequestHandler = ctx.json(%*{"error": "Empty query"}, 400) return - var reqCtx = getRequestDatabaseContext(server, request) - reqCtx.currentUser = userId - reqCtx.currentRole = role - let tokens = tokenize(queryStr) - let astNode = parse(tokens) - - if astNode.stmts.len == 0: - ctx.json(%*{"rows": [], "affectedRows": 0, "columns": []}) - return - - # Extract optional params from JSON body + # Extract optional params from JSON body (no storage access yet) var params: seq[WireValue] = @[] if "params" in body and body["params"].kind == JArray: for p in body["params"]: @@ -218,31 +209,50 @@ proc queryHandler(server: HttpServer): RequestHandler = of JString: params.add(WireValue(kind: fkString, strVal: p.getStr())) else: params.add(WireValue(kind: fkString, strVal: $p)) - let res = executor.executeQuery(reqCtx, astNode, params) + # StorageGate: serialize against TCP + other Hunos workers (ORC safety) + var success: bool + var jsonRows = newJArray() + var jsonCols = newJArray() + var affected = 0 + var msg = "" + var errMsg = "" + withStorageGate: + var reqCtx = getRequestDatabaseContext(server, request) + reqCtx.currentUser = userId + reqCtx.currentRole = role + let tokens = tokenize(queryStr) + let astNode = parse(tokens) + if astNode.stmts.len == 0: + success = true + else: + let res = executor.executeQuery(reqCtx, astNode, params) + success = res.success + if res.success: + affected = res.affectedRows + msg = res.message + for row in res.rows: + var jsonRow = newJObject() + for col in res.columns: + if col in row and row[col].kind != vkNull: + jsonRow[col] = %valueToString(row[col]) + else: + jsonRow[col] = newJNull() + jsonRows.add(jsonRow) + for c in res.columns: + jsonCols.add(%c) + else: + errMsg = res.message - if res.success: - var jsonRows = newJArray() - for row in res.rows: - var jsonRow = newJObject() - for col in res.columns: - let key = col - if key in row and row[key].kind != vkNull: - jsonRow[key] = %valueToString(row[key]) - else: - jsonRow[key] = newJNull() - jsonRows.add(jsonRow) - var jsonCols = newJArray() - for c in res.columns: - jsonCols.add(%c) + if success: ctx.json(%*{ "rows": jsonRows, - "affectedRows": res.affectedRows, + "affectedRows": affected, "columns": jsonCols, - "message": if res.message.len > 0: %res.message else: newJNull() + "message": if msg.len > 0: %msg else: newJNull() }) else: server.metrics.queryErrors += 1 - ctx.json(%*{"error": res.message}, 400) + ctx.json(%*{"error": errMsg}, 400) proc healthHandler(): RequestHandler = return proc(request: Request) {.gcsafe.} = @@ -376,15 +386,16 @@ proc tablesHandler(server: HttpServer): RequestHandler = let ctx = newContext(request) if not server.checkAuth(request, ctx): return - let reqCtx = getRequestDatabaseContext(server, request) var tables = newJArray() - for name, tbl in reqCtx.tables: - var cols = newJArray() - for col in tbl.columns: - cols.add(%*{"name": col.name, "type": col.colType, - "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique}) - tables.add(%*{"name": name, "columns": cols, - "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len}) + withStorageGate: + let reqCtx = getRequestDatabaseContext(server, request) + for name, tbl in reqCtx.tables: + var cols = newJArray() + for col in tbl.columns: + cols.add(%*{"name": col.name, "type": col.colType, + "pk": col.isPk, "notNull": col.isNotNull, "unique": col.isUnique}) + tables.add(%*{"name": name, "columns": cols, + "pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len}) ctx.json(%*{"tables": tables}) proc databasesHandler(server: HttpServer): RequestHandler = @@ -393,24 +404,25 @@ proc databasesHandler(server: HttpServer): RequestHandler = let ctx = newContext(request) if not server.checkAuth(request, ctx): return - let dbs = server.registry.listDatabases() var arr = newJArray() - for dbName in dbs: - var obj = newJObject() - obj["name"] = %dbName - try: - let dbInfo = getDatabaseInfo(server.registry, dbName) - if dbInfo != nil and dbInfo.ctx != nil: - let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx)) - obj["tables"] = %dbCtx.tables.len - obj["connections"] = %getConnectionCount(server.registry, dbName) - else: + withStorageGate: + let dbs = server.registry.listDatabases() + for dbName in dbs: + var obj = newJObject() + obj["name"] = %dbName + try: + let dbInfo = getDatabaseInfo(server.registry, dbName) + if dbInfo != nil and dbInfo.ctx != nil: + let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx)) + obj["tables"] = %dbCtx.tables.len + obj["connections"] = %getConnectionCount(server.registry, dbName) + else: + obj["tables"] = %0 + obj["connections"] = %0 + except CatchableError: obj["tables"] = %0 obj["connections"] = %0 - except CatchableError: - obj["tables"] = %0 - obj["connections"] = %0 - arr.add(obj) + arr.add(obj) ctx.json(%*{"databases": arr}) proc createDatabaseHandler(server: HttpServer): RequestHandler = @@ -428,7 +440,8 @@ proc createDatabaseHandler(server: HttpServer): RequestHandler = ctx.json(%*{"error": "Empty database name"}, 400) return try: - discard getOrCreateDatabase(server.registry, dbName) + withStorageGate: + discard getOrCreateDatabase(server.registry, dbName) ctx.json(%*{"success": true, "name": dbName, "message": "Database created"}) except CatchableError as e: ctx.json(%*{"error": e.msg}, 400) @@ -444,7 +457,9 @@ proc dropDatabaseHandler(server: HttpServer): RequestHandler = ctx.json(%*{"error": "Missing database name"}, 400) return try: - let ok = dropDatabase(server.registry, dbName) + var ok = false + withStorageGate: + ok = dropDatabase(server.registry, dbName) if ok: ctx.json(%*{"success": true, "name": dbName, "message": "Database dropped"}) else: @@ -470,13 +485,15 @@ proc backupHandler(server: HttpServer): RequestHandler = let compression = if body != nil and "level" in body: body["level"].getInt() else: 6 try: var ok = false - if allDatabases: - ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false) - elif dbName.len > 0: - let dbDir = dataRoot / dbName - ok = backupDataDir(dbDir, outputFile, @[], compression, false) - else: - ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false) + # Gate held so live writers/compactors don't mutate files mid-backup + withStorageGate: + if allDatabases: + ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false) + elif dbName.len > 0: + let dbDir = dataRoot / dbName + ok = backupDataDir(dbDir, outputFile, @[], compression, false) + else: + ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false) if ok: ctx.json(%*{"success": true, "output": outputFile, "message": "Backup created"}) else: @@ -541,18 +558,20 @@ proc restoreHandler(server: HttpServer): RequestHandler = let meta = readBackupMeta(inputFile) let isMultiDb = meta != nil and meta{"databases"} != nil var ok = false - if isMultiDb or allDatabases: - ok = restoreAllDatabases(inputFile, dataRoot, false, false) - elif dbName.len > 0: - let dbDir = dataRoot / dbName - ok = restoreDataDir(inputFile, dbDir, false, false) - else: - ok = restoreAllDatabases(inputFile, dataRoot, false, false) + withStorageGate: + if isMultiDb or allDatabases: + ok = restoreAllDatabases(inputFile, dataRoot, false, false) + elif dbName.len > 0: + let dbDir = dataRoot / dbName + ok = restoreDataDir(inputFile, dbDir, false, false) + else: + ok = restoreAllDatabases(inputFile, dataRoot, false, false) + if ok: + # Reload under same gate after files are restored + server.registry.loadExistingDatabases() logRestore(inputFile, dataRoot, ok) if ok: - # Reload databases after restore - server.registry.loadExistingDatabases() ctx.json(%*{"success": true, "message": "Restore completed"}) else: ctx.json(%*{"error": "Restore failed"}, 500) @@ -890,10 +909,14 @@ proc run*(server: HttpServer, port: int = 9470) = asyncCheck server.ws.run(port + 1) hunosServer.serve(Port(port)) -proc stop*(server: HttpServer) = +proc stop*(server: HttpServer, closeStorage: bool = false) = + ## Stop HTTP listeners. By default does **not** close the shared registry — + ## when HTTP is spawned alongside TCP they share one registry owned by main. server.running = false server.ws.stop() - if server.registry != nil: - server.registry.closeAll() - else: - server.db.close() + if closeStorage: + withStorageGate: + if server.registry != nil: + server.registry.closeAll() + elif server.db != nil: + server.db.close() diff --git a/src/barabadb/core/registry.nim b/src/barabadb/core/registry.nim index 5a139e5..154c05a 100644 --- a/src/barabadb/core/registry.nim +++ b/src/barabadb/core/registry.nim @@ -29,6 +29,17 @@ type const reservedDbNames* = ["system", "information_schema", "pg_catalog"] +proc openLsmForRegistry(reg: DatabaseRegistry, dbDir: string): LSMTree = + ## Open LSM with WAL durability settings from registry config. + let memBytes = max(1, reg.config.memtableSizeMb) * 1024 * 1024 + newLSMTree( + dbDir, + memMaxSize = memBytes, + walSyncMode = parseWalSyncMode(reg.config.walSyncMode), + walGroupEvery = reg.config.walGroupEvery, + walGroupIntervalMs = reg.config.walSyncIntervalMs, + ) + proc isValidDbName*(name: string): bool = if name.len == 0: return false if '/' in name or '\\' in name: return false @@ -63,7 +74,7 @@ proc loadExistingDatabases*(reg: DatabaseRegistry) = if dbName.len > 0 and isValidDbName(dbName): let dbDir = reg.dataRoot / dbName info("Loading database '" & dbName & "' from " & dbDir) - let db = newLSMTree(dbDir) + let db = openLsmForRegistry(reg, dbDir) let ctx = reg.ctxFactory(db, reg) acquire(reg.lock) reg.databases[dbName] = DatabaseInfo( @@ -89,7 +100,7 @@ proc ensureDefaultDatabase*(reg: DatabaseRegistry) = if not exists: let dbDir = reg.dataRoot / defaultDbName info("Creating default database at " & dbDir) - let db = newLSMTree(dbDir) + let db = openLsmForRegistry(reg, dbDir) let ctx = reg.ctxFactory(db, reg) acquire(reg.lock) reg.databases[defaultDbName] = DatabaseInfo( @@ -113,7 +124,7 @@ proc getOrCreateDatabase*(reg: DatabaseRegistry, name: string): DatabaseInfo = # Create new database let dbDir = reg.dataRoot / name info("Creating database '" & name & "' at " & dbDir) - let db = newLSMTree(dbDir) + let db = openLsmForRegistry(reg, dbDir) let ctx = reg.ctxFactory(db, reg) let info = DatabaseInfo(name: name, db: db, ctx: ctx, activeConnections: 0) reg.databases[name] = info diff --git a/src/barabadb/core/server.nim b/src/barabadb/core/server.nim index c5b6c41..3192c55 100644 --- a/src/barabadb/core/server.nim +++ b/src/barabadb/core/server.nim @@ -21,6 +21,7 @@ import ../query/parser import ../query/ast import ../query/executor import ../storage/lsm +import ../storage/gate import ../core/mvcc import ../core/disttxn import ../core/replication @@ -206,61 +207,64 @@ proc valueToWire(val: string, colType: string): WireValue = proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq[WireValue] = @[], replication: ReplicationManager = nil): (bool, QueryResult, string) = - try: - let tokens = tokenize(query) - let astNode = parse(tokens) + ## All storage access is under the global StorageGate so HTTP worker threads + ## and the TCP event loop never touch ORC-managed LSM/executor state concurrently. + withStorageGate: + try: + let tokens = tokenize(query) + let astNode = parse(tokens) - if astNode.stmts.len == 0: - return (true, QueryResult(), "") + if astNode.stmts.len == 0: + return (true, QueryResult(), "") - let res = executor.executeQuery(ctx, astNode, params) - if res.success: - # Ship written key-value pairs to replicas - if replication != nil and res.keyValuePairs.len > 0: - for (key, value) in res.keyValuePairs: - var data = newSeq[byte](key.len + 1 + value.len) - for i, c in key: data[i] = byte(c) - data[key.len] = byte(0) - for i, c in value: data[key.len + 1 + i] = c - discard replication.writeLsn(data) - var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len) - qr.columns = res.columns + let res = executor.executeQuery(ctx, astNode, params) + if res.success: + # Ship written key-value pairs to replicas + if replication != nil and res.keyValuePairs.len > 0: + for (key, value) in res.keyValuePairs: + var data = newSeq[byte](key.len + 1 + value.len) + for i, c in key: data[i] = byte(c) + data[key.len] = byte(0) + for i, c in value: data[key.len + 1 + i] = c + discard replication.writeLsn(data) + var qr = QueryResult(affectedRows: res.affectedRows, rowCount: res.rows.len) + qr.columns = res.columns - var colTypes: seq[string] = @[] - var tableName = "" - if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil: - tableName = astNode.stmts[0].selFrom.fromTable - elif astNode.stmts[0].kind == nkInsert: - tableName = astNode.stmts[0].insTarget - elif astNode.stmts[0].kind == nkUpdate: - tableName = astNode.stmts[0].updTarget + var colTypes: seq[string] = @[] + var tableName = "" + if astNode.stmts[0].kind == nkSelect and astNode.stmts[0].selFrom != nil: + tableName = astNode.stmts[0].selFrom.fromTable + elif astNode.stmts[0].kind == nkInsert: + tableName = astNode.stmts[0].insTarget + elif astNode.stmts[0].kind == nkUpdate: + tableName = astNode.stmts[0].updTarget - if tableName.len > 0 and tableName in ctx.tables: - let tbl = ctx.tables[tableName] - for col in res.columns: - var found = "" - for c in tbl.columns: - if c.name.toLower() == col.toLower(): - found = c.colType - break - colTypes.add(found) + if tableName.len > 0 and tableName in ctx.tables: + let tbl = ctx.tables[tableName] + for col in res.columns: + var found = "" + for c in tbl.columns: + if c.name.toLower() == col.toLower(): + found = c.colType + break + colTypes.add(found) + else: + colTypes = newSeq[string](res.columns.len) + + qr.columnTypes = colTypes.mapIt(typeToFieldKind(it)) + qr.rows = @[] + for row in res.rows: + var wireRow: seq[WireValue] = @[] + for i, col in res.columns: + let val = if col in row: valueToString(row[col]) else: "\\N" + let cType = if i < colTypes.len: colTypes[i] else: "" + wireRow.add(valueToWire(val, cType)) + qr.rows.add(wireRow) + return (true, qr, res.message) else: - colTypes = newSeq[string](res.columns.len) - - qr.columnTypes = colTypes.mapIt(typeToFieldKind(it)) - qr.rows = @[] - for row in res.rows: - var wireRow: seq[WireValue] = @[] - for i, col in res.columns: - let val = if col in row: valueToString(row[col]) else: "\\N" - let cType = if i < colTypes.len: colTypes[i] else: "" - wireRow.add(valueToWire(val, cType)) - qr.rows.add(wireRow) - return (true, qr, res.message) - else: - return (false, QueryResult(), res.message) - except Exception as e: - return (false, QueryResult(), e.msg) + return (false, QueryResult(), res.message) + except Exception as e: + return (false, QueryResult(), e.msg) # ---------------------------------------------------------------------- # Response Serialization diff --git a/src/barabadb/query/exec/README.md b/src/barabadb/query/exec/README.md new file mode 100644 index 0000000..3caf2e2 --- /dev/null +++ b/src/barabadb/query/exec/README.md @@ -0,0 +1,29 @@ +# Executor package (`query/exec/`) + +The original `executor.nim` was a ~5.8k-line god object. Shared pieces live here; +`../executor.nim` remains the main execution engine and **re-exports** this package +so existing `import barabadb/query/executor` keeps working. + +## Modules + +| Module | Responsibility | +|--------|----------------| +| `types.nim` | `ExecutionContext`, `TableDef`, `Row`, `ExecResult`, … | +| `values.nim` | Null/string conversion, row payload parse/escape, SQL escapes | +| `schema.nim` | Durable catalog (`_schema:tables:*`), restore, index rebuild | + +## Import rules + +- **No cycles:** `types` → nothing in `exec/`; `values` → `types`; `schema` → `types` + `values`. +- `executor.nim` imports all three and `export`s them. +- Prefer adding new shared helpers under `exec/` instead of growing `executor.nim`. + +## Sensible next extractions (not done yet) + +1. `dml.nim` — `execScan` / `execInsert` / `execUpdate` / `execDelete` (needs eval/triggers hooks) +2. `rls.nim` — row-level security + privileges +3. `lower.nim` — AST → IR (`lowerExpr` / `lowerSelect`) +4. `plan_exec.nim` — IR plan walker / window functions +5. `hybrid.nim` — hybrid vector+FTS search helpers + +Keep statement dispatch (`executeQueryImpl`) in `executor.nim` until those land. diff --git a/src/barabadb/query/exec/schema.nim b/src/barabadb/query/exec/schema.nim new file mode 100644 index 0000000..40c7092 --- /dev/null +++ b/src/barabadb/query/exec/schema.nim @@ -0,0 +1,231 @@ +## Schema catalog persistence — CREATE/DROP/ALTER survive restart +import std/strutils +import std/tables +import std/sequtils +import ../ast +import ../lexer as qlex +import ../parser as qpar +import ../../storage/lsm +import ../../storage/btree +import types +import values + +const + SchemaTablePrefix* = "_schema:tables:" + SchemaViewPrefix* = "_schema:views:" + SchemaTriggerPrefix* = "_schema:triggers:" + SchemaUserPrefix* = "_schema:users:" + SchemaPolicyPrefix* = "_schema:policies:" + ## Legacy CREATE TABLE keys (pre-fix) used a migrations: counter suffix + SchemaLegacyCreatePrefix* = "_schema:migrations:" + +proc tableSchemaKey*(tableName: string): string = + SchemaTablePrefix & tableName + +proc litToString(node: Node): string = + ## Evaluate simple literal defaults for schema materialization (no full expr engine). + if node == nil: return "" + case node.kind + of nkStringLit: return node.strVal + of nkIntLit: return $node.intVal + of nkFloatLit: return $node.floatVal + of nkBoolLit: return $node.boolVal + of nkNullLit: return "\\N" + else: return "" + +proc serializeTableDdl*(tbl: TableDef): string = + ## Stable DDL for a table definition (survives restart via LSM). + var colDefs: seq[string] = @[] + let multiPk = tbl.pkColumns.len > 1 + for col in tbl.columns: + var parts: seq[string] = @[col.name, col.colType] + if col.isPk and not multiPk: + parts.add("PRIMARY KEY") + if col.autoIncrement: + parts.add("AUTO_INCREMENT") + if col.isNotNull: + parts.add("NOT NULL") + if col.isUnique and not col.isPk: + parts.add("UNIQUE") + if col.defaultVal.len > 0: + parts.add("DEFAULT '" & sqlEscapeString(col.defaultVal) & "'") + if col.fkTable.len > 0: + parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")") + if col.fkOnDelete.len > 0: + parts.add("ON DELETE " & col.fkOnDelete) + if col.fkOnUpdate.len > 0: + parts.add("ON UPDATE " & col.fkOnUpdate) + colDefs.add(parts.join(" ")) + if multiPk: + colDefs.add("PRIMARY KEY (" & tbl.pkColumns.join(", ") & ")") + result = "CREATE TABLE " & tbl.name & " (" & colDefs.join(", ") & ")" + +proc persistTableSchema*(ctx: ExecutionContext, tbl: TableDef) = + ## Write table DDL under a stable key so restore finds it after flush/restart. + let ddl = serializeTableDdl(tbl) + ctx.db.put(tableSchemaKey(tbl.name), cast[seq[byte]](ddl)) + +proc dropTableSchema*(ctx: ExecutionContext, tableName: string) = + ctx.db.delete(tableSchemaKey(tableName)) + +proc applyCreateTableStmt*(ctx: ExecutionContext, stmt: Node) = + ## Materialize CREATE TABLE AST into ctx.tables + empty secondary indexes. + var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[], + foreignKeys: @[], checks: @[], triggers: @[]) + for col in stmt.crtColumns: + if col.kind == nkColumnDef: + var colDef = ColumnDef(name: col.cdName, colType: col.cdType) + colDef.autoIncrement = col.cdAutoIncrement + for cst in col.cdConstraints: + if cst.kind == nkConstraintDef: + case cst.cstType + of "pkey": + colDef.isPk = true + if col.cdName notin tbl.pkColumns: + tbl.pkColumns.add(col.cdName) + ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]() + of "notnull": colDef.isNotNull = true + of "unique": + colDef.isUnique = true + ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]() + of "default": + if cst.cstDefault != nil: + colDef.defaultVal = litToString(cst.cstDefault) + of "fkey": + colDef.fkTable = cst.cstRefTable + colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: "" + colDef.fkOnDelete = cst.cstOnDelete + colDef.fkOnUpdate = cst.cstOnUpdate + else: discard + tbl.columns.add(colDef) + # Table-level constraints + for cstNode in stmt.crtConstraints: + if cstNode.kind == nkConstraintDef: + if cstNode.cstType == "pkey": + for c in cstNode.cstColumns: + if c notin tbl.pkColumns: + tbl.pkColumns.add(c) + for i, col in tbl.columns: + if col.name == c: + tbl.columns[i].isPk = true + let idxName = stmt.crtName & "." & c + if idxName notin ctx.btrees: + ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]() + elif cstNode.cstType == "fkey": + tbl.foreignKeys.add(ForeignKeyDef( + refTable: cstNode.cstRefTable, + refColumn: if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: "", + onDelete: cstNode.cstOnDelete, + onUpdate: cstNode.cstOnUpdate)) + if cstNode.cstColumns.len > 0: + for i, c in tbl.columns: + if c.name in cstNode.cstColumns: + tbl.columns[i].fkTable = cstNode.cstRefTable + tbl.columns[i].fkColumn = if cstNode.cstRefColumns.len > 0: cstNode.cstRefColumns[0] else: "" + tbl.columns[i].fkOnDelete = cstNode.cstOnDelete + tbl.columns[i].fkOnUpdate = cstNode.cstOnUpdate + elif cstNode.cstType == "check": + tbl.checks.add(CheckDef(name: "check_" & $tbl.checks.len, checkNode: cstNode.cstCheck)) + ctx.tables[stmt.crtName] = tbl + +proc rebuildSecondaryIndexes*(ctx: ExecutionContext) = + ## Rebuild in-memory B-Tree indexes from durable row data after schema restore. + for tableName, tbl in ctx.tables.pairs: + for col in tbl.columns: + if col.isPk or col.isUnique: + let idxName = tableName & "." & col.name + if idxName notin ctx.btrees: + ctx.btrees[idxName] = newBTreeIndex[string, IndexEntry]() + let prefix = tableName & "." + for (key, value) in ctx.db.scanAll(): + if not key.startsWith(prefix): continue + if key.startsWith("_schema:"): continue + let valStr = cast[string](value) + let rest = key[prefix.len..^1] + var colVals = initTable[string, string]() + let eqPos = rest.find('=') + if eqPos >= 0 and ':' notin rest: + colVals[rest[0..= 0: + colVals[part[0.. 0 and not isNull(idxVal): + ctx.btrees[colName].insert(idxVal, IndexEntry(lsmKey: key, rowValue: valStr)) + +proc restoreSchema*(ctx: ExecutionContext) = + ## Load durable schema from LSM (memtable + SSTables). Stable keys only. + var tableDdls: seq[string] = @[] + var otherDdls: seq[string] = @[] + + for (key, value) in ctx.db.scanAll(): + if not key.startsWith("_schema:"): continue + let ddl = cast[string](value) + if ddl.len == 0: continue + if key.startsWith(SchemaTablePrefix): + tableDdls.add(ddl) + elif key.startsWith(SchemaLegacyCreatePrefix) and ddl.toUpperAscii().startsWith("CREATE TABLE"): + tableDdls.add(ddl) + elif key.startsWith(SchemaViewPrefix) or key.startsWith(SchemaTriggerPrefix) or + key.startsWith(SchemaUserPrefix) or key.startsWith(SchemaPolicyPrefix): + otherDdls.add(ddl) + elif ddl.toUpperAscii().startsWith("CREATE VIEW") or + ddl.toUpperAscii().startsWith("CREATE TRIGGER") or + ddl.toUpperAscii().startsWith("CREATE USER") or + ddl.toUpperAscii().startsWith("CREATE POLICY"): + otherDdls.add(ddl) + + for ddl in tableDdls: + try: + let tokens = qlex.tokenize(ddl) + let astNode = qpar.parse(tokens) + if astNode.stmts.len > 0 and astNode.stmts[0].kind == nkCreateTable: + applyCreateTableStmt(ctx, astNode.stmts[0]) + if astNode.stmts[0].crtName in ctx.tables: + persistTableSchema(ctx, ctx.tables[astNode.stmts[0].crtName]) + except CatchableError: + continue + + for ddl in otherDdls: + var astNode: Node + try: + let tokens = qlex.tokenize(ddl) + astNode = qpar.parse(tokens) + except CatchableError: + continue + if astNode.stmts.len == 0: continue + let stmt = astNode.stmts[0] + case stmt.kind + of nkCreateView: + ctx.views[stmt.cvName] = stmt.cvQuery + of nkCreateTrigger: + if stmt.trigTable in ctx.tables: + ctx.tables[stmt.trigTable].triggers.add(TriggerDef( + name: stmt.trigName, + timing: stmt.trigTiming, + event: stmt.trigEvent, + action: stmt.trigAction, + )) + of nkCreateUser: + ctx.users[stmt.cuName] = UserDef(name: stmt.cuName, + passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[]) + of nkCreatePolicy: + var pols = ctx.policies.getOrDefault(stmt.cpTable) + pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable, + command: stmt.cpCommand, usingExpr: stmt.cpUsing, + withCheckExpr: stmt.cpWithCheck)) + ctx.policies[stmt.cpTable] = pols + else: discard + + rebuildSecondaryIndexes(ctx) diff --git a/src/barabadb/query/exec/types.nim b/src/barabadb/query/exec/types.nim new file mode 100644 index 0000000..401af96 --- /dev/null +++ b/src/barabadb/query/exec/types.nim @@ -0,0 +1,143 @@ +## Executor types — shared by all exec/* modules and executor.nim +import std/tables +import std/locks +import ../ast +import ../ir +import ../../core/types +import ../../storage/lsm +import ../../storage/btree +import ../../core/mvcc +import ../../fts/engine as fts +import ../../vector/engine as vengine +import ../../graph/engine as gengine +import ../../ai/embed as embedmod +import ../../ai/llm as llmmod +import ../../core/registry + +type + IndexEntry* = ref object + lsmKey*: string + rowValue*: string + + ChangeKind* = enum + ckInsert, ckUpdate, ckDelete + + ChangeEvent* = object + table*: string + kind*: ChangeKind + key*: string + data*: string + + UserDef* = object + name*: string + passwordHash*: string + isSuperuser*: bool + roles*: seq[string] + + PrivilegeDef* = object + tableName*: string + command*: string # SELECT, INSERT, UPDATE, DELETE, ALL + + PolicyDef* = object + name*: string + tableName*: string + command*: string # ALL, SELECT, INSERT, UPDATE, DELETE + usingExpr*: Node # parsed USING expression + withCheckExpr*: Node # parsed WITH CHECK expression + + SharedLock* = ref object + lock*: Lock + + ForeignKeyDef* = object + refTable*: string + refColumn*: string + onDelete*: string # CASCADE, SET NULL, RESTRICT + onUpdate*: string # CASCADE, SET NULL, RESTRICT + + CheckDef* = object + name*: string + expr*: string # stored expression string + checkNode*: Node # AST for runtime evaluation + + TriggerDef* = object + name*: string + timing*: string # BEFORE, AFTER + event*: string # INSERT, UPDATE, DELETE + action*: Node # SQL statement AST + + ColumnDef* = object + name*: string + colType*: string + isPk*: bool + isNotNull*: bool + isUnique*: bool + defaultVal*: string + fkTable*: string + fkColumn*: string + fkOnDelete*: string + fkOnUpdate*: string + autoIncrement*: bool + + TableDef* = object + name*: string + columns*: seq[ColumnDef] + pkColumns*: seq[string] + foreignKeys*: seq[ForeignKeyDef] + checks*: seq[CheckDef] + triggers*: seq[TriggerDef] + + Row* = Table[string, Value] + + ExecutionContext* = ref object + db*: LSMTree + tables*: Table[string, TableDef] + btrees*: Table[string, BTreeIndex[string, IndexEntry]] + views*: Table[string, Node] # view name -> SELECT AST + cteTables*: Table[string, seq[Row]] # CTE name -> rows + ftsIndexes*: Table[string, fts.InvertedIndex] # table.col -> FTS index + vectorIndexes*: Table[string, vengine.HNSWIndex] # table.col -> HNSW index + graphs*: Table[string, gengine.Graph] # graph name -> Graph object + embedder*: embedmod.Embedder # optional embedding service client + llmClient*: llmmod.LLMClient # optional LLM client for NL->SQL + txnManager*: TxnManager + pendingTxn*: Transaction + onChange*: proc(ev: ChangeEvent) {.closure.} + users*: Table[string, UserDef] + policies*: Table[string, seq[PolicyDef]] # table name -> policies + currentUser*: string + currentRole*: string + sessionVars*: Table[string, string] + autoIncCounters*: Table[string, int64] + sequences*: Table[string, int64] + sharedLock*: SharedLock # shared across cloned contexts + outerRow*: Table[string, string] # outer query row for correlated subqueries + subqueryPlan*: IRPlan # current subquery plan being evaluated + currentDatabase*: string # name of the currently selected database + registry*: DatabaseRegistry # nil for single-DB mode + + MigrationRecord* = object + name*: string + checksum*: string + appliedAt*: int64 + appliedBy*: string + durationMs*: int + rolledBack*: bool + + ExecResult* = object + success*: bool + columns*: seq[string] + rows*: seq[Row] + affectedRows*: int + message*: string + keyValuePairs*: seq[(string, seq[byte])] + +proc `==`*(a, b: IndexEntry): bool = + a.lsmKey == b.lsmKey and a.rowValue == b.rowValue + +proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "", + kvPairs: seq[(string, seq[byte])] = @[]): ExecResult = + ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg, + keyValuePairs: kvPairs) + +proc errResult*(msg: string): ExecResult = + ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg) diff --git a/src/barabadb/query/exec/values.nim b/src/barabadb/query/exec/values.nim new file mode 100644 index 0000000..d0ede5a --- /dev/null +++ b/src/barabadb/query/exec/values.nim @@ -0,0 +1,119 @@ +## Value / row serialization helpers used across the executor +import std/strutils +import std/tables +import std/json +import ../../core/types +import types + +proc isNull*(value: string): bool = + value == "\\N" or value.toLower() == "null" + +proc valueToString*(v: Value): string = + case v.kind + of vkNull: return "\\N" + of vkString: return v.strVal + of vkInt64: return $v.int64Val + of vkFloat64: return $v.float64Val + of vkBool: return $v.boolVal + else: return "" + +proc `%`*(v: Value): JsonNode = + case v.kind + of vkNull: return newJNull() + of vkString: return %v.strVal + of vkInt64: return %v.int64Val + of vkFloat64: return %v.float64Val + of vkBool: return %v.boolVal + else: return newJNull() + +proc toString*(v: Value): string = valueToString(v) + +proc `[]=`*(t: var Row, key: string, val: string) = + t[key] = Value(kind: vkString, strVal: val) + +proc escapeRowVal*(v: string): string = + v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=") + +proc unescapeRowVal*(v: string): string = + result = "" + var i = 0 + while i < v.len: + if v[i] == '\\' and i + 1 < v.len: + case v[i+1] + of '\\', ',', '=': + result &= v[i+1] + i += 2 + continue + else: discard + result &= v[i] + inc i + +proc parseRowData*(valStr: string): Table[string, string] = + ## Parse "col1=val1,col2=val2" into a table + result = initTable[string, string]() + var i = 0 + var part = "" + while i < valStr.len: + if valStr[i] == '\\' and i + 1 < valStr.len: + part &= valStr[i] + part &= valStr[i+1] + i += 2 + continue + if valStr[i] == ',': + let eqPos = part.find('=') + if eqPos >= 0: + let k = part[0.. 0: + let eqPos = part.find('=') + if eqPos >= 0: + let k = part[0.. 0: result &= ", " + result &= "\"" & sqlEscapeIdent(col) & "\"" + result &= ") VALUES " + for ri, row in rows: + if ri > 0: result &= ", " + result &= "(" + for ci, val in row: + if ci > 0: result &= ", " + if val.len == 0 or val == "\\N": + result &= "NULL" + else: + result &= "'" & sqlEscapeString(val) & "'" + result &= ")" + +proc getValue*(values: seq[string], fields: seq[string], colName: string): string = + for i, f in fields: + if f.toLower() == colName.toLower(): + if i < values.len: return values[i] + return "\\N" + return "\\N" + +proc getTableDef*(ctx: ExecutionContext, tableName: string): TableDef = + if tableName in ctx.tables: return ctx.tables[tableName] + return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[]) diff --git a/src/barabadb/query/executor.nim b/src/barabadb/query/executor.nim index 71020fb..fe98d11 100644 --- a/src/barabadb/query/executor.nim +++ b/src/barabadb/query/executor.nim @@ -1,4 +1,7 @@ ## BaraQL Executor — AST lowering, IR compilation, and execution +## +## Shared types/helpers live under `exec/` (re-exported below for API stability). +## See `exec/README.md` for module map and further extraction plan. import std/os import std/strutils import std/tables @@ -53,140 +56,18 @@ import ../ai/embed as embedmod import ../ai/llm as llmmod import ../graph/cypher as cyphermod -type - IndexEntry* = ref object - lsmKey*: string - rowValue*: string - - ChangeKind* = enum - ckInsert, ckUpdate, ckDelete - - ChangeEvent* = object - table*: string - kind*: ChangeKind - key*: string - data*: string - - UserDef* = object - name*: string - passwordHash*: string - isSuperuser*: bool - roles*: seq[string] - - PrivilegeDef* = object - tableName*: string - command*: string # SELECT, INSERT, UPDATE, DELETE, ALL - - PolicyDef* = object - name*: string - tableName*: string - command*: string # ALL, SELECT, INSERT, UPDATE, DELETE - usingExpr*: Node # parsed USING expression - withCheckExpr*: Node # parsed WITH CHECK expression - - SharedLock* = ref object - lock*: Lock - - ExecutionContext* = ref object - db*: LSMTree - tables*: Table[string, TableDef] - btrees*: Table[string, BTreeIndex[string, IndexEntry]] - views*: Table[string, Node] # view name -> SELECT AST - cteTables*: Table[string, seq[Row]] # CTE name -> rows - ftsIndexes*: Table[string, fts.InvertedIndex] # table.col -> FTS index - vectorIndexes*: Table[string, vengine.HNSWIndex] # table.col -> HNSW index - graphs*: Table[string, gengine.Graph] # graph name -> Graph object - embedder*: embedmod.Embedder # optional embedding service client - llmClient*: llmmod.LLMClient # optional LLM client for NL->SQL - txnManager*: TxnManager - pendingTxn*: Transaction - onChange*: proc(ev: ChangeEvent) {.closure.} - users*: Table[string, UserDef] - policies*: Table[string, seq[PolicyDef]] # table name -> policies - currentUser*: string - currentRole*: string - sessionVars*: Table[string, string] - autoIncCounters*: Table[string, int64] - sequences*: Table[string, int64] - sharedLock*: SharedLock # shared across cloned contexts — protects tables, views, btrees, ftsIndexes, users, policies, autoIncCounters, sequences - outerRow*: Table[string, string] # outer query row for correlated subqueries - subqueryPlan*: IRPlan # current subquery plan being evaluated (for correlation in execScan) - currentDatabase*: string # name of the currently selected database - registry*: DatabaseRegistry # reference to the database registry (nil for single-DB mode) - - MigrationRecord* = object - name*: string - checksum*: string - appliedAt*: int64 - appliedBy*: string - durationMs*: int - rolledBack*: bool - - ForeignKeyDef* = object - refTable*: string - refColumn*: string - onDelete*: string # CASCADE, SET NULL, RESTRICT - onUpdate*: string # CASCADE, SET NULL, RESTRICT - - CheckDef* = object - name*: string - expr*: string # stored expression string - checkNode*: Node # AST for runtime evaluation - - TriggerDef* = object - name*: string - timing*: string # BEFORE, AFTER - event*: string # INSERT, UPDATE, DELETE - action*: Node # SQL statement AST - - TableDef* = object - name*: string - columns*: seq[ColumnDef] - pkColumns*: seq[string] - foreignKeys*: seq[ForeignKeyDef] - checks*: seq[CheckDef] - triggers*: seq[TriggerDef] - - ColumnDef* = object - name*: string - colType*: string - isPk*: bool - isNotNull*: bool - isUnique*: bool - defaultVal*: string - fkTable*: string - fkColumn*: string - fkOnDelete*: string - fkOnUpdate*: string - autoIncrement*: bool - - Row* = Table[string, Value] - - ExecResult* = object - success*: bool - columns*: seq[string] - rows*: seq[Row] - affectedRows*: int - message*: string - keyValuePairs*: seq[(string, seq[byte])] - -proc `==`*(a, b: IndexEntry): bool = - a.lsmKey == b.lsmKey and a.rowValue == b.rowValue - -proc okResult*(rows: seq[Row] = @[], cols: seq[string] = @[], affected: int = 0, msg: string = "", - kvPairs: seq[(string, seq[byte])] = @[]): ExecResult = - ExecResult(success: true, columns: cols, rows: rows, affectedRows: affected, message: msg, - keyValuePairs: kvPairs) - -proc errResult*(msg: string): ExecResult = - ExecResult(success: false, columns: @[], rows: @[], affectedRows: 0, message: msg) +import exec/types +import exec/values +import exec/schema +export types +export values +export schema # ---------------------------------------------------------------------- # Context management # ---------------------------------------------------------------------- proc evalNodeToString(node: Node): string -proc restoreSchema(ctx: ExecutionContext) proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): ExecutionContext = result = ExecutionContext(db: db, tables: initTable[string, TableDef](), @@ -213,32 +94,6 @@ proc newExecutionContext*(db: LSMTree, registry: DatabaseRegistry = nil): Execut # AST to SQL serializer (for VIEW DDL persistence) # ---------------------------------------------------------------------- -proc sqlEscapeIdent*(ident: string): string = - ## Escape SQL identifiers by doubling double-quotes. - result = ident.replace("\"", "\"\"") - -proc sqlEscapeString*(s: string): string = - ## Escape SQL string literals by doubling single-quotes. - result = s.replace("'", "''") - -proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string = - ## Build a multi-row INSERT statement for bulk import. - result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" (" - for i, col in columns: - if i > 0: result &= ", " - result &= "\"" & sqlEscapeIdent(col) & "\"" - result &= ") VALUES " - for ri, row in rows: - if ri > 0: result &= ", " - result &= "(" - for ci, val in row: - if ci > 0: result &= ", " - if val.len == 0 or val == "\\N": - result &= "NULL" - else: - result &= "'" & sqlEscapeString(val) & "'" - result &= ")" - proc exprToSql(node: Node): string = if node == nil: return "" @@ -343,76 +198,6 @@ proc selectToSql(node: Node): string = if node.selOffset != nil and node.selOffset.offsetExpr.kind == nkIntLit: result.add(" OFFSET " & $node.selOffset.offsetExpr.intVal) -# ---------------------------------------------------------------------- -# Schema restore -# ---------------------------------------------------------------------- - -proc restoreSchema(ctx: ExecutionContext) = - for entry in ctx.db.scanMemTable(): - if entry.deleted: continue - if not entry.key.startsWith("_schema:"): continue - let ddl = cast[string](entry.value) - if ddl.len == 0: continue - var astNode: Node - try: - let tokens = qlex.tokenize(ddl) - astNode = qpar.parse(tokens) - except: - # Skip corrupted schema entries during startup - continue - if astNode.stmts.len > 0: - let stmt = astNode.stmts[0] - case stmt.kind - of nkCreateTable: - var tbl = TableDef(name: stmt.crtName, columns: @[], pkColumns: @[], - foreignKeys: @[], checks: @[], triggers: @[]) - for col in stmt.crtColumns: - if col.kind == nkColumnDef: - var colDef = ColumnDef(name: col.cdName, colType: col.cdType) - colDef.autoIncrement = col.cdAutoIncrement - for cst in col.cdConstraints: - if cst.kind == nkConstraintDef: - case cst.cstType - of "pkey": - colDef.isPk = true - tbl.pkColumns.add(col.cdName) - ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]() - of "notnull": colDef.isNotNull = true - of "unique": - colDef.isUnique = true - ctx.btrees[stmt.crtName & "." & col.cdName] = newBTreeIndex[string, IndexEntry]() - of "default": - if cst.cstDefault != nil: - colDef.defaultVal = evalNodeToString(cst.cstDefault) - of "fkey": - colDef.fkTable = cst.cstRefTable - colDef.fkColumn = if cst.cstRefColumns.len > 0: cst.cstRefColumns[0] else: "" - colDef.fkOnDelete = cst.cstOnDelete - colDef.fkOnUpdate = cst.cstOnUpdate - else: discard - tbl.columns.add(colDef) - ctx.tables[stmt.crtName] = tbl - of nkCreateView: - ctx.views[stmt.cvName] = stmt.cvQuery - of nkCreateTrigger: - if stmt.trigTable in ctx.tables: - ctx.tables[stmt.trigTable].triggers.add(TriggerDef( - name: stmt.trigName, - timing: stmt.trigTiming, - event: stmt.trigEvent, - action: stmt.trigAction, - )) - of nkCreateUser: - ctx.users[stmt.cuName] = UserDef(name: stmt.cuName, - passwordHash: stmt.cuPassword, isSuperuser: stmt.cuSuperuser, roles: @[]) - of nkCreatePolicy: - var pols = ctx.policies.getOrDefault(stmt.cpTable) - pols.add(PolicyDef(name: stmt.cpName, tableName: stmt.cpTable, - command: stmt.cpCommand, usingExpr: stmt.cpUsing, - withCheckExpr: stmt.cpWithCheck)) - ctx.policies[stmt.cpTable] = pols - else: discard - proc cloneForConnection*(ctx: ExecutionContext): ExecutionContext = var svCopy = initTable[string, string]() for k, v in ctx.sessionVars: @@ -512,97 +297,6 @@ proc getMigrationBody(ctx: ExecutionContext, name: string): (bool, string, strin else: return (true, ddl, "") return (false, "", "") - -# ---------------------------------------------------------------------- -# Helpers -# ---------------------------------------------------------------------- - -proc getTableDef(ctx: ExecutionContext, tableName: string): TableDef = - if tableName in ctx.tables: return ctx.tables[tableName] - return TableDef(name: tableName, columns: @[], pkColumns: @[], foreignKeys: @[], checks: @[]) - -proc getValue(values: seq[string], fields: seq[string], colName: string): string = - for i, f in fields: - if f.toLower() == colName.toLower() and i < values.len: - return values[i] - return "\\N" - -proc isNull*(value: string): bool = - value == "\\N" or value.toLower() == "null" - -proc valueToString*(v: Value): string = - case v.kind - of vkNull: return "\\N" - of vkString: return v.strVal - of vkInt64: return $v.int64Val - of vkFloat64: return $v.float64Val - of vkBool: return $v.boolVal - else: return "" - -proc `%`*(v: Value): JsonNode = - case v.kind - of vkNull: return newJNull() - of vkString: return %v.strVal - of vkInt64: return %v.int64Val - of vkFloat64: return %v.float64Val - of vkBool: return %v.boolVal - else: return newJNull() - -proc toString*(v: Value): string = valueToString(v) - -proc `[]=`*(t: var Row, key: string, val: string) = - t[key] = Value(kind: vkString, strVal: val) - -proc escapeRowVal(v: string): string = - v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=") - -proc unescapeRowVal(v: string): string = - result = "" - var i = 0 - while i < v.len: - if v[i] == '\\' and i + 1 < v.len: - case v[i+1] - of '\\', ',', '=': - result &= v[i+1] - i += 2 - continue - else: discard - result &= v[i] - inc i - -proc parseRowData(valStr: string): Table[string, string] = - ## Parse "col1=val1,col2=val2" into a table - result = initTable[string, string]() - var i = 0 - var part = "" - while i < valStr.len: - if valStr[i] == '\\' and i + 1 < valStr.len: - part &= valStr[i] - part &= valStr[i+1] - i += 2 - continue - if valStr[i] == ',': - let eqPos = part.find('=') - if eqPos >= 0: - let k = part[0.. 0: - let eqPos = part.find('=') - if eqPos >= 0: - let k = part[0.. 0: parts.add("DEFAULT '" & col.defaultVal & "'") - if col.fkTable.len > 0: - parts.add("REFERENCES " & col.fkTable & "(" & col.fkColumn & ")") - colDefs.add(parts.join(" ")) - let schemaKey = "_schema:migrations:" & $ctx.tables.len - ctx.db.put(schemaKey, cast[seq[byte]]("CREATE TABLE " & stmt.crtName & " (" & colDefs.join(", ") & ")")) - + persistTableSchema(ctx, tbl) return okResult() of nkDropTable: - ctx.tables.del(stmt.drtName) + let dropName = stmt.drtName + ctx.tables.del(dropName) var toDelete: seq[string] = @[] for idxName in ctx.btrees.keys.toSeq(): - if idxName.startsWith(stmt.drtName & "."): toDelete.add(idxName) + if idxName.startsWith(dropName & "."): toDelete.add(idxName) for idxName in toDelete: ctx.btrees.del(idxName) + # Remove durable schema entry + dropTableSchema(ctx, dropName) + # Remove row data for this table + var dataKeys: seq[string] = @[] + let prefix = dropName & "." + for (key, _) in ctx.db.scanAll(): + if key.startsWith(prefix): + dataKeys.add(key) + for key in dataKeys: + ctx.db.delete(key) + # Drop orphan legacy schema keys that mentioned this table + var legacyKeys: seq[string] = @[] + for (key, value) in ctx.db.scanAll(): + if key.startsWith(SchemaLegacyCreatePrefix): + let ddl = cast[string](value) + if ddl.contains("CREATE TABLE " & dropName) or ddl.contains("CREATE TABLE \"" & dropName): + legacyKeys.add(key) + for key in legacyKeys: + ctx.db.delete(key) return okResult() of nkCreateGraph: @@ -5113,6 +4812,7 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu var colDef = ColumnDef(name: op.cdName, colType: op.cdType) tbl.columns.add(colDef) ctx.tables[stmt.altName] = tbl + persistTableSchema(ctx, tbl) return okResult(msg="ALTER TABLE " & stmt.altName & " executed") return errResult("Table '" & stmt.altName & "' does not exist") diff --git a/src/barabadb/storage/compaction.nim b/src/barabadb/storage/compaction.nim index 4e020c9..4dd2bdd 100644 --- a/src/barabadb/storage/compaction.nim +++ b/src/barabadb/storage/compaction.nim @@ -8,6 +8,7 @@ import ../storage/lsm const MaxLevel* = 7 LevelMultiplier* = 10 # each level is 10x the previous + ## L0 uses file-count trigger (overlapping ranges); lower levels use size. type SSTableMeta* = object @@ -29,20 +30,44 @@ type levels*: seq[seq[SSTableMeta]] dataDir*: string maxSizePerLevel*: seq[int] + l0FileLimit*: int -proc newCompactionStrategy*(dataDir: string): CompactionStrategy = +proc newCompactionStrategy*(dataDir: string, l0FileLimit: int = L0CompactionTrigger): CompactionStrategy = result = CompactionStrategy( levels: newSeq[seq[SSTableMeta]](MaxLevel), dataDir: dataDir, maxSizePerLevel: newSeq[int](MaxLevel), + l0FileLimit: l0FileLimit, ) for i in 0..= MaxLevel - 1: return false + if level == 0: + # L0 files can overlap — count-based trigger (RocksDB-style) + return cs.levels[0].len >= cs.l0FileLimit return cs.totalSize(level) > cs.maxSizePerLevel[level] proc pickTablesForCompaction*(cs: CompactionStrategy, level: int): seq[SSTableMeta] = diff --git a/src/barabadb/storage/gate.nim b/src/barabadb/storage/gate.nim new file mode 100644 index 0000000..a2d7bc6 --- /dev/null +++ b/src/barabadb/storage/gate.nim @@ -0,0 +1,41 @@ +## Global storage gate — exclusive multi-thread entry to LSM / executor. +## +## Why: Hunos HTTP runs handlers on a worker-thread pool (`spawn` + internal +## workers). The TCP server runs on the main async loop. Both share the same +## `LSMTree` / `ExecutionContext` refs. Nim's default ORC memory manager is not +## safe for concurrent refcount ops on the same objects from multiple OS threads. +## +## Holding this gate for the full duration of a query/compaction/DDL ensures +## only one thread mutates or reads GC-managed storage state at a time. +## +## Ordering: always acquire StorageGate **before** any per-DB `LSMTree.lock`. +## Call `initStorageGate()` once from main before accepting connections. +import std/locks + +var + gGate: Lock + gInited*: bool + +proc initStorageGate*() = + ## Idempotent when called from a single thread at startup. + if not gInited: + initLock(gGate) + gInited = true + +proc acquireStorageGate*() {.inline.} = + ## Prefer calling initStorageGate() once at process start (main). + ## Lazy-init is allowed for unit tests (single-threaded). + if not gInited: + initStorageGate() + acquire(gGate) + +proc releaseStorageGate*() {.inline.} = + release(gGate) + +template withStorageGate*(body: untyped) = + ## Exclusive ownership of the storage engine for `body`. + acquireStorageGate() + try: + body + finally: + releaseStorageGate() diff --git a/src/barabadb/storage/lsm.nim b/src/barabadb/storage/lsm.nim index ed8ed51..4327aee 100644 --- a/src/barabadb/storage/lsm.nim +++ b/src/barabadb/storage/lsm.nim @@ -13,6 +13,11 @@ import bloom import wal import mmap import crc32 +import rwlock + +# Re-export WAL durability knobs for callers of newLSMTree +export wal +export rwlock const SSTableMagic* = 0x53535442'u32 # "SSTB" @@ -21,6 +26,8 @@ const DefaultBloomFpRate* = 0.01 ManifestVersion* = 1 ManifestFileName* = "MANIFEST" + ## Trigger L0 compaction when this many L0 SSTables exist. + L0CompactionTrigger* = 4 type Entry* = object @@ -29,9 +36,10 @@ type timestamp*: uint64 deleted*: bool + ## Hash-table MemTable: O(1) put/get. Sorted only when flushing to SSTable. MemTable* = object - entries: seq[Entry] - size: int + map: Table[string, Entry] + size: int ## approximate byte size of live entries maxSize: int SSTable* = object @@ -56,55 +64,64 @@ type currentSeq: uint64 nextSSTableId*: int manifestSequence*: int64 - lock*: Lock + ## Reader-writer lock: concurrent gets; exclusive put/flush/compact. + ## `acquire(db.lock)` is exclusive (write) for backward compatibility. + lock*: RwLock walLock*: Lock + ## Set by flush when L0 file count hits L0CompactionTrigger (hint for compactors). + needsCompaction*: bool + ## When true, flushUnsafe skips WAL rewrite (recovery still holds the WAL file open). + recovering: bool proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable = - MemTable(entries: @[], size: 0, maxSize: maxSize) + MemTable(map: initTable[string, Entry](), size: 0, maxSize: maxSize) -proc len*(mt: MemTable): int = mt.entries.len +proc len*(mt: MemTable): int = mt.map.len + +proc byteSize*(mt: MemTable): int = mt.size proc put*(mt: var MemTable, key: string, value: seq[byte], timestamp: uint64, deleted: bool = false): bool = + ## O(1) average-case insert/update. Returns false if the new key would exceed maxSize. let entrySize = key.len + value.len + 16 if entrySize > mt.maxSize: return false let entry = Entry(key: key, value: value, timestamp: timestamp, deleted: deleted) - let pos = mt.entries.lowerBound(entry, proc(a, b: Entry): int = cmp(a.key, b.key)) - if pos < mt.entries.len and mt.entries[pos].key == key: - let oldSize = mt.entries[pos].key.len + mt.entries[pos].value.len + 16 - mt.entries[pos] = entry + if key in mt.map: + let old = mt.map[key] + # Only accept equal-or-newer timestamps (WAL recovery may replay older values) + if timestamp < old.timestamp: + return true + let oldSize = old.key.len + old.value.len + 16 + mt.map[key] = entry mt.size += entrySize - oldSize else: - if mt.size + entrySize > mt.maxSize and mt.entries.len > 0: + if mt.size + entrySize > mt.maxSize and mt.map.len > 0: return false - mt.entries.insert(entry, pos) + mt.map[key] = entry mt.size += entrySize return true proc get*(mt: MemTable, key: string): (bool, Entry) = - if mt.entries.len == 0: - return (false, Entry()) - var lo = 0 - var hi = mt.entries.len - 1 - while lo <= hi: - let mid = (lo + hi) div 2 - let c = cmp(mt.entries[mid].key, key) - if c == 0: - return (true, mt.entries[mid]) - elif c < 0: - lo = mid + 1 - else: - hi = mid - 1 + if key in mt.map: + return (true, mt.map[key]) return (false, Entry()) +proc sortedEntries*(mt: MemTable): seq[Entry] = + ## Materialize entries sorted by key — used for SSTable flush and ordered scans. + result = newSeqOfCap[Entry](mt.map.len) + for _, entry in mt.map: + result.add(entry) + result.sort(proc(a, b: Entry): int = cmp(a.key, b.key)) + proc scan*(mt: MemTable, startKey, endKey: string): seq[Entry] = result = @[] - for entry in mt.entries: - if entry.key >= startKey and entry.key <= endKey: + for key, entry in mt.map: + if key >= startKey and key <= endKey: result.add(entry) + result.sort(proc(a, b: Entry): int = cmp(a.key, b.key)) proc clear*(mt: var MemTable) = - mt.entries.setLen(0) + mt.map.clear() mt.size = 0 # ---------------------------------------------------------------------- @@ -600,8 +617,15 @@ proc checkStorageConsistency*(db: LSMTree): seq[string] = # ---------------------------------------------------------------------- proc flushUnsafe(db: LSMTree) {.gcsafe.} +proc countL0*(db: LSMTree): int -proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree = +proc newLSMTree*( + dir: string, + memMaxSize: int = DefaultMemTableSize, + walSyncMode: WalSyncMode = wsmGroup, + walGroupEvery: int = DefaultWalGroupEvery, + walGroupIntervalMs: int = 0, +): LSMTree = createDir(dir) createDir(dir / "sstables") @@ -641,21 +665,29 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree = echo "[INFO] Loaded ", sstables.len, " SSTable(s) from directory scan" new(result) - initLock(result.lock) + initRwLock(result.lock) initLock(result.walLock) result.dir = dir result.memTable = newMemTable(memMaxSize) result.immutableMem = newMemTable(0) result.sstables = sstables - result.wal = newWriteAheadLog(dir / "wal") + result.wal = newWriteAheadLog( + dir / "wal", + syncMode = walSyncMode, + groupEvery = walGroupEvery, + groupIntervalMs = walGroupIntervalMs, + ) result.memMaxSize = memMaxSize result.currentSeq = 0 result.nextSSTableId = nextId result.manifestSequence = manifestSeq + result.recovering = false + result.needsCompaction = result.countL0() >= L0CompactionTrigger # WAL crash recovery — replay unflushed entries into memTable let walPath = dir / "wal" / "wal.log" if fileExists(walPath): + result.recovering = true var stream: FileStream = nil try: stream = newFileStream(walPath, fmRead) @@ -697,11 +729,30 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree = finally: if stream != nil: stream.close() + result.recovering = false + # After recovery, shrink WAL to live unflushed state only + acquire(result.walLock) + try: + var liveKeys: seq[string] = @[] + var liveVals: seq[seq[byte]] = @[] + var liveTs: seq[uint64] = @[] + var liveDel: seq[bool] = @[] + for e in result.immutableMem.sortedEntries(): + liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted) + for e in result.memTable.sortedEntries(): + liveKeys.add(e.key); liveVals.add(e.value); liveTs.add(e.timestamp); liveDel.add(e.deleted) + if liveKeys.len == 0: + result.wal.truncate() + else: + result.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel) + finally: + release(result.walLock) proc put*(db: LSMTree, key: string, value: seq[byte]) = let ts = uint64(getMonoTime().ticks()) - acquire(db.lock) - defer: release(db.lock) + acquireWrite(db.lock) + defer: releaseWrite(db.lock) + # WAL then memtable under the same exclusive lock → crash recovery sees a total order acquire(db.walLock) db.wal.writePut(cast[seq[byte]](key), value, ts) release(db.walLock) @@ -716,8 +767,8 @@ proc put*(db: LSMTree, key: string, value: seq[byte]) = proc delete*(db: LSMTree, key: string) = let ts = uint64(getMonoTime().ticks()) - acquire(db.lock) - defer: release(db.lock) + acquireWrite(db.lock) + defer: releaseWrite(db.lock) acquire(db.walLock) db.wal.writeDelete(cast[seq[byte]](key), ts) release(db.walLock) @@ -732,8 +783,8 @@ proc delete*(db: LSMTree, key: string) = proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = false) = ## Direct LSM insert without WAL logging — used by recovery. let ts = uint64(getMonoTime().ticks()) - acquire(db.lock) - defer: release(db.lock) + acquireWrite(db.lock) + defer: releaseWrite(db.lock) if not db.memTable.put(key, value, ts, deleted): if db.immutableMem.len > 0: db.flushUnsafe() @@ -745,18 +796,26 @@ proc putUnsafe*(db: LSMTree, key: string, value: seq[byte], deleted: bool = fals proc deleteUnsafe*(db: LSMTree, key: string) = putUnsafe(db, key, @[], deleted = true) +proc copyBytes(s: seq[byte]): seq[byte] = + ## Deep copy so callers on other threads never share ORC-managed seq buffers. + result = newSeq[byte](s.len) + if s.len > 0: + copyMem(addr result[0], unsafeAddr s[0], s.len) + proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) = + ## Caller must hold at least a read lock. + ## Returned values are deep-copied for multi-thread ORC safety (HTTP + TCP share LSM). let (found, entry) = db.memTable.get(key) if found: if entry.deleted: return (false, @[]) - return (true, entry.value) + return (true, copyBytes(entry.value)) let (found2, entry2) = db.immutableMem.get(key) if found2: if entry2.deleted: return (false, @[]) - return (true, entry2.value) + return (true, copyBytes(entry2.value)) # Search SSTables from newest to oldest for i in countdown(db.sstables.high, db.sstables.low): @@ -769,21 +828,40 @@ proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) = if found3: if entry3.deleted: return (false, @[]) - return (true, entry3.value) + return (true, copyBytes(entry3.value)) return (false, @[]) proc get*(db: LSMTree, key: string): (bool, seq[byte]) = - acquire(db.lock) - defer: release(db.lock) + ## Thread-safe lookup. + ## Default: exclusive lock — required for Nim ORC when TCP + HTTP threads share the DB. + ## Compile with `-d:baraConcurrentReads` for shared read locks (needs multi-thread-safe MM + ## such as a future atomicArc build; unsafe with default ORC across OS threads). + when defined(baraConcurrentReads): + acquireRead(db.lock) + defer: releaseRead(db.lock) + else: + acquireWrite(db.lock) + defer: releaseWrite(db.lock) return getUnsafe(db, key) proc contains*(db: LSMTree, key: string): bool = - acquire(db.lock) - defer: release(db.lock) + when defined(baraConcurrentReads): + acquireRead(db.lock) + defer: releaseRead(db.lock) + else: + acquireWrite(db.lock) + defer: releaseWrite(db.lock) let (found, _) = getUnsafe(db, key) return found +proc countL0*(db: LSMTree): int = + ## Number of level-0 SSTables (newest, uncompacted). + result = 0 + for sst in db.sstables: + if sst.level == 0: + inc result + proc flushUnsafe(db: LSMTree) = if db.immutableMem.len == 0 and db.memTable.len == 0: return @@ -802,7 +880,8 @@ proc flushUnsafe(db: LSMTree) = let path = db.dir / "sstables" / ($db.nextSSTableId & ".sst") inc db.nextSSTableId - var sst = writeSSTable(toFlush.entries, path, level = 0) + # Sort once at flush time (O(n log n)) — put/get stay O(1) + var sst = writeSSTable(toFlush.sortedEntries(), path, level = 0) sst.id = db.nextSSTableId - 1 db.sstables.add(sst) # SSTables are kept in insertion order (newest last) so getUnsafe can search newest-first @@ -814,22 +893,43 @@ proc flushUnsafe(db: LSMTree) = except CatchableError as e: echo "[WARN] Failed to write MANIFEST: ", e.msg - acquire(db.walLock) - db.wal.writeCommit(uint64(getMonoTime().ticks())) - db.wal.maybeRotate() - db.wal.sync() - release(db.walLock) + # Rewrite WAL to contain only still-unflushed memtable entries. + # Skip during recovery — the WAL file is still open for reading. + if not db.recovering: + acquire(db.walLock) + var liveKeys: seq[string] = @[] + var liveVals: seq[seq[byte]] = @[] + var liveTs: seq[uint64] = @[] + var liveDel: seq[bool] = @[] + for e in db.immutableMem.sortedEntries(): + liveKeys.add(e.key) + liveVals.add(e.value) + liveTs.add(e.timestamp) + liveDel.add(e.deleted) + for e in db.memTable.sortedEntries(): + liveKeys.add(e.key) + liveVals.add(e.value) + liveTs.add(e.timestamp) + liveDel.add(e.deleted) + if liveKeys.len == 0: + db.wal.truncate() + else: + db.wal.rewriteLive(liveKeys, liveVals, liveTs, liveDel) + release(db.walLock) + + if db.countL0() >= L0CompactionTrigger: + db.needsCompaction = true proc flush*(db: LSMTree) = - acquire(db.lock) - defer: release(db.lock) + acquireWrite(db.lock) + defer: releaseWrite(db.lock) flushUnsafe(db) proc checkpoint*(db: LSMTree) = ## Create a consistent checkpoint: freeze memtable, flush to SSTable, ## rotate WAL, and write MANIFEST. This provides a clean boundary ## for online backup without stopping the server. - acquire(db.lock) + acquireWrite(db.lock) # Flush any pending immutable memtable first if db.immutableMem.len > 0: @@ -850,10 +950,10 @@ proc checkpoint*(db: LSMTree) = db.wal.sync() release(db.walLock) - release(db.lock) + releaseWrite(db.lock) proc close*(db: LSMTree) = - acquire(db.lock) + acquireWrite(db.lock) try: # Flush both memtables to avoid data loss while db.immutableMem.len > 0: @@ -863,61 +963,109 @@ proc close*(db: LSMTree) = sst.close() db.wal.close() finally: - release(db.lock) + releaseWrite(db.lock) + +template withDataLock(db: LSMTree, body: untyped) = + ## Shared or exclusive depending on baraConcurrentReads (see get*). + when defined(baraConcurrentReads): + acquireRead(db.lock) + try: + body + finally: + releaseRead(db.lock) + else: + acquireWrite(db.lock) + try: + body + finally: + releaseWrite(db.lock) proc memTableSize*(db: LSMTree): int = - acquire(db.lock) - defer: release(db.lock) - return db.memTable.len + withDataLock(db): + return db.memTable.len proc sstableCount*(db: LSMTree): int = - acquire(db.lock) - defer: release(db.lock) - return db.sstables.len + withDataLock(db): + return db.sstables.len proc dir*(db: LSMTree): string = - acquire(db.lock) - defer: release(db.lock) - return db.dir + withDataLock(db): + return db.dir proc scanMemTable*(db: LSMTree): seq[Entry] = - acquire(db.lock) - defer: release(db.lock) - ## Return all entries from memory (memTable + immutableMem) - result = @[] - for e in db.memTable.entries: - result.add(e) - for e in db.immutableMem.entries: - result.add(e) + ## Return all entries from memory (memTable + immutableMem), sorted by key. + ## Immutable wins over active memtable only when timestamps are newer (same key rare). + withDataLock(db): + var merged = initTable[string, Entry]() + for e in db.immutableMem.sortedEntries(): + merged[e.key] = e + for e in db.memTable.sortedEntries(): + if e.key notin merged or e.timestamp >= merged[e.key].timestamp: + merged[e.key] = e + result = newSeqOfCap[Entry](merged.len) + for _, e in merged: + result.add(e) + result.sort(proc(a, b: Entry): int = cmp(a.key, b.key)) + +proc scanRange*(db: LSMTree, startKey, endKey: string): seq[(string, seq[byte])] = + ## Inclusive key range scan over memtables + SSTables (newest wins). + withDataLock(db): + var best = initTable[string, Entry]() + + for e in db.memTable.scan(startKey, endKey): + best[e.key] = e + for e in db.immutableMem.scan(startKey, endKey): + if e.key notin best or e.timestamp > best[e.key].timestamp: + best[e.key] = e + + for i in countdown(db.sstables.high, db.sstables.low): + let sst = db.sstables[i] + if sst.maxKey < startKey or sst.minKey > endKey: + continue + for key, offset in sst.index: + if key < startKey or key > endKey: + continue + if key in best: + continue + let (found, entry) = readSSTableEntry(sst, key) + if found: + best[key] = entry + + var keys = newSeqOfCap[string](best.len) + for k in best.keys: + keys.add(k) + keys.sort(cmp) + for k in keys: + let e = best[k] + if not e.deleted: + result.add((e.key, e.value)) proc scanAll*(db: LSMTree): seq[(string, seq[byte])] = ## Scan all active (non-deleted) entries from memory and SSTables. ## Used for shard data migration. - acquire(db.lock) - defer: release(db.lock) + withDataLock(db): + var seen = initTable[string, bool]() - var seen = initTable[string, bool]() + # Scan memtable first (most recent) + for e in db.memTable.sortedEntries(): + if e.key notin seen: + seen[e.key] = true + if not e.deleted: + result.add((e.key, e.value)) - # Scan memtable first (most recent) - for e in db.memTable.entries: - if e.key notin seen: - seen[e.key] = true - if not e.deleted: - result.add((e.key, e.value)) + # Scan immutable memtable + for e in db.immutableMem.sortedEntries(): + if e.key notin seen: + seen[e.key] = true + if not e.deleted: + result.add((e.key, e.value)) - # Scan immutable memtable - for e in db.immutableMem.entries: - if e.key notin seen: - seen[e.key] = true - if not e.deleted: - result.add((e.key, e.value)) - - # Scan SSTables from newest to oldest - for i in countdown(db.sstables.high, db.sstables.low): - let sst = db.sstables[i] - for key, offset in sst.index: - if key notin seen: - seen[key] = true - let (found, entry) = readSSTableEntry(sst, key) - if found and not entry.deleted: - result.add((entry.key, entry.value)) + # Scan SSTables from newest to oldest + for i in countdown(db.sstables.high, db.sstables.low): + let sst = db.sstables[i] + for key, offset in sst.index: + if key notin seen: + seen[key] = true + let (found, entry) = readSSTableEntry(sst, key) + if found and not entry.deleted: + result.add((entry.key, entry.value)) diff --git a/src/barabadb/storage/rwlock.nim b/src/barabadb/storage/rwlock.nim new file mode 100644 index 0000000..27335d7 --- /dev/null +++ b/src/barabadb/storage/rwlock.nim @@ -0,0 +1,80 @@ +## Simple reader-writer lock for LSM concurrent reads. +## Multiple readers OR one writer. Writers are exclusive. +## `acquire` / `release` are write-side (backward compatible with Lock-style usage). +import std/locks + +type + RwLock* = object + mu: Lock + readers: int ## active readers + writer: bool ## writer holds exclusive access + waitingWriters: int ## prefer writers to avoid reader starvation of compact/flush + canRead: Cond + canWrite: Cond + +proc initRwLock*(rw: var RwLock) = + initLock(rw.mu) + initCond(rw.canRead) + initCond(rw.canWrite) + rw.readers = 0 + rw.writer = false + rw.waitingWriters = 0 + +proc deinitRwLock*(rw: var RwLock) = + deinitCond(rw.canRead) + deinitCond(rw.canWrite) + deinitLock(rw.mu) + +proc acquireRead*(rw: var RwLock) = + ## Shared read lock. Blocks while a writer is active or waiting (writer preference). + acquire(rw.mu) + while rw.writer or rw.waitingWriters > 0: + wait(rw.canRead, rw.mu) + inc rw.readers + release(rw.mu) + +proc releaseRead*(rw: var RwLock) = + acquire(rw.mu) + dec rw.readers + if rw.readers == 0: + # Wake one waiting writer + signal(rw.canWrite) + release(rw.mu) + +proc acquireWrite*(rw: var RwLock) = + ## Exclusive write lock. + acquire(rw.mu) + inc rw.waitingWriters + while rw.writer or rw.readers > 0: + wait(rw.canWrite, rw.mu) + dec rw.waitingWriters + rw.writer = true + release(rw.mu) + +proc releaseWrite*(rw: var RwLock) = + acquire(rw.mu) + rw.writer = false + # Prefer draining writers, else open the gate for readers + if rw.waitingWriters > 0: + signal(rw.canWrite) + else: + broadcast(rw.canRead) + release(rw.mu) + +# Lock-compatible names: default exclusive (used by compaction, put, flush) +proc acquire*(rw: var RwLock) {.inline.} = acquireWrite(rw) +proc release*(rw: var RwLock) {.inline.} = releaseWrite(rw) + +template withReadLock*(rw: var RwLock, body: untyped) = + acquireRead(rw) + try: + body + finally: + releaseRead(rw) + +template withWriteLock*(rw: var RwLock, body: untyped) = + acquireWrite(rw) + try: + body + finally: + releaseWrite(rw) diff --git a/src/barabadb/storage/wal.nim b/src/barabadb/storage/wal.nim index c7aeb55..d4e4fc9 100644 --- a/src/barabadb/storage/wal.nim +++ b/src/barabadb/storage/wal.nim @@ -4,12 +4,16 @@ import std/os import std/streams import std/strutils import std/posix +import std/monotimes +import std/times const WALMagic* = 0x42415241'u32 # "BARA" WALVersion* = 1'u32 DefaultMaxWalSegmentSize* = 64 * 1024 * 1024 # 64MB WalArchiveDir* = "wal_archive" + ## Default group-commit batch size (entries between fsyncs). + DefaultWalGroupEvery* = 64 type WalEntryKind* = enum @@ -18,6 +22,15 @@ type wekCheckpoint = 3 wekCommit = 4 + ## Durability policy for WAL writes. + ## - wsmNone: flush userspace buffer only; fsync on truncate/rewrite/close/explicit sync + ## - wsmGroup: group commit — fsync every N entries and/or every intervalMs (default) + ## - wsmEvery: fsync after every entry (strict, slow) + WalSyncMode* = enum + wsmNone = "none" + wsmGroup = "group" + wsmEvery = "every" + WalEntry* = object kind*: WalEntryKind timestamp*: uint64 @@ -34,14 +47,28 @@ type path: string stream: FileStream entryCount: uint64 - syncOnWrite: bool + syncMode*: WalSyncMode + groupEvery*: int ## entries between fsyncs when mode=group + groupIntervalMs*: int ## time-based fsync when mode=group (0 = off) + unsyncedEntries: int ## entries written since last fsync + lastSync: MonoTime maxSegmentSize: int64 currentSequence: int64 + ## Counters for observability / benchmarks + fsyncCount*: uint64 + bytesSinceSync: int proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry] proc listWalArchive*(dir: string): seq[WalSegment] proc maybeRotate*(wal: var WriteAheadLog) +proc parseWalSyncMode*(s: string): WalSyncMode = + case s.toLowerAscii() + of "none", "async", "off", "false", "0": wsmNone + of "every", "sync", "full", "true", "1": wsmEvery + of "group", "batch", "": wsmGroup + else: wsmGroup + proc parseWalSequence*(filename: string): int64 = ## Extract sequence from "wal.000042.log" try: @@ -74,6 +101,12 @@ proc nextWalSequence*(dir: string): int64 = return 1 return segments[^1].sequence + 1 +proc fsyncPath(path: string) = + let fd = posix.open(cstring(path), O_RDWR) + if fd != -1: + discard posix.fsync(fd) + discard posix.close(fd) + proc rotate*(wal: var WriteAheadLog) = ## Close current WAL and archive it, then start a new one. if wal.stream != nil: @@ -96,7 +129,12 @@ proc rotate*(wal: var WriteAheadLog) = wal.stream.write(WALMagic) wal.stream.write(WALVersion) wal.stream.flush() + fsyncPath(wal.path) wal.entryCount = 0 + wal.unsyncedEntries = 0 + wal.bytesSinceSync = 0 + wal.lastSync = getMonoTime() + inc wal.fsyncCount proc maybeRotate*(wal: var WriteAheadLog) = ## Rotate if current WAL exceeds max segment size. @@ -106,7 +144,16 @@ proc maybeRotate*(wal: var WriteAheadLog) = if currentSize >= wal.maxSegmentSize: wal.rotate() -proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog = +proc newWriteAheadLog*( + dir: string, + syncMode: WalSyncMode = wsmGroup, + groupEvery: int = DefaultWalGroupEvery, + groupIntervalMs: int = 0, + syncOnWrite: bool = false, +): WriteAheadLog = + ## Create a WAL. + ## - syncMode controls durability (see WalSyncMode). + ## - syncOnWrite=true is legacy and forces wsmEvery. createDir(dir) let path = dir / "wal.log" let exists = fileExists(path) @@ -125,18 +172,62 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog = for e in readEntries(path): inc count + let mode = if syncOnWrite: wsmEvery else: syncMode + let ge = if groupEvery <= 0: DefaultWalGroupEvery else: groupEvery let seqNum = nextWalSequence(dir) WriteAheadLog( dir: dir, path: path, stream: stream, entryCount: count, - syncOnWrite: syncOnWrite, + syncMode: mode, + groupEvery: ge, + groupIntervalMs: groupIntervalMs, + unsyncedEntries: 0, + lastSync: getMonoTime(), maxSegmentSize: DefaultMaxWalSegmentSize, currentSequence: seqNum, + fsyncCount: 0, + bytesSinceSync: 0, ) +proc setSyncMode*(wal: var WriteAheadLog, mode: WalSyncMode) = + wal.syncMode = mode + +proc setGroupEvery*(wal: var WriteAheadLog, n: int) = + wal.groupEvery = if n <= 0: DefaultWalGroupEvery else: n + +proc setGroupIntervalMs*(wal: var WriteAheadLog, ms: int) = + wal.groupIntervalMs = max(0, ms) + +proc markSynced(wal: var WriteAheadLog) = + wal.unsyncedEntries = 0 + wal.bytesSinceSync = 0 + wal.lastSync = getMonoTime() + inc wal.fsyncCount + +proc maybeGroupSync(wal: var WriteAheadLog, entryBytes: int) = + ## Apply durability policy after a buffered write. + case wal.syncMode + of wsmNone: + discard + of wsmEvery: + fsyncPath(wal.path) + wal.markSynced() + of wsmGroup: + inc wal.unsyncedEntries + wal.bytesSinceSync += entryBytes + var due = wal.unsyncedEntries >= wal.groupEvery + if not due and wal.groupIntervalMs > 0: + let elapsedMs = (getMonoTime() - wal.lastSync).inMilliseconds + if elapsedMs >= wal.groupIntervalMs: + due = true + if due: + fsyncPath(wal.path) + wal.markSynced() + proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) = + let entryBytes = 1 + 8 + 4 + entry.key.len + 4 + entry.value.len wal.stream.write(uint8(entry.kind)) wal.stream.write(entry.timestamp) wal.stream.write(uint32(entry.key.len)) @@ -145,8 +236,9 @@ proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) = wal.stream.write(uint32(entry.value.len)) if entry.value.len > 0: wal.stream.writeData(unsafeAddr entry.value[0], entry.value.len) - if wal.syncOnWrite: - wal.stream.flush() + # Always push to kernel page cache; durability policy decides fsync + wal.stream.flush() + wal.maybeGroupSync(entryBytes) inc wal.entryCount # Check rotation every 1000 entries to avoid stat on every write if wal.entryCount mod 1000 == 0: @@ -177,28 +269,87 @@ proc writeCommit*(wal: var WriteAheadLog, timestamp: uint64) = )) proc sync*(wal: var WriteAheadLog) = + ## Force durability of all buffered WAL data. wal.stream.flush() - # Re-open with O_RDWR so fsync operates on a write-capable fd. - # Not ideal (two fds for same file) but avoids accessing private - # FileStream internals that vary across Nim versions. - let fd = posix.open(cstring(wal.path), O_RDWR) - if fd != -1: - discard posix.fsync(fd) - discard posix.close(fd) + fsyncPath(wal.path) + wal.markSynced() + +proc truncate*(wal: var WriteAheadLog) = + ## Reset WAL to empty (header only). Safe only when all prior entries + ## are durable in SSTables and nothing remains only-in-memtable. + if wal.stream != nil: + wal.stream.flush() + wal.stream.close() + wal.stream = newFileStream(wal.path, fmWrite) + if wal.stream == nil: + raise newException(IOError, "Cannot truncate WAL: " & wal.path) + wal.stream.write(WALMagic) + wal.stream.write(WALVersion) + wal.stream.flush() + fsyncPath(wal.path) + wal.entryCount = 0 + wal.markSynced() + +proc rewriteLive*(wal: var WriteAheadLog, + keys: openArray[string], + values: openArray[seq[byte]], + timestamps: openArray[uint64], + deleted: openArray[bool]) = + ## Atomically replace WAL contents with a live memtable snapshot. + ## Used after a partial flush so unflushed keys remain recoverable. + doAssert keys.len == values.len and keys.len == timestamps.len and keys.len == deleted.len + if keys.len == 0: + wal.truncate() + return + + let tmpPath = wal.path & ".rewrite" + let s = newFileStream(tmpPath, fmWrite) + if s == nil: + raise newException(IOError, "Cannot create WAL rewrite file: " & tmpPath) + s.write(WALMagic) + s.write(WALVersion) + var count: uint64 = 0 + for i in 0 ..< keys.len: + let kind = if deleted[i]: wekDelete else: wekPut + s.write(uint8(kind)) + s.write(timestamps[i]) + s.write(uint32(keys[i].len)) + if keys[i].len > 0: + s.write(keys[i]) + s.write(uint32(values[i].len)) + if values[i].len > 0: + s.writeData(unsafeAddr values[i][0], values[i].len) + inc count + s.flush() + s.close() + fsyncPath(tmpPath) + + if wal.stream != nil: + wal.stream.close() + if fileExists(wal.path): + removeFile(wal.path) + moveFile(tmpPath, wal.path) + wal.stream = newFileStream(wal.path, fmAppend) + if wal.stream == nil: + raise newException(IOError, "Cannot reopen WAL after rewrite: " & wal.path) + wal.entryCount = count + wal.markSynced() proc setMaxSegmentSize*(wal: var WriteAheadLog, size: int64) = wal.maxSegmentSize = size proc close*(wal: var WriteAheadLog) = wal.stream.flush() - let fd = posix.open(cstring(wal.path), O_RDWR) - if fd != -1: - discard posix.fsync(fd) - discard posix.close(fd) + fsyncPath(wal.path) + wal.markSynced() wal.stream.close() proc entryCount*(wal: WriteAheadLog): uint64 = wal.entryCount proc path*(wal: WriteAheadLog): string = wal.path +proc unsyncedEntries*(wal: WriteAheadLog): int = wal.unsyncedEntries + +## Legacy alias — true maps to wsmEvery +proc syncOnWrite*(wal: WriteAheadLog): bool = wal.syncMode == wsmEvery proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry] = result = @[] diff --git a/src/baradadb.nim b/src/baradadb.nim index 1d1a129..aa2a420 100644 --- a/src/baradadb.nim +++ b/src/baradadb.nim @@ -15,6 +15,7 @@ import barabadb/core/config import barabadb/core/logging import barabadb/protocol/ssl import barabadb/storage/lsm +import barabadb/storage/gate import barabadb/storage/compaction import barabadb/core/raft import barabadb/query/executor @@ -36,58 +37,66 @@ type proc newCompactionManager*(db: LSMTree): CompactionManager = result = CompactionManager(db: db, strategy: compaction.newCompactionStrategy(db.dir)) - for sst in db.sstables: - let meta = compaction.SSTableMeta( - path: sst.path, - level: sst.level, - minKey: sst.minKey, - maxKey: sst.maxKey, - entryCount: sst.entryCount, - sizeBytes: sst.entryCount * 64, - createdAt: 0, - ) - result.strategy.addTable(meta) + result.strategy.rebuildFromLSM(db) + +proc applyCompactionResult(db: LSMTree, result: compaction.CompactionResult) = + ## Apply compaction output under the caller's lock: update sstables + MANIFEST. + ## On Linux, compact may already have unlinked inputs; we still close our mmaps. + if result.outputTables.len == 0: + return + + var newSSTables: seq[SSTable] = @[] + var removedPaths = initTable[string, bool]() + for t in result.inputTables: + removedPaths[t.path] = true + for sst in db.sstables.mitems: + if sst.path notin removedPaths: + newSSTables.add(sst) + else: + # Drop mmap after compact unlinked the path (fd remains valid until close) + sst.close() + + for meta in result.outputTables: + try: + var sst = loadSSTable(meta.path) + let name = splitFile(meta.path).name + # Prefer numeric id from filename; otherwise allocate + let parsed = try: parseInt(name) except: -1 + if parsed >= 0: + sst.id = parsed + else: + sst.id = db.nextSSTableId + inc db.nextSSTableId + sst.level = meta.level + newSSTables.add(sst) + db.nextSSTableId = max(db.nextSSTableId, sst.id + 1) + except CatchableError as e: + warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg) + + newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id)) + db.sstables = newSSTables + db.needsCompaction = db.countL0() >= L0CompactionTrigger + + inc db.manifestSequence + try: + writeManifest(db) + except CatchableError as e: + warn("Failed to write MANIFEST after compaction: " & e.msg) proc compact*(cm: CompactionManager) = - acquire(cm.db.lock) - defer: release(cm.db.lock) - for level in 0 ..< compaction.MaxLevel: - if cm.strategy.needsCompaction(level): - let result = cm.strategy.compact(level) - if result.outputTables.len == 0: - continue - - # Remove compacted input SSTables from LSMTree - var newSSTables: seq[SSTable] = @[] - var removedPaths = initTable[string, bool]() - for t in result.inputTables: - removedPaths[t.path] = true - for sst in cm.db.sstables: - if sst.path notin removedPaths: - newSSTables.add(sst) - - # Load and add output SSTables - for meta in result.outputTables: - try: - var sst = loadSSTable(meta.path) - let name = splitFile(meta.path).name - # Extract numeric id from filename if possible - sst.id = try: parseInt(name) except: cm.db.nextSSTableId - sst.level = meta.level - newSSTables.add(sst) - cm.db.nextSSTableId = max(cm.db.nextSSTableId, sst.id + 1) - except CatchableError as e: - warn("Compaction output SSTable failed to load: " & meta.path & " — " & e.msg) - - newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id)) - cm.db.sstables = newSSTables - - # Update MANIFEST - inc cm.db.manifestSequence - try: - writeManifest(cm.db) - except CatchableError as e: - warn("Failed to write MANIFEST after compaction: " & e.msg) + # Gate first (cross-thread), then per-DB write lock + withStorageGate: + acquire(cm.db.lock) + try: + # Always rebuild from LSM — flushes add L0 tables the strategy never registered + cm.strategy.rebuildFromLSM(cm.db) + for level in 0 ..< compaction.MaxLevel: + if cm.strategy.needsCompaction(level): + let result = cm.strategy.compact(level) + applyCompactionResult(cm.db, result) + cm.strategy.rebuildFromLSM(cm.db) + finally: + release(cm.db.lock) proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.async.} = while true: @@ -96,17 +105,11 @@ proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.asyn proc newMultiCompactionManager*(registry: DatabaseRegistry): MultiCompactionManager = result = MultiCompactionManager(registry: registry, strategies: initTable[string, compaction.CompactionStrategy]()) - - # Initialize strategies for each existing database for name in listDatabases(registry): let info = getDatabaseInfo(registry, name) if info != nil: result.strategies[name] = compaction.newCompactionStrategy(info.db.dir) - for sst in info.db.sstables: - let meta = compaction.SSTableMeta( - path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey, - entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0) - result.strategies[name].addTable(meta) + result.strategies[name].rebuildFromLSM(info.db) proc compactAll(mcm: MultiCompactionManager) = for name in listDatabases(mcm.registry): @@ -114,51 +117,21 @@ proc compactAll(mcm: MultiCompactionManager) = if info == nil: continue let db = info.db - # Initialize strategy if not already if name notin mcm.strategies: mcm.strategies[name] = compaction.newCompactionStrategy(db.dir) - for sst in db.sstables: - let meta = compaction.SSTableMeta( - path: sst.path, level: sst.level, minKey: sst.minKey, maxKey: sst.maxKey, - entryCount: sst.entryCount, sizeBytes: sst.entryCount * 64, createdAt: 0) - mcm.strategies[name].addTable(meta) let strategy = mcm.strategies[name] - acquire(db.lock) - try: - for level in 0 ..< compaction.MaxLevel: - if strategy.needsCompaction(level): - let result = strategy.compact(level) - if result.outputTables.len == 0: continue - - var newSSTables: seq[SSTable] = @[] - var removedPaths = initTable[string, bool]() - for t in result.inputTables: - removedPaths[t.path] = true - for sst in db.sstables: - if sst.path notin removedPaths: - newSSTables.add(sst) - - for meta in result.outputTables: - try: - var sst = loadSSTable(meta.path) - let sstName = splitFile(meta.path).name - sst.id = try: parseInt(sstName) except: db.nextSSTableId - sst.level = meta.level - newSSTables.add(sst) - db.nextSSTableId = max(db.nextSSTableId, sst.id + 1) - except CatchableError as e: - warn("Compaction output SSTable failed to load: " & meta.path & " - " & e.msg) - - newSSTables.sort(proc(a, b: SSTable): int = cmp(a.id, b.id)) - db.sstables = newSSTables - inc db.manifestSequence - try: - writeManifest(db) - except CatchableError as e: - warn("Failed to write MANIFEST after compaction: " & e.msg) - finally: - release(db.lock) + withStorageGate: + acquire(db.lock) + try: + strategy.rebuildFromLSM(db) + for level in 0 ..< compaction.MaxLevel: + if strategy.needsCompaction(level): + let result = strategy.compact(level) + applyCompactionResult(db, result) + strategy.rebuildFromLSM(db) + finally: + release(db.lock) proc startMultiCompactionLoop*(mcm: MultiCompactionManager, intervalMs: int = 60000) {.async.} = while true: @@ -303,10 +276,13 @@ proc main() = quit(0) var config = loadConfig() + # Global exclusive gate for multi-thread storage (HTTP workers + TCP + compact) + initStorageGate() # Init structured logger from config let logLvl = parseEnum[LogLevel]("ll" & capitalizeAscii(config.logLevel)) defaultLogger = newLogger(logLvl, config.logFile) info("BaraDB v1.1.6 — Multimodal Database Engine") + info("Storage gate initialized (serializes HTTP/TCP/compaction access)") # Security check: warn if JWT secret is not configured if config.jwtSecret.len == 0: @@ -358,12 +334,13 @@ proc main() = # Wire state machine to apply committed entries to the default database let defaultDbInfo = getDatabaseInfo(registry, "default") raftNode.applyCommand = proc(cmd: string, data: seq[byte]) {.gcsafe.} = - if cmd == "put": - let parts = cast[string](data).split("\x00") - if parts.len >= 2: - defaultDbInfo.db.put(parts[0], cast[seq[byte]](parts[1])) - elif cmd == "delete": - defaultDbInfo.db.delete(cast[string](data)) + withStorageGate: + if cmd == "put": + let parts = cast[string](data).split("\x00") + if parts.len >= 2: + defaultDbInfo.db.put(parts[0], cast[seq[byte]](parts[1])) + elif cmd == "delete": + defaultDbInfo.db.delete(cast[string](data)) # Wire RAFT ↔ DistTxn wireRaftDistTxn(raftNode, tcpServer) @@ -395,11 +372,13 @@ proc main() = # Start TCP wire protocol server on main thread with async event loop waitFor runTcpServer(config) - # Shutdown - httpServer.stop() + # Shutdown: stop listeners first, then close storage under the gate + httpServer.stop(closeStorage = false) tcpServer.stop() if tcpServer.gossipProtocol != nil: tcpServer.gossipProtocol.stop() + withStorageGate: + registry.closeAll() when isMainModule: main() diff --git a/tests/test_all.nim b/tests/test_all.nim index b988d6c..b4514dc 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -196,6 +196,232 @@ suite "MANIFEST Catalog": check issues[0].contains("Orphan") db.close() +suite "Core Storage Hardening": + test "MemTable overwrite keeps newest value (hash table)": + let testDir = "/tmp/baradb_test_memtable_hash" + removeDir(testDir) + var db = newLSMTree(testDir, 64 * 1024) + db.put("k", cast[seq[byte]]("v1")) + db.put("k", cast[seq[byte]]("v2")) + db.put("k", cast[seq[byte]]("v3")) + let (found, val) = db.get("k") + check found + check cast[string](val) == "v3" + check db.memTableSize() == 1 + db.close() + + test "Many distinct keys without O(n) insert collapse": + ## Hash MemTable should handle thousands of puts without quadratic cost. + let testDir = "/tmp/baradb_test_memtable_many" + removeDir(testDir) + var db = newLSMTree(testDir, 8 * 1024 * 1024) + let n = 5000 + for i in 0 ..< n: + db.put("key_" & align($i, 6, '0'), cast[seq[byte]]("val_" & $i)) + for i in [0, 1, n div 2, n - 1]: + let (found, val) = db.get("key_" & align($i, 6, '0')) + check found + check cast[string](val) == "val_" & $i + db.close() + + test "scanMemTable returns sorted unique keys": + let testDir = "/tmp/baradb_test_scan_sorted" + removeDir(testDir) + var db = newLSMTree(testDir, 64 * 1024) + db.put("c", cast[seq[byte]]("3")) + db.put("a", cast[seq[byte]]("1")) + db.put("b", cast[seq[byte]]("2")) + db.put("a", cast[seq[byte]]("1b")) + let mem = db.scanMemTable() + check mem.len == 3 + check mem[0].key == "a" + check cast[string](mem[0].value) == "1b" + check mem[1].key == "b" + check mem[2].key == "c" + db.close() + + test "WAL truncated after full flush — recovery stays small": + let testDir = "/tmp/baradb_test_wal_truncate" + removeDir(testDir) + var db = newLSMTree(testDir, 256) + for i in 0 ..< 20: + db.put("k" & $i, cast[seq[byte]]("v" & $i)) + db.flush() + # After flush both memtables empty → WAL should only have header (or tiny rewrite) + let walPath = testDir / "wal" / "wal.log" + check fileExists(walPath) + let sizeAfterFlush = getFileSize(walPath) + check sizeAfterFlush < 256 # header only, not all 20 puts + db.close() + # Reopen: data comes from SSTables, not a bloated WAL + var db2 = newLSMTree(testDir, 256) + for i in 0 ..< 20: + let (found, val) = db2.get("k" & $i) + check found + check cast[string](val) == "v" & $i + db2.close() + + test "Partial flush rewrites WAL with remaining live keys": + let testDir = "/tmp/baradb_test_wal_rewrite" + removeDir(testDir) + # Tiny memtable forces flush of first batch while second batch stays in memory + var db = newLSMTree(testDir, 64) + db.put("old1", cast[seq[byte]]("a")) + db.put("old2", cast[seq[byte]]("b")) + # Force flush + db.flush() + db.put("live1", cast[seq[byte]]("x")) + db.put("live2", cast[seq[byte]]("y")) + # Do not flush — close without flush would lose live without WAL; close flushes + # Instead: crash-simulate by reopening after putting live keys (WAL rewrite on prior flush + # left empty; new puts are in current WAL) + db.close() + var db2 = newLSMTree(testDir, 64) + let (f1, v1) = db2.get("live1") + let (f2, v2) = db2.get("live2") + let (f3, _) = db2.get("old1") + check f1 and cast[string](v1) == "x" + check f2 and cast[string](v2) == "y" + check f3 + db2.close() + + test "L0 count trigger and rebuildFromLSM sees flushed tables": + let testDir = "/tmp/baradb_test_l0_trigger" + removeDir(testDir) + var db = newLSMTree(testDir, 128) + for round in 0 ..< L0CompactionTrigger: + db.put("r" & $round, cast[seq[byte]]("v" & $round)) + db.flush() + check db.countL0() >= L0CompactionTrigger + check db.needsCompaction == true + var cs = newCompactionStrategy(testDir) + cs.rebuildFromLSM(db) + check cs.needsCompaction(0) == true + check cs.levels[0].len >= L0CompactionTrigger + # Compact L0 → L1 + let cr = cs.compact(0) + check cr.outputTables.len == 1 + check cr.outputTables[0].level == 1 + # Apply manually: remove inputs from db, add output + var removed = initTable[string, bool]() + for t in cr.inputTables: + removed[t.path] = true + var kept: seq[SSTable] = @[] + for sst in db.sstables.mitems: + if sst.path notin removed: + kept.add(sst) + else: + sst.close() + var outSst = loadSSTable(cr.outputTables[0].path) + outSst.level = 1 + outSst.id = db.nextSSTableId + inc db.nextSSTableId + kept.add(outSst) + db.sstables = kept + check db.countL0() < L0CompactionTrigger + for round in 0 ..< L0CompactionTrigger: + let (found, val) = db.get("r" & $round) + check found + check cast[string](val) == "v" & $round + db.close() + + test "Crash recovery: WAL-only clone recovers unflushed puts": + ## Simulate crash: copy WAL without SSTables / without clean close flush. + let srcDir = "/tmp/baradb_test_crash_src" + let dstDir = "/tmp/baradb_test_crash_dst" + removeDir(srcDir) + removeDir(dstDir) + var db = newLSMTree(srcDir, 1024 * 1024) + db.put("persist_me", cast[seq[byte]]("yes")) + db.put("and_me", cast[seq[byte]]("also")) + db.wal.sync() + createDir(dstDir / "wal") + createDir(dstDir / "sstables") + copyFile(srcDir / "wal" / "wal.log", dstDir / "wal" / "wal.log") + db.close() # cleans up src; dst has WAL-only crash image + var recovered = newLSMTree(dstDir, 1024 * 1024) + let (f1, v1) = recovered.get("persist_me") + let (f2, v2) = recovered.get("and_me") + check f1 and cast[string](v1) == "yes" + check f2 and cast[string](v2) == "also" + recovered.close() + + test "WAL group commit fsyncs roughly every N entries": + let testDir = "/tmp/baradb_test_wal_group" + removeDir(testDir) + const n = 200 + const ge = 50 + var db = newLSMTree(testDir, 8 * 1024 * 1024, + walSyncMode = wsmGroup, walGroupEvery = ge) + let base = db.wal.fsyncCount # open/recovery may fsync once + for i in 0 ..< n: + db.put("g" & $i, cast[seq[byte]]("v")) + let afterPuts = db.wal.fsyncCount - base + # Group every 50 → about n/ge fsyncs; partial group not yet synced + check afterPuts >= uint64(n div ge) + check afterPuts < uint64(n) # far fewer than one-per-write + db.wal.sync() + check db.wal.fsyncCount > base + afterPuts or afterPuts >= uint64(n div ge) + db.close() + + test "WAL every-mode fsyncs at least once per write": + let testDir = "/tmp/baradb_test_wal_every" + removeDir(testDir) + const n = 30 + var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmEvery) + let base = db.wal.fsyncCount + for i in 0 ..< n: + db.put("e" & $i, cast[seq[byte]]("v")) + check db.wal.fsyncCount - base >= uint64(n) + db.close() + + test "WAL none-mode does not fsync on each put": + let testDir = "/tmp/baradb_test_wal_none" + removeDir(testDir) + const n = 100 + var db = newLSMTree(testDir, 8 * 1024 * 1024, walSyncMode = wsmNone) + let base = db.wal.fsyncCount + for i in 0 ..< n: + db.put("n" & $i, cast[seq[byte]]("v")) + # Puts alone should not fsync + check db.wal.fsyncCount == base + db.wal.sync() + check db.wal.fsyncCount == base + 1 + db.close() + + test "parseWalSyncMode accepts aliases": + check parseWalSyncMode("group") == wsmGroup + check parseWalSyncMode("every") == wsmEvery + check parseWalSyncMode("none") == wsmNone + check parseWalSyncMode("async") == wsmNone + check parseWalSyncMode("full") == wsmEvery + check parseWalSyncMode("batch") == wsmGroup + + test "scanRange returns inclusive sorted keys": + let testDir = "/tmp/baradb_test_scan_range" + removeDir(testDir) + var db = newLSMTree(testDir, 256) + for ch in ['a', 'b', 'c', 'd', 'e']: + db.put($ch, cast[seq[byte]]("v" & $ch)) + db.flush() + db.put("c", cast[seq[byte]]("vC2")) # newer in memtable + let rows = db.scanRange("b", "d") + check rows.len == 3 + check rows[0][0] == "b" + check rows[1][0] == "c" + check cast[string](rows[1][1]) == "vC2" + check rows[2][0] == "d" + db.close() + + test "scanRange empty when no keys in range": + let testDir = "/tmp/baradb_test_scan_empty" + removeDir(testDir) + var db = newLSMTree(testDir, 1024) + db.put("m", cast[seq[byte]]("1")) + check db.scanRange("a", "c").len == 0 + check db.scanRange("m", "m").len == 1 + db.close() + suite "BaraQL Lexer": test "Tokenize simple SELECT": let tokens = lex.tokenize("SELECT name FROM users WHERE age > 18") diff --git a/tests/test_schema_persist.nim b/tests/test_schema_persist.nim new file mode 100644 index 0000000..e655bae --- /dev/null +++ b/tests/test_schema_persist.nim @@ -0,0 +1,116 @@ +## Schema persistence — CREATE TABLE / data survive reopen +import std/unittest +import std/os +import std/strutils +import std/tables +import barabadb/storage/lsm +import barabadb/query/executor +import barabadb/query/parser +import barabadb/query/ast + +proc execSql(ctx: ExecutionContext, sql: string): ExecResult = + let node = parse(sql) + result = executeQuery(ctx, node) + +suite "Schema persistence": + test "CREATE TABLE survives flush + reopen": + let dir = "/tmp/baradb_schema_persist_1" + removeDir(dir) + block: + var db = newLSMTree(dir, 1024) # small memtable → forces flush + var ctx = newExecutionContext(db) + let r = execSql(ctx, "CREATE TABLE users (id INT PRIMARY KEY, name TEXT NOT NULL)") + check r.success + check ctx.tables.hasKey("users") + check ctx.tables["users"].columns.len == 2 + discard execSql(ctx, "INSERT INTO users (id, name) VALUES (1, 'Alice')") + discard execSql(ctx, "INSERT INTO users (id, name) VALUES (2, 'Bob')") + db.flush() + # Schema key must be durable + let (found, _) = db.get(tableSchemaKey("users")) + check found + db.close() + + # Reopen fresh context (simulates process restart) + block: + var db2 = newLSMTree(dir, 1024) + var ctx2 = newExecutionContext(db2) + check ctx2.tables.hasKey("users") + check ctx2.tables["users"].columns.len == 2 + check ctx2.tables["users"].pkColumns.len == 1 + let sel = execSql(ctx2, "SELECT id, name FROM users ORDER BY id") + check sel.success + check sel.rows.len == 2 + db2.close() + + test "DROP TABLE removes schema and data": + let dir = "/tmp/baradb_schema_persist_drop" + removeDir(dir) + var db = newLSMTree(dir) + var ctx = newExecutionContext(db) + check execSql(ctx, "CREATE TABLE t (id INT PRIMARY KEY)").success + check execSql(ctx, "INSERT INTO t (id) VALUES (1)").success + check execSql(ctx, "DROP TABLE t").success + check not ctx.tables.hasKey("t") + let (found, _) = db.get(tableSchemaKey("t")) + check not found + # Reopen — table must not reappear + db.close() + var db2 = newLSMTree(dir) + var ctx2 = newExecutionContext(db2) + check not ctx2.tables.hasKey("t") + db2.close() + + test "ALTER TABLE ADD COLUMN is persisted": + let dir = "/tmp/baradb_schema_persist_alter" + removeDir(dir) + block: + var db = newLSMTree(dir) + var ctx = newExecutionContext(db) + check execSql(ctx, "CREATE TABLE items (id INT PRIMARY KEY)").success + check execSql(ctx, "ALTER TABLE items ADD COLUMN label TEXT").success + check ctx.tables["items"].columns.len == 2 + db.flush() + db.close() + block: + var db2 = newLSMTree(dir) + var ctx2 = newExecutionContext(db2) + check ctx2.tables.hasKey("items") + check ctx2.tables["items"].columns.len == 2 + var names: seq[string] = @[] + for c in ctx2.tables["items"].columns: + names.add(c.name) + check "label" in names + db2.close() + + test "Multiple tables all restored": + let dir = "/tmp/baradb_schema_persist_multi" + removeDir(dir) + block: + var db = newLSMTree(dir, 512) + var ctx = newExecutionContext(db) + check execSql(ctx, "CREATE TABLE a (id INT PRIMARY KEY)").success + check execSql(ctx, "CREATE TABLE b (id INT PRIMARY KEY, a_id INT)").success + check execSql(ctx, "CREATE TABLE c (name TEXT)").success + for i in 0..20: + discard execSql(ctx, "INSERT INTO a (id) VALUES (" & $i & ")") + db.flush() + db.close() + block: + var db2 = newLSMTree(dir) + var ctx2 = newExecutionContext(db2) + check ctx2.tables.hasKey("a") + check ctx2.tables.hasKey("b") + check ctx2.tables.hasKey("c") + let sel = execSql(ctx2, "SELECT id FROM a") + check sel.success + check sel.rows.len == 21 + db2.close() + + test "Stable schema key format": + check tableSchemaKey("users") == "_schema:tables:users" + check serializeTableDdl(TableDef( + name: "t", + columns: @[ColumnDef(name: "id", colType: "INT", isPk: true)], + pkColumns: @["id"], + )).contains("PRIMARY KEY") diff --git a/tests/test_storage_hardening.nim b/tests/test_storage_hardening.nim new file mode 100644 index 0000000..48155a6 --- /dev/null +++ b/tests/test_storage_hardening.nim @@ -0,0 +1,168 @@ +## Focused storage hardening tests (avoids full suite compile issues) +import std/unittest +import std/os +import std/strutils +import std/locks +import barabadb/storage/lsm +import barabadb/storage/rwlock +import barabadb/storage/gate + +suite "Core Storage Hardening": + test "MemTable overwrite keeps newest value": + let testDir = "/tmp/baradb_th_mem" + removeDir(testDir) + var db = newLSMTree(testDir, 64 * 1024) + db.put("k", cast[seq[byte]]("v1")) + db.put("k", cast[seq[byte]]("v3")) + let (found, val) = db.get("k") + check found + check cast[string](val) == "v3" + db.close() + + test "scanRange inclusive": + let testDir = "/tmp/baradb_th_range" + removeDir(testDir) + var db = newLSMTree(testDir, 256) + for ch in ['a', 'b', 'c', 'd', 'e']: + db.put($ch, cast[seq[byte]]("v" & $ch)) + db.flush() + db.put("c", cast[seq[byte]]("vC2")) + let rows = db.scanRange("b", "d") + check rows.len == 3 + check rows[0][0] == "b" + check rows[1][0] == "c" + check cast[string](rows[1][1]) == "vC2" + db.close() + + test "WAL group commit": + let testDir = "/tmp/baradb_th_group" + removeDir(testDir) + const n = 200 + const ge = 50 + var db = newLSMTree(testDir, 8 * 1024 * 1024, + walSyncMode = wsmGroup, walGroupEvery = ge) + let base = db.wal.fsyncCount + for i in 0 ..< n: + db.put("g" & $i, cast[seq[byte]]("v")) + let after = db.wal.fsyncCount - base + check after >= uint64(n div ge) + check after < uint64(n) + db.close() + + test "RwLock concurrent readers": + var rw: RwLock + initRwLock(rw) + var counter = 0 + var maxReaders = 0 + var curReaders = 0 + var metaLock: Lock + initLock(metaLock) + var bad = false + + type TArgs = object + rw: ptr RwLock + meta: ptr Lock + counter: ptr int + curReaders: ptr int + maxReaders: ptr int + bad: ptr bool + isWriter: bool + + proc worker(a: TArgs) {.thread, gcsafe.} = + for i in 0 ..< 200: + if a.isWriter: + acquireWrite(a.rw[]) + a.counter[] += 1 + acquire(a.meta[]) + if a.curReaders[] != 0: + a.bad[] = true + release(a.meta[]) + releaseWrite(a.rw[]) + else: + acquireRead(a.rw[]) + acquire(a.meta[]) + inc a.curReaders[] + if a.curReaders[] > a.maxReaders[]: + a.maxReaders[] = a.curReaders[] + release(a.meta[]) + var x = 0 + for k in 0 ..< 50: x += k + discard x + acquire(a.meta[]) + dec a.curReaders[] + release(a.meta[]) + releaseRead(a.rw[]) + + var threads: array[8, Thread[TArgs]] + for t in 0 ..< 8: + let args = TArgs( + rw: addr rw, meta: addr metaLock, + counter: addr counter, curReaders: addr curReaders, + maxReaders: addr maxReaders, bad: addr bad, + isWriter: t == 0 or t == 1, + ) + createThread(threads[t], worker, args) + for t in 0 ..< 8: + joinThread(threads[t]) + + check not bad + check counter == 400 + check maxReaders >= 2 + deinitLock(metaLock) + deinitRwLock(rw) + + test "Interleaved put/get/flush single-threaded stress": + ## ORC is not multi-thread-safe for shared refs; stress the exclusive path serially. + let testDir = "/tmp/baradb_th_stress" + removeDir(testDir) + var db = newLSMTree(testDir, 4 * 1024, walSyncMode = wsmGroup, walGroupEvery = 32) + for i in 0 ..< 2000: + db.put("k" & $i, cast[seq[byte]]("v" & $i)) + if i mod 100 == 0: + let (f, v) = db.get("k0") + check f and cast[string](v) == "v0" + if i mod 400 == 0: + db.flush() + for i in [0, 500, 1000, 1999]: + let (f, v) = db.get("k" & $i) + check f and cast[string](v) == "v" & $i + db.close() + + test "StorageGate serializes concurrent critical sections": + initStorageGate() + var counter = 0 + var bad = false + var meta: Lock + initLock(meta) + + type GArgs = object + n: int + counter: ptr int + bad: ptr bool + meta: ptr Lock + + proc worker(a: GArgs) {.thread, gcsafe.} = + for i in 0 ..< a.n: + withStorageGate: + # Under the gate, only one thread should touch counter + let before = a.counter[] + a.counter[] = before + 1 + # Simulate work + var x = 0 + for k in 0 ..< 20: x += k + discard x + if a.counter[] != before + 1: + acquire(a.meta[]) + a.bad[] = true + release(a.meta[]) + + var threads: array[6, Thread[GArgs]] + for t in 0 ..< 6: + createThread(threads[t], worker, GArgs( + n: 100, counter: addr counter, bad: addr bad, meta: addr meta)) + for t in 0 ..< 6: + joinThread(threads[t]) + + check not bad + check counter == 600 + deinitLock(meta) \ No newline at end of file diff --git a/tests/test_wire_insert_stress.nim b/tests/test_wire_insert_stress.nim new file mode 100644 index 0000000..56b52b4 --- /dev/null +++ b/tests/test_wire_insert_stress.nim @@ -0,0 +1,26 @@ +## Regression: sequential wire-style INSERTs must not crash the process. +## Root cause was ORC cycle collector (markGray SIGSEGV); project uses --mm:arc. +import std/unittest +import std/os +import barabadb/storage/lsm +import barabadb/query/executor +import barabadb/query/parser + +proc execSql(ctx: ExecutionContext, sql: string): ExecResult = + executeQuery(ctx, parse(sql)) + +suite "Wire insert stress (ARC regression)": + test "200 sequential INSERTs via executor survive": + ## Mirrors the wire path (executeQuery under StorageGate each time). + let dir = "/tmp/baradb_wire_stress" + removeDir(dir) + var db = newLSMTree(dir, walSyncMode = wsmNone) + var ctx = newExecutionContext(db) + check execSql(ctx, "CREATE TABLE stress (id INT PRIMARY KEY, v TEXT)").success + for i in 0 ..< 200: + let r = execSql(ctx, "INSERT INTO stress (id, v) VALUES (" & $i & ", 'v" & $i & "')") + check r.success + let sel = execSql(ctx, "SELECT id FROM stress") + check sel.success + check sel.rows.len == 200 + db.close()