From cf2aba104f2738cbd14cdb635b9ca28a3fbd09f8 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Mon, 18 May 2026 12:52:44 +0300 Subject: [PATCH] Phase 5 complete: B-tree property tests + benchmark regression suite - prop_test.nim: 11 new property-based B-Tree invariants (size accuracy, get roundtrip, scan ordering, range correctness, contains, remove, large order, duplicates, empty tree, interleaved ops) - bench_all.nim: JSON-based benchmark result tracking with regression comparison against previous runs; each benchmark reports ops/sec delta vs last run --- benchmarks/bench_all.nim | 168 ++++++++++++++++++++++++++++++++++----- tests/prop_test.nim | 143 ++++++++++++++++++++++++++++++++- 2 files changed, 292 insertions(+), 19 deletions(-) diff --git a/benchmarks/bench_all.nim b/benchmarks/bench_all.nim index b0d07ae..d97c8e4 100644 --- a/benchmarks/bench_all.nim +++ b/benchmarks/bench_all.nim @@ -4,6 +4,9 @@ import std/times import std/tables import std/random import std/strutils +import std/os +import std/osproc +import std/json import ../src/barabadb/storage/lsm import ../src/barabadb/storage/btree import ../src/barabadb/vector/engine as vengine @@ -11,6 +14,89 @@ import ../src/barabadb/vector/simd import ../src/barabadb/fts/engine as fts import ../src/barabadb/graph/engine as gengine +# ═══════════════════════════════════════════════════ +# Benchmark Result Tracking +# ═══════════════════════════════════════════════════ +type + BenchResult* = object + name*: string + ops*: int + seconds*: float64 + opsPerSec*: float64 + timestamp*: string + + BenchReport* = object + version*: string + gitSha*: string + results*: seq[BenchResult] + +const ResultsFile = "benchmark_results.json" + +proc loadPreviousResults(path: string): seq[BenchResult] = + result = @[] + if not fileExists(path): return + try: + let j = parseFile(path) + if j.hasKey("results"): + for item in j["results"]: + if item.hasKey("name") and item.hasKey("opsPerSec"): + result.add(BenchResult( + name: item["name"].getStr(), + ops: if item.hasKey("ops"): item["ops"].getInt() else: 0, + seconds: if item.hasKey("seconds"): item["seconds"].getFloat() else: 0.0, + opsPerSec: item["opsPerSec"].getFloat(), + timestamp: if item.hasKey("timestamp"): item["timestamp"].getStr() else: "", + )) + except: + discard + +proc saveResults(path: string, report: BenchReport) = + var j = newJObject() + j["version"] = %report.version + j["gitSha"] = %report.gitSha + var arr = newJArray() + for r in report.results: + var obj = newJObject() + obj["name"] = %r.name + obj["ops"] = %r.ops + obj["seconds"] = %r.seconds + obj["opsPerSec"] = %r.opsPerSec + obj["timestamp"] = %r.timestamp + arr.add(obj) + j["results"] = arr + writeFile(path, j.pretty()) + +proc compareResult(name: string, currentOpsPerSec: float64, previous: seq[BenchResult]): string = + for p in previous: + if p.name == name: + let delta = currentOpsPerSec - p.opsPerSec + let pct = if p.opsPerSec > 0: (delta / p.opsPerSec) * 100.0 else: 0.0 + if abs(pct) < 1.0: + return " (≈ same)" + elif pct > 0: + return " (+" & pct.formatFloat(ffDecimal, 1) & "% vs last)" + else: + return " (" & pct.formatFloat(ffDecimal, 1) & "% vs last)" + return " (new)" + +var currentResults: seq[BenchResult] = @[] +var previousResults = loadPreviousResults(ResultsFile) + +proc recordResult(name: string, ops: int, seconds: float64) = + currentResults.add(BenchResult( + name: name, + ops: ops, + seconds: seconds, + opsPerSec: if seconds > 0: float64(ops) / seconds else: 0.0, + timestamp: now().format("yyyy-MM-dd HH:mm:ss"), + )) + +proc gitSha(): string = + let (output, code) = execCmdEx("git rev-parse --short HEAD") + if code == 0: + return output.strip() + return "unknown" + proc elapsed(start: MonoTime): float64 = let ns = float64((getMonoTime() - start).inNanoseconds) return ns / 1_000_000_000.0 @@ -26,7 +112,9 @@ proc formatOps(ops: int, secs: float64): string = proc benchLSMTree() = echo "=== LSM-Tree Storage ===" - var db = newLSMTree("/tmp/baradb_bench_lsm") + let benchDir = getTempDir() / "baradb_bench_lsm" + removeDir(benchDir) + var db = newLSMTree(benchDir) # Write benchmark let n = 100_000 @@ -34,7 +122,9 @@ proc benchLSMTree() = for i in 0.. 0: inc found let getTime = elapsed(getStart) - echo " Get ", n, " keys: ", getTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, getTime), ") (", found, " found)" + let getLabel = "BTree-Get" + recordResult(getLabel, n, getTime) + echo " Get ", n, " keys: ", getTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, getTime), ") (", found, " found)", compareResult(getLabel, currentResults[^1].opsPerSec, previousResults) # Scan benchmark let scanStart = getMonoTime() let scanResults = btree.scan("key_1000", "key_2000") let scanTime = elapsed(scanStart) - echo " Scan 1000 range: ", scanTime.formatFloat(ffDecimal, 6), "s (", scanResults.len, " results)" + let scanLabel = "BTree-Scan" + recordResult(scanLabel, scanResults.len, scanTime) + echo " Scan 1000 range: ", scanTime.formatFloat(ffDecimal, 6), "s (", scanResults.len, " results)", compareResult(scanLabel, currentResults[^1].opsPerSec, previousResults) proc benchVectorSearch() = echo "=== Vector Engine (HNSW) ===" @@ -89,7 +187,9 @@ proc benchVectorSearch() = vec[d] = rand(1.0) vengine.insert(idx, uint64(i), vec) let insertTime = elapsed(start) - echo " Insert ", n, " vectors (dim=", dim, "): ", insertTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, insertTime), ")" + let insertLabel = "HNSW-Insert" + recordResult(insertLabel, n, insertTime) + echo " Insert ", n, " vectors (dim=", dim, "): ", insertTime.formatFloat(ffDecimal, 3), "s (", formatOps(n, insertTime), ")", compareResult(insertLabel, currentResults[^1].opsPerSec, previousResults) # Search benchmark var query = newSeq[float32](dim) @@ -99,7 +199,9 @@ proc benchVectorSearch() = let searchStart = getMonoTime() let results = vengine.search(idx, query, 10) let searchTime = elapsed(searchStart) - echo " Search top-10: ", (searchTime * 1000).formatFloat(ffDecimal, 3), "ms" + let searchLabel = "HNSW-Search" + recordResult(searchLabel, 1, searchTime) + echo " Search top-10: ", (searchTime * 1000).formatFloat(ffDecimal, 3), "ms", compareResult(searchLabel, currentResults[^1].opsPerSec, previousResults) proc benchVectorSIMD() = echo "=== Vector SIMD Operations ===" @@ -122,21 +224,27 @@ proc benchVectorSIMD() = for i in 0..= 50 + check k <= 100 + check inserted[k] + + test "B-Tree contains after insert": + var rng = initRand(1004) + var btree = newBTreeIndex[int, string](order = 8) + var keys: seq[int] = @[] + for i in 0..<100: + let k = randKey(rng) + btree.insert(k, "v") + keys.add(k) + for k in keys: + check btree.contains(k) + + test "B-Tree remove decreases size": + var rng = initRand(1005) + var btree = newBTreeIndex[int, string](order = 8) + var inserted = initTable[int, seq[string]]() + for i in 0..<200: + let k = randKey(rng, 0, 100) + let v = "v" & $i + btree.insert(k, v) + if k notin inserted: + inserted[k] = @[] + inserted[k].add(v) + let beforeSize = btree.len + var removedCount = 0 + for k, vals in inserted: + if vals.len > 0: + btree.remove(k, vals[0]) + inc removedCount + # Size should decrease by number of keys that had values removed + # (if all values removed, key is deleted) + check btree.len <= beforeSize + + test "B-Tree with large order handles many inserts": + var rng = initRand(1006) + var btree = newBTreeIndex[int, string](order = 64) + for i in 0..<2000: + btree.insert(i, "v" & $i) + check btree.len == 2000 + for i in 0..<2000: + check btree.contains(i) + + test "B-Tree duplicate inserts append values": + var rng = initRand(1007) + var btree = newBTreeIndex[int, string](order = 8) + let k = 42 + for i in 0..<50: + btree.insert(k, "v" & $i) + let vals = btree.get(k) + check vals.len == 50 + for i in 0..<50: + check vals[i] == "v" & $i + + test "B-Tree scan on empty tree returns empty": + var btree = newBTreeIndex[int, string]() + let result = btree.scan(0, 100) + check result.len == 0 + + test "B-Tree random interleaved insert/remove maintains invariants": + var rng = initRand(1008) + var btree = newBTreeIndex[int, string](order = 8) + var tracker = initTable[int, seq[string]]() + for i in 0..<300: + let op = rng.rand(0..2) + let k = randKey(rng, 0, 50) + case op + of 0, 1: # insert + let v = "v" & $i + btree.insert(k, v) + if k notin tracker: + tracker[k] = @[] + tracker[k].add(v) + of 2: # remove + if k in tracker and tracker[k].len > 0: + let v = tracker[k][0] + btree.remove(k, v) + tracker[k].del(0) + if tracker[k].len == 0: + tracker.del(k) + else: discard + # Verify all tracked keys are present + for k, vals in tracker: + let got = btree.get(k) + check got == vals