perf: optimize FTS and HNSW engines + real PostgreSQL benchmarks
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
FTS Engine (src/barabadb/fts/engine.nim): - Fix bm25Score doing O(n) linear scan per document - Cache IDF per token instead of recomputing for each doc - Use entry.termFreq directly instead of searching postings again - Result: FTS search +438% (249 -> 1360 queries/s) HNSW Vector Engine (src/barabadb/vector/engine.nim): - Optimize distance functions with float32 + 4x loop unrolling - Rewrite searchLayer: swap+pop instead of O(n) del, track worst-nearest instead of sorting nearest on every iteration - Result: HNSW insert +117% (245 -> 543 ops/s), search 2.2x faster Benchmarks: - Add real PostgreSQL comparison script (benchmarks/pg_bench.py) - Add report generator (benchmarks/generate_report.py) - Fix compare.nim cpuTime() bug (was dividing by 1M incorrectly) - Add nimble tasks: bench_pg, bench_report Docs: - Update README.md and docs/en/performance.md with real measured numbers - Add benchmarks/REAL_COMPARISON.md Version bump: 1.1.7 -> 1.1.8
This commit is contained in:
@@ -185,14 +185,12 @@ proc bm25ScoreUnsafe(idx: InvertedIndex, term: string, docId: uint64,
|
||||
return 0.0
|
||||
|
||||
var tf = 0
|
||||
var found = false
|
||||
for entry in idx.postings[term]:
|
||||
if entry.docId == docId:
|
||||
tf = entry.termFreq
|
||||
found = true
|
||||
break
|
||||
|
||||
if not found:
|
||||
if tf == 0:
|
||||
return 0.0
|
||||
|
||||
let idf = ln((float64(n) - float64(df) + 0.5) / (float64(df) + 0.5) + 1.0)
|
||||
@@ -201,6 +199,17 @@ proc bm25ScoreUnsafe(idx: InvertedIndex, term: string, docId: uint64,
|
||||
(float64(tf) + k1 * (1.0 - b + b * docLen / idx.avgDocLen))
|
||||
return idf * tfNorm
|
||||
|
||||
# Optimized BM25 score when tf is already known (avoids linear scan)
|
||||
proc bm25ScoreUnsafeTf(idx: InvertedIndex, term: string, docId: uint64,
|
||||
tf: int, idf: float64,
|
||||
k1: float64 = 1.2, b: float64 = 0.75): float64 =
|
||||
if tf == 0 or idx.docCount == 0:
|
||||
return 0.0
|
||||
let docLen = float64(idx.docLengths.getOrDefault(docId, 0))
|
||||
let tfNorm = (float64(tf) * (k1 + 1.0)) /
|
||||
(float64(tf) + k1 * (1.0 - b + b * docLen / idx.avgDocLen))
|
||||
return idf * tfNorm
|
||||
|
||||
proc bm25Score*(idx: InvertedIndex, term: string, docId: uint64,
|
||||
k1: float64 = 1.2, b: float64 = 0.75): float64 =
|
||||
acquire(idx.lock)
|
||||
@@ -223,16 +232,22 @@ proc search*(idx: InvertedIndex, query: string, limit: int = 10,
|
||||
for token in queryTokens:
|
||||
if token notin idx.postings:
|
||||
continue
|
||||
for entry in idx.postings[token]:
|
||||
let score = bm25ScoreUnsafe(idx, token, entry.docId)
|
||||
let postings = idx.postings[token]
|
||||
let df = postings.len
|
||||
let n = idx.docCount
|
||||
if df == 0 or n == 0:
|
||||
continue
|
||||
let idf = ln((float64(n) - float64(df) + 0.5) / (float64(df) + 0.5) + 1.0)
|
||||
for entry in postings:
|
||||
let score = bm25ScoreUnsafeTf(idx, token, entry.docId, entry.termFreq, idf)
|
||||
if entry.docId notin docScores:
|
||||
docScores[entry.docId] = 0.0
|
||||
docHighlights[entry.docId] = @[]
|
||||
docScores[entry.docId] += score
|
||||
for pos in entry.positions:
|
||||
let start = pos
|
||||
let stop = pos + token.len
|
||||
docHighlights[entry.docId].add((start, stop))
|
||||
# Only add highlights if we have positions (skip for performance if empty)
|
||||
if entry.positions.len > 0:
|
||||
for pos in entry.positions:
|
||||
docHighlights[entry.docId].add((pos, pos + token.len))
|
||||
|
||||
var results: seq[SearchResult] = @[]
|
||||
for docId, score in docScores:
|
||||
|
||||
Reference in New Issue
Block a user