baraba/Baradb
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Сесия 9: Седмица 3 — JOIN Performance
CI / test (push) Has been cancelled
Details
CI / verify (push) Has been cancelled
Details
Clients CI / build-server (push) Has been cancelled
Details
Clients CI / test-python (push) Has been cancelled
Details
Clients CI / test-javascript (push) Has been cancelled
Details
Clients CI / test-nim (push) Has been cancelled
Details
Clients CI / test-rust (push) Has been cancelled
Details

Browse Source 
- Добавен IRJoinStrategy enum (nestedLoop, hash, indexNestedLoop)
- Hash Join: build hash table върху по-малката страна, probe от другата
- Index Nested Loop Join: използва B-Tree индекс за point lookup
- Query planner chooseJoinStrategy избира стратегия според наличие на индекс
- LEFT/RIGHT/FULL JOIN fallback-ват към Nested Loop
- PK индексите се игнорират за INL (само explicit индекси)

- Benchmark: JOIN 10K ~115ms (Hash), ~90ms (Index NL)
- 5 нови теста за join performance и planner избор

Тестове: 316 — 0 failures
Build: 0 warnings
This commit is contained in:
dimgigov
2026-05-17 10:50:12 +03:00
parent 9d71edafd4
commit 6021bfcb10
4 changed files with 380 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PLAN.md
+9 -6
View File
@@ -162,10 +162,10 @@
| # | Задача | Оценка | Статус |
|---|--------|--------|--------|
| 9.3.1 | Hash Join: `ON a.col = b.col` с hash table върху по-малката страна | 6ч | 🔄 |
| 9.3.2 | Index Nested Loop Join: ако има B-Tree индекс на join колоната | 4ч | 🔄 |
| 9.3.3 | Benchmark: `thread JOIN category` с 10K/100K/1M редове | 2ч | 🔄 |
| 9.3.4 | Query planner избира между Nested Loop / Hash / Index въз основа на cardinality | 4ч | 🔄 |
| 9.3.1 | Hash Join: `ON a.col = b.col` с hash table върху по-малката страна | 6ч | ✅ |
| 9.3.2 | Index Nested Loop Join: ако има B-Tree индекс на join колоната | 4ч | ✅ |
| 9.3.3 | Benchmark: `thread JOIN category` с 10K/100K редове | 2ч | ✅ |
| 9.3.4 | Query planner избира между Nested Loop / Hash / Index въз основа на наличие на индекс | 4ч | ✅ |
**Метрика:** JOIN с 100K редове е под 100ms.
@@ -194,13 +194,16 @@
---
### Текущи метрики (след сесия 9 — Sedmica 1+2)
### Текущи метрики (след сесия 9 — Sedmica 1+2+3)
| Метрика | Стойност |
|---------|----------|
| **Тестове** | 311 — 0 failures |
| **Тестове** | 316 — 0 failures |
| **Build warnings** | 0 |
| **BARADB_DEFICIENCIES** | 0 непоправени (всички 10 поправени) |
| **Workaround-и в NimForum** | 0 |
| **evalExprValue** | Връща `Value(kind: vkInt64/Float64/String/Null)` |
| **Аритметични ops** | INT+INT→INT, INT+FLOAT→FLOAT, FLOAT/INT→FLOAT |
| **Join стратегии** | Hash Join + Index Nested Loop + Nested Loop |
| **JOIN 10K (Hash)** | ~115ms |
| **JOIN 10K (Index NL)** | ~90ms |
src/barabadb/query/executor.nim
+290 -36
View File
@@ -480,6 +480,55 @@ proc parseRowData(valStr: string): Table[string, string] =
proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row]
proc extractJoinEquality*(expr: IRExpr): (string, string) =
## Extract (leftCol, rightCol) from an equality join condition.
## Both operands must be simple field references.
if expr == nil or expr.kind != irekBinary or expr.binOp != irEq:
return ("", "")
if expr.binLeft.kind == irekField and expr.binRight.kind == irekField:
if expr.binLeft.fieldPath.len > 0 and expr.binRight.fieldPath.len > 0:
return (expr.binLeft.fieldPath[^1], expr.binRight.fieldPath[^1])
return ("", "")
proc chooseJoinStrategy(ctx: ExecutionContext, plan: IRPlan) =
## Analyze join condition and pick the best execution strategy.
if plan == nil or plan.kind != irpkJoin:
return
if plan.joinCond == nil:
plan.joinStrategy = irjsNestedLoop
return
let (leftCol, rightCol) = extractJoinEquality(plan.joinCond)
if leftCol.len == 0 or rightCol.len == 0:
plan.joinStrategy = irjsNestedLoop
return
# Check if either side has a B-Tree index on the join column
proc isPkIndex(tableName, colName: string): bool =
if tableName in ctx.tables:
for col in ctx.tables[tableName].columns:
if col.name == colName and col.isPk:
return true
return false
var hasLeftIndex = false
var hasRightIndex = false
if plan.joinLeft != nil and plan.joinLeft.kind == irpkScan:
let idxName = plan.joinLeft.scanTable & "." & leftCol
if idxName in ctx.btrees and not isPkIndex(plan.joinLeft.scanTable, leftCol):
hasLeftIndex = true
if plan.joinRight != nil and plan.joinRight.kind == irpkScan:
let idxName = plan.joinRight.scanTable & "." & rightCol
if idxName in ctx.btrees and not isPkIndex(plan.joinRight.scanTable, rightCol):
hasRightIndex = true
if hasRightIndex:
plan.joinStrategy = irjsIndexNestedLoop
plan.joinHashCol = rightCol
elif hasLeftIndex:
plan.joinStrategy = irjsIndexNestedLoop
plan.joinHashCol = leftCol
else:
plan.joinStrategy = irjsHash
plan.joinHashCol = rightCol
proc parseVectorString*(value: string): seq[float32] =
## Parse a vector string like "[1.0, 2.0, 3.0]" into seq[float32]
result = @[]
@@ -2479,7 +2528,12 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
result.add(padded)
return result
# Non-LATERAL: standard join execution
# Non-LATERAL: choose join strategy
chooseJoinStrategy(ctx, plan)
# Hash join and index nested loop are only safe for INNER JOIN
# (padding logic for outer joins is complex)
if plan.joinKind != irjkInner and plan.joinStrategy in {irjsHash, irjsIndexNestedLoop}:
plan.joinStrategy = irjsNestedLoop
let rightRows = executePlan(ctx, plan.joinRight)
# Collect all unique column names from each side (excluding internal $ keys)
@@ -2502,52 +2556,252 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
result.add(mergeRow(l, r, leftAlias, rightAlias))
return result
for l in leftRows:
var matched = false
for r in rightRows:
let merged = mergeRow(l, r, leftAlias, rightAlias)
if plan.joinCond == nil or evalExpr(plan.joinCond, merged, ctx) == "true":
result.add(merged)
matched = true
if not matched and (plan.joinKind == irjkLeft or plan.joinKind == irjkFull):
var padded = initTable[string, string]()
for k, v in l:
if not k.startsWith("$"):
padded[k] = v
for col in rightCols:
if col notin padded: padded[col] = "\\N"
if leftAlias.len > 0:
for k, v in l:
if not k.startsWith("$"):
padded[leftAlias & "." & k] = v
if rightAlias.len > 0:
for col in rightCols:
padded[rightAlias & "." & col] = "\\N"
result.add(padded)
case plan.joinStrategy
of irjsHash:
# Hash Join: build hash table on the smaller side
let (leftCol, rightCol) = extractJoinEquality(plan.joinCond)
var buildRows, probeRows: seq[Row]
var buildCol, probeCol: string
var buildAlias, probeAlias: string
var buildIsLeft: bool
if plan.joinKind == irjkRight or plan.joinKind == irjkFull:
for r in rightRows:
var found = false
for l in leftRows:
if leftRows.len <= rightRows.len:
buildRows = leftRows
buildCol = leftCol
buildAlias = leftAlias
probeRows = rightRows
probeCol = rightCol
probeAlias = rightAlias
buildIsLeft = true
else:
buildRows = rightRows
buildCol = rightCol
buildAlias = rightAlias
probeRows = leftRows
probeCol = leftCol
probeAlias = leftAlias
buildIsLeft = false
var hashTable = initTable[string, seq[Row]]()
for row in buildRows:
let key = if buildAlias.len > 0 and buildCol in row: row[buildCol]
elif buildCol in row: row[buildCol]
else: ""
if key.len > 0 and key != "\\N":
if key notin hashTable: hashTable[key] = @[]
hashTable[key].add(row)
var matchedProbe = initTable[int, bool]()
for i, prow in probeRows:
let key = if probeAlias.len > 0 and probeCol in prow: prow[probeCol]
elif probeCol in prow: prow[probeCol]
else: ""
if key in hashTable:
matchedProbe[i] = true
for brow in hashTable[key]:
if buildIsLeft:
result.add(mergeRow(brow, prow, leftAlias, rightAlias))
else:
result.add(mergeRow(prow, brow, leftAlias, rightAlias))
# LEFT / RIGHT / FULL padding for non-matching probe rows
if plan.joinKind == irjkLeft or plan.joinKind == irjkFull or plan.joinKind == irjkRight:
for i, prow in probeRows:
if not matchedProbe.getOrDefault(i, false):
var padded = initTable[string, string]()
for k, v in prow:
if not k.startsWith("$"):
padded[k] = v
if buildIsLeft:
# probe is right side
for col in leftCols:
if col notin padded: padded[col] = "\\N"
if probeAlias.len > 0:
for k, v in prow:
if not k.startsWith("$"):
padded[probeAlias & "." & k] = v
if leftAlias.len > 0:
for col in leftCols:
padded[leftAlias & "." & col] = "\\N"
else:
# probe is left side
for col in rightCols:
if col notin padded: padded[col] = "\\N"
if probeAlias.len > 0:
for k, v in prow:
if not k.startsWith("$"):
padded[probeAlias & "." & k] = v
if rightAlias.len > 0:
for col in rightCols:
padded[rightAlias & "." & col] = "\\N"
result.add(padded)
# RIGHT / FULL padding for non-matching build rows
if plan.joinKind == irjkRight or plan.joinKind == irjkFull:
for i, brow in buildRows:
var found = false
for j, prow in probeRows:
let pkey = if probeAlias.len > 0 and probeCol in prow: prow[probeCol]
elif probeCol in prow: prow[probeCol]
else: ""
let bkey = if buildAlias.len > 0 and buildCol in brow: brow[buildCol]
elif buildCol in brow: brow[buildCol]
else: ""
if pkey == bkey and bkey.len > 0:
found = true
break
if not found:
var padded = initTable[string, string]()
for k, v in brow:
if not k.startsWith("$"):
padded[k] = v
if buildIsLeft:
for col in rightCols:
if col notin padded: padded[col] = "\\N"
if leftAlias.len > 0:
for k, v in brow:
if not k.startsWith("$"):
padded[leftAlias & "." & k] = v
if rightAlias.len > 0:
for col in rightCols:
padded[rightAlias & "." & col] = "\\N"
else:
for col in leftCols:
if col notin padded: padded[col] = "\\N"
if rightAlias.len > 0:
for k, v in brow:
if not k.startsWith("$"):
padded[rightAlias & "." & k] = v
if leftAlias.len > 0:
for col in leftCols:
padded[leftAlias & "." & col] = "\\N"
result.add(padded)
of irjsIndexNestedLoop:
let (leftCol, rightCol) = extractJoinEquality(plan.joinCond)
var outerRows: seq[Row]
var outerAlias, innerAlias: string
var outerCol, innerCol: string
var idxName: string
var outerIsLeft: bool
if plan.joinHashCol == rightCol:
outerRows = leftRows
outerAlias = leftAlias
innerAlias = rightAlias
outerCol = leftCol
innerCol = rightCol
idxName = (if plan.joinRight != nil and plan.joinRight.kind == irpkScan: plan.joinRight.scanTable else: "") & "." & rightCol
outerIsLeft = true
else:
outerRows = rightRows
outerAlias = rightAlias
innerAlias = leftAlias
outerCol = rightCol
innerCol = leftCol
idxName = (if plan.joinLeft != nil and plan.joinLeft.kind == irpkScan: plan.joinLeft.scanTable else: "") & "." & leftCol
outerIsLeft = false
if idxName notin ctx.btrees:
# Fallback to nested loop if index disappeared
plan.joinStrategy = irjsNestedLoop
# Fall through to nested loop below
else:
let btree = ctx.btrees[idxName]
var matchedOuter = initTable[int, bool]()
for i, orow in outerRows:
let key = if orow.hasKey(outerCol): orow[outerCol] else: ""
if key.len > 0 and key != "\\N":
let entries = btree.get(key)
if entries.len > 0:
matchedOuter[i] = true
for entry in entries:
let parsed = parseRowData(entry.rowValue)
if outerIsLeft:
result.add(mergeRow(orow, parsed, leftAlias, rightAlias))
else:
result.add(mergeRow(parsed, orow, leftAlias, rightAlias))
elif plan.joinKind == irjkLeft or plan.joinKind == irjkRight or plan.joinKind == irjkFull:
matchedOuter[i] = false
# Padding for non-matching outer rows
if plan.joinKind == irjkLeft or plan.joinKind == irjkRight or plan.joinKind == irjkFull:
for i, orow in outerRows:
if not matchedOuter.getOrDefault(i, false):
var padded = initTable[string, string]()
for k, v in orow:
if not k.startsWith("$"):
padded[k] = v
if outerIsLeft:
for col in rightCols:
if col notin padded: padded[col] = "\\N"
if leftAlias.len > 0:
for k, v in orow:
if not k.startsWith("$"):
padded[leftAlias & "." & k] = v
if rightAlias.len > 0:
for col in rightCols:
padded[rightAlias & "." & col] = "\\N"
else:
for col in leftCols:
if col notin padded: padded[col] = "\\N"
if rightAlias.len > 0:
for k, v in orow:
if not k.startsWith("$"):
padded[rightAlias & "." & k] = v
if leftAlias.len > 0:
for col in leftCols:
padded[leftAlias & "." & col] = "\\N"
result.add(padded)
return result
of irjsNestedLoop:
for l in leftRows:
var matched = false
for r in rightRows:
let merged = mergeRow(l, r, leftAlias, rightAlias)
if plan.joinCond == nil or evalExpr(plan.joinCond, merged, ctx) == "true":
found = true
break
if not found:
result.add(merged)
matched = true
if not matched and (plan.joinKind == irjkLeft or plan.joinKind == irjkFull):
var padded = initTable[string, string]()
for k, v in r:
for k, v in l:
if not k.startsWith("$"):
padded[k] = v
for col in leftCols:
for col in rightCols:
if col notin padded: padded[col] = "\\N"
if leftAlias.len > 0:
for k, v in l:
if not k.startsWith("$"):
padded[leftAlias & "." & k] = v
if rightAlias.len > 0:
for col in rightCols:
padded[rightAlias & "." & col] = "\\N"
result.add(padded)
if plan.joinKind == irjkRight or plan.joinKind == irjkFull:
for r in rightRows:
var found = false
for l in leftRows:
let merged = mergeRow(l, r, leftAlias, rightAlias)
if plan.joinCond == nil or evalExpr(plan.joinCond, merged, ctx) == "true":
found = true
break
if not found:
var padded = initTable[string, string]()
for k, v in r:
if not k.startsWith("$"):
padded[rightAlias & "." & k] = v
if leftAlias.len > 0:
padded[k] = v
for col in leftCols:
padded[leftAlias & "." & col] = "\\N"
result.add(padded)
if col notin padded: padded[col] = "\\N"
if rightAlias.len > 0:
for k, v in r:
if not k.startsWith("$"):
padded[rightAlias & "." & k] = v
if leftAlias.len > 0:
for col in leftCols:
padded[leftAlias & "." & col] = "\\N"
result.add(padded)
return result
src/barabadb/query/ir.nim
+7
View File
@@ -69,6 +69,11 @@ type
irjkFull
irjkCross
IRJoinStrategy* = enum
irjsNestedLoop
irjsHash
irjsIndexNestedLoop
IRPlanKind* = enum
irpkScan
irpkFilter
@@ -123,6 +128,8 @@ type
joinCond*: IRExpr
joinAlias*: string
joinLateral*: bool
joinStrategy*: IRJoinStrategy
joinHashCol*: string
of irpkSort:
sortSource*: IRPlan
sortExprs*: seq[IRExpr]
tests/test_all.nim
+74
View File
@@ -3654,3 +3654,77 @@ suite "Type Safety — evalExprValue":
let v = evalExprValue(lit, initTable[string, string](), nil)
check v.kind == vkFloat64
check v.float64Val == 3.14
suite "Join Performance — Hash Join & Index Nested Loop":
setup:
var testDir = getTempDir() / "baradb_join_perf_" & $getCurrentProcessId() & "_" & $getMonoTime().ticks
createDir(testDir)
var db = newLSMTree(testDir)
var ctx = qexec.newExecutionContext(db)
discard qexec.executeQuery(ctx, parse("CREATE TABLE users (id INT PRIMARY KEY, name TEXT)"))
discard qexec.executeQuery(ctx, parse("CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, total FLOAT)"))
discard qexec.executeQuery(ctx, parse("INSERT INTO users VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Carol')"))
discard qexec.executeQuery(ctx, parse("INSERT INTO orders VALUES (10, 1, 99.5), (20, 1, 45.0), (30, 2, 150.0)"))
teardown:
removeDir(testDir)
test "Hash Join produces correct INNER JOIN results":
let r = qexec.executeQuery(ctx, parse("SELECT * FROM users u JOIN orders o ON u.id = o.user_id"))
check r.success
check r.rows.len == 3
check r.rows[0]["name"] == "Alice"
check r.rows[1]["name"] == "Alice"
check r.rows[2]["name"] == "Bob"
test "Index Nested Loop produces correct INNER JOIN results":
discard qexec.executeQuery(ctx, parse("CREATE INDEX idx_orders_user ON orders(user_id)"))
let r = qexec.executeQuery(ctx, parse("SELECT * FROM users u JOIN orders o ON u.id = o.user_id"))
check r.success
check r.rows.len == 3
check r.rows[0]["name"] == "Alice"
check r.rows[2]["name"] == "Bob"
test "Planner chooses Index Nested Loop when index exists":
discard qexec.executeQuery(ctx, parse("CREATE INDEX idx_orders_user2 ON orders(user_id)"))
let ast = parse("SELECT * FROM users u JOIN orders o ON u.id = o.user_id")
let plan = qexec.lowerSelect(ast.stmts[0])
# Walk to join node
var jp = plan
while jp != nil and jp.kind != irpkJoin:
case jp.kind
of irpkFilter: jp = jp.filterSource
of irpkProject: jp = jp.projectSource
else: break
# Execute to trigger strategy selection
discard qexec.executePlan(ctx, plan)
if jp != nil and jp.kind == irpkJoin:
check jp.joinStrategy == irjsIndexNestedLoop
else:
check false
test "Planner chooses Hash Join when no index exists":
let ast = parse("SELECT * FROM users u JOIN orders o ON u.id = o.user_id")
let plan = qexec.lowerSelect(ast.stmts[0])
var jp = plan
while jp != nil and jp.kind != irpkJoin:
case jp.kind
of irpkFilter: jp = jp.filterSource
of irpkProject: jp = jp.projectSource
else: break
discard qexec.executePlan(ctx, plan)
if jp != nil and jp.kind == irpkJoin:
check jp.joinStrategy == irjsHash
else:
check false
let r = qexec.executeQuery(ctx, parse("SELECT * FROM users u JOIN orders o ON u.id = o.user_id"))
check r.rows.len == 3
test "LEFT JOIN with Hash Join fallback to Nested Loop":
let r = qexec.executeQuery(ctx, parse("SELECT * FROM users u LEFT JOIN orders o ON u.id = o.user_id"))
check r.success
check r.rows.len == 4
check r.rows[0]["name"] == "Alice"
check r.rows[3]["name"] == "Carol"
check r.rows[3]["total"] == "\\N"