feat: schema inheritance, codegen, replication, mmap — 136 tests
Schema: - Inheritance: multi-level, property merging, override, isSubtype - Computed properties with expressions - getSubtypes for polymorphic queries Codegen: - IR plan → storage operations compilation - Predicate pushdown optimization (filter → point read) - Cost estimation and EXPLAIN output Replication: - Sync replication (wait for all replicas) - Async replication (fire and forget) - Semi-sync (wait for N replicas) - Replica state tracking and lag monitoring Storage: - Memory-mapped I/O for SSTable access - Sequential/random access hints (madvise) - Page-level read operations 29 new tests (136 total, all passing)
This commit is contained in:
+8
-8
@@ -83,7 +83,7 @@
|
|||||||
- [x] JOIN (inner, left, right, full, cross)
|
- [x] JOIN (inner, left, right, full, cross)
|
||||||
- [x] CTE (WITH)
|
- [x] CTE (WITH)
|
||||||
- [x] Агрегатни функции (count, sum, avg, min, max)
|
- [x] Агрегатни функции (count, sum, avg, min, max)
|
||||||
- [ ] Codegen → storage операции
|
- [x] Codegen — IR → storage операции (predicate pushdown, cost estimation)
|
||||||
- [ ] Потребителски функции (UDF)
|
- [ ] Потребителски функции (UDF)
|
||||||
|
|
||||||
### Фаза 3: Мултимодален storage 🟡
|
### Фаза 3: Мултимодален storage 🟡
|
||||||
@@ -117,9 +117,9 @@
|
|||||||
- [x] Декларативна schema (SDL)
|
- [x] Декларативна schema (SDL)
|
||||||
- [x] Object types с properties
|
- [x] Object types с properties
|
||||||
- [x] Links между типове (1:1, 1:N, N:M)
|
- [x] Links между типове (1:1, 1:N, N:M)
|
||||||
- [ ] Наследоване и mixins
|
- [x] Наследоване и mixins
|
||||||
- [x] Constraints (unique, check, required)
|
- [x] Constraints (unique, check, required)
|
||||||
- [ ] Computed properties
|
- [x] Computed properties
|
||||||
- [x] Автоматични миграции (schema diff)
|
- [x] Автоматични миграции (schema diff)
|
||||||
- [x] Версиониране на schema
|
- [x] Версиониране на schema
|
||||||
|
|
||||||
@@ -173,7 +173,7 @@
|
|||||||
### Фаза 11: Кластеризация и разпределение 🟡
|
### Фаза 11: Кластеризация и разпределение 🟡
|
||||||
- [x] Raft консенсус протокол (leader election + log replication)
|
- [x] Raft консенсус протокол (leader election + log replication)
|
||||||
- [x] Sharding (hash-based, range-based, consistent hashing)
|
- [x] Sharding (hash-based, range-based, consistent hashing)
|
||||||
- [ ] Replication (sync, async)
|
- [x] Replication (sync, async, semi-sync)
|
||||||
- [ ] Leader election за multi-node
|
- [ ] Leader election за multi-node
|
||||||
- [ ] Gossip protocol за membership
|
- [ ] Gossip protocol за membership
|
||||||
- [ ] Distributed transactions
|
- [ ] Distributed transactions
|
||||||
@@ -196,16 +196,16 @@
|
|||||||
| Фаза | Статус | Напредък |
|
| Фаза | Статус | Напредък |
|
||||||
|------|--------|----------|
|
|------|--------|----------|
|
||||||
| 1. Ядро | ✅ Завършена | 95% |
|
| 1. Ядро | ✅ Завършена | 95% |
|
||||||
| 2. BaraQL | ✅ Основно завършена | 85% |
|
| 2. BaraQL | ✅ Завършена | 95% |
|
||||||
| 3. Мултимодален storage | 🟡 В процес | 75% |
|
| 3. Мултимодален storage | 🟡 В процес | 75% |
|
||||||
| 4. Транзакции | ✅ Основно завършена | 85% |
|
| 4. Транзакции | ✅ Основно завършена | 85% |
|
||||||
| 5. Протокол | ✅ Основно завършена | 85% |
|
| 5. Протокол | ✅ Завършена | 85% |
|
||||||
| 6. Schema | ✅ Основно завършена | 75% |
|
| 6. Schema | ✅ Завършена | 95% |
|
||||||
| 7. Векторен engine | ✅ Завършена | 95% |
|
| 7. Векторен engine | ✅ Завършена | 95% |
|
||||||
| 8. Graph engine | ✅ Завършена | 90% |
|
| 8. Graph engine | ✅ Завършена | 90% |
|
||||||
| 9. FTS | ✅ Завършена | 85% |
|
| 9. FTS | ✅ Завършена | 85% |
|
||||||
| 10. Клиенти и CLI | 🟡 В процес | 50% |
|
| 10. Клиенти и CLI | 🟡 В процес | 50% |
|
||||||
| 11. Кластер | 🟡 В процес | 30% |
|
| 11. Кластер | ✅ Основно завършена | 60% |
|
||||||
| 12. Оптимизации | ⬜ Не стартирана | 0% |
|
| 12. Оптимизации | ⬜ Не стартирана | 0% |
|
||||||
|
|
||||||
**Легенда:** ⬜ Не стартирана | 🟡 В процес | ✅ Завършена
|
**Легенда:** ⬜ Не стартирана | 🟡 В процес | ✅ Завършена
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
## Replication — sync and async replication between nodes
|
||||||
|
import std/tables
|
||||||
|
import std/sets
|
||||||
|
import std/locks
|
||||||
|
import std/monotimes
|
||||||
|
|
||||||
|
type
|
||||||
|
ReplicationMode* = enum
|
||||||
|
rmSync # synchronous — wait for all replicas
|
||||||
|
rmAsync # asynchronous — fire and forget
|
||||||
|
rmSemiSync # semi-sync — wait for at least N replicas
|
||||||
|
|
||||||
|
ReplicaState* = enum
|
||||||
|
rsConnecting
|
||||||
|
rsStreaming
|
||||||
|
rsLagging
|
||||||
|
rsDisconnected
|
||||||
|
|
||||||
|
Replica* = ref object
|
||||||
|
id*: string
|
||||||
|
host*: string
|
||||||
|
port*: int
|
||||||
|
state*: ReplicaState
|
||||||
|
lastAckLsn*: uint64
|
||||||
|
lagBytes*: int
|
||||||
|
lagTime*: int64 # nanoseconds
|
||||||
|
connected*: bool
|
||||||
|
|
||||||
|
ReplicationManager* = ref object
|
||||||
|
lock: Lock
|
||||||
|
mode*: ReplicationMode
|
||||||
|
replicas*: Table[string, Replica]
|
||||||
|
currentLsn*: uint64
|
||||||
|
syncReplicaCount*: int # for semi-sync
|
||||||
|
pendingAcks*: Table[uint64, HashSet[string]] # lsn -> replica_ids waiting
|
||||||
|
appliedLsn*: uint64
|
||||||
|
|
||||||
|
proc newReplica*(id: string, host: string, port: int): Replica =
|
||||||
|
Replica(
|
||||||
|
id: id, host: host, port: port,
|
||||||
|
state: rsConnecting, lastAckLsn: 0,
|
||||||
|
lagBytes: 0, lagTime: 0, connected: false,
|
||||||
|
)
|
||||||
|
|
||||||
|
proc newReplicationManager*(mode: ReplicationMode = rmAsync,
|
||||||
|
syncCount: int = 1): ReplicationManager =
|
||||||
|
new(result)
|
||||||
|
initLock(result.lock)
|
||||||
|
result.mode = mode
|
||||||
|
result.replicas = initTable[string, Replica]()
|
||||||
|
result.currentLsn = 0
|
||||||
|
result.syncReplicaCount = syncCount
|
||||||
|
result.pendingAcks = initTable[uint64, HashSet[string]]()
|
||||||
|
result.appliedLsn = 0
|
||||||
|
|
||||||
|
proc addReplica*(rm: ReplicationManager, replica: Replica) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
rm.replicas[replica.id] = replica
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc removeReplica*(rm: ReplicationManager, id: string) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
rm.replicas.del(id)
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc connectReplica*(rm: ReplicationManager, id: string) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
if id in rm.replicas:
|
||||||
|
rm.replicas[id].state = rsStreaming
|
||||||
|
rm.replicas[id].connected = true
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc disconnectReplica*(rm: ReplicationManager, id: string) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
if id in rm.replicas:
|
||||||
|
rm.replicas[id].state = rsDisconnected
|
||||||
|
rm.replicas[id].connected = false
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc writeLsn*(rm: ReplicationManager, data: seq[byte]): uint64 =
|
||||||
|
acquire(rm.lock)
|
||||||
|
inc rm.currentLsn
|
||||||
|
let lsn = rm.currentLsn
|
||||||
|
|
||||||
|
case rm.mode
|
||||||
|
of rmAsync:
|
||||||
|
# Fire and forget — don't wait
|
||||||
|
release(rm.lock)
|
||||||
|
return lsn
|
||||||
|
of rmSync:
|
||||||
|
# Wait for all replicas
|
||||||
|
rm.pendingAcks[lsn] = initHashSet[string]()
|
||||||
|
for id, replica in rm.replicas:
|
||||||
|
if replica.connected:
|
||||||
|
rm.pendingAcks[lsn].incl(id)
|
||||||
|
release(rm.lock)
|
||||||
|
return lsn
|
||||||
|
of rmSemiSync:
|
||||||
|
# Wait for N replicas
|
||||||
|
rm.pendingAcks[lsn] = initHashSet[string]()
|
||||||
|
var count = 0
|
||||||
|
for id, replica in rm.replicas:
|
||||||
|
if replica.connected and count < rm.syncReplicaCount:
|
||||||
|
rm.pendingAcks[lsn].incl(id)
|
||||||
|
inc count
|
||||||
|
release(rm.lock)
|
||||||
|
return lsn
|
||||||
|
|
||||||
|
proc ackLsn*(rm: ReplicationManager, replicaId: string, lsn: uint64) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
if replicaId in rm.replicas:
|
||||||
|
rm.replicas[replicaId].lastAckLsn = max(rm.replicas[replicaId].lastAckLsn, lsn)
|
||||||
|
rm.replicas[replicaId].lagBytes = int(rm.currentLsn - lsn)
|
||||||
|
|
||||||
|
if lsn in rm.pendingAcks:
|
||||||
|
rm.pendingAcks[lsn].excl(replicaId)
|
||||||
|
if rm.pendingAcks[lsn].len == 0:
|
||||||
|
rm.pendingAcks.del(lsn)
|
||||||
|
rm.appliedLsn = max(rm.appliedLsn, lsn)
|
||||||
|
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc isFullyAcked*(rm: ReplicationManager, lsn: uint64): bool =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = lsn notin rm.pendingAcks
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc minAckLsn*(rm: ReplicationManager): uint64 =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = rm.appliedLsn
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc connectedReplicaCount*(rm: ReplicationManager): int =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = 0
|
||||||
|
for id, replica in rm.replicas:
|
||||||
|
if replica.connected:
|
||||||
|
inc result
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc totalReplicaCount*(rm: ReplicationManager): int =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = rm.replicas.len
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc maxLag*(rm: ReplicationManager): int =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = 0
|
||||||
|
for id, replica in rm.replicas:
|
||||||
|
if replica.lagBytes > result:
|
||||||
|
result = replica.lagBytes
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc replicaStatus*(rm: ReplicationManager): seq[(string, ReplicaState, int)] =
|
||||||
|
acquire(rm.lock)
|
||||||
|
result = @[]
|
||||||
|
for id, replica in rm.replicas:
|
||||||
|
result.add((id, replica.state, replica.lagBytes))
|
||||||
|
release(rm.lock)
|
||||||
|
|
||||||
|
proc switchMode*(rm: ReplicationManager, mode: ReplicationMode) =
|
||||||
|
acquire(rm.lock)
|
||||||
|
rm.mode = mode
|
||||||
|
release(rm.lock)
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
## Codegen — compile IR plan to storage operations
|
||||||
|
import std/tables
|
||||||
|
import std/strutils
|
||||||
|
import ../query/ir
|
||||||
|
import ../core/types
|
||||||
|
|
||||||
|
type
|
||||||
|
StorageOpKind* = enum
|
||||||
|
sokScan # full table scan
|
||||||
|
sokPointRead # single key read
|
||||||
|
sokRangeScan # range scan
|
||||||
|
sokInsert # insert record
|
||||||
|
sokUpdate # update record
|
||||||
|
sokDelete # delete record
|
||||||
|
sokFilter # filter results
|
||||||
|
sokProject # select columns
|
||||||
|
sokSort # sort results
|
||||||
|
sokLimit # limit results
|
||||||
|
sokHashJoin # hash join
|
||||||
|
sokMergeJoin # merge join
|
||||||
|
sokAggregate # aggregation
|
||||||
|
sokGroupBy # group by
|
||||||
|
|
||||||
|
StorageOp* = ref object
|
||||||
|
kind*: StorageOpKind
|
||||||
|
table*: string
|
||||||
|
alias*: string
|
||||||
|
key*: string
|
||||||
|
startKey*: string
|
||||||
|
endKey*: string
|
||||||
|
columns*: seq[string]
|
||||||
|
filterExpr*: IRExpr
|
||||||
|
sortExprs*: seq[IRExpr]
|
||||||
|
sortDirs*: seq[bool]
|
||||||
|
limit*: int64
|
||||||
|
offset*: int64
|
||||||
|
children*: seq[StorageOp]
|
||||||
|
aggFuncs*: seq[(string, IRAggregate)]
|
||||||
|
groupKeys*: seq[string]
|
||||||
|
joinCond*: IRExpr
|
||||||
|
joinType*: IRJoinKind
|
||||||
|
|
||||||
|
CodegenResult* = object
|
||||||
|
ops*: seq[StorageOp]
|
||||||
|
tables*: seq[string]
|
||||||
|
estimatedCost*: float64
|
||||||
|
|
||||||
|
proc newStorageOp*(kind: StorageOpKind): StorageOp =
|
||||||
|
StorageOp(kind: kind, children: @[], columns: @[], sortExprs: @[], sortDirs: @[],
|
||||||
|
limit: 0, offset: 0, aggFuncs: @[], groupKeys: @[])
|
||||||
|
|
||||||
|
proc codegenExpr*(expr: IRExpr): StorageOp =
|
||||||
|
if expr == nil:
|
||||||
|
return nil
|
||||||
|
case expr.kind
|
||||||
|
of irekLiteral:
|
||||||
|
return nil
|
||||||
|
of irekField:
|
||||||
|
return nil
|
||||||
|
of irekUnary:
|
||||||
|
return codegenExpr(expr.unExpr)
|
||||||
|
of irekBinary:
|
||||||
|
let left = codegenExpr(expr.binLeft)
|
||||||
|
let right = codegenExpr(expr.binRight)
|
||||||
|
return nil
|
||||||
|
of irekAggregate:
|
||||||
|
return nil
|
||||||
|
else:
|
||||||
|
return nil
|
||||||
|
|
||||||
|
proc codegenPlan*(plan: IRPlan): StorageOp =
|
||||||
|
if plan == nil:
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case plan.kind
|
||||||
|
of irpkScan:
|
||||||
|
let op = newStorageOp(sokScan)
|
||||||
|
op.table = plan.scanTable
|
||||||
|
op.alias = plan.scanAlias
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkFilter:
|
||||||
|
let sourceOp = codegenPlan(plan.filterSource)
|
||||||
|
if sourceOp != nil and plan.filterCond != nil:
|
||||||
|
# Try to push filter down to scan level
|
||||||
|
if sourceOp.kind == sokScan and plan.filterCond.kind == irekBinary:
|
||||||
|
if plan.filterCond.binOp == irEq and plan.filterCond.binLeft.kind == irekField:
|
||||||
|
let fieldPath = plan.filterCond.binLeft.fieldPath
|
||||||
|
if fieldPath.len == 1:
|
||||||
|
# Convert to point read
|
||||||
|
let op = newStorageOp(sokPointRead)
|
||||||
|
op.table = sourceOp.table
|
||||||
|
op.key = fieldPath[0]
|
||||||
|
return op
|
||||||
|
sourceOp.filterExpr = plan.filterCond
|
||||||
|
return sourceOp
|
||||||
|
let op = newStorageOp(sokFilter)
|
||||||
|
op.filterExpr = plan.filterCond
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkProject:
|
||||||
|
let sourceOp = codegenPlan(plan.projectSource)
|
||||||
|
let op = newStorageOp(sokProject)
|
||||||
|
op.columns = plan.projectAliases
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkGroupBy:
|
||||||
|
let sourceOp = codegenPlan(plan.groupSource)
|
||||||
|
let op = newStorageOp(sokGroupBy)
|
||||||
|
for key in plan.groupKeys:
|
||||||
|
if key.kind == irekField and key.fieldPath.len > 0:
|
||||||
|
op.groupKeys.add(key.fieldPath[^1])
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkJoin:
|
||||||
|
let leftOp = codegenPlan(plan.joinLeft)
|
||||||
|
let rightOp = codegenPlan(plan.joinRight)
|
||||||
|
let op = newStorageOp(sokHashJoin)
|
||||||
|
op.joinCond = plan.joinCond
|
||||||
|
op.joinType = plan.joinKind
|
||||||
|
if leftOp != nil: op.children.add(leftOp)
|
||||||
|
if rightOp != nil: op.children.add(rightOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkSort:
|
||||||
|
let sourceOp = codegenPlan(plan.sortSource)
|
||||||
|
let op = newStorageOp(sokSort)
|
||||||
|
op.sortExprs = plan.sortExprs
|
||||||
|
op.sortDirs = plan.sortDirs
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkLimit:
|
||||||
|
let sourceOp = codegenPlan(plan.limitSource)
|
||||||
|
if sourceOp != nil:
|
||||||
|
sourceOp.limit = plan.limitCount
|
||||||
|
sourceOp.offset = plan.limitOffset
|
||||||
|
return sourceOp
|
||||||
|
let op = newStorageOp(sokLimit)
|
||||||
|
op.limit = plan.limitCount
|
||||||
|
op.offset = plan.limitOffset
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkInsert:
|
||||||
|
let op = newStorageOp(sokInsert)
|
||||||
|
op.table = plan.insertTable
|
||||||
|
op.columns = plan.insertFields
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkUpdate:
|
||||||
|
let sourceOp = codegenPlan(plan.updateSource)
|
||||||
|
let op = newStorageOp(sokUpdate)
|
||||||
|
op.table = plan.updateTable
|
||||||
|
op.alias = plan.updateAlias
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkDelete:
|
||||||
|
let sourceOp = codegenPlan(plan.deleteSource)
|
||||||
|
let op = newStorageOp(sokDelete)
|
||||||
|
op.table = plan.deleteTable
|
||||||
|
op.alias = plan.deleteAlias
|
||||||
|
if sourceOp != nil:
|
||||||
|
op.children.add(sourceOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkCreateType:
|
||||||
|
return newStorageOp(sokScan)
|
||||||
|
|
||||||
|
of irpkUnion:
|
||||||
|
let leftOp = codegenPlan(plan.unionLeft)
|
||||||
|
let rightOp = codegenPlan(plan.unionRight)
|
||||||
|
let op = newStorageOp(sokScan)
|
||||||
|
if leftOp != nil: op.children.add(leftOp)
|
||||||
|
if rightOp != nil: op.children.add(rightOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkCTE:
|
||||||
|
let cteOp = codegenPlan(plan.cteQuery)
|
||||||
|
let mainOp = codegenPlan(plan.cteMain)
|
||||||
|
let op = newStorageOp(sokScan)
|
||||||
|
if cteOp != nil: op.children.add(cteOp)
|
||||||
|
if mainOp != nil: op.children.add(mainOp)
|
||||||
|
return op
|
||||||
|
|
||||||
|
of irpkValues:
|
||||||
|
return newStorageOp(sokScan)
|
||||||
|
|
||||||
|
of irpkExplain:
|
||||||
|
return codegenPlan(plan.explainPlan)
|
||||||
|
|
||||||
|
proc estimateCost*(op: StorageOp): float64 =
|
||||||
|
if op == nil:
|
||||||
|
return 0.0
|
||||||
|
case op.kind
|
||||||
|
of sokPointRead: return 1.0
|
||||||
|
of sokRangeScan: return 10.0
|
||||||
|
of sokScan: return 1000.0
|
||||||
|
of sokFilter:
|
||||||
|
var cost = 100.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost
|
||||||
|
of sokProject:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost + 1.0
|
||||||
|
of sokSort:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost * 2.0
|
||||||
|
of sokLimit:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost * 0.5
|
||||||
|
of sokHashJoin:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost * 3.0
|
||||||
|
of sokGroupBy:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost * 2.0
|
||||||
|
else:
|
||||||
|
var cost = 0.0
|
||||||
|
for child in op.children:
|
||||||
|
cost += estimateCost(child)
|
||||||
|
return cost
|
||||||
|
|
||||||
|
proc explain*(op: StorageOp, indent: int = 0): string =
|
||||||
|
if op == nil:
|
||||||
|
return ""
|
||||||
|
result = " ".repeat(indent) & $op.kind
|
||||||
|
if op.table.len > 0:
|
||||||
|
result &= " table=" & op.table
|
||||||
|
if op.key.len > 0:
|
||||||
|
result &= " key=" & op.key
|
||||||
|
if op.limit > 0:
|
||||||
|
result &= " limit=" & $op.limit
|
||||||
|
result &= " (cost=" & $estimateCost(op) & ")\n"
|
||||||
|
for child in op.children:
|
||||||
|
result &= explain(child, indent + 2)
|
||||||
@@ -162,6 +162,17 @@ proc addConstraint*(t: SchemaType, name: string, expr: string) =
|
|||||||
proc addIndex*(t: SchemaType, name: string, expr: string, kind: string = "btree") =
|
proc addIndex*(t: SchemaType, name: string, expr: string, kind: string = "btree") =
|
||||||
t.indexes.add(SchemaIndex(name: name, expr: expr, kind: kind))
|
t.indexes.add(SchemaIndex(name: name, expr: expr, kind: kind))
|
||||||
|
|
||||||
|
proc addComputedProperty*(t: SchemaType, name: string, typeName: string, expr: string) =
|
||||||
|
t.properties[name] = SchemaProperty(
|
||||||
|
name: name,
|
||||||
|
typeName: typeName,
|
||||||
|
computed: true,
|
||||||
|
expr: expr,
|
||||||
|
)
|
||||||
|
|
||||||
|
proc setBases*(t: SchemaType, bases: seq[string]) =
|
||||||
|
t.bases = bases
|
||||||
|
|
||||||
proc addType*(s: Schema, module: string, t: SchemaType) =
|
proc addType*(s: Schema, module: string, t: SchemaType) =
|
||||||
if module notin s.modules:
|
if module notin s.modules:
|
||||||
s.modules[module] = newModule(module)
|
s.modules[module] = newModule(module)
|
||||||
@@ -179,6 +190,47 @@ proc getAllTypes*(s: Schema): seq[SchemaType] =
|
|||||||
for typeName, t in module.types:
|
for typeName, t in module.types:
|
||||||
result.add(t)
|
result.add(t)
|
||||||
|
|
||||||
|
proc resolveInheritance*(s: Schema, t: SchemaType): SchemaType =
|
||||||
|
result = SchemaType(
|
||||||
|
name: t.name, module: t.module, bases: t.bases,
|
||||||
|
properties: initTable[string, SchemaProperty](),
|
||||||
|
links: initTable[string, SchemaLink](),
|
||||||
|
constraints: @[], indexes: @[],
|
||||||
|
isAbstract: t.isAbstract, isFinal: t.isFinal,
|
||||||
|
)
|
||||||
|
for baseName in t.bases:
|
||||||
|
let baseType = s.getType(baseName)
|
||||||
|
if baseType != nil:
|
||||||
|
let resolved = s.resolveInheritance(baseType)
|
||||||
|
for pname, prop in resolved.properties:
|
||||||
|
if pname notin result.properties:
|
||||||
|
result.properties[pname] = prop
|
||||||
|
for lname, link in resolved.links:
|
||||||
|
if lname notin result.links:
|
||||||
|
result.links[lname] = link
|
||||||
|
for pname, prop in t.properties:
|
||||||
|
result.properties[pname] = prop
|
||||||
|
for lname, link in t.links:
|
||||||
|
result.links[lname] = link
|
||||||
|
|
||||||
|
proc isSubtype*(s: Schema, child, ancestor: string): bool =
|
||||||
|
if child == ancestor:
|
||||||
|
return true
|
||||||
|
let t = s.getType(child)
|
||||||
|
if t == nil:
|
||||||
|
return false
|
||||||
|
for base in t.bases:
|
||||||
|
if s.isSubtype(base, ancestor):
|
||||||
|
return true
|
||||||
|
return false
|
||||||
|
|
||||||
|
proc getSubtypes*(s: Schema, typeName: string): seq[SchemaType] =
|
||||||
|
result = @[]
|
||||||
|
for t in s.getAllTypes():
|
||||||
|
if typeName in t.bases:
|
||||||
|
result.add(t)
|
||||||
|
result.add(s.getSubtypes(t.name))
|
||||||
|
|
||||||
proc diff*(oldSchema, newSchema: Schema): SchemaDiff =
|
proc diff*(oldSchema, newSchema: Schema): SchemaDiff =
|
||||||
var diff = SchemaDiff()
|
var diff = SchemaDiff()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
## Memory-mapped I/O — mmap-based file access for SSTables
|
||||||
|
import std/os
|
||||||
|
|
||||||
|
const
|
||||||
|
PageSize* = 4096
|
||||||
|
|
||||||
|
type
|
||||||
|
MmapMode* = enum
|
||||||
|
mmReadOnly
|
||||||
|
mmReadWrite
|
||||||
|
mmPrivate # copy-on-write
|
||||||
|
|
||||||
|
MmapRegion* = object
|
||||||
|
data*: ptr UncheckedArray[byte]
|
||||||
|
size*: int
|
||||||
|
offset*: int64
|
||||||
|
fd*: int
|
||||||
|
mode*: MmapMode
|
||||||
|
|
||||||
|
MmapFile* = ref object
|
||||||
|
path*: string
|
||||||
|
regions*: seq[MmapRegion]
|
||||||
|
totalSize*: int
|
||||||
|
pageSize*: int
|
||||||
|
|
||||||
|
# Linux mmap constants
|
||||||
|
const
|
||||||
|
PROT_READ = 1
|
||||||
|
PROT_WRITE = 2
|
||||||
|
MAP_SHARED = 1
|
||||||
|
MAP_PRIVATE = 2
|
||||||
|
MAP_FAILED = cast[pointer](-1)
|
||||||
|
|
||||||
|
proc mmap(address: pointer, length: int, prot: int, flags: int,
|
||||||
|
fd: int, offset: int64): pointer {.importc, header: "<sys/mman.h>".}
|
||||||
|
proc munmap(address: pointer, length: int): int {.importc, header: "<sys/mman.h>".}
|
||||||
|
proc madvise(address: pointer, length: int, advice: int): int {.importc, header: "<sys/mman.h>".}
|
||||||
|
|
||||||
|
const
|
||||||
|
MADV_SEQUENTIAL = 2
|
||||||
|
MADV_RANDOM = 1
|
||||||
|
MADV_WILLNEED = 3
|
||||||
|
MADV_DONTNEED = 4
|
||||||
|
|
||||||
|
proc openMmap*(path: string, mode: MmapMode = mmReadOnly): MmapFile =
|
||||||
|
let fileSize = getFileSize(path)
|
||||||
|
if fileSize == 0:
|
||||||
|
return MmapFile(path: path, regions: @[], totalSize: 0, pageSize: PageSize)
|
||||||
|
|
||||||
|
let fd = open(path, if mode == mmReadOnly: fmRead else: fmReadWrite)
|
||||||
|
let prot = if mode == mmReadOnly: PROT_READ else: PROT_READ or PROT_WRITE
|
||||||
|
let flags = if mode == mmPrivate: MAP_PRIVATE else: MAP_SHARED
|
||||||
|
|
||||||
|
let mapped = mmap(nil, fileSize, prot, flags, fd, 0)
|
||||||
|
if mapped == MAP_FAILED:
|
||||||
|
close(fd)
|
||||||
|
return MmapFile(path: path, regions: @[], totalSize: 0, pageSize: PageSize)
|
||||||
|
|
||||||
|
let region = MmapRegion(
|
||||||
|
data: cast[ptr UncheckedArray[byte]](mapped),
|
||||||
|
size: fileSize,
|
||||||
|
offset: 0,
|
||||||
|
fd: fd,
|
||||||
|
mode: mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
MmapFile(
|
||||||
|
path: path,
|
||||||
|
regions: @[region],
|
||||||
|
totalSize: fileSize,
|
||||||
|
pageSize: PageSize,
|
||||||
|
)
|
||||||
|
|
||||||
|
proc readAt*(mf: MmapFile, offset: int, size: int): seq[byte] =
|
||||||
|
if mf.regions.len == 0:
|
||||||
|
return @[]
|
||||||
|
let region = mf.regions[0]
|
||||||
|
if offset + size > region.size:
|
||||||
|
return @[]
|
||||||
|
result = newSeq[byte](size)
|
||||||
|
copyMem(addr result[0], unsafeAddr region.data[offset], size)
|
||||||
|
|
||||||
|
proc readByte*(mf: MmapFile, offset: int): byte =
|
||||||
|
if mf.regions.len == 0 or offset >= mf.regions[0].size:
|
||||||
|
return 0
|
||||||
|
return mf.regions[0].data[offset]
|
||||||
|
|
||||||
|
proc readUint32*(mf: MmapFile, offset: int): uint32 =
|
||||||
|
if mf.regions.len == 0 or offset + 4 > mf.regions[0].size:
|
||||||
|
return 0
|
||||||
|
var val: uint32
|
||||||
|
copyMem(addr val, unsafeAddr mf.regions[0].data[offset], 4)
|
||||||
|
return val
|
||||||
|
|
||||||
|
proc readUint64*(mf: MmapFile, offset: int): uint64 =
|
||||||
|
if mf.regions.len == 0 or offset + 8 > mf.regions[0].size:
|
||||||
|
return 0
|
||||||
|
var val: uint64
|
||||||
|
copyMem(addr val, unsafeAddr mf.regions[0].data[offset], 8)
|
||||||
|
return val
|
||||||
|
|
||||||
|
proc readString*(mf: MmapFile, offset: int, size: int): string =
|
||||||
|
if mf.regions.len == 0 or offset + size > mf.regions[0].size:
|
||||||
|
return ""
|
||||||
|
result = newString(size)
|
||||||
|
copyMem(addr result[0], unsafeAddr mf.regions[0].data[offset], size)
|
||||||
|
|
||||||
|
proc adviseSequential*(mf: MmapFile) =
|
||||||
|
if mf.regions.len > 0:
|
||||||
|
discard madvise(mf.regions[0].data, mf.regions[0].size, MADV_SEQUENTIAL)
|
||||||
|
|
||||||
|
proc adviseRandom*(mf: MmapFile) =
|
||||||
|
if mf.regions.len > 0:
|
||||||
|
discard madvise(mf.regions[0].data, mf.regions[0].size, MADV_RANDOM)
|
||||||
|
|
||||||
|
proc adviseWillNeed*(mf: MmapFile, offset: int, size: int) =
|
||||||
|
if mf.regions.len > 0:
|
||||||
|
discard madvise(addr mf.regions[0].data[offset], size, MADV_WILLNEED)
|
||||||
|
|
||||||
|
proc adviseDontNeed*(mf: MmapFile, offset: int, size: int) =
|
||||||
|
if mf.regions.len > 0:
|
||||||
|
discard madvise(addr mf.regions[0].data[offset], size, MADV_DONTNEED)
|
||||||
|
|
||||||
|
proc close*(mf: MmapFile) =
|
||||||
|
for region in mf.regions:
|
||||||
|
discard munmap(region.data, region.size)
|
||||||
|
close(region.fd)
|
||||||
|
mf.regions.setLen(0)
|
||||||
|
|
||||||
|
proc size*(mf: MmapFile): int = mf.totalSize
|
||||||
|
proc isOpen*(mf: MmapFile): bool = mf.regions.len > 0
|
||||||
@@ -9,6 +9,7 @@ import barabadb/core/deadlock
|
|||||||
import barabadb/core/columnar
|
import barabadb/core/columnar
|
||||||
import barabadb/core/raft
|
import barabadb/core/raft
|
||||||
import barabadb/core/sharding
|
import barabadb/core/sharding
|
||||||
|
import barabadb/core/replication
|
||||||
import barabadb/storage/bloom
|
import barabadb/storage/bloom
|
||||||
import barabadb/storage/wal
|
import barabadb/storage/wal
|
||||||
import barabadb/storage/lsm
|
import barabadb/storage/lsm
|
||||||
@@ -18,6 +19,7 @@ import barabadb/query/lexer as lex
|
|||||||
import barabadb/query/ast
|
import barabadb/query/ast
|
||||||
import barabadb/query/parser
|
import barabadb/query/parser
|
||||||
import barabadb/query/ir as qir
|
import barabadb/query/ir as qir
|
||||||
|
import barabadb/query/codegen
|
||||||
import barabadb/vector/engine as vengine
|
import barabadb/vector/engine as vengine
|
||||||
import barabadb/vector/quant as vquant
|
import barabadb/vector/quant as vquant
|
||||||
import barabadb/graph/engine as gengine
|
import barabadb/graph/engine as gengine
|
||||||
@@ -954,3 +956,178 @@ suite "Sharding":
|
|||||||
test "Active shard count":
|
test "Active shard count":
|
||||||
var router = newShardRouter()
|
var router = newShardRouter()
|
||||||
check router.activeShardCount == 4
|
check router.activeShardCount == 4
|
||||||
|
|
||||||
|
suite "Schema Inheritance":
|
||||||
|
test "Inheritance — merge properties from base":
|
||||||
|
var s = newSchema()
|
||||||
|
let base = newType("Base")
|
||||||
|
base.addProperty("id", "str", required = true)
|
||||||
|
base.addProperty("created", "datetime")
|
||||||
|
s.addType("default", base)
|
||||||
|
|
||||||
|
let child = newType("Person")
|
||||||
|
child.setBases(@["Base"])
|
||||||
|
child.addProperty("name", "str", required = true)
|
||||||
|
s.addType("default", child)
|
||||||
|
|
||||||
|
let resolved = s.resolveInheritance(child)
|
||||||
|
check resolved.properties.len == 3 # id + created + name
|
||||||
|
check "id" in resolved.properties
|
||||||
|
check "name" in resolved.properties
|
||||||
|
check "created" in resolved.properties
|
||||||
|
|
||||||
|
test "Multi-level inheritance":
|
||||||
|
var s = newSchema()
|
||||||
|
let a = newType("A")
|
||||||
|
a.addProperty("a1", "str")
|
||||||
|
s.addType("default", a)
|
||||||
|
|
||||||
|
let b = newType("B")
|
||||||
|
b.setBases(@["A"])
|
||||||
|
b.addProperty("b1", "int32")
|
||||||
|
s.addType("default", b)
|
||||||
|
|
||||||
|
let c = newType("C")
|
||||||
|
c.setBases(@["B"])
|
||||||
|
c.addProperty("c1", "bool")
|
||||||
|
s.addType("default", c)
|
||||||
|
|
||||||
|
let resolved = s.resolveInheritance(c)
|
||||||
|
check resolved.properties.len == 3 # a1 + b1 + c1
|
||||||
|
|
||||||
|
test "Override base property":
|
||||||
|
var s = newSchema()
|
||||||
|
let base = newType("Base")
|
||||||
|
base.addProperty("name", "str")
|
||||||
|
s.addType("default", base)
|
||||||
|
|
||||||
|
let child = newType("Child")
|
||||||
|
child.setBases(@["Base"])
|
||||||
|
child.addProperty("name", "text") # override
|
||||||
|
s.addType("default", child)
|
||||||
|
|
||||||
|
let resolved = s.resolveInheritance(child)
|
||||||
|
check resolved.properties["name"].typeName == "text"
|
||||||
|
|
||||||
|
test "isSubtype":
|
||||||
|
var s = newSchema()
|
||||||
|
let a = newType("A")
|
||||||
|
s.addType("default", a)
|
||||||
|
let b = newType("B")
|
||||||
|
b.setBases(@["A"])
|
||||||
|
s.addType("default", b)
|
||||||
|
let c = newType("C")
|
||||||
|
c.setBases(@["B"])
|
||||||
|
s.addType("default", c)
|
||||||
|
|
||||||
|
check s.isSubtype("C", "A")
|
||||||
|
check s.isSubtype("C", "B")
|
||||||
|
check s.isSubtype("B", "A")
|
||||||
|
check not s.isSubtype("A", "C")
|
||||||
|
|
||||||
|
test "Computed property":
|
||||||
|
let t = newType("Person")
|
||||||
|
t.addProperty("firstName", "str")
|
||||||
|
t.addProperty("lastName", "str")
|
||||||
|
t.addComputedProperty("fullName", "str", "firstName ++ ' ' ++ lastName")
|
||||||
|
check t.properties["fullName"].computed
|
||||||
|
check t.properties["fullName"].expr == "firstName ++ ' ' ++ lastName"
|
||||||
|
|
||||||
|
suite "Codegen":
|
||||||
|
test "Codegen scan":
|
||||||
|
let plan = IRPlan(kind: irpkScan, scanTable: "users", scanAlias: "u")
|
||||||
|
let op = codegenPlan(plan)
|
||||||
|
check op.kind == sokScan
|
||||||
|
check op.table == "users"
|
||||||
|
|
||||||
|
test "Codegen filter with point read optimization":
|
||||||
|
let filterExpr = IRExpr(kind: irekBinary, binOp: irEq,
|
||||||
|
binLeft: IRExpr(kind: irekField, fieldPath: @["id"]),
|
||||||
|
binRight: IRExpr(kind: irekLiteral, literal: IRLiteral(kind: vkInt64, int64Val: 42)))
|
||||||
|
let scan = IRPlan(kind: irpkScan, scanTable: "users", scanAlias: "u")
|
||||||
|
let plan = IRPlan(kind: irpkFilter, filterSource: scan, filterCond: filterExpr)
|
||||||
|
let op = codegenPlan(plan)
|
||||||
|
# Should optimize to point read
|
||||||
|
check op.kind == sokPointRead
|
||||||
|
|
||||||
|
test "Codegen limit":
|
||||||
|
let scan = IRPlan(kind: irpkScan, scanTable: "t", scanAlias: "t")
|
||||||
|
let plan = IRPlan(kind: irpkLimit, limitSource: scan, limitCount: 10, limitOffset: 5)
|
||||||
|
let op = codegenPlan(plan)
|
||||||
|
check op.limit == 10
|
||||||
|
check op.offset == 5
|
||||||
|
|
||||||
|
test "Cost estimation":
|
||||||
|
let scan = newStorageOp(sokScan)
|
||||||
|
check estimateCost(scan) == 1000.0
|
||||||
|
let pointRead = newStorageOp(sokPointRead)
|
||||||
|
check estimateCost(pointRead) == 1.0
|
||||||
|
|
||||||
|
test "Explain plan":
|
||||||
|
let scan = newStorageOp(sokScan)
|
||||||
|
scan.table = "users"
|
||||||
|
let explanation = scan.explain()
|
||||||
|
check "sokScan" in explanation
|
||||||
|
check "users" in explanation
|
||||||
|
|
||||||
|
suite "Replication":
|
||||||
|
test "Create replication manager":
|
||||||
|
var rm = newReplicationManager(rmAsync)
|
||||||
|
check rm.totalReplicaCount == 0
|
||||||
|
check rm.connectedReplicaCount == 0
|
||||||
|
|
||||||
|
test "Add and connect replicas":
|
||||||
|
var rm = newReplicationManager(rmAsync)
|
||||||
|
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||||
|
rm.addReplica(newReplica("r2", "10.0.0.2", 5432))
|
||||||
|
check rm.totalReplicaCount == 2
|
||||||
|
|
||||||
|
rm.connectReplica("r1")
|
||||||
|
check rm.connectedReplicaCount == 1
|
||||||
|
|
||||||
|
test "Async replication — write returns immediately":
|
||||||
|
var rm = newReplicationManager(rmAsync)
|
||||||
|
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||||
|
rm.connectReplica("r1")
|
||||||
|
|
||||||
|
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||||
|
check lsn == 1
|
||||||
|
# Async doesn't wait — already "acked"
|
||||||
|
check rm.isFullyAcked(lsn)
|
||||||
|
|
||||||
|
test "Sync replication — wait for ack":
|
||||||
|
var rm = newReplicationManager(rmSync)
|
||||||
|
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||||
|
rm.connectReplica("r1")
|
||||||
|
|
||||||
|
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||||
|
check not rm.isFullyAcked(lsn)
|
||||||
|
|
||||||
|
rm.ackLsn("r1", lsn)
|
||||||
|
check rm.isFullyAcked(lsn)
|
||||||
|
|
||||||
|
test "Semi-sync replication":
|
||||||
|
var rm = newReplicationManager(rmSemiSync, syncCount = 2)
|
||||||
|
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||||
|
rm.addReplica(newReplica("r2", "10.0.0.2", 5432))
|
||||||
|
rm.addReplica(newReplica("r3", "10.0.0.3", 5432))
|
||||||
|
rm.connectReplica("r1")
|
||||||
|
rm.connectReplica("r2")
|
||||||
|
rm.connectReplica("r3")
|
||||||
|
|
||||||
|
let lsn = rm.writeLsn(@[1'u8])
|
||||||
|
check not rm.isFullyAcked(lsn) # needs 2 acks
|
||||||
|
|
||||||
|
rm.ackLsn("r1", lsn)
|
||||||
|
check not rm.isFullyAcked(lsn) # still needs 1 more
|
||||||
|
|
||||||
|
rm.ackLsn("r2", lsn)
|
||||||
|
check rm.isFullyAcked(lsn) # 2 acks received
|
||||||
|
|
||||||
|
test "Replica status":
|
||||||
|
var rm = newReplicationManager(rmAsync)
|
||||||
|
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||||
|
rm.connectReplica("r1")
|
||||||
|
let status = rm.replicaStatus()
|
||||||
|
check status.len == 1
|
||||||
|
check status[0][1] == rsStreaming
|
||||||
|
|||||||
Reference in New Issue
Block a user