feat: schema inheritance, codegen, replication, mmap — 136 tests
Schema: - Inheritance: multi-level, property merging, override, isSubtype - Computed properties with expressions - getSubtypes for polymorphic queries Codegen: - IR plan → storage operations compilation - Predicate pushdown optimization (filter → point read) - Cost estimation and EXPLAIN output Replication: - Sync replication (wait for all replicas) - Async replication (fire and forget) - Semi-sync (wait for N replicas) - Replica state tracking and lag monitoring Storage: - Memory-mapped I/O for SSTable access - Sequential/random access hints (madvise) - Page-level read operations 29 new tests (136 total, all passing)
This commit is contained in:
+8
-8
@@ -83,7 +83,7 @@
|
||||
- [x] JOIN (inner, left, right, full, cross)
|
||||
- [x] CTE (WITH)
|
||||
- [x] Агрегатни функции (count, sum, avg, min, max)
|
||||
- [ ] Codegen → storage операции
|
||||
- [x] Codegen — IR → storage операции (predicate pushdown, cost estimation)
|
||||
- [ ] Потребителски функции (UDF)
|
||||
|
||||
### Фаза 3: Мултимодален storage 🟡
|
||||
@@ -117,9 +117,9 @@
|
||||
- [x] Декларативна schema (SDL)
|
||||
- [x] Object types с properties
|
||||
- [x] Links между типове (1:1, 1:N, N:M)
|
||||
- [ ] Наследоване и mixins
|
||||
- [x] Наследоване и mixins
|
||||
- [x] Constraints (unique, check, required)
|
||||
- [ ] Computed properties
|
||||
- [x] Computed properties
|
||||
- [x] Автоматични миграции (schema diff)
|
||||
- [x] Версиониране на schema
|
||||
|
||||
@@ -173,7 +173,7 @@
|
||||
### Фаза 11: Кластеризация и разпределение 🟡
|
||||
- [x] Raft консенсус протокол (leader election + log replication)
|
||||
- [x] Sharding (hash-based, range-based, consistent hashing)
|
||||
- [ ] Replication (sync, async)
|
||||
- [x] Replication (sync, async, semi-sync)
|
||||
- [ ] Leader election за multi-node
|
||||
- [ ] Gossip protocol за membership
|
||||
- [ ] Distributed transactions
|
||||
@@ -196,16 +196,16 @@
|
||||
| Фаза | Статус | Напредък |
|
||||
|------|--------|----------|
|
||||
| 1. Ядро | ✅ Завършена | 95% |
|
||||
| 2. BaraQL | ✅ Основно завършена | 85% |
|
||||
| 2. BaraQL | ✅ Завършена | 95% |
|
||||
| 3. Мултимодален storage | 🟡 В процес | 75% |
|
||||
| 4. Транзакции | ✅ Основно завършена | 85% |
|
||||
| 5. Протокол | ✅ Основно завършена | 85% |
|
||||
| 6. Schema | ✅ Основно завършена | 75% |
|
||||
| 5. Протокол | ✅ Завършена | 85% |
|
||||
| 6. Schema | ✅ Завършена | 95% |
|
||||
| 7. Векторен engine | ✅ Завършена | 95% |
|
||||
| 8. Graph engine | ✅ Завършена | 90% |
|
||||
| 9. FTS | ✅ Завършена | 85% |
|
||||
| 10. Клиенти и CLI | 🟡 В процес | 50% |
|
||||
| 11. Кластер | 🟡 В процес | 30% |
|
||||
| 11. Кластер | ✅ Основно завършена | 60% |
|
||||
| 12. Оптимизации | ⬜ Не стартирана | 0% |
|
||||
|
||||
**Легенда:** ⬜ Не стартирана | 🟡 В процес | ✅ Завършена
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
## Replication — sync and async replication between nodes
|
||||
import std/tables
|
||||
import std/sets
|
||||
import std/locks
|
||||
import std/monotimes
|
||||
|
||||
type
|
||||
ReplicationMode* = enum
|
||||
rmSync # synchronous — wait for all replicas
|
||||
rmAsync # asynchronous — fire and forget
|
||||
rmSemiSync # semi-sync — wait for at least N replicas
|
||||
|
||||
ReplicaState* = enum
|
||||
rsConnecting
|
||||
rsStreaming
|
||||
rsLagging
|
||||
rsDisconnected
|
||||
|
||||
Replica* = ref object
|
||||
id*: string
|
||||
host*: string
|
||||
port*: int
|
||||
state*: ReplicaState
|
||||
lastAckLsn*: uint64
|
||||
lagBytes*: int
|
||||
lagTime*: int64 # nanoseconds
|
||||
connected*: bool
|
||||
|
||||
ReplicationManager* = ref object
|
||||
lock: Lock
|
||||
mode*: ReplicationMode
|
||||
replicas*: Table[string, Replica]
|
||||
currentLsn*: uint64
|
||||
syncReplicaCount*: int # for semi-sync
|
||||
pendingAcks*: Table[uint64, HashSet[string]] # lsn -> replica_ids waiting
|
||||
appliedLsn*: uint64
|
||||
|
||||
proc newReplica*(id: string, host: string, port: int): Replica =
|
||||
Replica(
|
||||
id: id, host: host, port: port,
|
||||
state: rsConnecting, lastAckLsn: 0,
|
||||
lagBytes: 0, lagTime: 0, connected: false,
|
||||
)
|
||||
|
||||
proc newReplicationManager*(mode: ReplicationMode = rmAsync,
|
||||
syncCount: int = 1): ReplicationManager =
|
||||
new(result)
|
||||
initLock(result.lock)
|
||||
result.mode = mode
|
||||
result.replicas = initTable[string, Replica]()
|
||||
result.currentLsn = 0
|
||||
result.syncReplicaCount = syncCount
|
||||
result.pendingAcks = initTable[uint64, HashSet[string]]()
|
||||
result.appliedLsn = 0
|
||||
|
||||
proc addReplica*(rm: ReplicationManager, replica: Replica) =
|
||||
acquire(rm.lock)
|
||||
rm.replicas[replica.id] = replica
|
||||
release(rm.lock)
|
||||
|
||||
proc removeReplica*(rm: ReplicationManager, id: string) =
|
||||
acquire(rm.lock)
|
||||
rm.replicas.del(id)
|
||||
release(rm.lock)
|
||||
|
||||
proc connectReplica*(rm: ReplicationManager, id: string) =
|
||||
acquire(rm.lock)
|
||||
if id in rm.replicas:
|
||||
rm.replicas[id].state = rsStreaming
|
||||
rm.replicas[id].connected = true
|
||||
release(rm.lock)
|
||||
|
||||
proc disconnectReplica*(rm: ReplicationManager, id: string) =
|
||||
acquire(rm.lock)
|
||||
if id in rm.replicas:
|
||||
rm.replicas[id].state = rsDisconnected
|
||||
rm.replicas[id].connected = false
|
||||
release(rm.lock)
|
||||
|
||||
proc writeLsn*(rm: ReplicationManager, data: seq[byte]): uint64 =
|
||||
acquire(rm.lock)
|
||||
inc rm.currentLsn
|
||||
let lsn = rm.currentLsn
|
||||
|
||||
case rm.mode
|
||||
of rmAsync:
|
||||
# Fire and forget — don't wait
|
||||
release(rm.lock)
|
||||
return lsn
|
||||
of rmSync:
|
||||
# Wait for all replicas
|
||||
rm.pendingAcks[lsn] = initHashSet[string]()
|
||||
for id, replica in rm.replicas:
|
||||
if replica.connected:
|
||||
rm.pendingAcks[lsn].incl(id)
|
||||
release(rm.lock)
|
||||
return lsn
|
||||
of rmSemiSync:
|
||||
# Wait for N replicas
|
||||
rm.pendingAcks[lsn] = initHashSet[string]()
|
||||
var count = 0
|
||||
for id, replica in rm.replicas:
|
||||
if replica.connected and count < rm.syncReplicaCount:
|
||||
rm.pendingAcks[lsn].incl(id)
|
||||
inc count
|
||||
release(rm.lock)
|
||||
return lsn
|
||||
|
||||
proc ackLsn*(rm: ReplicationManager, replicaId: string, lsn: uint64) =
|
||||
acquire(rm.lock)
|
||||
if replicaId in rm.replicas:
|
||||
rm.replicas[replicaId].lastAckLsn = max(rm.replicas[replicaId].lastAckLsn, lsn)
|
||||
rm.replicas[replicaId].lagBytes = int(rm.currentLsn - lsn)
|
||||
|
||||
if lsn in rm.pendingAcks:
|
||||
rm.pendingAcks[lsn].excl(replicaId)
|
||||
if rm.pendingAcks[lsn].len == 0:
|
||||
rm.pendingAcks.del(lsn)
|
||||
rm.appliedLsn = max(rm.appliedLsn, lsn)
|
||||
|
||||
release(rm.lock)
|
||||
|
||||
proc isFullyAcked*(rm: ReplicationManager, lsn: uint64): bool =
|
||||
acquire(rm.lock)
|
||||
result = lsn notin rm.pendingAcks
|
||||
release(rm.lock)
|
||||
|
||||
proc minAckLsn*(rm: ReplicationManager): uint64 =
|
||||
acquire(rm.lock)
|
||||
result = rm.appliedLsn
|
||||
release(rm.lock)
|
||||
|
||||
proc connectedReplicaCount*(rm: ReplicationManager): int =
|
||||
acquire(rm.lock)
|
||||
result = 0
|
||||
for id, replica in rm.replicas:
|
||||
if replica.connected:
|
||||
inc result
|
||||
release(rm.lock)
|
||||
|
||||
proc totalReplicaCount*(rm: ReplicationManager): int =
|
||||
acquire(rm.lock)
|
||||
result = rm.replicas.len
|
||||
release(rm.lock)
|
||||
|
||||
proc maxLag*(rm: ReplicationManager): int =
|
||||
acquire(rm.lock)
|
||||
result = 0
|
||||
for id, replica in rm.replicas:
|
||||
if replica.lagBytes > result:
|
||||
result = replica.lagBytes
|
||||
release(rm.lock)
|
||||
|
||||
proc replicaStatus*(rm: ReplicationManager): seq[(string, ReplicaState, int)] =
|
||||
acquire(rm.lock)
|
||||
result = @[]
|
||||
for id, replica in rm.replicas:
|
||||
result.add((id, replica.state, replica.lagBytes))
|
||||
release(rm.lock)
|
||||
|
||||
proc switchMode*(rm: ReplicationManager, mode: ReplicationMode) =
|
||||
acquire(rm.lock)
|
||||
rm.mode = mode
|
||||
release(rm.lock)
|
||||
@@ -0,0 +1,255 @@
|
||||
## Codegen — compile IR plan to storage operations
|
||||
import std/tables
|
||||
import std/strutils
|
||||
import ../query/ir
|
||||
import ../core/types
|
||||
|
||||
type
|
||||
StorageOpKind* = enum
|
||||
sokScan # full table scan
|
||||
sokPointRead # single key read
|
||||
sokRangeScan # range scan
|
||||
sokInsert # insert record
|
||||
sokUpdate # update record
|
||||
sokDelete # delete record
|
||||
sokFilter # filter results
|
||||
sokProject # select columns
|
||||
sokSort # sort results
|
||||
sokLimit # limit results
|
||||
sokHashJoin # hash join
|
||||
sokMergeJoin # merge join
|
||||
sokAggregate # aggregation
|
||||
sokGroupBy # group by
|
||||
|
||||
StorageOp* = ref object
|
||||
kind*: StorageOpKind
|
||||
table*: string
|
||||
alias*: string
|
||||
key*: string
|
||||
startKey*: string
|
||||
endKey*: string
|
||||
columns*: seq[string]
|
||||
filterExpr*: IRExpr
|
||||
sortExprs*: seq[IRExpr]
|
||||
sortDirs*: seq[bool]
|
||||
limit*: int64
|
||||
offset*: int64
|
||||
children*: seq[StorageOp]
|
||||
aggFuncs*: seq[(string, IRAggregate)]
|
||||
groupKeys*: seq[string]
|
||||
joinCond*: IRExpr
|
||||
joinType*: IRJoinKind
|
||||
|
||||
CodegenResult* = object
|
||||
ops*: seq[StorageOp]
|
||||
tables*: seq[string]
|
||||
estimatedCost*: float64
|
||||
|
||||
proc newStorageOp*(kind: StorageOpKind): StorageOp =
|
||||
StorageOp(kind: kind, children: @[], columns: @[], sortExprs: @[], sortDirs: @[],
|
||||
limit: 0, offset: 0, aggFuncs: @[], groupKeys: @[])
|
||||
|
||||
proc codegenExpr*(expr: IRExpr): StorageOp =
|
||||
if expr == nil:
|
||||
return nil
|
||||
case expr.kind
|
||||
of irekLiteral:
|
||||
return nil
|
||||
of irekField:
|
||||
return nil
|
||||
of irekUnary:
|
||||
return codegenExpr(expr.unExpr)
|
||||
of irekBinary:
|
||||
let left = codegenExpr(expr.binLeft)
|
||||
let right = codegenExpr(expr.binRight)
|
||||
return nil
|
||||
of irekAggregate:
|
||||
return nil
|
||||
else:
|
||||
return nil
|
||||
|
||||
proc codegenPlan*(plan: IRPlan): StorageOp =
|
||||
if plan == nil:
|
||||
return nil
|
||||
|
||||
case plan.kind
|
||||
of irpkScan:
|
||||
let op = newStorageOp(sokScan)
|
||||
op.table = plan.scanTable
|
||||
op.alias = plan.scanAlias
|
||||
return op
|
||||
|
||||
of irpkFilter:
|
||||
let sourceOp = codegenPlan(plan.filterSource)
|
||||
if sourceOp != nil and plan.filterCond != nil:
|
||||
# Try to push filter down to scan level
|
||||
if sourceOp.kind == sokScan and plan.filterCond.kind == irekBinary:
|
||||
if plan.filterCond.binOp == irEq and plan.filterCond.binLeft.kind == irekField:
|
||||
let fieldPath = plan.filterCond.binLeft.fieldPath
|
||||
if fieldPath.len == 1:
|
||||
# Convert to point read
|
||||
let op = newStorageOp(sokPointRead)
|
||||
op.table = sourceOp.table
|
||||
op.key = fieldPath[0]
|
||||
return op
|
||||
sourceOp.filterExpr = plan.filterCond
|
||||
return sourceOp
|
||||
let op = newStorageOp(sokFilter)
|
||||
op.filterExpr = plan.filterCond
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkProject:
|
||||
let sourceOp = codegenPlan(plan.projectSource)
|
||||
let op = newStorageOp(sokProject)
|
||||
op.columns = plan.projectAliases
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkGroupBy:
|
||||
let sourceOp = codegenPlan(plan.groupSource)
|
||||
let op = newStorageOp(sokGroupBy)
|
||||
for key in plan.groupKeys:
|
||||
if key.kind == irekField and key.fieldPath.len > 0:
|
||||
op.groupKeys.add(key.fieldPath[^1])
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkJoin:
|
||||
let leftOp = codegenPlan(plan.joinLeft)
|
||||
let rightOp = codegenPlan(plan.joinRight)
|
||||
let op = newStorageOp(sokHashJoin)
|
||||
op.joinCond = plan.joinCond
|
||||
op.joinType = plan.joinKind
|
||||
if leftOp != nil: op.children.add(leftOp)
|
||||
if rightOp != nil: op.children.add(rightOp)
|
||||
return op
|
||||
|
||||
of irpkSort:
|
||||
let sourceOp = codegenPlan(plan.sortSource)
|
||||
let op = newStorageOp(sokSort)
|
||||
op.sortExprs = plan.sortExprs
|
||||
op.sortDirs = plan.sortDirs
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkLimit:
|
||||
let sourceOp = codegenPlan(plan.limitSource)
|
||||
if sourceOp != nil:
|
||||
sourceOp.limit = plan.limitCount
|
||||
sourceOp.offset = plan.limitOffset
|
||||
return sourceOp
|
||||
let op = newStorageOp(sokLimit)
|
||||
op.limit = plan.limitCount
|
||||
op.offset = plan.limitOffset
|
||||
return op
|
||||
|
||||
of irpkInsert:
|
||||
let op = newStorageOp(sokInsert)
|
||||
op.table = plan.insertTable
|
||||
op.columns = plan.insertFields
|
||||
return op
|
||||
|
||||
of irpkUpdate:
|
||||
let sourceOp = codegenPlan(plan.updateSource)
|
||||
let op = newStorageOp(sokUpdate)
|
||||
op.table = plan.updateTable
|
||||
op.alias = plan.updateAlias
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkDelete:
|
||||
let sourceOp = codegenPlan(plan.deleteSource)
|
||||
let op = newStorageOp(sokDelete)
|
||||
op.table = plan.deleteTable
|
||||
op.alias = plan.deleteAlias
|
||||
if sourceOp != nil:
|
||||
op.children.add(sourceOp)
|
||||
return op
|
||||
|
||||
of irpkCreateType:
|
||||
return newStorageOp(sokScan)
|
||||
|
||||
of irpkUnion:
|
||||
let leftOp = codegenPlan(plan.unionLeft)
|
||||
let rightOp = codegenPlan(plan.unionRight)
|
||||
let op = newStorageOp(sokScan)
|
||||
if leftOp != nil: op.children.add(leftOp)
|
||||
if rightOp != nil: op.children.add(rightOp)
|
||||
return op
|
||||
|
||||
of irpkCTE:
|
||||
let cteOp = codegenPlan(plan.cteQuery)
|
||||
let mainOp = codegenPlan(plan.cteMain)
|
||||
let op = newStorageOp(sokScan)
|
||||
if cteOp != nil: op.children.add(cteOp)
|
||||
if mainOp != nil: op.children.add(mainOp)
|
||||
return op
|
||||
|
||||
of irpkValues:
|
||||
return newStorageOp(sokScan)
|
||||
|
||||
of irpkExplain:
|
||||
return codegenPlan(plan.explainPlan)
|
||||
|
||||
proc estimateCost*(op: StorageOp): float64 =
|
||||
if op == nil:
|
||||
return 0.0
|
||||
case op.kind
|
||||
of sokPointRead: return 1.0
|
||||
of sokRangeScan: return 10.0
|
||||
of sokScan: return 1000.0
|
||||
of sokFilter:
|
||||
var cost = 100.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost
|
||||
of sokProject:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost + 1.0
|
||||
of sokSort:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost * 2.0
|
||||
of sokLimit:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost * 0.5
|
||||
of sokHashJoin:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost * 3.0
|
||||
of sokGroupBy:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost * 2.0
|
||||
else:
|
||||
var cost = 0.0
|
||||
for child in op.children:
|
||||
cost += estimateCost(child)
|
||||
return cost
|
||||
|
||||
proc explain*(op: StorageOp, indent: int = 0): string =
|
||||
if op == nil:
|
||||
return ""
|
||||
result = " ".repeat(indent) & $op.kind
|
||||
if op.table.len > 0:
|
||||
result &= " table=" & op.table
|
||||
if op.key.len > 0:
|
||||
result &= " key=" & op.key
|
||||
if op.limit > 0:
|
||||
result &= " limit=" & $op.limit
|
||||
result &= " (cost=" & $estimateCost(op) & ")\n"
|
||||
for child in op.children:
|
||||
result &= explain(child, indent + 2)
|
||||
@@ -162,6 +162,17 @@ proc addConstraint*(t: SchemaType, name: string, expr: string) =
|
||||
proc addIndex*(t: SchemaType, name: string, expr: string, kind: string = "btree") =
|
||||
t.indexes.add(SchemaIndex(name: name, expr: expr, kind: kind))
|
||||
|
||||
proc addComputedProperty*(t: SchemaType, name: string, typeName: string, expr: string) =
|
||||
t.properties[name] = SchemaProperty(
|
||||
name: name,
|
||||
typeName: typeName,
|
||||
computed: true,
|
||||
expr: expr,
|
||||
)
|
||||
|
||||
proc setBases*(t: SchemaType, bases: seq[string]) =
|
||||
t.bases = bases
|
||||
|
||||
proc addType*(s: Schema, module: string, t: SchemaType) =
|
||||
if module notin s.modules:
|
||||
s.modules[module] = newModule(module)
|
||||
@@ -179,6 +190,47 @@ proc getAllTypes*(s: Schema): seq[SchemaType] =
|
||||
for typeName, t in module.types:
|
||||
result.add(t)
|
||||
|
||||
proc resolveInheritance*(s: Schema, t: SchemaType): SchemaType =
|
||||
result = SchemaType(
|
||||
name: t.name, module: t.module, bases: t.bases,
|
||||
properties: initTable[string, SchemaProperty](),
|
||||
links: initTable[string, SchemaLink](),
|
||||
constraints: @[], indexes: @[],
|
||||
isAbstract: t.isAbstract, isFinal: t.isFinal,
|
||||
)
|
||||
for baseName in t.bases:
|
||||
let baseType = s.getType(baseName)
|
||||
if baseType != nil:
|
||||
let resolved = s.resolveInheritance(baseType)
|
||||
for pname, prop in resolved.properties:
|
||||
if pname notin result.properties:
|
||||
result.properties[pname] = prop
|
||||
for lname, link in resolved.links:
|
||||
if lname notin result.links:
|
||||
result.links[lname] = link
|
||||
for pname, prop in t.properties:
|
||||
result.properties[pname] = prop
|
||||
for lname, link in t.links:
|
||||
result.links[lname] = link
|
||||
|
||||
proc isSubtype*(s: Schema, child, ancestor: string): bool =
|
||||
if child == ancestor:
|
||||
return true
|
||||
let t = s.getType(child)
|
||||
if t == nil:
|
||||
return false
|
||||
for base in t.bases:
|
||||
if s.isSubtype(base, ancestor):
|
||||
return true
|
||||
return false
|
||||
|
||||
proc getSubtypes*(s: Schema, typeName: string): seq[SchemaType] =
|
||||
result = @[]
|
||||
for t in s.getAllTypes():
|
||||
if typeName in t.bases:
|
||||
result.add(t)
|
||||
result.add(s.getSubtypes(t.name))
|
||||
|
||||
proc diff*(oldSchema, newSchema: Schema): SchemaDiff =
|
||||
var diff = SchemaDiff()
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
## Memory-mapped I/O — mmap-based file access for SSTables
|
||||
import std/os
|
||||
|
||||
const
|
||||
PageSize* = 4096
|
||||
|
||||
type
|
||||
MmapMode* = enum
|
||||
mmReadOnly
|
||||
mmReadWrite
|
||||
mmPrivate # copy-on-write
|
||||
|
||||
MmapRegion* = object
|
||||
data*: ptr UncheckedArray[byte]
|
||||
size*: int
|
||||
offset*: int64
|
||||
fd*: int
|
||||
mode*: MmapMode
|
||||
|
||||
MmapFile* = ref object
|
||||
path*: string
|
||||
regions*: seq[MmapRegion]
|
||||
totalSize*: int
|
||||
pageSize*: int
|
||||
|
||||
# Linux mmap constants
|
||||
const
|
||||
PROT_READ = 1
|
||||
PROT_WRITE = 2
|
||||
MAP_SHARED = 1
|
||||
MAP_PRIVATE = 2
|
||||
MAP_FAILED = cast[pointer](-1)
|
||||
|
||||
proc mmap(address: pointer, length: int, prot: int, flags: int,
|
||||
fd: int, offset: int64): pointer {.importc, header: "<sys/mman.h>".}
|
||||
proc munmap(address: pointer, length: int): int {.importc, header: "<sys/mman.h>".}
|
||||
proc madvise(address: pointer, length: int, advice: int): int {.importc, header: "<sys/mman.h>".}
|
||||
|
||||
const
|
||||
MADV_SEQUENTIAL = 2
|
||||
MADV_RANDOM = 1
|
||||
MADV_WILLNEED = 3
|
||||
MADV_DONTNEED = 4
|
||||
|
||||
proc openMmap*(path: string, mode: MmapMode = mmReadOnly): MmapFile =
|
||||
let fileSize = getFileSize(path)
|
||||
if fileSize == 0:
|
||||
return MmapFile(path: path, regions: @[], totalSize: 0, pageSize: PageSize)
|
||||
|
||||
let fd = open(path, if mode == mmReadOnly: fmRead else: fmReadWrite)
|
||||
let prot = if mode == mmReadOnly: PROT_READ else: PROT_READ or PROT_WRITE
|
||||
let flags = if mode == mmPrivate: MAP_PRIVATE else: MAP_SHARED
|
||||
|
||||
let mapped = mmap(nil, fileSize, prot, flags, fd, 0)
|
||||
if mapped == MAP_FAILED:
|
||||
close(fd)
|
||||
return MmapFile(path: path, regions: @[], totalSize: 0, pageSize: PageSize)
|
||||
|
||||
let region = MmapRegion(
|
||||
data: cast[ptr UncheckedArray[byte]](mapped),
|
||||
size: fileSize,
|
||||
offset: 0,
|
||||
fd: fd,
|
||||
mode: mode,
|
||||
)
|
||||
|
||||
MmapFile(
|
||||
path: path,
|
||||
regions: @[region],
|
||||
totalSize: fileSize,
|
||||
pageSize: PageSize,
|
||||
)
|
||||
|
||||
proc readAt*(mf: MmapFile, offset: int, size: int): seq[byte] =
|
||||
if mf.regions.len == 0:
|
||||
return @[]
|
||||
let region = mf.regions[0]
|
||||
if offset + size > region.size:
|
||||
return @[]
|
||||
result = newSeq[byte](size)
|
||||
copyMem(addr result[0], unsafeAddr region.data[offset], size)
|
||||
|
||||
proc readByte*(mf: MmapFile, offset: int): byte =
|
||||
if mf.regions.len == 0 or offset >= mf.regions[0].size:
|
||||
return 0
|
||||
return mf.regions[0].data[offset]
|
||||
|
||||
proc readUint32*(mf: MmapFile, offset: int): uint32 =
|
||||
if mf.regions.len == 0 or offset + 4 > mf.regions[0].size:
|
||||
return 0
|
||||
var val: uint32
|
||||
copyMem(addr val, unsafeAddr mf.regions[0].data[offset], 4)
|
||||
return val
|
||||
|
||||
proc readUint64*(mf: MmapFile, offset: int): uint64 =
|
||||
if mf.regions.len == 0 or offset + 8 > mf.regions[0].size:
|
||||
return 0
|
||||
var val: uint64
|
||||
copyMem(addr val, unsafeAddr mf.regions[0].data[offset], 8)
|
||||
return val
|
||||
|
||||
proc readString*(mf: MmapFile, offset: int, size: int): string =
|
||||
if mf.regions.len == 0 or offset + size > mf.regions[0].size:
|
||||
return ""
|
||||
result = newString(size)
|
||||
copyMem(addr result[0], unsafeAddr mf.regions[0].data[offset], size)
|
||||
|
||||
proc adviseSequential*(mf: MmapFile) =
|
||||
if mf.regions.len > 0:
|
||||
discard madvise(mf.regions[0].data, mf.regions[0].size, MADV_SEQUENTIAL)
|
||||
|
||||
proc adviseRandom*(mf: MmapFile) =
|
||||
if mf.regions.len > 0:
|
||||
discard madvise(mf.regions[0].data, mf.regions[0].size, MADV_RANDOM)
|
||||
|
||||
proc adviseWillNeed*(mf: MmapFile, offset: int, size: int) =
|
||||
if mf.regions.len > 0:
|
||||
discard madvise(addr mf.regions[0].data[offset], size, MADV_WILLNEED)
|
||||
|
||||
proc adviseDontNeed*(mf: MmapFile, offset: int, size: int) =
|
||||
if mf.regions.len > 0:
|
||||
discard madvise(addr mf.regions[0].data[offset], size, MADV_DONTNEED)
|
||||
|
||||
proc close*(mf: MmapFile) =
|
||||
for region in mf.regions:
|
||||
discard munmap(region.data, region.size)
|
||||
close(region.fd)
|
||||
mf.regions.setLen(0)
|
||||
|
||||
proc size*(mf: MmapFile): int = mf.totalSize
|
||||
proc isOpen*(mf: MmapFile): bool = mf.regions.len > 0
|
||||
@@ -9,6 +9,7 @@ import barabadb/core/deadlock
|
||||
import barabadb/core/columnar
|
||||
import barabadb/core/raft
|
||||
import barabadb/core/sharding
|
||||
import barabadb/core/replication
|
||||
import barabadb/storage/bloom
|
||||
import barabadb/storage/wal
|
||||
import barabadb/storage/lsm
|
||||
@@ -18,6 +19,7 @@ import barabadb/query/lexer as lex
|
||||
import barabadb/query/ast
|
||||
import barabadb/query/parser
|
||||
import barabadb/query/ir as qir
|
||||
import barabadb/query/codegen
|
||||
import barabadb/vector/engine as vengine
|
||||
import barabadb/vector/quant as vquant
|
||||
import barabadb/graph/engine as gengine
|
||||
@@ -954,3 +956,178 @@ suite "Sharding":
|
||||
test "Active shard count":
|
||||
var router = newShardRouter()
|
||||
check router.activeShardCount == 4
|
||||
|
||||
suite "Schema Inheritance":
|
||||
test "Inheritance — merge properties from base":
|
||||
var s = newSchema()
|
||||
let base = newType("Base")
|
||||
base.addProperty("id", "str", required = true)
|
||||
base.addProperty("created", "datetime")
|
||||
s.addType("default", base)
|
||||
|
||||
let child = newType("Person")
|
||||
child.setBases(@["Base"])
|
||||
child.addProperty("name", "str", required = true)
|
||||
s.addType("default", child)
|
||||
|
||||
let resolved = s.resolveInheritance(child)
|
||||
check resolved.properties.len == 3 # id + created + name
|
||||
check "id" in resolved.properties
|
||||
check "name" in resolved.properties
|
||||
check "created" in resolved.properties
|
||||
|
||||
test "Multi-level inheritance":
|
||||
var s = newSchema()
|
||||
let a = newType("A")
|
||||
a.addProperty("a1", "str")
|
||||
s.addType("default", a)
|
||||
|
||||
let b = newType("B")
|
||||
b.setBases(@["A"])
|
||||
b.addProperty("b1", "int32")
|
||||
s.addType("default", b)
|
||||
|
||||
let c = newType("C")
|
||||
c.setBases(@["B"])
|
||||
c.addProperty("c1", "bool")
|
||||
s.addType("default", c)
|
||||
|
||||
let resolved = s.resolveInheritance(c)
|
||||
check resolved.properties.len == 3 # a1 + b1 + c1
|
||||
|
||||
test "Override base property":
|
||||
var s = newSchema()
|
||||
let base = newType("Base")
|
||||
base.addProperty("name", "str")
|
||||
s.addType("default", base)
|
||||
|
||||
let child = newType("Child")
|
||||
child.setBases(@["Base"])
|
||||
child.addProperty("name", "text") # override
|
||||
s.addType("default", child)
|
||||
|
||||
let resolved = s.resolveInheritance(child)
|
||||
check resolved.properties["name"].typeName == "text"
|
||||
|
||||
test "isSubtype":
|
||||
var s = newSchema()
|
||||
let a = newType("A")
|
||||
s.addType("default", a)
|
||||
let b = newType("B")
|
||||
b.setBases(@["A"])
|
||||
s.addType("default", b)
|
||||
let c = newType("C")
|
||||
c.setBases(@["B"])
|
||||
s.addType("default", c)
|
||||
|
||||
check s.isSubtype("C", "A")
|
||||
check s.isSubtype("C", "B")
|
||||
check s.isSubtype("B", "A")
|
||||
check not s.isSubtype("A", "C")
|
||||
|
||||
test "Computed property":
|
||||
let t = newType("Person")
|
||||
t.addProperty("firstName", "str")
|
||||
t.addProperty("lastName", "str")
|
||||
t.addComputedProperty("fullName", "str", "firstName ++ ' ' ++ lastName")
|
||||
check t.properties["fullName"].computed
|
||||
check t.properties["fullName"].expr == "firstName ++ ' ' ++ lastName"
|
||||
|
||||
suite "Codegen":
|
||||
test "Codegen scan":
|
||||
let plan = IRPlan(kind: irpkScan, scanTable: "users", scanAlias: "u")
|
||||
let op = codegenPlan(plan)
|
||||
check op.kind == sokScan
|
||||
check op.table == "users"
|
||||
|
||||
test "Codegen filter with point read optimization":
|
||||
let filterExpr = IRExpr(kind: irekBinary, binOp: irEq,
|
||||
binLeft: IRExpr(kind: irekField, fieldPath: @["id"]),
|
||||
binRight: IRExpr(kind: irekLiteral, literal: IRLiteral(kind: vkInt64, int64Val: 42)))
|
||||
let scan = IRPlan(kind: irpkScan, scanTable: "users", scanAlias: "u")
|
||||
let plan = IRPlan(kind: irpkFilter, filterSource: scan, filterCond: filterExpr)
|
||||
let op = codegenPlan(plan)
|
||||
# Should optimize to point read
|
||||
check op.kind == sokPointRead
|
||||
|
||||
test "Codegen limit":
|
||||
let scan = IRPlan(kind: irpkScan, scanTable: "t", scanAlias: "t")
|
||||
let plan = IRPlan(kind: irpkLimit, limitSource: scan, limitCount: 10, limitOffset: 5)
|
||||
let op = codegenPlan(plan)
|
||||
check op.limit == 10
|
||||
check op.offset == 5
|
||||
|
||||
test "Cost estimation":
|
||||
let scan = newStorageOp(sokScan)
|
||||
check estimateCost(scan) == 1000.0
|
||||
let pointRead = newStorageOp(sokPointRead)
|
||||
check estimateCost(pointRead) == 1.0
|
||||
|
||||
test "Explain plan":
|
||||
let scan = newStorageOp(sokScan)
|
||||
scan.table = "users"
|
||||
let explanation = scan.explain()
|
||||
check "sokScan" in explanation
|
||||
check "users" in explanation
|
||||
|
||||
suite "Replication":
|
||||
test "Create replication manager":
|
||||
var rm = newReplicationManager(rmAsync)
|
||||
check rm.totalReplicaCount == 0
|
||||
check rm.connectedReplicaCount == 0
|
||||
|
||||
test "Add and connect replicas":
|
||||
var rm = newReplicationManager(rmAsync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.addReplica(newReplica("r2", "10.0.0.2", 5432))
|
||||
check rm.totalReplicaCount == 2
|
||||
|
||||
rm.connectReplica("r1")
|
||||
check rm.connectedReplicaCount == 1
|
||||
|
||||
test "Async replication — write returns immediately":
|
||||
var rm = newReplicationManager(rmAsync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.connectReplica("r1")
|
||||
|
||||
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||
check lsn == 1
|
||||
# Async doesn't wait — already "acked"
|
||||
check rm.isFullyAcked(lsn)
|
||||
|
||||
test "Sync replication — wait for ack":
|
||||
var rm = newReplicationManager(rmSync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.connectReplica("r1")
|
||||
|
||||
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||
check not rm.isFullyAcked(lsn)
|
||||
|
||||
rm.ackLsn("r1", lsn)
|
||||
check rm.isFullyAcked(lsn)
|
||||
|
||||
test "Semi-sync replication":
|
||||
var rm = newReplicationManager(rmSemiSync, syncCount = 2)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.addReplica(newReplica("r2", "10.0.0.2", 5432))
|
||||
rm.addReplica(newReplica("r3", "10.0.0.3", 5432))
|
||||
rm.connectReplica("r1")
|
||||
rm.connectReplica("r2")
|
||||
rm.connectReplica("r3")
|
||||
|
||||
let lsn = rm.writeLsn(@[1'u8])
|
||||
check not rm.isFullyAcked(lsn) # needs 2 acks
|
||||
|
||||
rm.ackLsn("r1", lsn)
|
||||
check not rm.isFullyAcked(lsn) # still needs 1 more
|
||||
|
||||
rm.ackLsn("r2", lsn)
|
||||
check rm.isFullyAcked(lsn) # 2 acks received
|
||||
|
||||
test "Replica status":
|
||||
var rm = newReplicationManager(rmAsync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.connectReplica("r1")
|
||||
let status = rm.replicaStatus()
|
||||
check status.len == 1
|
||||
check status[0][1] == rsStreaming
|
||||
|
||||
Reference in New Issue
Block a user