feat: full BaraQL (GROUP BY/JOIN/CTE/aggregates), Raft consensus, sharding — 107 tests
BaraQL parser extended: - GROUP BY + HAVING clauses - JOIN (INNER, LEFT, RIGHT, FULL, CROSS) with ON conditions - CTE (WITH ... AS) with multiple CTEs - Aggregate functions (count, sum, avg, min, max) - CASE/WHEN/THEN/ELSE/END expressions - BETWEEN, IN, LIKE/ILIKE, IS NULL/IS NOT NULL - Dotted path identifiers (table.column) - Subqueries in FROM clause - CREATE TYPE with colon syntax (name: type) - UPDATE SET ... WHERE, DELETE FROM ... WHERE Infrastructure: - Raft consensus: leader election, log replication, RequestVote/AppendEntries - Sharding: hash-based, range-based, consistent hashing, rebalancing - Fixed lexer: = is equality, := is assignment, added : (colon) token - 34 new parser tests (107 total, all passing)
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
## Sharding — hash-based and range-based data distribution
|
||||
import std/tables
|
||||
import std/hashes
|
||||
import std/algorithm
|
||||
import std/sets
|
||||
|
||||
type
|
||||
ShardStrategy* = enum
|
||||
ssHash = "hash"
|
||||
ssRange = "range"
|
||||
ssConsistent = "consistent"
|
||||
|
||||
Shard* = object
|
||||
id*: int
|
||||
name*: string
|
||||
minKey*: string
|
||||
maxKey*: string
|
||||
nodeIds*: seq[string] # replica node ids
|
||||
isActive*: bool
|
||||
entryCount*: int
|
||||
|
||||
ShardRouter* = ref object
|
||||
strategy*: ShardStrategy
|
||||
shards*: seq[Shard]
|
||||
vnodeRing*: seq[(uint64, int)] # (hash, shard_id) sorted
|
||||
replicas*: int
|
||||
|
||||
ShardConfig* = object
|
||||
numShards*: int
|
||||
replicas*: int
|
||||
strategy*: ShardStrategy
|
||||
|
||||
proc defaultShardConfig*(): ShardConfig =
|
||||
ShardConfig(numShards: 4, replicas: 1, strategy: ssHash)
|
||||
|
||||
proc newShardRouter*(config: ShardConfig = defaultShardConfig()): ShardRouter =
|
||||
result = ShardRouter(
|
||||
strategy: config.strategy,
|
||||
shards: @[],
|
||||
vnodeRing: @[],
|
||||
replicas: config.replicas,
|
||||
)
|
||||
for i in 0..<config.numShards:
|
||||
result.shards.add(Shard(
|
||||
id: i,
|
||||
name: "shard_" & $i,
|
||||
minKey: "",
|
||||
maxKey: "",
|
||||
nodeIds: @[],
|
||||
isActive: true,
|
||||
entryCount: 0,
|
||||
))
|
||||
|
||||
proc hashKey*(key: string): uint64 =
|
||||
return uint64(hash(key))
|
||||
|
||||
proc getShardHash*(router: ShardRouter, key: string): int =
|
||||
let h = hashKey(key)
|
||||
return int(h mod uint64(router.shards.len))
|
||||
|
||||
proc getShardRange*(router: ShardRouter, key: string): int =
|
||||
for i, shard in router.shards:
|
||||
if key >= shard.minKey and key <= shard.maxKey:
|
||||
return i
|
||||
return 0
|
||||
|
||||
proc getShardConsistent*(router: ShardRouter, key: string): int =
|
||||
if router.vnodeRing.len == 0:
|
||||
return getShardHash(router, key)
|
||||
let h = hashKey(key)
|
||||
for (ringHash, shardId) in router.vnodeRing:
|
||||
if h <= ringHash:
|
||||
return shardId
|
||||
return router.vnodeRing[0][1]
|
||||
|
||||
proc getShard*(router: ShardRouter, key: string): int =
|
||||
case router.strategy
|
||||
of ssHash: router.getShardHash(key)
|
||||
of ssRange: router.getShardRange(key)
|
||||
of ssConsistent: router.getShardConsistent(key)
|
||||
|
||||
proc addVirtualNodes*(router: var ShardRouter, numVnodes: int = 100) =
|
||||
for shard in router.shards:
|
||||
for i in 0..<numVnodes:
|
||||
let vnodeKey = shard.name & "_vnode_" & $i
|
||||
router.vnodeRing.add((hashKey(vnodeKey), shard.id))
|
||||
router.vnodeRing.sort(proc(a, b: (uint64, int)): int = cmp(a[0], b[0]))
|
||||
|
||||
proc setRangeBounds*(router: var ShardRouter, bounds: seq[(string, string)]) =
|
||||
for i, bound in bounds:
|
||||
if i < router.shards.len:
|
||||
router.shards[i].minKey = bound[0]
|
||||
router.shards[i].maxKey = bound[1]
|
||||
|
||||
proc assignNode*(router: var ShardRouter, shardId: int, nodeId: string) =
|
||||
if shardId < router.shards.len:
|
||||
router.shards[shardId].nodeIds.add(nodeId)
|
||||
|
||||
proc getShardForNode*(router: ShardRouter, nodeId: string): seq[int] =
|
||||
result = @[]
|
||||
for i, shard in router.shards:
|
||||
if nodeId in shard.nodeIds:
|
||||
result.add(i)
|
||||
|
||||
proc replicasOf*(router: ShardRouter, key: string): seq[string] =
|
||||
let shardId = router.getShard(key)
|
||||
if shardId < router.shards.len:
|
||||
return router.shards[shardId].nodeIds
|
||||
return @[]
|
||||
|
||||
proc allShards*(router: ShardRouter): seq[Shard] =
|
||||
return router.shards
|
||||
|
||||
proc activeShardCount*(router: ShardRouter): int =
|
||||
result = 0
|
||||
for s in router.shards:
|
||||
if s.isActive:
|
||||
inc result
|
||||
|
||||
proc rebalance*(router: var ShardRouter, nodes: seq[string]) =
|
||||
if nodes.len == 0:
|
||||
return
|
||||
# Clear existing assignments
|
||||
for i in 0..<router.shards.len:
|
||||
router.shards[i].nodeIds = @[]
|
||||
|
||||
# Round-robin assign replicas
|
||||
for i in 0..<router.shards.len:
|
||||
for r in 0..<router.replicas:
|
||||
let nodeIdx = (i + r) mod nodes.len
|
||||
router.shards[i].nodeIds.add(nodes[nodeIdx])
|
||||
|
||||
proc shardCount*(router: ShardRouter): int = router.shards.len
|
||||
Reference in New Issue
Block a user