feat: complete Phase 3 + new production roadmap for Web/ERP
- Thread-safety: locks in LSMTree and Graph engines - Raft network transport: async TCP, serialization, heartbeat, 3-node election test - CI/CD: GitHub Actions workflow - Cleanup: remove dead code, unused imports, build artifacts - New PLAN.md targeting production Web/ERP readiness - 216 tests passing
This commit is contained in:
@@ -1,6 +0,0 @@
|
||||
import barabadb/core/config
|
||||
import barabadb/core/server
|
||||
import barabadb/core/types
|
||||
import barabadb/storage/lsm
|
||||
import barabadb/storage/wal
|
||||
import barabadb/storage/bloom
|
||||
@@ -1,7 +1,6 @@
|
||||
## BaraDB CLI — interactive query shell
|
||||
import std/terminal
|
||||
import std/strutils
|
||||
import std/tables
|
||||
import ../query/lexer
|
||||
import ../query/parser
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import std/asyncdispatch
|
||||
import std/asyncnet
|
||||
import std/strutils
|
||||
import std/json
|
||||
import ../protocol/wire
|
||||
|
||||
type
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
## Import/Export — JSON, CSV, Parquet-like formats
|
||||
import std/tables
|
||||
import std/strutils
|
||||
import std/sequtils
|
||||
import ../core/types
|
||||
|
||||
type
|
||||
ExportFormat* = enum
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import std/tables
|
||||
import std/os
|
||||
import std/sequtils
|
||||
import ../core/types
|
||||
import ../storage/lsm
|
||||
import ../vector/engine as vengine
|
||||
import ../graph/engine as gengine
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
## Distributed Transactions — cross-node atomic operations
|
||||
import std/tables
|
||||
import std/sets
|
||||
import std/locks
|
||||
import std/monotimes
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
## Gossip Protocol — membership and failure detection
|
||||
import std/tables
|
||||
import std/sets
|
||||
import std/random
|
||||
import std/monotimes
|
||||
|
||||
|
||||
+228
-7
@@ -5,6 +5,12 @@ import std/deques
|
||||
import std/algorithm
|
||||
import std/random
|
||||
import std/monotimes
|
||||
import std/asyncdispatch
|
||||
import std/asyncnet
|
||||
import std/streams
|
||||
import std/strutils
|
||||
import std/endians
|
||||
import ../protocol/wire
|
||||
|
||||
type
|
||||
RaftState* = enum
|
||||
@@ -36,6 +42,8 @@ type
|
||||
electionTimeout*: int
|
||||
heartbeatTimeout*: int
|
||||
votesReceived*: HashSet[string]
|
||||
peerAddrs*: Table[string, tuple[host: string, port: int]]
|
||||
raftPort*: int
|
||||
|
||||
RaftMessageKind* = enum
|
||||
rmkRequestVote
|
||||
@@ -63,7 +71,7 @@ type
|
||||
nodes*: Table[string, RaftNode]
|
||||
messageQueue*: Deque[RaftMessage]
|
||||
|
||||
proc newRaftNode*(id: string, peers: seq[string]): RaftNode =
|
||||
proc newRaftNode*(id: string, peers: seq[string], raftPort: int = 0): RaftNode =
|
||||
randomize()
|
||||
RaftNode(
|
||||
id: id,
|
||||
@@ -80,6 +88,8 @@ proc newRaftNode*(id: string, peers: seq[string]): RaftNode =
|
||||
electionTimeout: 150 + rand(150),
|
||||
heartbeatTimeout: 50,
|
||||
votesReceived: initHashSet[string](),
|
||||
peerAddrs: initTable[string, tuple[host: string, port: int]](),
|
||||
raftPort: raftPort,
|
||||
)
|
||||
|
||||
proc newRaftCluster*(): RaftCluster =
|
||||
@@ -305,23 +315,234 @@ proc checkTimeout*(timer: ElectionTimer): bool =
|
||||
let elapsed = (getMonoTime().ticks() - timer.lastHeartbeat) div 1_000_000
|
||||
return elapsed > timer.timeoutMs
|
||||
|
||||
proc startElection*(timer: ElectionTimer) =
|
||||
proc stop*(timer: ElectionTimer) =
|
||||
timer.running = false
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network Transport — async TCP communication for Raft
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
RaftMagic = "RAFT"
|
||||
RaftProtoVersion = 1'u32
|
||||
|
||||
proc writeString(s: Stream, str: string) =
|
||||
s.write(uint32(str.len))
|
||||
if str.len > 0:
|
||||
s.writeData(str[0].unsafeAddr, str.len)
|
||||
|
||||
proc readString(s: Stream): string =
|
||||
let len = int(s.readUint32())
|
||||
if len > 0:
|
||||
result = newString(len)
|
||||
discard s.readData(result[0].addr, len)
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc writeLogEntry(s: Stream, entry: LogEntry) =
|
||||
s.write(entry.term)
|
||||
s.write(entry.index)
|
||||
s.writeString(entry.command)
|
||||
s.write(uint32(entry.data.len))
|
||||
if entry.data.len > 0:
|
||||
for b in entry.data:
|
||||
s.write(char(b))
|
||||
|
||||
proc readLogEntry(s: Stream): LogEntry =
|
||||
result.term = s.readUint64()
|
||||
result.index = s.readUint64()
|
||||
result.command = s.readString()
|
||||
let dataLen = int(s.readUint32())
|
||||
result.data = newSeq[byte](dataLen)
|
||||
for i in 0 ..< dataLen:
|
||||
result.data[i] = byte(s.readChar())
|
||||
|
||||
proc serialize*(msg: RaftMessage): seq[byte] =
|
||||
let stream = newStringStream()
|
||||
stream.write(RaftMagic)
|
||||
stream.write(RaftProtoVersion)
|
||||
stream.write(uint32(ord(msg.kind)))
|
||||
stream.write(msg.term)
|
||||
stream.writeString(msg.senderId)
|
||||
stream.write(msg.lastLogIndex)
|
||||
stream.write(msg.lastLogTerm)
|
||||
stream.write(msg.prevLogIndex)
|
||||
stream.write(msg.prevLogTerm)
|
||||
stream.write(uint32(msg.entries.len))
|
||||
for entry in msg.entries:
|
||||
stream.writeLogEntry(entry)
|
||||
stream.write(msg.leaderCommit)
|
||||
stream.write(char(if msg.success: 1 else: 0))
|
||||
stream.write(msg.matchIdx)
|
||||
let strData = stream.data
|
||||
result = newSeq[byte](strData.len)
|
||||
for i in 0 ..< strData.len:
|
||||
result[i] = byte(strData[i])
|
||||
stream.close()
|
||||
|
||||
proc deserializeRaftMessage*(data: seq[byte]): RaftMessage =
|
||||
let stream = newStringStream(cast[string](data))
|
||||
let magic = stream.readStr(4)
|
||||
if magic != RaftMagic:
|
||||
raise newException(ValueError, "Invalid Raft magic bytes")
|
||||
let version = stream.readUint32()
|
||||
if version != RaftProtoVersion:
|
||||
raise newException(ValueError, "Unsupported Raft protocol version")
|
||||
result.kind = RaftMessageKind(stream.readUint32())
|
||||
result.term = stream.readUint64()
|
||||
result.senderId = stream.readString()
|
||||
result.lastLogIndex = stream.readUint64()
|
||||
result.lastLogTerm = stream.readUint64()
|
||||
result.prevLogIndex = stream.readUint64()
|
||||
result.prevLogTerm = stream.readUint64()
|
||||
let entryCount = int(stream.readUint32())
|
||||
result.entries = newSeq[LogEntry](entryCount)
|
||||
for i in 0 ..< entryCount:
|
||||
result.entries[i] = stream.readLogEntry()
|
||||
result.leaderCommit = stream.readUint64()
|
||||
result.success = stream.readChar() != '\0'
|
||||
result.matchIdx = stream.readUint64()
|
||||
stream.close()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RaftNetwork — async TCP transport
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
RaftNetwork* = ref object
|
||||
node*: RaftNode
|
||||
socket*: AsyncSocket
|
||||
running*: bool
|
||||
peerSockets*: Table[string, AsyncSocket]
|
||||
|
||||
proc newRaftNetwork*(node: RaftNode): RaftNetwork =
|
||||
RaftNetwork(
|
||||
node: node,
|
||||
running: false,
|
||||
peerSockets: initTable[string, AsyncSocket](),
|
||||
)
|
||||
|
||||
proc connectToPeer(net: RaftNetwork, peerId: string) {.async.} =
|
||||
if peerId notin net.node.peerAddrs:
|
||||
return
|
||||
let (host, port) = net.node.peerAddrs[peerId]
|
||||
try:
|
||||
let sock = newAsyncSocket()
|
||||
await sock.connect(host, Port(port))
|
||||
net.peerSockets[peerId] = sock
|
||||
except:
|
||||
discard
|
||||
|
||||
proc send*(net: RaftNetwork, peerId: string, msg: RaftMessage) {.async.} =
|
||||
if peerId notin net.peerSockets:
|
||||
await net.connectToPeer(peerId)
|
||||
if peerId in net.peerSockets:
|
||||
let data = serialize(msg)
|
||||
let payloadLen = uint32(data.len)
|
||||
var header = newSeq[byte](4)
|
||||
bigEndian32(addr header[0], unsafeAddr payloadLen)
|
||||
try:
|
||||
await net.peerSockets[peerId].send(cast[string](header) & cast[string](data))
|
||||
except:
|
||||
net.peerSockets.del(peerId)
|
||||
|
||||
proc broadcast*(net: RaftNetwork, msgs: seq[RaftMessage]) {.async.} =
|
||||
for i, peer in net.node.peers:
|
||||
if i < msgs.len:
|
||||
await net.send(peer, msgs[i])
|
||||
|
||||
proc processMessage(net: RaftNetwork, msg: RaftMessage) {.async.} =
|
||||
case msg.kind
|
||||
of rmkRequestVote:
|
||||
let reply = net.node.handleRequestVote(msg)
|
||||
await net.send(msg.senderId, reply)
|
||||
of rmkRequestVoteReply:
|
||||
net.node.handleVoteReply(msg)
|
||||
of rmkAppendEntries:
|
||||
let reply = net.node.handleAppendEntries(msg)
|
||||
await net.send(msg.senderId, reply)
|
||||
of rmkAppendEntriesReply:
|
||||
net.node.handleAppendReply(msg.senderId, msg)
|
||||
|
||||
proc receiveLoop(net: RaftNetwork, client: AsyncSocket) {.async.} =
|
||||
try:
|
||||
while net.running:
|
||||
let lenData = await client.recv(4)
|
||||
if lenData.len < 4:
|
||||
break
|
||||
var pos = 0
|
||||
let payloadLen = int(readUint32(cast[seq[byte]](lenData), pos))
|
||||
let payloadStr = await client.recv(payloadLen)
|
||||
if payloadStr.len < payloadLen:
|
||||
break
|
||||
var payload = newSeq[byte](payloadLen)
|
||||
for i in 0 ..< payloadLen:
|
||||
payload[i] = byte(payloadStr[i])
|
||||
let msg = deserializeRaftMessage(payload)
|
||||
asyncCheck net.processMessage(msg)
|
||||
except:
|
||||
discard
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
proc heartbeatLoop(net: RaftNetwork) {.async.} =
|
||||
while net.running:
|
||||
if net.node.state == rsLeader:
|
||||
for peer in net.node.peers:
|
||||
let msg = net.node.appendEntries(peer)
|
||||
await net.send(peer, msg)
|
||||
await sleepAsync(net.node.heartbeatTimeout)
|
||||
|
||||
proc run*(net: RaftNetwork) {.async.} =
|
||||
net.socket = newAsyncSocket()
|
||||
net.socket.setSockOpt(OptReuseAddr, true)
|
||||
net.socket.bindAddr(Port(net.node.raftPort), "127.0.0.1")
|
||||
net.socket.listen()
|
||||
net.running = true
|
||||
asyncCheck net.heartbeatLoop()
|
||||
while net.running:
|
||||
try:
|
||||
let client = await net.socket.accept()
|
||||
asyncCheck net.receiveLoop(client)
|
||||
except:
|
||||
break
|
||||
|
||||
proc stop*(net: RaftNetwork) =
|
||||
net.running = false
|
||||
if net.socket != nil:
|
||||
net.socket.close()
|
||||
for peerId, sock in net.peerSockets:
|
||||
sock.close()
|
||||
net.peerSockets.clear()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ElectionTimer integration with network transport
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc startElection*(timer: ElectionTimer, net: RaftNetwork) =
|
||||
if timer.node.state != rsCandidate:
|
||||
timer.node.becomeCandidate()
|
||||
if net != nil:
|
||||
let msgs = timer.node.requestVote()
|
||||
for i, peer in timer.node.peers:
|
||||
if i < msgs.len:
|
||||
asyncCheck net.send(peer, msgs[i])
|
||||
|
||||
proc tick*(timer: ElectionTimer) =
|
||||
proc tick*(timer: ElectionTimer, net: RaftNetwork = nil) =
|
||||
case timer.node.state
|
||||
of rsFollower:
|
||||
if timer.checkTimeout():
|
||||
timer.startElection()
|
||||
timer.startElection(net)
|
||||
timer.resetTimeout()
|
||||
of rsCandidate:
|
||||
if timer.checkTimeout():
|
||||
# Election timed out — restart
|
||||
timer.node.becomeCandidate()
|
||||
if net != nil:
|
||||
let msgs = timer.node.requestVote()
|
||||
for i, peer in timer.node.peers:
|
||||
if i < msgs.len:
|
||||
asyncCheck net.send(peer, msgs[i])
|
||||
timer.resetTimeout()
|
||||
of rsLeader:
|
||||
timer.resetTimeout() # Keep alive
|
||||
|
||||
proc stop*(timer: ElectionTimer) =
|
||||
timer.running = false
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import std/tables
|
||||
import std/sets
|
||||
import std/locks
|
||||
import std/monotimes
|
||||
|
||||
type
|
||||
ReplicationMode* = enum
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import std/asyncdispatch
|
||||
import std/asyncnet
|
||||
import std/strutils
|
||||
import std/sequtils
|
||||
import std/re
|
||||
import std/os
|
||||
import std/endians
|
||||
@@ -100,7 +99,7 @@ proc execSelect(db: LSMTree, astNode: Node): QueryResult =
|
||||
result.rows.add(row)
|
||||
result.rowCount = result.rows.len
|
||||
|
||||
proc execInsert(db: var LSMTree, query: string): QueryResult =
|
||||
proc execInsert(db: LSMTree, query: string): QueryResult =
|
||||
result = QueryResult()
|
||||
# Manual parsing for simple INSERT: INSERT table { field := 'value' }
|
||||
# We use the value as the key for simple KV semantics
|
||||
@@ -121,7 +120,7 @@ proc execInsert(db: var LSMTree, query: string): QueryResult =
|
||||
db.put(key, cast[seq[byte]](value))
|
||||
result.affectedRows = 1
|
||||
|
||||
proc execDelete(db: var LSMTree, astNode: Node): QueryResult =
|
||||
proc execDelete(db: LSMTree, astNode: Node): QueryResult =
|
||||
result = QueryResult()
|
||||
var keyFilter = ""
|
||||
if astNode.delWhere != nil and astNode.delWhere.whereExpr != nil:
|
||||
@@ -130,7 +129,7 @@ proc execDelete(db: var LSMTree, astNode: Node): QueryResult =
|
||||
db.delete(keyFilter)
|
||||
result.affectedRows = 1
|
||||
|
||||
proc executeQuery(db: var LSMTree, query: string): (bool, QueryResult, string) =
|
||||
proc executeQuery(db: LSMTree, query: string): (bool, QueryResult, string) =
|
||||
try:
|
||||
let tokens = tokenize(query)
|
||||
let astNode = parse(tokens)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
## Sharding — hash-based and range-based data distribution
|
||||
import std/tables
|
||||
import std/hashes
|
||||
import std/algorithm
|
||||
import std/sets
|
||||
|
||||
type
|
||||
ShardStrategy* = enum
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
## Multi-Language FTS — tokenizers for different languages
|
||||
import std/tables
|
||||
import std/unicode
|
||||
import std/strutils
|
||||
import std/sets
|
||||
|
||||
+177
-24
@@ -5,6 +5,8 @@ import std/algorithm
|
||||
import std/math
|
||||
import std/sets
|
||||
import std/hashes
|
||||
import std/streams
|
||||
import std/locks
|
||||
|
||||
type
|
||||
EdgeId* = distinct uint64
|
||||
@@ -36,6 +38,7 @@ type
|
||||
reverseAdj*: Table[NodeId, seq[AdjacencyEntry]] # incoming
|
||||
nextNodeId: uint64
|
||||
nextEdgeId: uint64
|
||||
lock: Lock
|
||||
|
||||
proc `==`*(a, b: EdgeId): bool = uint64(a) == uint64(b)
|
||||
proc `==`*(a, b: NodeId): bool = uint64(a) == uint64(b)
|
||||
@@ -43,16 +46,18 @@ proc hash*(x: EdgeId): Hash = hash(uint64(x))
|
||||
proc hash*(x: NodeId): Hash = hash(uint64(x))
|
||||
|
||||
proc newGraph*(): Graph =
|
||||
Graph(
|
||||
nodes: initTable[NodeId, GraphNode](),
|
||||
edges: initTable[EdgeId, Edge](),
|
||||
adjacency: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||
reverseAdj: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||
nextNodeId: 1,
|
||||
nextEdgeId: 1,
|
||||
)
|
||||
new(result)
|
||||
initLock(result.lock)
|
||||
result.nodes = initTable[NodeId, GraphNode]()
|
||||
result.edges = initTable[EdgeId, Edge]()
|
||||
result.adjacency = initTable[NodeId, seq[AdjacencyEntry]]()
|
||||
result.reverseAdj = initTable[NodeId, seq[AdjacencyEntry]]()
|
||||
result.nextNodeId = 1
|
||||
result.nextEdgeId = 1
|
||||
|
||||
proc addNode*(g: Graph, label: string, properties: Table[string, string] = initTable[string, string]()): NodeId =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
let id = NodeId(g.nextNodeId)
|
||||
inc g.nextNodeId
|
||||
g.nodes[id] = GraphNode(id: id, label: label, properties: properties)
|
||||
@@ -63,6 +68,8 @@ proc addNode*(g: Graph, label: string, properties: Table[string, string] = initT
|
||||
proc addEdge*(g: Graph, src, dst: NodeId, label: string = "",
|
||||
properties: Table[string, string] = initTable[string, string](),
|
||||
weight: float64 = 1.0): EdgeId =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
let id = EdgeId(g.nextEdgeId)
|
||||
inc g.nextEdgeId
|
||||
g.edges[id] = Edge(id: id, src: src, dst: dst, label: label,
|
||||
@@ -72,22 +79,32 @@ proc addEdge*(g: Graph, src, dst: NodeId, label: string = "",
|
||||
return id
|
||||
|
||||
proc getNode*(g: Graph, id: NodeId): GraphNode =
|
||||
g.nodes[id]
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
return g.nodes[id]
|
||||
|
||||
proc getEdge*(g: Graph, id: EdgeId): Edge =
|
||||
g.edges[id]
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
return g.edges[id]
|
||||
|
||||
proc neighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = @[]
|
||||
for entry in g.adjacency.getOrDefault(nodeId, @[]):
|
||||
result.add(entry.neighbor)
|
||||
|
||||
proc inNeighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = @[]
|
||||
for entry in g.reverseAdj.getOrDefault(nodeId, @[]):
|
||||
result.add(entry.neighbor)
|
||||
|
||||
proc removeNode*(g: Graph, nodeId: NodeId) =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
if nodeId notin g.nodes:
|
||||
return
|
||||
|
||||
@@ -112,6 +129,8 @@ proc removeNode*(g: Graph, nodeId: NodeId) =
|
||||
g.reverseAdj.del(nodeId)
|
||||
|
||||
proc bfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = @[]
|
||||
var visited = initHashSet[NodeId]()
|
||||
var queue = initDeque[(NodeId, int)]()
|
||||
@@ -123,12 +142,14 @@ proc bfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||
result.add(node)
|
||||
if maxDepth >= 0 and depth >= maxDepth:
|
||||
continue
|
||||
for neighbor in g.neighbors(node):
|
||||
if neighbor notin visited:
|
||||
visited.incl(neighbor)
|
||||
queue.addLast((neighbor, depth + 1))
|
||||
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||
if entry.neighbor notin visited:
|
||||
visited.incl(entry.neighbor)
|
||||
queue.addLast((entry.neighbor, depth + 1))
|
||||
|
||||
proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = @[]
|
||||
var visited = initHashSet[NodeId]()
|
||||
var stack: seq[(NodeId, int)] = @[(start, 0)]
|
||||
@@ -141,11 +162,13 @@ proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||
result.add(node)
|
||||
if maxDepth >= 0 and depth >= maxDepth:
|
||||
continue
|
||||
for neighbor in g.neighbors(node):
|
||||
if neighbor notin visited:
|
||||
stack.add((neighbor, depth + 1))
|
||||
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||
if entry.neighbor notin visited:
|
||||
stack.add((entry.neighbor, depth + 1))
|
||||
|
||||
proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
var visited = initHashSet[NodeId]()
|
||||
var parent = initTable[NodeId, NodeId]()
|
||||
var queue = initDeque[NodeId]()
|
||||
@@ -163,15 +186,17 @@ proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] =
|
||||
path.reverse()
|
||||
return path
|
||||
|
||||
for neighbor in g.neighbors(node):
|
||||
if neighbor notin visited:
|
||||
visited.incl(neighbor)
|
||||
parent[neighbor] = node
|
||||
queue.addLast(neighbor)
|
||||
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||
if entry.neighbor notin visited:
|
||||
visited.incl(entry.neighbor)
|
||||
parent[entry.neighbor] = node
|
||||
queue.addLast(entry.neighbor)
|
||||
|
||||
return @[]
|
||||
|
||||
proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = initTable[NodeId, float64]()
|
||||
var visited = initHashSet[NodeId]()
|
||||
|
||||
@@ -196,6 +221,8 @@ proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] =
|
||||
result[entry.neighbor] = newDist
|
||||
|
||||
proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): Table[NodeId, float64] =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
result = initTable[NodeId, float64]()
|
||||
let n = g.nodes.len
|
||||
if n == 0:
|
||||
@@ -227,5 +254,131 @@ proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): T
|
||||
|
||||
result = newRanks
|
||||
|
||||
proc nodeCount*(g: Graph): int = g.nodes.len
|
||||
proc edgeCount*(g: Graph): int = g.edges.len
|
||||
proc nodeCount*(g: Graph): int =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
return g.nodes.len
|
||||
|
||||
proc edgeCount*(g: Graph): int =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
return g.edges.len
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence — binary save/load
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
GraphFileMagic = "BGRF"
|
||||
GraphFileVersion = 1'u32
|
||||
|
||||
proc writeString(s: Stream, str: string) =
|
||||
s.write(uint32(str.len))
|
||||
if str.len > 0:
|
||||
s.writeData(str[0].unsafeAddr, str.len)
|
||||
|
||||
proc readString(s: Stream): string =
|
||||
let len = s.readUint32()
|
||||
if len > 0:
|
||||
result = newString(int(len))
|
||||
discard s.readData(result[0].addr, int(len))
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc saveToFile*(g: Graph, path: string) =
|
||||
acquire(g.lock)
|
||||
defer: release(g.lock)
|
||||
let s = newFileStream(path, fmWrite)
|
||||
if s.isNil:
|
||||
raise newException(IOError, "Cannot open graph file for writing: " & path)
|
||||
|
||||
s.write(GraphFileMagic)
|
||||
s.write(GraphFileVersion)
|
||||
s.write(uint32(g.nodes.len))
|
||||
s.write(uint32(g.edges.len))
|
||||
s.write(g.nextNodeId)
|
||||
s.write(g.nextEdgeId)
|
||||
|
||||
for nodeId, node in g.nodes:
|
||||
s.write(uint64(nodeId))
|
||||
s.writeString(node.label)
|
||||
s.write(uint32(node.properties.len))
|
||||
for key, val in node.properties:
|
||||
s.writeString(key)
|
||||
s.writeString(val)
|
||||
|
||||
for edgeId, edge in g.edges:
|
||||
s.write(uint64(edgeId))
|
||||
s.write(uint64(edge.src))
|
||||
s.write(uint64(edge.dst))
|
||||
s.writeString(edge.label)
|
||||
s.write(edge.weight)
|
||||
s.write(uint32(edge.properties.len))
|
||||
for key, val in edge.properties:
|
||||
s.writeString(key)
|
||||
s.writeString(val)
|
||||
|
||||
s.close()
|
||||
|
||||
proc loadFromFile*(path: string): Graph =
|
||||
let s = newFileStream(path, fmRead)
|
||||
if s.isNil:
|
||||
raise newException(IOError, "Cannot open graph file for reading: " & path)
|
||||
|
||||
let magic = s.readStr(4)
|
||||
if magic != GraphFileMagic:
|
||||
raise newException(ValueError, "Invalid graph file magic bytes")
|
||||
|
||||
let version = s.readUint32()
|
||||
if version != GraphFileVersion:
|
||||
raise newException(ValueError, "Unsupported graph file version: " & $version)
|
||||
|
||||
let nodeCount = int(s.readUint32())
|
||||
let edgeCount = int(s.readUint32())
|
||||
let nextNodeId = s.readUint64()
|
||||
let nextEdgeId = s.readUint64()
|
||||
|
||||
result = Graph(
|
||||
nodes: initTable[NodeId, GraphNode](),
|
||||
edges: initTable[EdgeId, Edge](),
|
||||
adjacency: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||
reverseAdj: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||
nextNodeId: nextNodeId,
|
||||
nextEdgeId: nextEdgeId,
|
||||
lock: Lock(),
|
||||
)
|
||||
initLock(result.lock)
|
||||
|
||||
for i in 0 ..< nodeCount:
|
||||
let id = NodeId(s.readUint64())
|
||||
let label = s.readString()
|
||||
let propCount = int(s.readUint32())
|
||||
var props = initTable[string, string]()
|
||||
for j in 0 ..< propCount:
|
||||
let key = s.readString()
|
||||
let val = s.readString()
|
||||
props[key] = val
|
||||
result.nodes[id] = GraphNode(id: id, label: label, properties: props)
|
||||
result.adjacency[id] = @[]
|
||||
result.reverseAdj[id] = @[]
|
||||
|
||||
for i in 0 ..< edgeCount:
|
||||
let id = EdgeId(s.readUint64())
|
||||
let src = NodeId(s.readUint64())
|
||||
let dst = NodeId(s.readUint64())
|
||||
let label = s.readString()
|
||||
let weight = s.readFloat64()
|
||||
let propCount = int(s.readUint32())
|
||||
var props = initTable[string, string]()
|
||||
for j in 0 ..< propCount:
|
||||
let key = s.readString()
|
||||
let val = s.readString()
|
||||
props[key] = val
|
||||
result.edges[id] = Edge(id: id, src: src, dst: dst, label: label,
|
||||
properties: props, weight: weight)
|
||||
result.adjacency[src].add(AdjacencyEntry(edgeId: id, neighbor: dst,
|
||||
weight: weight, label: label))
|
||||
result.reverseAdj[dst].add(AdjacencyEntry(edgeId: id, neighbor: src,
|
||||
weight: weight, label: label))
|
||||
|
||||
s.close()
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
## HTTP/REST API — JSON endpoint
|
||||
import std/asynchttpserver
|
||||
import std/asyncdispatch
|
||||
import std/json
|
||||
import std/strutils
|
||||
import std/tables
|
||||
|
||||
type
|
||||
HttpMethod* = enum
|
||||
hmGet = "GET"
|
||||
hmPost = "POST"
|
||||
hmPut = "PUT"
|
||||
hmDelete = "DELETE"
|
||||
hmPatch = "PATCH"
|
||||
hmOptions = "OPTIONS"
|
||||
|
||||
RouteHandler* = proc(req: Request): Future[JsonNode] {.gcsafe.}
|
||||
|
||||
HttpRouter* = ref object
|
||||
routes: Table[string, Table[string, RouteHandler]] # method -> path -> handler
|
||||
middlewares: seq[RouteHandler]
|
||||
port*: int
|
||||
address*: string
|
||||
|
||||
Request* = ref object
|
||||
httpMethod*: HttpMethod
|
||||
path*: string
|
||||
query*: Table[string, string]
|
||||
headers*: Table[string, string]
|
||||
body*: string
|
||||
contentType*: string
|
||||
|
||||
Response* = object
|
||||
status*: int
|
||||
body*: JsonNode
|
||||
headers*: Table[string, string]
|
||||
|
||||
proc newHttpRouter*(port: int = 8080, address: string = "0.0.0.0"): HttpRouter =
|
||||
HttpRouter(
|
||||
routes: initTable[string, Table[string, RouteHandler]](),
|
||||
middlewares: @[],
|
||||
port: port,
|
||||
address: address,
|
||||
)
|
||||
|
||||
proc addRoute*(router: HttpRouter, meth: HttpMethod, path: string, handler: RouteHandler) =
|
||||
let m = $meth
|
||||
if m notin router.routes:
|
||||
router.routes[m] = initTable[string, RouteHandler]()
|
||||
router.routes[m][path] = handler
|
||||
|
||||
proc get*(router: HttpRouter, path: string, handler: RouteHandler) =
|
||||
router.addRoute(hmGet, path, handler)
|
||||
|
||||
proc post*(router: HttpRouter, path: string, handler: RouteHandler) =
|
||||
router.addRoute(hmPost, path, handler)
|
||||
|
||||
proc put*(router: HttpRouter, path: string, handler: RouteHandler) =
|
||||
router.addRoute(hmPut, path, handler)
|
||||
|
||||
proc delete*(router: HttpRouter, path: string, handler: RouteHandler) =
|
||||
router.addRoute(hmDelete, path, handler)
|
||||
|
||||
proc jsonResponse*(status: int, data: JsonNode, headers: Table[string, string] = initTable[string, string]()): Response =
|
||||
Response(status: status, body: data, headers: headers)
|
||||
|
||||
proc errorResponse*(status: int, message: string): Response =
|
||||
jsonResponse(status, %*{"error": message})
|
||||
|
||||
proc successResponse*(data: JsonNode): Response =
|
||||
jsonResponse(200, data)
|
||||
|
||||
proc parseQuery*(queryString: string): Table[string, string] =
|
||||
result = initTable[string, string]()
|
||||
if queryString.len == 0:
|
||||
return
|
||||
for pair in queryString.split("&"):
|
||||
let parts = pair.split("=", 1)
|
||||
if parts.len == 2:
|
||||
result[parts[0]] = parts[1]
|
||||
elif parts.len == 1:
|
||||
result[parts[0]] = ""
|
||||
|
||||
proc matchPath*(pattern, path: string): Table[string, string] =
|
||||
result = initTable[string, string]()
|
||||
let patternParts = pattern.split("/")
|
||||
let pathParts = path.split("/")
|
||||
if patternParts.len != pathParts.len:
|
||||
return
|
||||
for i in 0..<patternParts.len:
|
||||
if patternParts[i].startsWith(":"):
|
||||
result[patternParts[i][1..^1]] = pathParts[i]
|
||||
elif patternParts[i] != pathParts[i]:
|
||||
result.clear()
|
||||
return
|
||||
|
||||
proc corsHeaders*(): Table[string, string] =
|
||||
result = initTable[string, string]()
|
||||
result["Access-Control-Allow-Origin"] = "*"
|
||||
result["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, PATCH, OPTIONS"
|
||||
result["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
|
||||
|
||||
proc jsonHeaders*(): Table[string, string] =
|
||||
result = initTable[string, string]()
|
||||
result["Content-Type"] = "application/json"
|
||||
|
||||
proc handleCors*(req: Request): Response =
|
||||
if req.httpMethod == hmOptions:
|
||||
return jsonResponse(204, newJNull(), corsHeaders())
|
||||
return nil
|
||||
|
||||
proc parseRequest*(httpMethod: string, path: string, headers: Table[string, string], body: string): Request =
|
||||
let methodMap = {
|
||||
"GET": hmGet, "POST": hmPost, "PUT": hmPut,
|
||||
"DELETE": hmDelete, "PATCH": hmPatch, "OPTIONS": hmOptions,
|
||||
}.toTable
|
||||
|
||||
var query = initTable[string, string]()
|
||||
var cleanPath = path
|
||||
let qPos = path.find('?')
|
||||
if qPos >= 0:
|
||||
cleanPath = path[0..<qPos]
|
||||
query = parseQuery(path[qPos+1..^1])
|
||||
|
||||
Request(
|
||||
httpMethod: methodMap.getOrDefault(httpMethod, hmGet),
|
||||
path: cleanPath,
|
||||
query: query,
|
||||
headers: headers,
|
||||
body: body,
|
||||
contentType: headers.getOrDefault("Content-Type", ""),
|
||||
)
|
||||
|
||||
proc handleRequest*(router: HttpRouter, req: Request): Future[Response] {.async.} =
|
||||
# CORS
|
||||
let corsResp = handleCors(req)
|
||||
if corsResp != nil:
|
||||
return corsResp
|
||||
|
||||
let methodStr = $req.httpMethod
|
||||
if methodStr in router.routes:
|
||||
for pattern, handler in router.routes[methodStr]:
|
||||
let params = matchPath(pattern, req.path)
|
||||
if params.len > 0 or pattern == req.path:
|
||||
try:
|
||||
return jsonResponse(200, await handler(req), jsonHeaders())
|
||||
except CatchableError as e:
|
||||
return errorResponse(500, e.msg)
|
||||
|
||||
return errorResponse(404, "Not found: " & req.path)
|
||||
@@ -1,100 +0,0 @@
|
||||
## TLS/SSL — transport layer security wrapper
|
||||
import std/os
|
||||
import std/strutils
|
||||
|
||||
type
|
||||
TLSVersion* = enum
|
||||
tls12 = "TLSv1.2"
|
||||
tls13 = "TLSv1.3"
|
||||
|
||||
TLSConfig* = object
|
||||
certFile*: string
|
||||
keyFile*: string
|
||||
caFile*: string
|
||||
minVersion*: TLSVersion
|
||||
verifyPeer*: bool
|
||||
cipherSuites*: seq[string]
|
||||
|
||||
TLSState* = enum
|
||||
tsDisconnected
|
||||
tsHandshaking
|
||||
tsConnected
|
||||
tsError
|
||||
|
||||
TLSConnection* = ref object
|
||||
config*: TLSConfig
|
||||
state*: TLSState
|
||||
host*: string
|
||||
port*: int
|
||||
|
||||
proc defaultTLSConfig*(): TLSConfig =
|
||||
TLSConfig(
|
||||
certFile: "",
|
||||
keyFile: "",
|
||||
caFile: "",
|
||||
minVersion: tls12,
|
||||
verifyPeer: false,
|
||||
cipherSuites: @[
|
||||
"TLS_AES_256_GCM_SHA384",
|
||||
"TLS_CHACHA20_POLY1305_SHA256",
|
||||
"TLS_AES_128_GCM_SHA256",
|
||||
],
|
||||
)
|
||||
|
||||
proc newTLSConfig*(certFile, keyFile: string, caFile: string = "",
|
||||
minVersion: TLSVersion = tls12,
|
||||
verifyPeer: bool = false): TLSConfig =
|
||||
TLSConfig(
|
||||
certFile: certFile,
|
||||
keyFile: keyFile,
|
||||
caFile: caFile,
|
||||
minVersion: minVersion,
|
||||
verifyPeer: verifyPeer,
|
||||
cipherSuites: @[
|
||||
"TLS_AES_256_GCM_SHA384",
|
||||
"TLS_CHACHA20_POLY1305_SHA256",
|
||||
"TLS_AES_128_GCM_SHA256",
|
||||
],
|
||||
)
|
||||
|
||||
proc validateConfig*(config: TLSConfig): seq[string] =
|
||||
result = @[]
|
||||
if config.certFile.len == 0:
|
||||
result.add("Certificate file not specified")
|
||||
elif not fileExists(config.certFile):
|
||||
result.add("Certificate file not found: " & config.certFile)
|
||||
if config.keyFile.len == 0:
|
||||
result.add("Key file not specified")
|
||||
elif not fileExists(config.keyFile):
|
||||
result.add("Key file not found: " & config.keyFile)
|
||||
if config.caFile.len > 0 and not fileExists(config.caFile):
|
||||
result.add("CA file not found: " & config.caFile)
|
||||
|
||||
proc isValid*(config: TLSConfig): bool =
|
||||
return config.validateConfig().len == 0
|
||||
|
||||
proc newTLSConnection*(config: TLSConfig, host: string, port: int): TLSConnection =
|
||||
TLSConnection(config: config, state: tsDisconnected, host: host, port: port)
|
||||
|
||||
proc state*(conn: TLSConnection): TLSState = conn.state
|
||||
|
||||
# Self-signed certificate generation helper
|
||||
proc generateSelfSignedCert*(outputDir: string): (string, string) =
|
||||
let certPath = outputDir / "server.crt"
|
||||
let keyPath = outputDir / "server.key"
|
||||
|
||||
createDir(outputDir)
|
||||
# Use openssl to generate self-signed cert
|
||||
let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & keyPath &
|
||||
" -out " & certPath & " -days 365 -nodes -subj '/CN=localhost' 2>/dev/null"
|
||||
discard execShellCmd(cmd)
|
||||
|
||||
return (certPath, keyPath)
|
||||
|
||||
proc certificateInfo*(certPath: string): Table[string, string] =
|
||||
result = initTable[string, string]()
|
||||
if not fileExists(certPath):
|
||||
return
|
||||
# Would use openssl to parse cert in production
|
||||
result["path"] = certPath
|
||||
result["exists"] = "true"
|
||||
@@ -1,216 +0,0 @@
|
||||
## WebSocket — streaming protocol support
|
||||
import std/asyncdispatch
|
||||
import std/asyncnet
|
||||
import std/strutils
|
||||
import std/base64
|
||||
import std/sha1
|
||||
import std/hashes
|
||||
import std/tables
|
||||
|
||||
const
|
||||
WS_FIN* = 0x80'u8
|
||||
WS_TEXT* = 0x01'u8
|
||||
WS_BINARY* = 0x02'u8
|
||||
WS_CLOSE* = 0x08'u8
|
||||
WS_PING* = 0x09'u8
|
||||
WS_PONG* = 0x0A'u8
|
||||
WS_MAX_FRAME* = 65536
|
||||
|
||||
type
|
||||
WsFrame* = object
|
||||
fin*: bool
|
||||
opcode*: uint8
|
||||
payload*: seq[byte]
|
||||
masked*: bool
|
||||
|
||||
WebSocket* = ref object
|
||||
socket: AsyncSocket
|
||||
connected*: bool
|
||||
onMessage*: proc(data: seq[byte]) {.gcsafe.}
|
||||
onClose*: proc() {.gcsafe.}
|
||||
onPing*: proc(data: seq[byte]) {.gcsafe.}
|
||||
onPong*: proc(data: seq[byte]) {.gcsafe.}
|
||||
|
||||
WsServer* = ref object
|
||||
socket: AsyncSocket
|
||||
port: int
|
||||
address: string
|
||||
clients*: seq[WebSocket]
|
||||
onConnect*: proc(ws: WebSocket) {.gcsafe.}
|
||||
onDisconnect*: proc(ws: WebSocket) {.gcsafe.}
|
||||
onMessage*: proc(ws: WebSocket, data: seq[byte]) {.gcsafe.}
|
||||
|
||||
proc newWebSocket*(socket: AsyncSocket): WebSocket =
|
||||
WebSocket(
|
||||
socket: socket,
|
||||
connected: true,
|
||||
onMessage: nil,
|
||||
onClose: nil,
|
||||
onPing: nil,
|
||||
onPong: nil,
|
||||
)
|
||||
|
||||
proc newWsServer*(port: int = 8081, address: string = "0.0.0.0"): WsServer =
|
||||
WsServer(
|
||||
socket: newAsyncSocket(),
|
||||
port: port,
|
||||
address: address,
|
||||
clients: @[],
|
||||
onConnect: nil,
|
||||
onDisconnect: nil,
|
||||
onMessage: nil,
|
||||
)
|
||||
|
||||
proc wsHandshakeKey(clientKey: string): string =
|
||||
let magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
||||
let combined = clientKey & magic
|
||||
let hash = computeSHA1(combined)
|
||||
return encode(hash)
|
||||
|
||||
proc sendFrame*(ws: WebSocket, opcode: uint8, data: openArray[byte]) {.async.} =
|
||||
var frame: seq[byte] = @[]
|
||||
frame.add(opcode or WS_FIN)
|
||||
|
||||
if data.len < 126:
|
||||
frame.add(byte(data.len))
|
||||
elif data.len < 65536:
|
||||
frame.add(126'u8)
|
||||
frame.add(byte((data.len shr 8) and 0xFF))
|
||||
frame.add(byte(data.len and 0xFF))
|
||||
else:
|
||||
frame.add(127'u8)
|
||||
for i in 0..7:
|
||||
frame.add(byte((data.len shr (56 - i * 8)) and 0xFF))
|
||||
|
||||
for b in data:
|
||||
frame.add(b)
|
||||
|
||||
await ws.socket.send(cast[string](frame))
|
||||
|
||||
proc sendText*(ws: WebSocket, text: string) {.async.} =
|
||||
await ws.sendFrame(WS_TEXT, cast[seq[byte]](text))
|
||||
|
||||
proc sendBinary*(ws: WebSocket, data: seq[byte]) {.async.} =
|
||||
await ws.sendFrame(WS_BINARY, data)
|
||||
|
||||
proc sendPing*(ws: WebSocket, data: seq[byte] = @[]) {.async.} =
|
||||
await ws.sendFrame(WS_PING, data)
|
||||
|
||||
proc sendPong*(ws: WebSocket, data: seq[byte] = @[]) {.async.} =
|
||||
await ws.sendFrame(WS_PONG, data)
|
||||
|
||||
proc close*(ws: WebSocket) {.async.} =
|
||||
if ws.connected:
|
||||
ws.connected = false
|
||||
await ws.sendFrame(WS_CLOSE, @[])
|
||||
ws.socket.close()
|
||||
if ws.onClose != nil:
|
||||
ws.onClose()
|
||||
|
||||
proc readFrame*(ws: WebSocket): Future[WsFrame] {.async.} =
|
||||
var header: array[2, byte]
|
||||
let read1 = await ws.socket.recv(2)
|
||||
if read1.len < 2:
|
||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
||||
|
||||
header[0] = byte(read1[0])
|
||||
header[1] = byte(read1[1])
|
||||
|
||||
result.fin = (header[0] and WS_FIN) != 0
|
||||
result.opcode = header[0] and 0x0F
|
||||
result.masked = (header[1] and 0x80) != 0
|
||||
var payloadLen = int(header[1] and 0x7F)
|
||||
|
||||
if payloadLen == 126:
|
||||
let ext = await ws.socket.recv(2)
|
||||
if ext.len < 2:
|
||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
||||
payloadLen = (int(byte(ext[0])) shl 8) or int(byte(ext[1]))
|
||||
elif payloadLen == 127:
|
||||
let ext = await ws.socket.recv(8)
|
||||
if ext.len < 8:
|
||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
||||
payloadLen = 0
|
||||
for i in 0..7:
|
||||
payloadLen = (payloadLen shl 8) or int(byte(ext[i]))
|
||||
|
||||
var maskKey: array[4, byte] = [0'u8, 0, 0, 0]
|
||||
if result.masked:
|
||||
let mk = await ws.socket.recv(4)
|
||||
if mk.len < 4:
|
||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
||||
for i in 0..3:
|
||||
maskKey[i] = byte(mk[i])
|
||||
|
||||
let payloadData = await ws.socket.recv(payloadLen)
|
||||
if payloadData.len < payloadLen:
|
||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
||||
|
||||
result.payload = newSeq[byte](payloadLen)
|
||||
for i in 0..<payloadLen:
|
||||
if result.masked:
|
||||
result.payload[i] = byte(payloadData[i]) xor maskKey[i mod 4]
|
||||
else:
|
||||
result.payload[i] = byte(payloadData[i])
|
||||
|
||||
proc handleUpgrade*(client: AsyncSocket, requestHeaders: Table[string, string]): Future[WebSocket] {.async.} =
|
||||
let wsKey = requestHeaders.getOrDefault("Sec-WebSocket-Key", "")
|
||||
if wsKey.len == 0:
|
||||
return nil
|
||||
|
||||
let acceptKey = wsHandshakeKey(wsKey)
|
||||
let response = "HTTP/1.1 101 Switching Protocols\r\L" &
|
||||
"Upgrade: websocket\r\L" &
|
||||
"Connection: Upgrade\r\L" &
|
||||
"Sec-WebSocket-Accept: " & acceptKey & "\r\L\r\L"
|
||||
await client.send(response)
|
||||
return newWebSocket(client)
|
||||
|
||||
proc run*(server: WsServer) {.async.} =
|
||||
server.socket.setSockOpt(OptReuseAddr, true)
|
||||
server.socket.bindAddr(Port(server.port), server.address)
|
||||
server.socket.listen()
|
||||
|
||||
while true:
|
||||
let client = await server.socket.accept()
|
||||
let ws = newWebSocket(client)
|
||||
server.clients.add(ws)
|
||||
|
||||
if server.onConnect != nil:
|
||||
server.onConnect(ws)
|
||||
|
||||
# Read loop
|
||||
try:
|
||||
while ws.connected:
|
||||
let frame = await ws.readFrame()
|
||||
case frame.opcode
|
||||
of WS_TEXT, WS_BINARY:
|
||||
if server.onMessage != nil:
|
||||
server.onMessage(ws, frame.payload)
|
||||
of WS_PING:
|
||||
await ws.sendPong(frame.payload)
|
||||
of WS_CLOSE:
|
||||
ws.connected = false
|
||||
of WS_PONG:
|
||||
discard
|
||||
else:
|
||||
discard
|
||||
except:
|
||||
discard
|
||||
finally:
|
||||
ws.connected = false
|
||||
server.clients = server.clients.filterIt(it != ws)
|
||||
if server.onDisconnect != nil:
|
||||
server.onDisconnect(ws)
|
||||
|
||||
proc broadcast*(server: WsServer, data: seq[byte]) {.async.} =
|
||||
for client in server.clients:
|
||||
if client.connected:
|
||||
await client.sendBinary(data)
|
||||
|
||||
proc broadcastText*(server: WsServer, text: string) {.async.} =
|
||||
for client in server.clients:
|
||||
if client.connected:
|
||||
await client.sendText(text)
|
||||
|
||||
proc clientCount*(server: WsServer): int = server.clients.len
|
||||
@@ -1,7 +1,6 @@
|
||||
## Adaptive Query Execution — runtime query plan adaptation
|
||||
import std/tables
|
||||
import std/monotimes
|
||||
import std/algorithm
|
||||
import std/strutils
|
||||
|
||||
type
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
## Codegen — compile IR plan to storage operations
|
||||
import std/tables
|
||||
import std/strutils
|
||||
import ../query/ir
|
||||
import ../core/types
|
||||
|
||||
type
|
||||
StorageOpKind* = enum
|
||||
|
||||
@@ -6,6 +6,7 @@ import std/strutils
|
||||
import std/tables
|
||||
import std/monotimes
|
||||
import std/streams
|
||||
import std/locks
|
||||
import bloom
|
||||
import wal
|
||||
import mmap
|
||||
@@ -38,7 +39,7 @@ type
|
||||
entryCount: int
|
||||
mmapFile: MmapFile
|
||||
|
||||
LSMTree* = object
|
||||
LSMTree* = ref object
|
||||
dir: string
|
||||
memTable: MemTable
|
||||
immutableMem: MemTable
|
||||
@@ -46,8 +47,8 @@ type
|
||||
wal: WriteAheadLog
|
||||
memMaxSize: int
|
||||
currentSeq: uint64
|
||||
readLocks: int
|
||||
nextSSTableId: int
|
||||
lock: Lock
|
||||
|
||||
proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable =
|
||||
MemTable(entries: @[], size: 0, maxSize: maxSize)
|
||||
@@ -312,19 +313,20 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
|
||||
|
||||
sstables.sort(proc(a, b: SSTable): int = cmp(a.minKey, b.minKey))
|
||||
|
||||
LSMTree(
|
||||
dir: dir,
|
||||
memTable: newMemTable(memMaxSize),
|
||||
immutableMem: newMemTable(0),
|
||||
sstables: sstables,
|
||||
wal: newWriteAheadLog(dir / "wal"),
|
||||
memMaxSize: memMaxSize,
|
||||
currentSeq: 0,
|
||||
readLocks: 0,
|
||||
nextSSTableId: nextId,
|
||||
)
|
||||
new(result)
|
||||
initLock(result.lock)
|
||||
result.dir = dir
|
||||
result.memTable = newMemTable(memMaxSize)
|
||||
result.immutableMem = newMemTable(0)
|
||||
result.sstables = sstables
|
||||
result.wal = newWriteAheadLog(dir / "wal")
|
||||
result.memMaxSize = memMaxSize
|
||||
result.currentSeq = 0
|
||||
result.nextSSTableId = nextId
|
||||
|
||||
proc put*(db: var LSMTree, key: string, value: seq[byte]) =
|
||||
proc put*(db: LSMTree, key: string, value: seq[byte]) =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
let ts = uint64(getMonoTime().ticks())
|
||||
db.wal.writePut(cast[seq[byte]](key), value, ts)
|
||||
if not db.memTable.put(key, value, ts):
|
||||
@@ -332,12 +334,14 @@ proc put*(db: var LSMTree, key: string, value: seq[byte]) =
|
||||
db.memTable = newMemTable(db.memMaxSize)
|
||||
discard db.memTable.put(key, value, ts)
|
||||
|
||||
proc delete*(db: var LSMTree, key: string) =
|
||||
proc delete*(db: LSMTree, key: string) =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
let ts = uint64(getMonoTime().ticks())
|
||||
db.wal.writeDelete(cast[seq[byte]](key), ts)
|
||||
discard db.memTable.put(key, @[], ts, deleted = true)
|
||||
|
||||
proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||
proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||
let (found, entry) = db.memTable.get(key)
|
||||
if found:
|
||||
if entry.deleted:
|
||||
@@ -365,11 +369,18 @@ proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||
|
||||
return (false, @[])
|
||||
|
||||
proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
return getUnsafe(db, key)
|
||||
|
||||
proc contains*(db: LSMTree, key: string): bool =
|
||||
let (found, _) = db.get(key)
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
let (found, _) = getUnsafe(db, key)
|
||||
return found
|
||||
|
||||
proc flush*(db: var LSMTree) =
|
||||
proc flushUnsafe(db: LSMTree) =
|
||||
if db.immutableMem.len == 0 and db.memTable.len == 0:
|
||||
return
|
||||
|
||||
@@ -394,17 +405,37 @@ proc flush*(db: var LSMTree) =
|
||||
db.wal.writeCommit(uint64(getMonoTime().ticks()))
|
||||
db.wal.sync()
|
||||
|
||||
proc close*(db: var LSMTree) =
|
||||
db.flush()
|
||||
proc flush*(db: LSMTree) =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
flushUnsafe(db)
|
||||
|
||||
proc close*(db: LSMTree) =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
flushUnsafe(db)
|
||||
for sst in db.sstables.mitems:
|
||||
sst.close()
|
||||
db.wal.close()
|
||||
|
||||
proc memTableSize*(db: LSMTree): int = db.memTable.len
|
||||
proc sstableCount*(db: LSMTree): int = db.sstables.len
|
||||
proc dir*(db: LSMTree): string = db.dir
|
||||
proc memTableSize*(db: LSMTree): int =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
return db.memTable.len
|
||||
|
||||
proc sstableCount*(db: LSMTree): int =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
return db.sstables.len
|
||||
|
||||
proc dir*(db: LSMTree): string =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
return db.dir
|
||||
|
||||
proc scanMemTable*(db: LSMTree): seq[Entry] =
|
||||
acquire(db.lock)
|
||||
defer: release(db.lock)
|
||||
## Return all entries from memory (memTable + immutableMem)
|
||||
result = @[]
|
||||
for e in db.memTable.entries:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
## Crash Recovery — WAL replay with REDO/UNDO
|
||||
import std/streams
|
||||
import std/os
|
||||
import std/tables
|
||||
import ../storage/wal
|
||||
|
||||
type
|
||||
|
||||
Reference in New Issue
Block a user