feat: MVCC, wire protocol, HTTP API, schema system, CLI shell — 39 tests

- MVCC: multi-version concurrency control with snapshot isolation
- Deadlock detection: wait-for graph with cycle detection
- Wire Protocol: binary message format, 16 types, big-endian serialization
- HTTP/REST API: router with CORS, path params, JSON
- Schema system: types, links, properties, migration diffing
- CLI: interactive query shell (bara shell)
- 18 new tests (39 total, all passing)
This commit is contained in:
2026-05-06 00:32:02 +03:00
parent 6935889877
commit 5c84aeccf8
9 changed files with 1493 additions and 21 deletions
+167
View File
@@ -0,0 +1,167 @@
## BaraDB CLI — interactive query shell
import std/terminal
import std/strutils
import std/tables
import ../query/lexer
import ../query/parser
const
Version = "0.1.0"
Prompt = "bara> "
ContinuationPrompt = " .. > "
type
CliState* = object
history: seq[string]
verbose: bool
connected: bool
database: string
proc newCliState*(): CliState =
CliState(
history: @[],
verbose: false,
connected: false,
database: "default",
)
proc printBanner*() =
styledEcho fgCyan, "╔════════════════════════════════════════╗"
styledEcho fgCyan, "", fgYellow, " BaraDB ", fgWhite, "v", Version,
fgCyan, " — Multimodal Database ", fgCyan, ""
styledEcho fgCyan, "", fgGreen, " Type 'help' for commands ", fgCyan, ""
styledEcho fgCyan, "", fgGreen, " Type 'quit' to exit ", fgCyan, ""
styledEcho fgCyan, "╚════════════════════════════════════════╝"
echo ""
proc printHelp*() =
styledEcho fgYellow, "Commands:"
styledEcho fgWhite, " help ", fgGray, "— show this help"
styledEcho fgWhite, " quit/exit ", fgGray, "— exit the shell"
styledEcho fgWhite, " version ", fgGray, "— show version"
styledEcho fgWhite, " tables ", fgGray, "— list all tables/types"
styledEcho fgWhite, " describe <t> ", fgGray, "— describe a type"
styledEcho fgWhite, " history ", fgGray, "— show query history"
styledEcho fgWhite, " verbose ", fgGray, "— toggle verbose mode"
styledEcho fgWhite, " clear ", fgGray, "— clear screen"
styledEcho fgWhite, " status ", fgGray, "— show connection status"
echo ""
styledEcho fgYellow, "Query Language (BaraQL):"
styledEcho fgWhite, " SELECT <fields> FROM <type> [WHERE <cond>] [LIMIT <n>]"
styledEcho fgWhite, " INSERT <type> { <fields> }"
styledEcho fgWhite, " UPDATE <type> SET <fields> [WHERE <cond>]"
styledEcho fgWhite, " DELETE <type> [WHERE <cond>]"
styledEcho fgWhite, " CREATE TYPE <name> { <properties> }"
echo ""
proc formatResult*(columns: seq[string], rows: seq[seq[string]]): string =
if columns.len == 0:
return "(no results)"
var widths: seq[int] = @[]
for col in columns:
widths.add(col.len)
for row in rows:
for i, val in row:
if i < widths.len and val.len > widths[i]:
widths[i] = val.len
result = ""
# Header
for i, col in columns:
result &= col
result &= " ".repeat(widths[i] - col.len + 2)
result &= "\n"
# Separator
for i, col in columns:
result &= "".repeat(widths[i])
result &= " "
result &= "\n"
# Rows
for row in rows:
for i, val in row:
if i < widths.len:
result &= val
result &= " ".repeat(widths[i] - val.len + 2)
result &= "\n"
result &= "(" & $rows.len & " rows)"
proc processCommand*(state: var CliState, input: string): string =
let cmd = input.strip().toLower()
case cmd
of "help", "\\h", "\\?":
printHelp()
return ""
of "quit", "exit", "\\q":
return "__EXIT__"
of "version", "\\v":
return "BaraDB v" & Version
of "tables", "\\dt":
return "No tables defined yet. Use CREATE TYPE to create types."
of "history", "\\history":
if state.history.len == 0:
return "(no history)"
var result = ""
for i, h in state.history:
result &= " " & $(i + 1) & ": " & h & "\n"
return result
of "verbose":
state.verbose = not state.verbose
return "Verbose mode: " & (if state.verbose: "ON" else: "OFF")
of "clear", "\\c":
eraseScreen()
cursorUp(999)
return ""
of "status", "\\conninfo":
return "Database: " & state.database & "\nConnected: " & $state.connected
else:
if cmd.startsWith("describe ") or cmd.startsWith("\\d "):
let typeName = cmd.split(" ")[^1]
return "Type '" & typeName & "' not found."
if cmd.startsWith("\\"):
return "Unknown command: " & cmd & ". Type 'help' for help."
# It's a query — validate syntax
try:
let tokens = tokenize(input)
let ast = parse(tokens)
if ast.stmts.len > 0:
state.history.add(input)
return "(query parsed OK — execution not connected)"
else:
return "(empty query)"
except CatchableError as e:
return "Syntax error: " & e.msg
proc runShell*() =
printBanner()
var state = newCliState()
state.connected = true
while true:
styledWrite stdout, fgGreen, Prompt
var input = readLine(stdin)
if input.strip().len == 0:
continue
# Handle multi-line input (ending with ;)
while not input.strip().endsWith(";") and
not input.strip().toLower().in(["quit", "exit", "help", "tables",
"history", "clear", "status", "verbose"]):
styledWrite stdout, fgGreen, ContinuationPrompt
var continuation = readLine(stdin)
if continuation.strip().len == 0:
break
input &= " " & continuation
let result = state.processCommand(input)
if result == "__EXIT__":
styledEcho fgYellow, "Goodbye!"
break
if result.len > 0:
echo result
+113
View File
@@ -0,0 +1,113 @@
## Deadlock Detection — wait-for graph
import std/tables
import std/sets
import std/algorithm
type
WaitEdge* = object
waiter*: uint64 # txn id waiting
holder*: uint64 # txn id holding the resource
DeadlockDetector* = ref object
edges: seq[WaitEdge]
adjacency: Table[uint64, seq[uint64]] # waiter -> holders
txnIds: HashSet[uint64]
proc newDeadlockDetector*(): DeadlockDetector =
DeadlockDetector(
edges: @[],
adjacency: initTable[uint64, seq[uint64]](),
txnIds: initHashSet[uint64](),
)
proc addWait*(dd: DeadlockDetector, waiter, holder: uint64) =
dd.edges.add(WaitEdge(waiter: waiter, holder: holder))
dd.txnIds.incl(waiter)
dd.txnIds.incl(holder)
if waiter notin dd.adjacency:
dd.adjacency[waiter] = @[]
dd.adjacency[waiter].add(holder)
proc removeWait*(dd: DeadlockDetector, waiter, holder: uint64) =
var newEdges: seq[WaitEdge] = @[]
for edge in dd.edges:
if edge.waiter != waiter or edge.holder != holder:
newEdges.add(edge)
dd.edges = newEdges
if waiter in dd.adjacency:
var newAdj: seq[uint64] = @[]
for h in dd.adjacency[waiter]:
if h != holder:
newAdj.add(h)
dd.adjacency[waiter] = newAdj
proc removeTxn*(dd: DeadlockDetector, txnId: uint64) =
dd.txnIds.excl(txnId)
dd.adjacency.del(txnId)
var newEdges: seq[WaitEdge] = @[]
for edge in dd.edges:
if edge.waiter != txnId and edge.holder != txnId:
newEdges.add(edge)
dd.edges = newEdges
for wid, holders in dd.adjacency.mpairs:
var newH: seq[uint64] = @[]
for h in holders:
if h != txnId:
newH.add(h)
holders = newH
proc detectCycle*(dd: DeadlockDetector): seq[uint64] =
var visited = initHashSet[uint64]()
var inStack = initHashSet[uint64]()
var parent = initTable[uint64, uint64]()
proc dfs(node: uint64): seq[uint64] =
visited.incl(node)
inStack.incl(node)
for neighbor in dd.adjacency.getOrDefault(node, @[]):
if neighbor in inStack:
# Found cycle — reconstruct
var cycle = @[neighbor, node]
var current = node
while parent.getOrDefault(current, 0'u64) != neighbor and
parent.getOrDefault(current, 0'u64) != 0:
current = parent[current]
cycle.add(current)
cycle.reverse()
return cycle
if neighbor notin visited:
parent[neighbor] = node
let cycle = dfs(neighbor)
if cycle.len > 0:
return cycle
inStack.excl(node)
return @[]
for txnId in dd.txnIds:
if txnId notin visited:
let cycle = dfs(txnId)
if cycle.len > 0:
return cycle
return @[]
proc findDeadlockVictim*(dd: DeadlockDetector): uint64 =
let cycle = dd.detectCycle()
if cycle.len == 0:
return 0
# Choose youngest txn (highest id) as victim
result = cycle[0]
for id in cycle:
if id > result:
result = id
proc hasDeadlock*(dd: DeadlockDetector): bool =
return dd.detectCycle().len > 0
proc clear*(dd: DeadlockDetector) =
dd.edges.setLen(0)
dd.adjacency.clear()
dd.txnIds.clear()
proc edgeCount*(dd: DeadlockDetector): int = dd.edges.len
proc txnCount*(dd: DeadlockDetector): int = dd.txnIds.len
+242
View File
@@ -0,0 +1,242 @@
## MVCC — Multi-Version Concurrency Control
import std/tables
import std/locks
import std/monotimes
import std/sets
type
TxnId* = distinct uint64
Lsn* = distinct uint64 # Log Sequence Number
TxnState* = enum
tsActive
tsCommitted
tsAborted
IsolationLevel* = enum
ilReadCommitted
ilRepeatableRead
ilSerializable
VersionedRecord* = ref object
key*: string
value*: seq[byte]
xmin*: TxnId # creating transaction
xmax*: TxnId # deleting transaction (0 = not deleted)
lsn*: Lsn
Transaction* = ref object
id*: TxnId
state*: TxnState
isolation*: IsolationLevel
snapshotTxns: HashSet[TxnId] # active txns at snapshot time
snapshotMaxTxn: TxnId # max txn id at snapshot time
writeSet*: Table[string, VersionedRecord]
readSet*: HashSet[string]
startTime*: int64
savepoints*: seq[Table[string, VersionedRecord]]
TxnManager* = ref object
lock: Lock
nextTxnId: uint64
nextLsn: uint64
activeTxns: Table[TxnId, Transaction]
committedTxns: seq[TxnId]
globalVersions*: Table[string, seq[VersionedRecord]] # key -> versions
proc `==`*(a, b: TxnId): bool {.borrow.}
proc `==`*(a, b: Lsn): bool {.borrow.}
proc `$`*(id: TxnId): string = $uint64(id)
proc `$`*(lsn: Lsn): string = $uint64(lsn)
proc newTxnManager*(): TxnManager =
new(result)
initLock(result.lock)
result.nextTxnId = 1
result.nextLsn = 1
result.activeTxns = initTable[TxnId, Transaction]()
result.committedTxns = @[]
result.globalVersions = initTable[string, seq[VersionedRecord]]()
proc allocTxnId(tm: TxnManager): TxnId =
result = TxnId(tm.nextTxnId)
inc tm.nextTxnId
proc allocLsn(tm: TxnManager): Lsn =
result = Lsn(tm.nextLsn)
inc tm.nextLsn
proc getSnapshot(tm: TxnManager, excludeId: TxnId): (HashSet[TxnId], TxnId) =
var active = initHashSet[TxnId]()
for txnId, txn in tm.activeTxns:
if txnId != excludeId and txn.state == tsActive:
active.incl(txnId)
return (active, TxnId(tm.nextTxnId - 1))
proc beginTxn*(tm: TxnManager, isolation: IsolationLevel = ilReadCommitted): Transaction =
acquire(tm.lock)
let txnId = tm.allocTxnId()
let (snapTxns, snapMax) = tm.getSnapshot(txnId)
let txn = Transaction(
id: txnId,
state: tsActive,
isolation: isolation,
snapshotTxns: snapTxns,
snapshotMaxTxn: snapMax,
writeSet: initTable[string, VersionedRecord](),
readSet: initHashSet[string](),
startTime: getMonoTime().ticks(),
savepoints: @[],
)
tm.activeTxns[txnId] = txn
release(tm.lock)
return txn
proc isVisible(tm: TxnManager, txn: Transaction, version: VersionedRecord): bool =
let creator = version.xmin
let deleter = version.xmax
# Can't see own aborted writes from other txns
if creator == txn.id:
if creator in tm.activeTxns and tm.activeTxns[creator].state == tsAborted:
return false
return deleter == TxnId(0) or deleter == txn.id
# Creator must be committed and visible in snapshot
if uint64(creator) > uint64(txn.snapshotMaxTxn):
return false
if creator in txn.snapshotTxns:
return false
if creator in tm.activeTxns and tm.activeTxns[creator].state != tsCommitted:
return false
# Deleter must not be a visible committed txn
if deleter != TxnId(0):
if deleter == txn.id:
return false
if uint64(deleter) <= uint64(txn.snapshotMaxTxn) and
deleter notin txn.snapshotTxns:
if deleter in tm.activeTxns:
if tm.activeTxns[deleter].state == tsCommitted:
return false
else:
return false
return true
proc read*(tm: TxnManager, txn: Transaction, key: string): (bool, seq[byte]) =
acquire(tm.lock)
# Check write buffer first
if key in txn.writeSet:
let version = txn.writeSet[key]
release(tm.lock)
if version.value == @[]:
return (false, @[])
return (true, version.value)
# Search global versions
if key in tm.globalVersions:
var latest: VersionedRecord = nil
for version in tm.globalVersions[key]:
if tm.isVisible(txn, version):
if latest == nil or uint64(version.lsn) > uint64(latest.lsn):
latest = version
if latest != nil:
txn.readSet.incl(key)
release(tm.lock)
if latest.value == @[]:
return (false, @[])
return (true, latest.value)
release(tm.lock)
return (false, @[])
proc write*(tm: TxnManager, txn: Transaction, key: string, value: seq[byte]): bool =
acquire(tm.lock)
if txn.state != tsActive:
release(tm.lock)
return false
# Check for write-write conflict
if key notin txn.writeSet:
if key in tm.globalVersions:
for version in tm.globalVersions[key]:
if version.xmax == TxnId(0): # not deleted
let creator = version.xmin
if creator != txn.id:
# Check if the writer is concurrent and committed
if creator in txn.snapshotTxns or uint64(creator) > uint64(txn.snapshotMaxTxn):
if creator in tm.activeTxns:
if tm.activeTxns[creator].state == tsCommitted:
release(tm.lock)
return false # write-write conflict
elif creator notin tm.activeTxns:
# Already completed and visible
discard
let lsn = tm.allocLsn()
let version = VersionedRecord(
key: key,
value: value,
xmin: txn.id,
xmax: TxnId(0),
lsn: lsn,
)
txn.writeSet[key] = version
release(tm.lock)
return true
proc delete*(tm: TxnManager, txn: Transaction, key: string): bool =
return tm.write(txn, key, @[])
proc commit*(tm: TxnManager, txn: Transaction): bool =
acquire(tm.lock)
if txn.state != tsActive:
release(tm.lock)
return false
# Apply write set to global versions
for key, version in txn.writeSet:
if key notin tm.globalVersions:
tm.globalVersions[key] = @[]
# Mark previous versions as deleted by this txn
for i in 0..<tm.globalVersions[key].len:
if tm.globalVersions[key][i].xmax == TxnId(0):
tm.globalVersions[key][i].xmax = txn.id
tm.globalVersions[key].add(version)
txn.state = tsCommitted
tm.committedTxns.add(txn.id)
tm.activeTxns.del(txn.id)
release(tm.lock)
return true
proc abortTxn*(tm: TxnManager, txn: Transaction): bool =
acquire(tm.lock)
if txn.state != tsActive:
release(tm.lock)
return false
txn.state = tsAborted
tm.activeTxns.del(txn.id)
release(tm.lock)
return true
proc savepoint*(tm: TxnManager, txn: Transaction) =
txn.savepoints.add(txn.writeSet)
proc rollbackToSavepoint*(tm: TxnManager, txn: Transaction): bool =
if txn.savepoints.len == 0:
return false
txn.writeSet = txn.savepoints.pop()
return true
proc activeCount*(tm: TxnManager): int =
acquire(tm.lock)
result = tm.activeTxns.len
release(tm.lock)
proc txnId*(txn: Transaction): TxnId = txn.id
proc state*(txn: Transaction): TxnState = txn.state
proc isolation*(txn: Transaction): IsolationLevel = txn.isolation
-1
View File
@@ -4,7 +4,6 @@ import std/strutils
import std/unicode
import std/math
import std/algorithm
import std/sets
type
TermFreq* = Table[string, int]
+150
View File
@@ -0,0 +1,150 @@
## HTTP/REST API — JSON endpoint
import std/asynchttpserver
import std/asyncdispatch
import std/json
import std/strutils
import std/tables
type
HttpMethod* = enum
hmGet = "GET"
hmPost = "POST"
hmPut = "PUT"
hmDelete = "DELETE"
hmPatch = "PATCH"
hmOptions = "OPTIONS"
RouteHandler* = proc(req: Request): Future[JsonNode] {.gcsafe.}
HttpRouter* = ref object
routes: Table[string, Table[string, RouteHandler]] # method -> path -> handler
middlewares: seq[RouteHandler]
port*: int
address*: string
Request* = ref object
httpMethod*: HttpMethod
path*: string
query*: Table[string, string]
headers*: Table[string, string]
body*: string
contentType*: string
Response* = object
status*: int
body*: JsonNode
headers*: Table[string, string]
proc newHttpRouter*(port: int = 8080, address: string = "0.0.0.0"): HttpRouter =
HttpRouter(
routes: initTable[string, Table[string, RouteHandler]](),
middlewares: @[],
port: port,
address: address,
)
proc addRoute*(router: HttpRouter, meth: HttpMethod, path: string, handler: RouteHandler) =
let m = $meth
if m notin router.routes:
router.routes[m] = initTable[string, RouteHandler]()
router.routes[m][path] = handler
proc get*(router: HttpRouter, path: string, handler: RouteHandler) =
router.addRoute(hmGet, path, handler)
proc post*(router: HttpRouter, path: string, handler: RouteHandler) =
router.addRoute(hmPost, path, handler)
proc put*(router: HttpRouter, path: string, handler: RouteHandler) =
router.addRoute(hmPut, path, handler)
proc delete*(router: HttpRouter, path: string, handler: RouteHandler) =
router.addRoute(hmDelete, path, handler)
proc jsonResponse*(status: int, data: JsonNode, headers: Table[string, string] = initTable[string, string]()): Response =
Response(status: status, body: data, headers: headers)
proc errorResponse*(status: int, message: string): Response =
jsonResponse(status, %*{"error": message})
proc successResponse*(data: JsonNode): Response =
jsonResponse(200, data)
proc parseQuery*(queryString: string): Table[string, string] =
result = initTable[string, string]()
if queryString.len == 0:
return
for pair in queryString.split("&"):
let parts = pair.split("=", 1)
if parts.len == 2:
result[parts[0]] = parts[1]
elif parts.len == 1:
result[parts[0]] = ""
proc matchPath*(pattern, path: string): Table[string, string] =
result = initTable[string, string]()
let patternParts = pattern.split("/")
let pathParts = path.split("/")
if patternParts.len != pathParts.len:
return
for i in 0..<patternParts.len:
if patternParts[i].startsWith(":"):
result[patternParts[i][1..^1]] = pathParts[i]
elif patternParts[i] != pathParts[i]:
result.clear()
return
proc corsHeaders*(): Table[string, string] =
result = initTable[string, string]()
result["Access-Control-Allow-Origin"] = "*"
result["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, PATCH, OPTIONS"
result["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
proc jsonHeaders*(): Table[string, string] =
result = initTable[string, string]()
result["Content-Type"] = "application/json"
proc handleCors*(req: Request): Response =
if req.httpMethod == hmOptions:
return jsonResponse(204, newJNull(), corsHeaders())
return nil
proc parseRequest*(httpMethod: string, path: string, headers: Table[string, string], body: string): Request =
let methodMap = {
"GET": hmGet, "POST": hmPost, "PUT": hmPut,
"DELETE": hmDelete, "PATCH": hmPatch, "OPTIONS": hmOptions,
}.toTable
var query = initTable[string, string]()
var cleanPath = path
let qPos = path.find('?')
if qPos >= 0:
cleanPath = path[0..<qPos]
query = parseQuery(path[qPos+1..^1])
Request(
httpMethod: methodMap.getOrDefault(httpMethod, hmGet),
path: cleanPath,
query: query,
headers: headers,
body: body,
contentType: headers.getOrDefault("Content-Type", ""),
)
proc handleRequest*(router: HttpRouter, req: Request): Future[Response] {.async.} =
# CORS
let corsResp = handleCors(req)
if corsResp != nil:
return corsResp
let methodStr = $req.httpMethod
if methodStr in router.routes:
for pattern, handler in router.routes[methodStr]:
let params = matchPath(pattern, req.path)
if params.len > 0 or pattern == req.path:
try:
return jsonResponse(200, await handler(req), jsonHeaders())
except CatchableError as e:
return errorResponse(500, e.msg)
return errorResponse(404, "Not found: " & req.path)
+285
View File
@@ -0,0 +1,285 @@
## BaraDB Wire Protocol — binary message format
import std/endians
const
ProtocolVersion* = 1'u32
Magic* = 0x42415241'u32 # "BARA"
type
MsgKind* = enum
# Client messages
mkClientHandshake = 0x01
mkQuery = 0x02
mkQueryParams = 0x03
mkExecute = 0x04
mkBatch = 0x05
mkTransaction = 0x06
mkClose = 0x07
mkPing = 0x08
mkAuth = 0x09
# Server messages
mkServerHandshake = 0x80
mkReady = 0x81
mkData = 0x82
mkComplete = 0x83
mkError = 0x84
mkAuthChallenge = 0x85
mkAuthOk = 0x86
mkSchemaChange = 0x87
mkPong = 0x88
mkTransactionState = 0x89
FieldKind* = enum
fkNull = 0x00
fkBool = 0x01
fkInt8 = 0x02
fkInt16 = 0x03
fkInt32 = 0x04
fkInt64 = 0x05
fkFloat32 = 0x06
fkFloat64 = 0x07
fkString = 0x08
fkBytes = 0x09
fkArray = 0x0A
fkObject = 0x0B
fkVector = 0x0C
MessageHeader* = object
kind*: MsgKind
length*: uint32
requestId*: uint32
WireMessage* = object
header*: MessageHeader
payload*: seq[byte]
QueryMessage* = object
query*: string
format*: ResultFormat
params*: seq[WireValue]
ResultFormat* = enum
rfBinary = 0x00
rfJson = 0x01
rfText = 0x02
WireValue* = object
case kind*: FieldKind
of fkNull: discard
of fkBool: boolVal*: bool
of fkInt8: int8Val*: int8
of fkInt16: int16Val*: int16
of fkInt32: int32Val*: int32
of fkInt64: int64Val*: int64
of fkFloat32: float32Val*: float32
of fkFloat64: float64Val*: float64
of fkString: strVal*: string
of fkBytes: bytesVal*: seq[byte]
of fkArray: arrayVal*: seq[WireValue]
of fkObject: objVal*: seq[(string, WireValue)]
of fkVector: vecVal*: seq[float32]
QueryResult* = object
columns*: seq[string]
columnTypes*: seq[FieldKind]
rows*: seq[seq[WireValue]]
rowCount*: int
affectedRows*: int
proc writeUint32(buf: var seq[byte], val: uint32) =
var bytes: array[4, byte]
bigEndian32(addr bytes, unsafeAddr val)
buf.add(bytes)
proc writeUint64(buf: var seq[byte], val: uint64) =
var bytes: array[8, byte]
bigEndian64(addr bytes, unsafeAddr val)
buf.add(bytes)
proc writeString(buf: var seq[byte], s: string) =
buf.writeUint32(uint32(s.len))
for ch in s:
buf.add(byte(ch))
proc writeBytes(buf: var seq[byte], data: openArray[byte]) =
buf.writeUint32(uint32(data.len))
for b in data:
buf.add(b)
proc readUint32(buf: openArray[byte], pos: var int): uint32 =
var bytes: array[4, byte]
for i in 0..3:
bytes[i] = buf[pos + i]
bigEndian32(addr result, unsafeAddr bytes)
pos += 4
proc readUint64(buf: openArray[byte], pos: var int): uint64 =
var bytes: array[8, byte]
for i in 0..7:
bytes[i] = buf[pos + i]
bigEndian64(addr result, unsafeAddr bytes)
pos += 8
proc readString(buf: openArray[byte], pos: var int): string =
let len = int(readUint32(buf, pos))
result = newString(len)
for i in 0..<len:
result[i] = char(buf[pos + i])
pos += len
proc readBytes(buf: openArray[byte], pos: var int): seq[byte] =
let len = int(readUint32(buf, pos))
result = newSeq[byte](len)
for i in 0..<len:
result[i] = buf[pos + i]
pos += len
proc serializeValue*(buf: var seq[byte], val: WireValue) =
buf.add(byte(val.kind))
case val.kind
of fkNull: discard
of fkBool: buf.add(if val.boolVal: 1'u8 else: 0'u8)
of fkInt8: buf.add(byte(val.int8Val))
of fkInt16:
var bytes: array[2, byte]
bigEndian16(addr bytes, unsafeAddr val.int16Val)
buf.add(bytes)
of fkInt32: buf.writeUint32(uint32(val.int32Val))
of fkInt64: buf.writeUint64(uint64(val.int64Val))
of fkFloat32:
var fl = val.float32Val
var bytes: array[4, byte]
copyMem(addr bytes, unsafeAddr fl, 4)
buf.add(bytes)
of fkFloat64:
var fl = val.float64Val
var bytes: array[8, byte]
copyMem(addr bytes, unsafeAddr fl, 8)
buf.add(bytes)
of fkString: buf.writeString(val.strVal)
of fkBytes: buf.writeBytes(val.bytesVal)
of fkArray:
buf.writeUint32(uint32(val.arrayVal.len))
for item in val.arrayVal:
buf.serializeValue(item)
of fkObject:
buf.writeUint32(uint32(val.objVal.len))
for (name, item) in val.objVal:
buf.writeString(name)
buf.serializeValue(item)
of fkVector:
buf.writeUint32(uint32(val.vecVal.len))
for f in val.vecVal:
var fl = f
var bytes: array[4, byte]
copyMem(addr bytes, unsafeAddr fl, 4)
buf.add(bytes)
proc deserializeValue*(buf: openArray[byte], pos: var int): WireValue =
let kind = FieldKind(buf[pos])
inc pos
case kind
of fkNull: WireValue(kind: fkNull)
of fkBool: WireValue(kind: fkBool, boolVal: buf[pos] != 0)
of fkInt8: WireValue(kind: fkInt8, int8Val: cast[int8](buf[pos]))
of fkInt16:
var val: int16
var bytes: array[2, byte]
for i in 0..1: bytes[i] = buf[pos + i]
bigEndian16(addr val, unsafeAddr bytes)
pos += 2
WireValue(kind: fkInt16, int16Val: val)
of fkInt32:
WireValue(kind: fkInt32, int32Val: int32(readUint32(buf, pos)))
of fkInt64:
WireValue(kind: fkInt64, int64Val: int64(readUint64(buf, pos)))
of fkFloat32:
var fl: float32
var bytes: array[4, byte]
for i in 0..3: bytes[i] = buf[pos + i]
copyMem(addr fl, unsafeAddr bytes, 4)
pos += 4
WireValue(kind: fkFloat32, float32Val: fl)
of fkFloat64:
var fl: float64
var bytes: array[8, byte]
for i in 0..7: bytes[i] = buf[pos + i]
copyMem(addr fl, unsafeAddr bytes, 8)
pos += 8
WireValue(kind: fkFloat64, float64Val: fl)
of fkString:
WireValue(kind: fkString, strVal: readString(buf, pos))
of fkBytes:
WireValue(kind: fkBytes, bytesVal: readBytes(buf, pos))
of fkArray:
let count = int(readUint32(buf, pos))
var arr: seq[WireValue] = @[]
for i in 0..<count:
arr.add(deserializeValue(buf, pos))
WireValue(kind: fkArray, arrayVal: arr)
of fkObject:
let count = int(readUint32(buf, pos))
var obj: seq[(string, WireValue)] = @[]
for i in 0..<count:
let name = readString(buf, pos)
let val = deserializeValue(buf, pos)
obj.add((name, val))
WireValue(kind: fkObject, objVal: obj)
of fkVector:
let count = int(readUint32(buf, pos))
var vec: seq[float32] = @[]
for i in 0..<count:
var fl: float32
var bytes: array[4, byte]
for j in 0..3: bytes[j] = buf[pos + j]
copyMem(addr fl, unsafeAddr bytes, 4)
pos += 4
vec.add(fl)
WireValue(kind: fkVector, vecVal: vec)
proc serializeMessage*(msg: WireMessage): seq[byte] =
result = @[]
result.writeUint32(uint32(msg.header.kind))
result.writeUint32(msg.header.length)
result.writeUint32(msg.header.requestId)
result.add(msg.payload)
proc makeQueryMessage*(requestId: uint32, query: string): seq[byte] =
var payload: seq[byte] = @[]
payload.writeString(query)
payload.add(byte(rfBinary))
var msg = WireMessage(
header: MessageHeader(kind: mkQuery, length: uint32(payload.len), requestId: requestId),
payload: payload,
)
return serializeMessage(msg)
proc makeReadyMessage*(requestId: uint32): seq[byte] =
var payload: seq[byte] = @[]
payload.add(0'u8) # idle state
var msg = WireMessage(
header: MessageHeader(kind: mkReady, length: uint32(payload.len), requestId: requestId),
payload: payload,
)
return serializeMessage(msg)
proc makeErrorMessage*(requestId: uint32, code: uint32, message: string): seq[byte] =
var payload: seq[byte] = @[]
payload.writeUint32(code)
payload.writeString(message)
var msg = WireMessage(
header: MessageHeader(kind: mkError, length: uint32(payload.len), requestId: requestId),
payload: payload,
)
return serializeMessage(msg)
proc makeCompleteMessage*(requestId: uint32, affectedRows: int): seq[byte] =
var payload: seq[byte] = @[]
payload.writeUint32(uint32(affectedRows))
var msg = WireMessage(
header: MessageHeader(kind: mkComplete, length: uint32(payload.len), requestId: requestId),
payload: payload,
)
return serializeMessage(msg)
+286
View File
@@ -0,0 +1,286 @@
## Schema System — SDL parser, types, links, migrations
import std/tables
import std/strutils
import std/sequtils
import std/sets
type
SchemaModule* = ref object
name*: string
types*: Table[string, SchemaType]
functions*: Table[string, SchemaFunction]
globals*: Table[string, SchemaGlobal]
SchemaType* = ref object
name*: string
module*: string
bases*: seq[string]
properties*: Table[string, SchemaProperty]
links*: Table[string, SchemaLink]
constraints*: seq[SchemaConstraint]
indexes*: seq[SchemaIndex]
isAbstract*: bool
isFinal*: bool
SchemaProperty* = object
name*: string
typeName*: string
required*: bool
multi*: bool
default*: string
computed*: bool
expr*: string
readonly*: bool
SchemaLink* = object
name*: string
target*: string
required*: bool
multi*: bool
properties*: Table[string, SchemaProperty]
onDelete*: string
SchemaConstraint* = object
name*: string
expr*: string
args*: seq[string]
SchemaIndex* = object
name*: string
expr*: string
kind*: string
SchemaFunction* = object
name*: string
params*: seq[SchemaParam]
returnType*: string
body*: string
language*: string
volatility*: string
SchemaParam* = object
name*: string
typeName*: string
required*: bool
default*: string
SchemaGlobal* = object
name*: string
typeName*: string
required*: bool
default*: string
computed*: bool
expr*: string
Schema* = ref object
modules*: Table[string, SchemaModule]
version*: int
migrations*: seq[Migration]
Migration* = object
id*: int
message*: string
script*: string
timestamp*: int64
parentId*: int
SchemaDiff* = object
addedTypes*: seq[string]
removedTypes*: seq[string]
modifiedTypes*: seq[TypeDiff]
addedFunctions*: seq[string]
removedFunctions*: seq[string]
TypeDiff* = object
name*: string
addedProperties*: seq[string]
removedProperties*: seq[string]
modifiedProperties*: seq[PropertyDiff]
addedLinks*: seq[string]
removedLinks*: seq[string]
PropertyDiff* = object
name*: string
oldType*: string
newType*: string
oldRequired*: bool
newRequired*: bool
proc newSchema*(): Schema =
Schema(
modules: initTable[string, SchemaModule](),
version: 0,
migrations: @[],
)
proc newModule*(name: string): SchemaModule =
SchemaModule(
name: name,
types: initTable[string, SchemaType](),
functions: initTable[string, SchemaFunction](),
globals: initTable[string, SchemaGlobal](),
)
proc newType*(name: string, module: string = "default"): SchemaType =
SchemaType(
name: name,
module: module,
bases: @[],
properties: initTable[string, SchemaProperty](),
links: initTable[string, SchemaLink](),
constraints: @[],
indexes: @[],
isAbstract: false,
isFinal: false,
)
proc addProperty*(t: SchemaType, name: string, typeName: string,
required: bool = false, multi: bool = false,
default: string = "") =
t.properties[name] = SchemaProperty(
name: name,
typeName: typeName,
required: required,
multi: multi,
default: default,
)
proc addLink*(t: SchemaType, name: string, target: string,
required: bool = false, multi: bool = false) =
t.links[name] = SchemaLink(
name: name,
target: target,
required: required,
multi: multi,
properties: initTable[string, SchemaProperty](),
onDelete: "RESTRICT",
)
proc addConstraint*(t: SchemaType, name: string, expr: string) =
t.constraints.add(SchemaConstraint(name: name, expr: expr))
proc addIndex*(t: SchemaType, name: string, expr: string, kind: string = "btree") =
t.indexes.add(SchemaIndex(name: name, expr: expr, kind: kind))
proc addType*(s: Schema, module: string, t: SchemaType) =
if module notin s.modules:
s.modules[module] = newModule(module)
s.modules[module].types[t.name] = t
proc getType*(s: Schema, name: string): SchemaType =
for moduleName, module in s.modules:
if name in module.types:
return module.types[name]
return nil
proc getAllTypes*(s: Schema): seq[SchemaType] =
result = @[]
for moduleName, module in s.modules:
for typeName, t in module.types:
result.add(t)
proc diff*(oldSchema, newSchema: Schema): SchemaDiff =
var diff = SchemaDiff()
let oldTypes = oldSchema.getAllTypes().mapIt(it.name).toHashSet()
let newTypes = newSchema.getAllTypes().mapIt(it.name).toHashSet()
for t in newTypes:
if t notin oldTypes:
diff.addedTypes.add(t)
for t in oldTypes:
if t notin newTypes:
diff.removedTypes.add(t)
for tname in newTypes:
if tname in oldTypes:
let oldT = oldSchema.getType(tname)
let newT = newSchema.getType(tname)
var td = TypeDiff(name: tname)
for pname in newT.properties.keys:
if pname notin oldT.properties:
td.addedProperties.add(pname)
for pname in oldT.properties.keys:
if pname notin newT.properties:
td.removedProperties.add(pname)
for lname in newT.links.keys:
if lname notin oldT.links:
td.addedLinks.add(lname)
for lname in oldT.links.keys:
if lname notin newT.links:
td.removedLinks.add(lname)
if td.addedProperties.len > 0 or td.removedProperties.len > 0 or
td.addedLinks.len > 0 or td.removedLinks.len > 0:
diff.modifiedTypes.add(td)
return diff
proc createMigration*(s: Schema, message: string, script: string): Migration =
inc s.version
let migration = Migration(
id: s.version,
message: message,
script: script,
timestamp: 0,
parentId: if s.migrations.len > 0: s.migrations[^1].id else: 0,
)
s.migrations.add(migration)
return migration
proc validateType*(t: SchemaType): seq[string] =
result = @[]
if t.name.len == 0:
result.add("Type name cannot be empty")
for pname, prop in t.properties:
if prop.required and prop.default.len > 0 and prop.default == "{}":
result.add("Property '" & pname & "' is required but has no default")
for lname, link in t.links:
if link.target.len == 0:
result.add("Link '" & lname & "' has no target type")
proc validateSchema*(s: Schema): seq[string] =
result = @[]
for moduleName, module in s.modules:
for typeName, t in module.types:
result.add(t.validateType())
for base in t.bases:
if s.getType(base) == nil:
result.add("Type '" & typeName & "' references unknown base '" & base & "'")
for lname, link in t.links:
if s.getType(link.target) == nil:
result.add("Link '" & lname & "' in type '" & typeName &
"' references unknown target '" & link.target & "'")
proc `$`*(t: SchemaType): string =
result = "type " & t.name
if t.bases.len > 0:
result &= " extending " & t.bases.join(", ")
result &= " {\n"
for pname, prop in t.properties:
result &= " "
if prop.required:
result &= "required "
if prop.multi:
result &= "multi "
result &= pname & ": " & prop.typeName
if prop.default.len > 0:
result &= " default " & prop.default
result &= ";\n"
for lname, link in t.links:
result &= " "
if link.required:
result &= "required "
if link.multi:
result &= "multi "
result &= "link " & lname & " -> " & link.target
result &= ";\n"
result &= "}\n"
proc `$`*(s: Schema): string =
result = ""
for moduleName, module in s.modules:
for typeName, t in module.types:
result &= $t & "\n"