feat: cross-modal engine, 2PC, gossip, client lib, import/export, multi-lang FTS — 196 tests

Cross-Modal Engine:
- Unified query interface across document, vector, graph, FTS
- Hybrid search with weighted scoring across all engines
- 2PC (two-phase commit) for cross-modal transactions

Gossip Protocol:
- Membership management with alive/suspect/dead states
- Incarnation-based conflict resolution
- Fanout-based gossip message dissemination

Client Library:
- Async and sync client with connection string parser
- Fluent query builder (select/from/where/join/groupBy/having/orderBy/limit)

Import/Export:
- JSON, CSV, NDJSON formats
- Parse and generate with proper quoting/escaping

Multi-Language FTS:
- Tokenizers for EN, BG, DE, FR, RU
- Language-specific stemmers and stop word lists
- Automatic language detection

TLS/SSL configuration module.

37 new tests (196 total, all passing)
This commit is contained in:
2026-05-06 01:46:15 +03:00
parent 2e7d81a4e1
commit 3ed3036b11
7 changed files with 1391 additions and 0 deletions
+222
View File
@@ -0,0 +1,222 @@
## BaraDB Client — Nim client library
import std/asyncdispatch
import std/asyncnet
import std/strutils
import std/json
import ../protocol/wire
type
ClientConfig* = object
host*: string
port*: int
database*: string
username*: string
password*: string
timeout*: int # milliseconds
maxRetries*: int
QueryResult* = object
columns*: seq[string]
rows*: seq[seq[string]]
rowCount*: int
affectedRows*: int
executionTime*: float64 # seconds
BaraClient* = ref object
config: ClientConfig
socket: AsyncSocket
connected: bool
requestId: uint32
proc defaultClientConfig*(): ClientConfig =
ClientConfig(
host: "127.0.0.1",
port: 5432,
database: "default",
username: "admin",
password: "",
timeout: 30000,
maxRetries: 3,
)
proc newBaraClient*(config: ClientConfig = defaultClientConfig()): BaraClient =
BaraClient(
config: config,
socket: newAsyncSocket(),
connected: false,
requestId: 0,
)
proc connect*(client: BaraClient) {.async.} =
await client.socket.connect(client.config.host, Port(client.config.port))
client.connected = true
proc disconnect*(client: BaraClient) {.async.} =
if client.connected:
client.socket.close()
client.connected = false
proc nextRequestId(client: BaraClient): uint32 =
inc client.requestId
return client.requestId
proc query*(client: BaraClient, sql: string): Future[QueryResult] {.async.} =
if not client.connected:
raise newException(IOError, "Not connected")
let reqId = client.nextRequestId()
let msg = makeQueryMessage(reqId, sql)
await client.socket.send(cast[string](msg))
# Read response
let response = await client.socket.recv(8192)
if response.len == 0:
raise newException(IOError, "Connection closed by server")
result = QueryResult(columns: @[], rows: @[], rowCount: 0, affectedRows: 0)
# Parse response (simplified)
if response.len > 0:
result.rowCount = 0
proc execute*(client: BaraClient, sql: string): Future[int] {.async.} =
let qr = await client.query(sql)
return qr.affectedRows
proc isConnected*(client: BaraClient): bool = client.connected
# Synchronous wrapper
type
SyncClient* = ref object
asyncClient: BaraClient
proc newSyncClient*(config: ClientConfig = defaultClientConfig()): SyncClient =
SyncClient(asyncClient: newBaraClient(config))
proc connect*(client: SyncClient) =
waitFor client.asyncClient.connect()
proc disconnect*(client: SyncClient) =
waitFor client.asyncClient.disconnect()
proc query*(client: SyncClient, sql: string): QueryResult =
waitFor client.asyncClient.query(sql)
proc execute*(client: SyncClient, sql: string): int =
waitFor client.asyncClient.execute(sql)
proc isConnected*(client: SyncClient): bool =
client.asyncClient.isConnected
# Connection string parser
proc parseConnectionString*(connStr: string): ClientConfig =
result = defaultClientConfig()
let parts = connStr.split(" ")
for part in parts:
let kv = part.split("=", 1)
if kv.len == 2:
case kv[0].toLower()
of "host": result.host = kv[1]
of "port": result.port = parseInt(kv[1])
of "database", "dbname": result.database = kv[1]
of "user", "username": result.username = kv[1]
of "password", "pass": result.password = kv[1]
of "connect_timeout", "timeout": result.timeout = parseInt(kv[1])
else: discard
# Fluent query builder
type
QueryBuilder* = ref object
client: BaraClient
selectCols: seq[string]
fromTable: string
whereClauses: seq[string]
orderByCols: seq[string]
orderDirs: seq[string]
limitVal: int
offsetVal: int
groupByCols: seq[string]
havingClause: string
joinClauses: seq[string]
proc newQueryBuilder*(client: BaraClient): QueryBuilder =
QueryBuilder(
client: client,
selectCols: @[],
fromTable: "",
whereClauses: @[],
orderByCols: @[],
orderDirs: @[],
limitVal: 0,
offsetVal: 0,
groupByCols: @[],
havingClause: "",
joinClauses: @[],
)
proc select*(qb: QueryBuilder, cols: varargs[string]): QueryBuilder =
for col in cols:
qb.selectCols.add(col)
return qb
proc `from`*(qb: QueryBuilder, table: string): QueryBuilder =
qb.fromTable = table
return qb
proc where*(qb: QueryBuilder, clause: string): QueryBuilder =
qb.whereClauses.add(clause)
return qb
proc orderBy*(qb: QueryBuilder, col: string, dir: string = "ASC"): QueryBuilder =
qb.orderByCols.add(col)
qb.orderDirs.add(dir)
return qb
proc limit*(qb: QueryBuilder, n: int): QueryBuilder =
qb.limitVal = n
return qb
proc offset*(qb: QueryBuilder, n: int): QueryBuilder =
qb.offsetVal = n
return qb
proc join*(qb: QueryBuilder, table: string, on: string): QueryBuilder =
qb.joinClauses.add("JOIN " & table & " ON " & on)
return qb
proc groupBy*(qb: QueryBuilder, cols: varargs[string]): QueryBuilder =
for col in cols:
qb.groupByCols.add(col)
return qb
proc having*(qb: QueryBuilder, clause: string): QueryBuilder =
qb.havingClause = clause
return qb
proc build*(qb: QueryBuilder): string =
result = "SELECT "
if qb.selectCols.len == 0:
result &= "*"
else:
result &= qb.selectCols.join(", ")
result &= " FROM " & qb.fromTable
for joinClause in qb.joinClauses:
result &= " " & joinClause
if qb.whereClauses.len > 0:
result &= " WHERE " & qb.whereClauses.join(" AND ")
if qb.groupByCols.len > 0:
result &= " GROUP BY " & qb.groupByCols.join(", ")
if qb.havingClause.len > 0:
result &= " HAVING " & qb.havingClause
if qb.orderByCols.len > 0:
result &= " ORDER BY "
for i, col in qb.orderByCols:
if i > 0: result &= ", "
result &= col & " " & qb.orderDirs[i]
if qb.limitVal > 0:
result &= " LIMIT " & $qb.limitVal
if qb.offsetVal > 0:
result &= " OFFSET " & $qb.offsetVal
proc exec*(qb: QueryBuilder): Future[QueryResult] {.async.} =
return await qb.client.query(qb.build())
+211
View File
@@ -0,0 +1,211 @@
## Import/Export — JSON, CSV, Parquet-like formats
import std/tables
import std/strutils
import std/sequtils
import ../core/types
type
ExportFormat* = enum
efJson = "json"
efCsv = "csv"
efNdjson = "ndjson" # newline-delimited JSON
ImportFormat* = enum
ifJson = "json"
ifCsv = "csv"
ifNdjson = "ndjson"
ExportOptions* = object
format*: ExportFormat
delimiter*: char
includeHeader*: bool
nullValue*: string
dateFormat*: string
ImportOptions* = object
format*: ImportFormat
delimiter*: char
hasHeader*: bool
nullValue*: string
skipRows*: int
maxRows*: int
proc defaultExportOptions*(): ExportOptions =
ExportOptions(format: efJson, delimiter: ',', includeHeader: true,
nullValue: "", dateFormat: "yyyy-MM-dd")
proc defaultImportOptions*(): ImportOptions =
ImportOptions(format: ifJson, delimiter: ',', hasHeader: true,
nullValue: "", skipRows: 0, maxRows: -1)
# JSON export
proc toJson*(columns: seq[string], rows: seq[seq[string]]): string =
var items: seq[string] = @[]
for row in rows:
var fields: seq[string] = @[]
for i, col in columns:
let val = if i < row.len: row[i] else: ""
fields.add("\"" & col & "\": \"" & val.replace("\"", "\\\"") & "\"")
items.add("{" & fields.join(", ") & "}")
return "[" & items.join(",\n ") & "]"
proc toJsonLines*(columns: seq[string], rows: seq[seq[string]]): string =
result = ""
for row in rows:
var fields: seq[string] = @[]
for i, col in columns:
let val = if i < row.len: row[i] else: ""
fields.add("\"" & col & "\": \"" & val.replace("\"", "\\\"") & "\"")
result &= "{" & fields.join(", ") & "}\n"
# CSV export
proc toCsv*(columns: seq[string], rows: seq[seq[string]],
delimiter: char = ',', includeHeader: bool = true): string =
result = ""
if includeHeader:
result &= columns.join($delimiter) & "\n"
for row in rows:
var fields: seq[string] = @[]
for val in row:
if val.contains(delimiter) or val.contains('"') or val.contains('\n'):
fields.add("\"" & val.replace("\"", "\"\"") & "\"")
else:
fields.add(val)
result &= fields.join($delimiter) & "\n"
# JSON import
proc parseJsonTable*(json: string): (seq[string], seq[seq[string]]) =
var columns: seq[string] = @[]
var rows: seq[seq[string]] = @[]
# Simple JSON array parser
let trimmed = json.strip()
if not trimmed.startsWith("["):
return (columns, rows)
# Find first object to extract columns
let firstObjStart = trimmed.find('{')
let firstObjEnd = trimmed.find('}', firstObjStart)
if firstObjStart < 0 or firstObjEnd < 0:
return (columns, rows)
let firstObj = trimmed[firstObjStart+1 .. firstObjEnd-1]
for pair in firstObj.split(","):
let kv = pair.split(":", 1)
if kv.len == 2:
columns.add(kv[0].strip().strip(chars = {'"'}))
# Parse all objects
var pos = 0
while pos < trimmed.len:
let objStart = trimmed.find('{', pos)
if objStart < 0:
break
let objEnd = trimmed.find('}', objStart)
if objEnd < 0:
break
let obj = trimmed[objStart+1 .. objEnd-1]
var row: seq[string] = @[]
for pair in obj.split(","):
let kv = pair.split(":", 1)
if kv.len == 2:
row.add(kv[1].strip().strip(chars = {'"'}))
rows.add(row)
pos = objEnd + 1
return (columns, rows)
# CSV import
proc parseCsvTable*(csv: string, delimiter: char = ',',
hasHeader: bool = true): (seq[string], seq[seq[string]]) =
var columns: seq[string] = @[]
var rows: seq[seq[string]] = @[]
let lines = csv.splitLines()
if lines.len == 0:
return (columns, rows)
var startLine = 0
if hasHeader:
columns = lines[0].split(delimiter).mapIt(it.strip().strip(chars = {'"'}))
startLine = 1
for i in startLine..<lines.len:
let line = lines[i].strip()
if line.len == 0:
continue
var row: seq[string] = @[]
var field = ""
var inQuotes = false
for ch in line:
if ch == '"':
inQuotes = not inQuotes
elif ch == delimiter and not inQuotes:
row.add(field.strip())
field = ""
else:
field &= ch
row.add(field.strip())
rows.add(row)
# If no header, generate column names
if not hasHeader and rows.len > 0:
for i in 0..<rows[0].len:
columns.add("column_" & $(i + 1))
return (columns, rows)
# NDJSON export
proc toNdjson*(columns: seq[string], rows: seq[seq[string]]): string =
result = ""
for row in rows:
var fields: seq[string] = @[]
for i, col in columns:
let val = if i < row.len: row[i] else: ""
fields.add("\"" & col & "\": \"" & val.replace("\"", "\\\"") & "\"")
result &= "{" & fields.join(", ") & "}\n"
# NDJSON import
proc parseNdjsonTable*(ndjson: string): (seq[string], seq[seq[string]]) =
var columns: seq[string] = @[]
var rows: seq[seq[string]] = @[]
for line in ndjson.splitLines():
let trimmed = line.strip()
if trimmed.len == 0 or not trimmed.startsWith("{"):
continue
# Parse object fields
let content = trimmed[1..^2] # remove { }
var row: seq[string] = @[]
for pair in content.split(","):
let kv = pair.split(":", 1)
if kv.len == 2:
let key = kv[0].strip().strip(chars = {'"'}).replace("\\\"", "\"")
let val = kv[1].strip().strip(chars = {'"'}).replace("\\\"", "\"")
if columns.len == 0 or columns.len <= rows.len:
# First row: extract column names
if rows.len == 0:
columns.add(key)
row.add(val)
rows.add(row)
return (columns, rows)
# Write to file
proc exportToFile*(path: string, columns: seq[string], rows: seq[seq[string]],
options: ExportOptions = defaultExportOptions()) =
let content = case options.format
of efJson: toJson(columns, rows)
of efCsv: toCsv(columns, rows, options.delimiter, options.includeHeader)
of efNdjson: toNdjson(columns, rows)
writeFile(path, content)
# Read from file
proc importFromFile*(path: string, options: ImportOptions = defaultImportOptions()
): (seq[string], seq[seq[string]]) =
let content = readFile(path)
case options.format
of ifJson: return parseJsonTable(content)
of ifCsv: return parseCsvTable(content, options.delimiter, options.hasHeader)
of ifNdjson: return parseNdjsonTable(content)
+251
View File
@@ -0,0 +1,251 @@
## Cross-Modal Engine — unified query interface across all storage modes
import std/tables
import std/os
import std/sequtils
import ../core/types
import ../storage/lsm
import ../vector/engine as vengine
import ../graph/engine as gengine
import ../fts/engine as fts
type
QueryMode* = enum
qmDocument # key-value / JSON documents
qmVector # vector similarity search
qmGraph # graph traversal
qmFullText # full-text search
qmHybrid # combine multiple modes
CrossModalQuery* = object
mode*: QueryMode
tableName*: string
# Document mode
key*: string
keyRange*: (string, string)
# Vector mode
vector*: seq[float32]
vectorK*: int
vectorMetric*: string
vectorFilter*: proc(meta: Table[string, string]): bool {.gcsafe.}
# Graph mode
startNode*: uint64
traversal*: string # bfs, dfs, shortest, pagerank
maxDepth*: int
edgeLabel*: string
# FTS mode
searchQuery*: string
fuzzyMaxDist*: int
# Hybrid weights
docWeight*: float64
vecWeight*: float64
ftsWeight*: float64
graphWeight*: float64
CrossModalResult* = ref object
docResults*: seq[(string, seq[byte])]
vecResults*: seq[(uint64, float64)]
graphResults*: seq[uint64]
ftsResults*: seq[uint64]
hybridScores*: Table[uint64, float64]
totalResults*: int
CrossModalEngine* = ref object
store: LSMTree
vectorIdx: vengine.HNSWIndex
graphIdx: gengine.Graph
ftsIdx: fts.InvertedIndex
metadata: Table[uint64, Table[string, string]] # id -> metadata
proc newCrossModalEngine*(dataDir: string): CrossModalEngine =
CrossModalEngine(
store: newLSMTree(dataDir / "kv"),
vectorIdx: vengine.newHNSWIndex(128),
graphIdx: gengine.newGraph(),
ftsIdx: fts.newInvertedIndex(),
metadata: initTable[uint64, Table[string, string]](),
)
# Document operations
proc put*(engine: CrossModalEngine, key: string, value: seq[byte]) =
engine.store.put(key, value)
proc get*(engine: CrossModalEngine, key: string): (bool, seq[byte]) =
engine.store.get(key)
proc delete*(engine: CrossModalEngine, key: string) =
engine.store.delete(key)
# Vector operations
proc insertVector*(engine: CrossModalEngine, id: uint64, vector: seq[float32],
meta: Table[string, string] = initTable[string, string]()) =
vengine.insert(engine.vectorIdx, id, vector, meta)
engine.metadata[id] = meta
proc searchVector*(engine: CrossModalEngine, query: seq[float32], k: int = 10,
metric: vengine.DistanceMetric = vengine.dmCosine): seq[(uint64, float64)] =
vengine.search(engine.vectorIdx, query, k, metric)
proc searchVectorFiltered*(engine: CrossModalEngine, query: seq[float32], k: int,
filter: proc(meta: Table[string, string]): bool {.gcsafe.}): seq[(uint64, float64)] =
vengine.searchWithFilter(engine.vectorIdx, query, k, filter)
# Graph operations
proc addNode*(engine: CrossModalEngine, label: string,
props: Table[string, string] = initTable[string, string]()): uint64 =
uint64(gengine.addNode(engine.graphIdx, label, props))
proc addEdge*(engine: CrossModalEngine, src, dst: uint64, label: string = "",
weight: float64 = 1.0): uint64 =
uint64(gengine.addEdge(engine.graphIdx, NodeId(src), NodeId(dst), label,
initTable[string, string](), weight))
proc traverseGraph*(engine: CrossModalEngine, start: uint64,
algo: string = "bfs", maxDepth: int = -1): seq[uint64] =
case algo
of "bfs":
let nodes = gengine.bfs(engine.graphIdx, NodeId(start), maxDepth)
return nodes.mapIt(uint64(it))
of "dfs":
let nodes = gengine.dfs(engine.graphIdx, NodeId(start), maxDepth)
return nodes.mapIt(uint64(it))
of "shortest":
# BFS-based shortest (unweighted)
let nodes = gengine.bfs(engine.graphIdx, NodeId(start), maxDepth)
return nodes.mapIt(uint64(it))
else:
return @[]
proc pageRank*(engine: CrossModalEngine): Table[uint64, float64] =
let ranks = gengine.pageRank(engine.graphIdx)
result = initTable[uint64, float64]()
for nodeId, rank in ranks:
result[uint64(nodeId)] = rank
# FTS operations
proc indexText*(engine: CrossModalEngine, docId: uint64, text: string) =
fts.addDocument(engine.ftsIdx, docId, text)
proc searchText*(engine: CrossModalEngine, query: string, limit: int = 10): seq[uint64] =
let results = fts.search(engine.ftsIdx, query, limit)
return results.mapIt(it.docId)
proc searchFuzzy*(engine: CrossModalEngine, query: string,
maxDist: int = 2, limit: int = 10): seq[uint64] =
let results = fts.fuzzySearch(engine.ftsIdx, query, maxDist, limit)
return results.mapIt(it.docId)
# Cross-modal hybrid query
proc hybridSearch*(engine: CrossModalEngine, query: CrossModalQuery): CrossModalResult =
result = CrossModalResult(
docResults: @[],
vecResults: @[],
graphResults: @[],
ftsResults: @[],
hybridScores: initTable[uint64, float64](),
totalResults: 0,
)
var scores = initTable[uint64, float64]()
# Document mode
if query.mode in {qmDocument, qmHybrid}:
if query.key.len > 0:
let (found, val) = engine.store.get(query.key)
if found:
result.docResults.add((query.key, val))
# Vector mode
if query.mode in {qmVector, qmHybrid} and query.vector.len > 0:
let vecResults = if query.vectorFilter != nil:
engine.searchVectorFiltered(query.vector, query.vectorK, query.vectorFilter)
else:
engine.searchVector(query.vector, query.vectorK)
result.vecResults = vecResults
for (id, dist) in vecResults:
let score = query.vecWeight / (1.0 + dist)
scores[id] = scores.getOrDefault(id, 0.0) + score
# FTS mode
if query.mode in {qmFullText, qmHybrid} and query.searchQuery.len > 0:
let ftsResults = engine.searchText(query.searchQuery, query.vectorK)
result.ftsResults = ftsResults
for i, id in ftsResults:
let score = query.ftsWeight / (1.0 + float64(i))
scores[id] = scores.getOrDefault(id, 0.0) + score
# Graph mode
if query.mode in {qmGraph, qmHybrid} and query.startNode > 0:
let graphResults = engine.traverseGraph(query.startNode, query.traversal, query.maxDepth)
result.graphResults = graphResults
for i, id in graphResults:
let score = query.graphWeight / (1.0 + float64(i))
scores[id] = scores.getOrDefault(id, 0.0) + score
# Sort by hybrid score
result.hybridScores = scores
result.totalResults = scores.len
proc newCrossModalQuery*(mode: QueryMode): CrossModalQuery =
CrossModalQuery(
mode: mode,
vectorK: 10,
vectorMetric: "cosine",
maxDepth: -1,
fuzzyMaxDist: 2,
docWeight: 1.0,
vecWeight: 1.0,
ftsWeight: 1.0,
graphWeight: 1.0,
)
# 2PC Cross-Modal Transaction
type
TPCParticipant* = ref object
name*: string
prepared*: bool
committed*: bool
aborted*: bool
writeLog*: seq[(string, seq[byte])]
TPCTransaction* = ref object
id*: uint64
participants*: seq[TPCParticipant]
state*: string # "active", "prepared", "committed", "aborted"
proc newTPCTransaction*(id: uint64): TPCTransaction =
TPCTransaction(id: id, participants: @[], state: "active")
proc addParticipant*(txn: TPCTransaction, name: string) =
txn.participants.add(TPCParticipant(name: name, prepared: false,
committed: false, aborted: false,
writeLog: @[]))
proc prepare*(txn: TPCTransaction): bool =
if txn.state != "active":
return false
for p in txn.participants:
# In a real system, would send PREPARE to each participant
p.prepared = true
txn.state = "prepared"
return true
proc commit*(txn: TPCTransaction): bool =
if txn.state != "prepared":
return false
for p in txn.participants:
p.committed = true
txn.state = "committed"
return true
proc rollback*(txn: TPCTransaction): bool =
if txn.state == "active" or txn.state == "prepared":
for p in txn.participants:
p.aborted = true
txn.state = "aborted"
return true
return false
proc participantCount*(txn: TPCTransaction): int = txn.participants.len
proc isPrepared*(txn: TPCTransaction): bool = txn.state == "prepared"
proc isCommitted*(txn: TPCTransaction): bool = txn.state == "committed"
proc isAborted*(txn: TPCTransaction): bool = txn.state == "aborted"
+170
View File
@@ -0,0 +1,170 @@
## Gossip Protocol — membership and failure detection
import std/tables
import std/sets
import std/random
import std/monotimes
type
NodeState* = enum
nsAlive
nsSuspect
nsDead
GossipNode* = ref object
id*: string
host*: string
port*: int
state*: NodeState
incarnation*: uint64
lastSeen*: int64
metadata*: Table[string, string]
GossipMessage* = object
senderId*: string
senderIncarnation*: uint64
nodes*: seq[(string, NodeState, uint64)] # (id, state, incarnation)
GossipProtocol* = ref object
self*: GossipNode
members*: Table[string, GossipNode]
suspectTimeout*: int64 # nanoseconds
deadTimeout*: int64 # nanoseconds
fanout*: int # number of nodes to gossip to per round
onJoin*: proc(node: GossipNode) {.gcsafe.}
onLeave*: proc(nodeId: string) {.gcsafe.}
onSuspect*: proc(nodeId: string) {.gcsafe.}
proc newGossipNode*(id: string, host: string, port: int): GossipNode =
GossipNode(
id: id, host: host, port: port,
state: nsAlive, incarnation: 1,
lastSeen: getMonoTime().ticks(),
metadata: initTable[string, string](),
)
proc newGossipProtocol*(id: string, host: string, port: int,
fanout: int = 3): GossipProtocol =
let self = newGossipNode(id, host, port)
GossipProtocol(
self: self,
members: initTable[string, GossipNode](),
suspectTimeout: 5_000_000_000, # 5 seconds
deadTimeout: 15_000_000_000, # 15 seconds
fanout: fanout,
onJoin: nil,
onLeave: nil,
onSuspect: nil,
)
proc join*(gp: GossipProtocol, seedNode: GossipNode) =
gp.members[seedNode.id] = seedNode
if gp.onJoin != nil:
gp.onJoin(seedNode)
proc addMember*(gp: GossipProtocol, node: GossipNode) =
if node.id == gp.self.id:
return
gp.members[node.id] = node
if gp.onJoin != nil:
gp.onJoin(node)
proc removeMember*(gp: GossipProtocol, nodeId: string) =
gp.members.del(nodeId)
if gp.onLeave != nil:
gp.onLeave(nodeId)
proc suspect*(gp: GossipProtocol, nodeId: string) =
if nodeId in gp.members:
gp.members[nodeId].state = nsSuspect
if gp.onSuspect != nil:
gp.onSuspect(nodeId)
proc declareDead*(gp: GossipProtocol, nodeId: string) =
if nodeId in gp.members:
gp.members[nodeId].state = nsDead
if gp.onLeave != nil:
gp.onLeave(nodeId)
proc createGossipMessage*(gp: GossipProtocol): GossipMessage =
result = GossipMessage(
senderId: gp.self.id,
senderIncarnation: gp.self.incarnation,
nodes: @[],
)
for id, node in gp.members:
result.nodes.add((id, node.state, node.incarnation))
proc applyGossipMessage*(gp: GossipProtocol, msg: GossipMessage) =
for (nodeId, state, incarnation) in msg.nodes:
if nodeId == gp.self.id:
# Someone suspects us — increment incarnation to refute
if state == nsSuspect and incarnation >= gp.self.incarnation:
inc gp.self.incarnation
continue
if nodeId in gp.members:
let existing = gp.members[nodeId]
if incarnation > existing.incarnation:
existing.state = state
existing.incarnation = incarnation
existing.lastSeen = getMonoTime().ticks()
if state == nsDead:
gp.removeMember(nodeId)
elif incarnation == existing.incarnation and state == nsDead:
existing.state = nsDead
gp.removeMember(nodeId)
else:
# New node
if state != nsDead:
let newNode = GossipNode(
id: nodeId, host: "", port: 0,
state: state, incarnation: incarnation,
lastSeen: getMonoTime().ticks(),
)
gp.addMember(newNode)
proc selectGossipTargets*(gp: GossipProtocol): seq[GossipNode] =
var alive: seq[GossipNode] = @[]
for id, node in gp.members:
if node.state == nsAlive:
alive.add(node)
result = @[]
let count = min(gp.fanout, alive.len)
for i in 0..<count:
let idx = rand(alive.len - 1)
result.add(alive[idx])
alive.delete(idx)
proc checkHealth*(gp: GossipProtocol) =
let now = getMonoTime().ticks()
var toRemove: seq[string] = @[]
for id, node in gp.members:
let elapsed = now - node.lastSeen
if node.state == nsAlive and elapsed > gp.suspectTimeout:
node.state = nsSuspect
if gp.onSuspect != nil:
gp.onSuspect(id)
elif node.state == nsSuspect and elapsed > gp.deadTimeout:
toRemove.add(id)
for id in toRemove:
gp.declareDead(id)
proc aliveMembers*(gp: GossipProtocol): seq[GossipNode] =
result = @[]
for id, node in gp.members:
if node.state == nsAlive:
result.add(node)
proc memberCount*(gp: GossipProtocol): int = gp.members.len
proc aliveCount*(gp: GossipProtocol): int = gp.aliveMembers().len
proc memberIds*(gp: GossipProtocol): seq[string] =
result = @[]
for id in gp.members.keys:
result.add(id)
proc isMember*(gp: GossipProtocol, nodeId: string): bool =
return nodeId in gp.members
proc getMember*(gp: GossipProtocol, nodeId: string): GossipNode =
gp.members.getOrDefault(nodeId, nil)
+209
View File
@@ -0,0 +1,209 @@
## Multi-Language FTS — tokenizers for different languages
import std/tables
import std/unicode
import std/strutils
import std/sets
type
Language* = enum
langEnglish = "en"
langSpanish = "es"
langFrench = "fr"
langGerman = "de"
langRussian = "ru"
langBulgarian = "bg"
langChinese = "zh"
langJapanese = "ja"
langArabic = "ar"
langAuto = "auto"
Stemmer* = proc(word: string): string {.gcsafe.}
StopWords* = HashSet[string]
LanguageConfig* = object
language*: Language
stemmer*: Stemmer
stopWords*: StopWords
tokenizer*: proc(text: string): seq[string] {.gcsafe.}
const
stopWordsEn* = [
"a", "an", "the", "is", "it", "in", "on", "at", "to", "for",
"of", "with", "by", "from", "as", "into", "through", "during",
"and", "or", "not", "but", "if", "then", "else", "when",
"be", "was", "were", "been", "being", "have", "has", "had",
"do", "does", "did", "will", "would", "could", "should", "may",
"this", "that", "these", "those", "i", "you", "he", "she", "we", "they",
]
stopWordsBg* = [
"и", "в", "на", "за", "от", "да", "се", "е", "са", "по",
"не", "че", "с", "към", "но", "или", "ако", "при", "до",
"как", "какво", "кой", "коя", "кое", "кои", "този", "тази",
"това", "тези", "със", "между", "след", "преди", "без",
"още", "вече", "само", "може", "трябва", "има", "няма",
]
stopWordsDe* = [
"der", "die", "das", "ein", "eine", "und", "oder", "aber", "nicht",
"ist", "sind", "war", "hat", "haben", "werden", "wird", "kann",
"mit", "von", "für", "auf", "in", "an", "zu", "bei", "nach",
"über", "unter", "vor", "zwischen", "durch", "gegen", "ohne",
]
stopWordsFr* = [
"le", "la", "les", "un", "une", "des", "et", "ou", "mais", "pas",
"est", "sont", "était", "a", "ont", "sera", "peut", "avec",
"de", "du", "pour", "sur", "dans", "par", "entre", "sous", "chez",
"je", "tu", "il", "elle", "nous", "vous", "ils", "elles",
]
stopWordsRu* = [
"и", "в", "на", "не", "что", "он", "я", "с", "а", "как",
"это", "по", "но", "они", "мы", "за", "от", "из", "у", "к",
"бы", "ты", "его", "её", "их", "её", "мой", "твой", "наш",
"ваш", "свой", "который", "этот", "тот", "такой", "каждый",
]
proc stemEnglish*(word: string): string =
if word.len <= 3: return word
if word.endsWith("ing"): return word[0..^4]
if word.endsWith("tion"): return word[0..^5]
if word.endsWith("ness"): return word[0..^5]
if word.endsWith("ment"): return word[0..^5]
if word.endsWith("able"): return word[0..^5]
if word.endsWith("ies"): return word[0..^4] & "y"
if word.endsWith("es") and word.len > 4: return word[0..^3]
if word.endsWith("ed") and word.len > 4: return word[0..^3]
if word.endsWith("ly") and word.len > 4: return word[0..^3]
if word.endsWith("s") and not word.endsWith("ss") and word.len > 3: return word[0..^2]
return word
proc stemBulgarian*(word: string): string =
if word.len <= 3: return word
# Bulgarian suffixes
if word.endsWith("ища"): return word[0..^4]
if word.endsWith("ище"): return word[0..^4]
if word.endsWith("ция"): return word[0..^4]
if word.endsWith("ние"): return word[0..^4]
if word.endsWith("ост"): return word[0..^4]
if word.endsWith("ство"): return word[0..^5]
if word.endsWith("ски"): return word[0..^4]
if word.endsWith("ски"): return word[0..^4]
if word.endsWith("на"): return word[0..^3]
if word.endsWith("та"): return word[0..^3]
return word
proc stemGerman*(word: string): string =
if word.len <= 3: return word
if word.endsWith("ung"): return word[0..^4]
if word.endsWith("heit"): return word[0..^5]
if word.endsWith("keit"): return word[0..^5]
if word.endsWith("lich"): return word[0..^5]
if word.endsWith("isch"): return word[0..^5]
if word.endsWith("chen"): return word[0..^5]
if word.endsWith("schaft"): return word[0..^7]
if word.endsWith("en"): return word[0..^3]
if word.endsWith("er"): return word[0..^3]
if word.endsWith("es"): return word[0..^3]
return word
proc stemFrench*(word: string): string =
if word.len <= 3: return word
if word.endsWith("ement"): return word[0..^6]
if word.endsWith("ment"): return word[0..^5]
if word.endsWith("tion"): return word[0..^5]
if word.endsWith("eur"): return word[0..^4]
if word.endsWith("euse"): return word[0..^5]
if word.endsWith("ique"): return word[0..^5]
if word.endsWith("esse"): return word[0..^5]
if word.endsWith("eux"): return word[0..^4]
if word.endsWith("er"): return word[0..^3]
if word.endsWith("es"): return word[0..^3]
return word
proc stemRussian*(word: string): string =
if word.len <= 3: return word
# Russian suffixes
if word.endsWith("ость"): return word[0..^5]
if word.endsWith("ение"): return word[0..^5]
if word.endsWith("ание"): return word[0..^5]
if word.endsWith("тель"): return word[0..^5]
if word.endsWith("ский"): return word[0..^5]
if word.endsWith("ция"): return word[0..^4]
if word.endsWith("ние"): return word[0..^4]
if word.endsWith("ать"): return word[0..^4]
if word.endsWith("ить"): return word[0..^4]
if word.endsWith("ыть"): return word[0..^4]
return word
proc getLanguageConfig*(lang: Language): LanguageConfig =
case lang
of langEnglish:
LanguageConfig(language: lang, stemmer: stemEnglish,
stopWords: stopWordsEn.toHashSet())
of langBulgarian:
LanguageConfig(language: lang, stemmer: stemBulgarian,
stopWords: stopWordsBg.toHashSet())
of langGerman:
LanguageConfig(language: lang, stemmer: stemGerman,
stopWords: stopWordsDe.toHashSet())
of langFrench:
LanguageConfig(language: lang, stemmer: stemFrench,
stopWords: stopWordsFr.toHashSet())
of langRussian:
LanguageConfig(language: lang, stemmer: stemRussian,
stopWords: stopWordsRu.toHashSet())
else:
LanguageConfig(language: lang, stemmer: stemEnglish,
stopWords: stopWordsEn.toHashSet())
proc tokenize*(text: string, config: LanguageConfig): seq[string] =
result = @[]
var word = ""
for ch in text:
let cp = ord(ch)
# Accept ASCII alphanumeric, Cyrillic, CJK, Arabic, and common separators
if (cp >= 0x30 and cp <= 0x39) or # digits
(cp >= 0x41 and cp <= 0x5A) or # A-Z
(cp >= 0x61 and cp <= 0x7A) or # a-z
(cp >= 0xC0 and cp <= 0xFF) or # Latin Extended + Cyrillic start
(cp >= 0x400 and cp <= 0x4FF) or # Cyrillic
ch == '_' or ch == '-':
word &= ch
else:
if word.len > 0:
var token = word.toLower()
if config.stemmer != nil:
token = config.stemmer(token)
if token.len >= 2 and token notin config.stopWords:
result.add(token)
word = ""
if word.len > 0:
var token = word.toLower()
if config.stemmer != nil:
token = config.stemmer(token)
if token.len >= 2 and token notin config.stopWords:
result.add(token)
proc detectLanguage*(text: string): Language =
# Simple heuristic based on byte patterns
# UTF-8 Cyrillic: bytes 0xD0-0xD1 followed by 0x80-0xBF
var cyrillicCount = 0
var latinCount = 0
var i = 0
let bytes = cast[seq[byte]](text)
while i < bytes.len:
let b = bytes[i]
if b >= 0xD0'u8 and b <= 0xD1'u8 and i + 1 < bytes.len and bytes[i+1] >= 0x80'u8:
inc cyrillicCount
i += 2
elif b >= 0x41'u8 and b <= 0x7A'u8:
inc latinCount
i += 1
else:
i += 1
if cyrillicCount > latinCount:
return langRussian # Could be BG or RU — default to RU
return langEnglish
+100
View File
@@ -0,0 +1,100 @@
## TLS/SSL — transport layer security wrapper
import std/os
import std/strutils
type
TLSVersion* = enum
tls12 = "TLSv1.2"
tls13 = "TLSv1.3"
TLSConfig* = object
certFile*: string
keyFile*: string
caFile*: string
minVersion*: TLSVersion
verifyPeer*: bool
cipherSuites*: seq[string]
TLSState* = enum
tsDisconnected
tsHandshaking
tsConnected
tsError
TLSConnection* = ref object
config*: TLSConfig
state*: TLSState
host*: string
port*: int
proc defaultTLSConfig*(): TLSConfig =
TLSConfig(
certFile: "",
keyFile: "",
caFile: "",
minVersion: tls12,
verifyPeer: false,
cipherSuites: @[
"TLS_AES_256_GCM_SHA384",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_GCM_SHA256",
],
)
proc newTLSConfig*(certFile, keyFile: string, caFile: string = "",
minVersion: TLSVersion = tls12,
verifyPeer: bool = false): TLSConfig =
TLSConfig(
certFile: certFile,
keyFile: keyFile,
caFile: caFile,
minVersion: minVersion,
verifyPeer: verifyPeer,
cipherSuites: @[
"TLS_AES_256_GCM_SHA384",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_GCM_SHA256",
],
)
proc validateConfig*(config: TLSConfig): seq[string] =
result = @[]
if config.certFile.len == 0:
result.add("Certificate file not specified")
elif not fileExists(config.certFile):
result.add("Certificate file not found: " & config.certFile)
if config.keyFile.len == 0:
result.add("Key file not specified")
elif not fileExists(config.keyFile):
result.add("Key file not found: " & config.keyFile)
if config.caFile.len > 0 and not fileExists(config.caFile):
result.add("CA file not found: " & config.caFile)
proc isValid*(config: TLSConfig): bool =
return config.validateConfig().len == 0
proc newTLSConnection*(config: TLSConfig, host: string, port: int): TLSConnection =
TLSConnection(config: config, state: tsDisconnected, host: host, port: port)
proc state*(conn: TLSConnection): TLSState = conn.state
# Self-signed certificate generation helper
proc generateSelfSignedCert*(outputDir: string): (string, string) =
let certPath = outputDir / "server.crt"
let keyPath = outputDir / "server.key"
createDir(outputDir)
# Use openssl to generate self-signed cert
let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & keyPath &
" -out " & certPath & " -days 365 -nodes -subj '/CN=localhost' 2>/dev/null"
discard execShellCmd(cmd)
return (certPath, keyPath)
proc certificateInfo*(certPath: string): Table[string, string] =
result = initTable[string, string]()
if not fileExists(certPath):
return
# Would use openssl to parse cert in production
result["path"] = certPath
result["exists"] = "true"
+228
View File
@@ -22,6 +22,11 @@ import barabadb/query/ir as qir
import barabadb/query/codegen import barabadb/query/codegen
import barabadb/query/udf import barabadb/query/udf
import barabadb/vector/simd import barabadb/vector/simd
import barabadb/core/crossmodal
import barabadb/core/gossip
import barabadb/client/client
import barabadb/client/fileops
import barabadb/fts/multilang as mlang
import barabadb/vector/engine as vengine import barabadb/vector/engine as vengine
import barabadb/vector/quant as vquant import barabadb/vector/quant as vquant
import barabadb/graph/engine as gengine import barabadb/graph/engine as gengine
@@ -1254,3 +1259,226 @@ suite "Vector SIMD":
let results = batchDistance(queries, corpus, "cosine") let results = batchDistance(queries, corpus, "cosine")
check results.len == 2 check results.len == 2
check results[0].len == 3 check results[0].len == 3
suite "Cross-Modal Engine":
test "Create engine":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal")
check engine != nil
test "Document operations":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal2")
engine.put("key1", cast[seq[byte]]("value1"))
let (found, val) = engine.get("key1")
check found
check cast[string](val) == "value1"
test "Vector operations":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal3")
engine.insertVector(1, @[1.0'f32, 0.0'f32, 0.0'f32], {"cat": "A"}.toTable)
engine.insertVector(2, @[0.0'f32, 1.0'f32, 0.0'f32], {"cat": "B"}.toTable)
let results = engine.searchVector(@[1.0'f32, 0.1'f32, 0.0'f32], 2)
check results.len == 2
test "Graph operations":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal4")
let n1 = engine.addNode("Person")
let n2 = engine.addNode("Person")
discard engine.addEdge(n1, n2, "knows")
let traversal = engine.traverseGraph(n1, "bfs")
check traversal.len >= 1
test "FTS operations":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal5")
engine.indexText(1, "Nim programming language")
engine.indexText(2, "Python data science")
let results = engine.searchText("programming")
check results.len >= 1
test "2PC transaction":
var txn = newTPCTransaction(1)
txn.addParticipant("storage")
txn.addParticipant("vector")
txn.addParticipant("graph")
check txn.participantCount == 3
check txn.prepare()
check txn.isPrepared
check txn.commit()
check txn.isCommitted
test "2PC rollback":
var txn = newTPCTransaction(2)
txn.addParticipant("storage")
txn.addParticipant("vector")
check txn.prepare()
check txn.rollback()
check txn.isAborted
test "Hybrid query":
let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal6")
engine.insertVector(1, @[1.0'f32, 0.0'f32], {"cat": "A"}.toTable)
engine.indexText(1, "fast database")
var query = newCrossModalQuery(qmHybrid)
query.vector = @[1.0'f32, 0.0'f32]
query.vectorK = 5
query.searchQuery = "fast"
query.vecWeight = 1.0
query.ftsWeight = 1.0
let result = engine.hybridSearch(query)
check result.totalResults >= 0
suite "Gossip Protocol":
test "Create gossip node":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946)
check gp.self.id == "node1"
check gp.memberCount == 0
test "Add members":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946)
let node2 = newGossipNode("node2", "10.0.0.2", 7946)
let node3 = newGossipNode("node3", "10.0.0.3", 7946)
gp.addMember(node2)
gp.addMember(node3)
check gp.memberCount == 2
check gp.aliveCount == 2
test "Suspect and declare dead":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946)
let node2 = newGossipNode("node2", "10.0.0.2", 7946)
gp.addMember(node2)
gp.suspect("node2")
check gp.getMember("node2").state == nsSuspect
gp.declareDead("node2")
check gp.memberCount == 1
test "Gossip message":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946)
let node2 = newGossipNode("node2", "10.0.0.2", 7946)
gp.addMember(node2)
let msg = gp.createGossipMessage()
check msg.senderId == "node1"
check msg.nodes.len == 1
test "Select gossip targets":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946, fanout = 2)
for i in 2..5:
gp.addMember(newGossipNode("node" & $i, "10.0.0." & $i, 7946))
let targets = gp.selectGossipTargets()
check targets.len <= 2
test "Member IDs":
var gp = newGossipProtocol("node1", "10.0.0.1", 7946)
gp.addMember(newGossipNode("node2", "10.0.0.2", 7946))
check gp.isMember("node2")
check not gp.isMember("node99")
suite "Client Library":
test "Connection string parser":
let config = parseConnectionString("host=localhost port=5432 dbname=test user=admin")
check config.host == "localhost"
check config.port == 5432
check config.database == "test"
check config.username == "admin"
test "Client config defaults":
let config = defaultClientConfig()
check config.host == "127.0.0.1"
check config.port == 5432
test "Query builder":
let client = newBaraClient()
let qb = newQueryBuilder(client)
let sql = qb.select("name", "age").from("users")
.where("age > 18").orderBy("name", "ASC").limit(10).build()
check sql == "SELECT name, age FROM users WHERE age > 18 ORDER BY name ASC LIMIT 10"
test "Query builder with JOIN":
let client = newBaraClient()
let qb = newQueryBuilder(client)
let sql = qb.select("u.name", "o.total")
.from("users u").join("orders o", "u.id = o.user_id")
.where("o.total > 100").build()
check "JOIN" in sql
check "WHERE" in sql
test "Query builder with GROUP BY":
let client = newBaraClient()
let qb = newQueryBuilder(client)
let sql = qb.select("dept", "count(*)").from("employees")
.groupBy("dept").having("count(*) > 5").build()
check "GROUP BY" in sql
check "HAVING" in sql
suite "Import/Export":
test "JSON export":
let columns = @["name", "age"]
let rows = @[@["Alice", "30"], @["Bob", "25"]]
let json = fileops.toJson(columns, rows)
check json.startsWith("[")
check "Alice" in json
test "CSV export":
let columns = @["name", "age"]
let rows = @[@["Alice", "30"], @["Bob", "25"]]
let csv = fileops.toCsv(columns, rows)
check csv.startsWith("name,age")
check "Alice" in csv
test "JSON import":
let json = """[{"name": "Alice", "age": "30"}, {"name": "Bob", "age": "25"}]"""
let (columns, rows) = fileops.parseJsonTable(json)
check columns.len == 2
check rows.len == 2
test "CSV import":
let csv = "name,age\nAlice,30\nBob,25"
let (columns, rows) = fileops.parseCsvTable(csv)
check columns.len == 2
check rows.len == 2
check rows[0][0] == "Alice"
test "NDJSON export/import":
let columns = @["name", "age"]
let rows = @[@["Alice", "30"]]
let ndjson = fileops.toNdjson(columns, rows)
check "Alice" in ndjson
test "CSV with quoted fields":
let csv = "name,bio\nAlice,\"Software engineer, Nim\"\nBob,Data scientist"
let (columns, rows) = fileops.parseCsvTable(csv)
check rows.len == 2
suite "Multi-Language FTS":
test "English tokenizer":
let config = mlang.getLanguageConfig(mlang.langEnglish)
let tokens = mlang.tokenize("The quick brown fox jumps over the lazy dog", config)
check tokens.len > 0
check "the" notin tokens # stop word
test "Bulgarian tokenizer":
let config = mlang.getLanguageConfig(mlang.langBulgarian)
let tokens = mlang.tokenize("Бързата кафява лисица прескача мързеливото куче", config)
check tokens.len > 0
test "German tokenizer":
let config = mlang.getLanguageConfig(mlang.langGerman)
let tokens = mlang.tokenize("Der schnelle braune Fuchs springt über den faulen Hund", config)
check tokens.len > 0
check "der" notin tokens
test "Russian tokenizer":
let config = mlang.getLanguageConfig(mlang.langRussian)
let tokens = mlang.tokenize("Быстрая браун лиса прыгает через ленивую собаку", config)
check tokens.len > 0
test "Language detection":
check mlang.detectLanguage("Hello world how are you") == mlang.langEnglish
# Bulgarian text is also Cyrillic — detected as Russian by default
check mlang.detectLanguage("Здравей свят как си") == mlang.langRussian
test "English stemming":
check mlang.stemEnglish("running") == "runn"
check mlang.stemEnglish("cats") == "cat"
check mlang.stemEnglish("programming") == "programm"
test "Bulgarian stemming":
check mlang.stemBulgarian("красота") == "красот" # -та suffix