diff --git a/src/barabadb/client/client.nim b/src/barabadb/client/client.nim new file mode 100644 index 0000000..ac27b68 --- /dev/null +++ b/src/barabadb/client/client.nim @@ -0,0 +1,222 @@ +## BaraDB Client — Nim client library +import std/asyncdispatch +import std/asyncnet +import std/strutils +import std/json +import ../protocol/wire + +type + ClientConfig* = object + host*: string + port*: int + database*: string + username*: string + password*: string + timeout*: int # milliseconds + maxRetries*: int + + QueryResult* = object + columns*: seq[string] + rows*: seq[seq[string]] + rowCount*: int + affectedRows*: int + executionTime*: float64 # seconds + + BaraClient* = ref object + config: ClientConfig + socket: AsyncSocket + connected: bool + requestId: uint32 + +proc defaultClientConfig*(): ClientConfig = + ClientConfig( + host: "127.0.0.1", + port: 5432, + database: "default", + username: "admin", + password: "", + timeout: 30000, + maxRetries: 3, + ) + +proc newBaraClient*(config: ClientConfig = defaultClientConfig()): BaraClient = + BaraClient( + config: config, + socket: newAsyncSocket(), + connected: false, + requestId: 0, + ) + +proc connect*(client: BaraClient) {.async.} = + await client.socket.connect(client.config.host, Port(client.config.port)) + client.connected = true + +proc disconnect*(client: BaraClient) {.async.} = + if client.connected: + client.socket.close() + client.connected = false + +proc nextRequestId(client: BaraClient): uint32 = + inc client.requestId + return client.requestId + +proc query*(client: BaraClient, sql: string): Future[QueryResult] {.async.} = + if not client.connected: + raise newException(IOError, "Not connected") + + let reqId = client.nextRequestId() + let msg = makeQueryMessage(reqId, sql) + await client.socket.send(cast[string](msg)) + + # Read response + let response = await client.socket.recv(8192) + if response.len == 0: + raise newException(IOError, "Connection closed by server") + + result = QueryResult(columns: @[], rows: @[], rowCount: 0, affectedRows: 0) + + # Parse response (simplified) + if response.len > 0: + result.rowCount = 0 + +proc execute*(client: BaraClient, sql: string): Future[int] {.async.} = + let qr = await client.query(sql) + return qr.affectedRows + +proc isConnected*(client: BaraClient): bool = client.connected + +# Synchronous wrapper +type + SyncClient* = ref object + asyncClient: BaraClient + +proc newSyncClient*(config: ClientConfig = defaultClientConfig()): SyncClient = + SyncClient(asyncClient: newBaraClient(config)) + +proc connect*(client: SyncClient) = + waitFor client.asyncClient.connect() + +proc disconnect*(client: SyncClient) = + waitFor client.asyncClient.disconnect() + +proc query*(client: SyncClient, sql: string): QueryResult = + waitFor client.asyncClient.query(sql) + +proc execute*(client: SyncClient, sql: string): int = + waitFor client.asyncClient.execute(sql) + +proc isConnected*(client: SyncClient): bool = + client.asyncClient.isConnected + +# Connection string parser +proc parseConnectionString*(connStr: string): ClientConfig = + result = defaultClientConfig() + let parts = connStr.split(" ") + for part in parts: + let kv = part.split("=", 1) + if kv.len == 2: + case kv[0].toLower() + of "host": result.host = kv[1] + of "port": result.port = parseInt(kv[1]) + of "database", "dbname": result.database = kv[1] + of "user", "username": result.username = kv[1] + of "password", "pass": result.password = kv[1] + of "connect_timeout", "timeout": result.timeout = parseInt(kv[1]) + else: discard + +# Fluent query builder +type + QueryBuilder* = ref object + client: BaraClient + selectCols: seq[string] + fromTable: string + whereClauses: seq[string] + orderByCols: seq[string] + orderDirs: seq[string] + limitVal: int + offsetVal: int + groupByCols: seq[string] + havingClause: string + joinClauses: seq[string] + +proc newQueryBuilder*(client: BaraClient): QueryBuilder = + QueryBuilder( + client: client, + selectCols: @[], + fromTable: "", + whereClauses: @[], + orderByCols: @[], + orderDirs: @[], + limitVal: 0, + offsetVal: 0, + groupByCols: @[], + havingClause: "", + joinClauses: @[], + ) + +proc select*(qb: QueryBuilder, cols: varargs[string]): QueryBuilder = + for col in cols: + qb.selectCols.add(col) + return qb + +proc `from`*(qb: QueryBuilder, table: string): QueryBuilder = + qb.fromTable = table + return qb + +proc where*(qb: QueryBuilder, clause: string): QueryBuilder = + qb.whereClauses.add(clause) + return qb + +proc orderBy*(qb: QueryBuilder, col: string, dir: string = "ASC"): QueryBuilder = + qb.orderByCols.add(col) + qb.orderDirs.add(dir) + return qb + +proc limit*(qb: QueryBuilder, n: int): QueryBuilder = + qb.limitVal = n + return qb + +proc offset*(qb: QueryBuilder, n: int): QueryBuilder = + qb.offsetVal = n + return qb + +proc join*(qb: QueryBuilder, table: string, on: string): QueryBuilder = + qb.joinClauses.add("JOIN " & table & " ON " & on) + return qb + +proc groupBy*(qb: QueryBuilder, cols: varargs[string]): QueryBuilder = + for col in cols: + qb.groupByCols.add(col) + return qb + +proc having*(qb: QueryBuilder, clause: string): QueryBuilder = + qb.havingClause = clause + return qb + +proc build*(qb: QueryBuilder): string = + result = "SELECT " + if qb.selectCols.len == 0: + result &= "*" + else: + result &= qb.selectCols.join(", ") + result &= " FROM " & qb.fromTable + for joinClause in qb.joinClauses: + result &= " " & joinClause + if qb.whereClauses.len > 0: + result &= " WHERE " & qb.whereClauses.join(" AND ") + if qb.groupByCols.len > 0: + result &= " GROUP BY " & qb.groupByCols.join(", ") + if qb.havingClause.len > 0: + result &= " HAVING " & qb.havingClause + if qb.orderByCols.len > 0: + result &= " ORDER BY " + for i, col in qb.orderByCols: + if i > 0: result &= ", " + result &= col & " " & qb.orderDirs[i] + if qb.limitVal > 0: + result &= " LIMIT " & $qb.limitVal + if qb.offsetVal > 0: + result &= " OFFSET " & $qb.offsetVal + +proc exec*(qb: QueryBuilder): Future[QueryResult] {.async.} = + return await qb.client.query(qb.build()) diff --git a/src/barabadb/client/fileops.nim b/src/barabadb/client/fileops.nim new file mode 100644 index 0000000..3c48657 --- /dev/null +++ b/src/barabadb/client/fileops.nim @@ -0,0 +1,211 @@ +## Import/Export — JSON, CSV, Parquet-like formats +import std/tables +import std/strutils +import std/sequtils +import ../core/types + +type + ExportFormat* = enum + efJson = "json" + efCsv = "csv" + efNdjson = "ndjson" # newline-delimited JSON + + ImportFormat* = enum + ifJson = "json" + ifCsv = "csv" + ifNdjson = "ndjson" + + ExportOptions* = object + format*: ExportFormat + delimiter*: char + includeHeader*: bool + nullValue*: string + dateFormat*: string + + ImportOptions* = object + format*: ImportFormat + delimiter*: char + hasHeader*: bool + nullValue*: string + skipRows*: int + maxRows*: int + +proc defaultExportOptions*(): ExportOptions = + ExportOptions(format: efJson, delimiter: ',', includeHeader: true, + nullValue: "", dateFormat: "yyyy-MM-dd") + +proc defaultImportOptions*(): ImportOptions = + ImportOptions(format: ifJson, delimiter: ',', hasHeader: true, + nullValue: "", skipRows: 0, maxRows: -1) + +# JSON export +proc toJson*(columns: seq[string], rows: seq[seq[string]]): string = + var items: seq[string] = @[] + for row in rows: + var fields: seq[string] = @[] + for i, col in columns: + let val = if i < row.len: row[i] else: "" + fields.add("\"" & col & "\": \"" & val.replace("\"", "\\\"") & "\"") + items.add("{" & fields.join(", ") & "}") + return "[" & items.join(",\n ") & "]" + +proc toJsonLines*(columns: seq[string], rows: seq[seq[string]]): string = + result = "" + for row in rows: + var fields: seq[string] = @[] + for i, col in columns: + let val = if i < row.len: row[i] else: "" + fields.add("\"" & col & "\": \"" & val.replace("\"", "\\\"") & "\"") + result &= "{" & fields.join(", ") & "}\n" + +# CSV export +proc toCsv*(columns: seq[string], rows: seq[seq[string]], + delimiter: char = ',', includeHeader: bool = true): string = + result = "" + if includeHeader: + result &= columns.join($delimiter) & "\n" + for row in rows: + var fields: seq[string] = @[] + for val in row: + if val.contains(delimiter) or val.contains('"') or val.contains('\n'): + fields.add("\"" & val.replace("\"", "\"\"") & "\"") + else: + fields.add(val) + result &= fields.join($delimiter) & "\n" + +# JSON import +proc parseJsonTable*(json: string): (seq[string], seq[seq[string]]) = + var columns: seq[string] = @[] + var rows: seq[seq[string]] = @[] + + # Simple JSON array parser + let trimmed = json.strip() + if not trimmed.startsWith("["): + return (columns, rows) + + # Find first object to extract columns + let firstObjStart = trimmed.find('{') + let firstObjEnd = trimmed.find('}', firstObjStart) + if firstObjStart < 0 or firstObjEnd < 0: + return (columns, rows) + + let firstObj = trimmed[firstObjStart+1 .. firstObjEnd-1] + for pair in firstObj.split(","): + let kv = pair.split(":", 1) + if kv.len == 2: + columns.add(kv[0].strip().strip(chars = {'"'})) + + # Parse all objects + var pos = 0 + while pos < trimmed.len: + let objStart = trimmed.find('{', pos) + if objStart < 0: + break + let objEnd = trimmed.find('}', objStart) + if objEnd < 0: + break + let obj = trimmed[objStart+1 .. objEnd-1] + var row: seq[string] = @[] + for pair in obj.split(","): + let kv = pair.split(":", 1) + if kv.len == 2: + row.add(kv[1].strip().strip(chars = {'"'})) + rows.add(row) + pos = objEnd + 1 + + return (columns, rows) + +# CSV import +proc parseCsvTable*(csv: string, delimiter: char = ',', + hasHeader: bool = true): (seq[string], seq[seq[string]]) = + var columns: seq[string] = @[] + var rows: seq[seq[string]] = @[] + + let lines = csv.splitLines() + if lines.len == 0: + return (columns, rows) + + var startLine = 0 + if hasHeader: + columns = lines[0].split(delimiter).mapIt(it.strip().strip(chars = {'"'})) + startLine = 1 + + for i in startLine.. 0: + for i in 0.. metadata + +proc newCrossModalEngine*(dataDir: string): CrossModalEngine = + CrossModalEngine( + store: newLSMTree(dataDir / "kv"), + vectorIdx: vengine.newHNSWIndex(128), + graphIdx: gengine.newGraph(), + ftsIdx: fts.newInvertedIndex(), + metadata: initTable[uint64, Table[string, string]](), + ) + +# Document operations +proc put*(engine: CrossModalEngine, key: string, value: seq[byte]) = + engine.store.put(key, value) + +proc get*(engine: CrossModalEngine, key: string): (bool, seq[byte]) = + engine.store.get(key) + +proc delete*(engine: CrossModalEngine, key: string) = + engine.store.delete(key) + +# Vector operations +proc insertVector*(engine: CrossModalEngine, id: uint64, vector: seq[float32], + meta: Table[string, string] = initTable[string, string]()) = + vengine.insert(engine.vectorIdx, id, vector, meta) + engine.metadata[id] = meta + +proc searchVector*(engine: CrossModalEngine, query: seq[float32], k: int = 10, + metric: vengine.DistanceMetric = vengine.dmCosine): seq[(uint64, float64)] = + vengine.search(engine.vectorIdx, query, k, metric) + +proc searchVectorFiltered*(engine: CrossModalEngine, query: seq[float32], k: int, + filter: proc(meta: Table[string, string]): bool {.gcsafe.}): seq[(uint64, float64)] = + vengine.searchWithFilter(engine.vectorIdx, query, k, filter) + +# Graph operations +proc addNode*(engine: CrossModalEngine, label: string, + props: Table[string, string] = initTable[string, string]()): uint64 = + uint64(gengine.addNode(engine.graphIdx, label, props)) + +proc addEdge*(engine: CrossModalEngine, src, dst: uint64, label: string = "", + weight: float64 = 1.0): uint64 = + uint64(gengine.addEdge(engine.graphIdx, NodeId(src), NodeId(dst), label, + initTable[string, string](), weight)) + +proc traverseGraph*(engine: CrossModalEngine, start: uint64, + algo: string = "bfs", maxDepth: int = -1): seq[uint64] = + case algo + of "bfs": + let nodes = gengine.bfs(engine.graphIdx, NodeId(start), maxDepth) + return nodes.mapIt(uint64(it)) + of "dfs": + let nodes = gengine.dfs(engine.graphIdx, NodeId(start), maxDepth) + return nodes.mapIt(uint64(it)) + of "shortest": + # BFS-based shortest (unweighted) + let nodes = gengine.bfs(engine.graphIdx, NodeId(start), maxDepth) + return nodes.mapIt(uint64(it)) + else: + return @[] + +proc pageRank*(engine: CrossModalEngine): Table[uint64, float64] = + let ranks = gengine.pageRank(engine.graphIdx) + result = initTable[uint64, float64]() + for nodeId, rank in ranks: + result[uint64(nodeId)] = rank + +# FTS operations +proc indexText*(engine: CrossModalEngine, docId: uint64, text: string) = + fts.addDocument(engine.ftsIdx, docId, text) + +proc searchText*(engine: CrossModalEngine, query: string, limit: int = 10): seq[uint64] = + let results = fts.search(engine.ftsIdx, query, limit) + return results.mapIt(it.docId) + +proc searchFuzzy*(engine: CrossModalEngine, query: string, + maxDist: int = 2, limit: int = 10): seq[uint64] = + let results = fts.fuzzySearch(engine.ftsIdx, query, maxDist, limit) + return results.mapIt(it.docId) + +# Cross-modal hybrid query +proc hybridSearch*(engine: CrossModalEngine, query: CrossModalQuery): CrossModalResult = + result = CrossModalResult( + docResults: @[], + vecResults: @[], + graphResults: @[], + ftsResults: @[], + hybridScores: initTable[uint64, float64](), + totalResults: 0, + ) + + var scores = initTable[uint64, float64]() + + # Document mode + if query.mode in {qmDocument, qmHybrid}: + if query.key.len > 0: + let (found, val) = engine.store.get(query.key) + if found: + result.docResults.add((query.key, val)) + + # Vector mode + if query.mode in {qmVector, qmHybrid} and query.vector.len > 0: + let vecResults = if query.vectorFilter != nil: + engine.searchVectorFiltered(query.vector, query.vectorK, query.vectorFilter) + else: + engine.searchVector(query.vector, query.vectorK) + result.vecResults = vecResults + for (id, dist) in vecResults: + let score = query.vecWeight / (1.0 + dist) + scores[id] = scores.getOrDefault(id, 0.0) + score + + # FTS mode + if query.mode in {qmFullText, qmHybrid} and query.searchQuery.len > 0: + let ftsResults = engine.searchText(query.searchQuery, query.vectorK) + result.ftsResults = ftsResults + for i, id in ftsResults: + let score = query.ftsWeight / (1.0 + float64(i)) + scores[id] = scores.getOrDefault(id, 0.0) + score + + # Graph mode + if query.mode in {qmGraph, qmHybrid} and query.startNode > 0: + let graphResults = engine.traverseGraph(query.startNode, query.traversal, query.maxDepth) + result.graphResults = graphResults + for i, id in graphResults: + let score = query.graphWeight / (1.0 + float64(i)) + scores[id] = scores.getOrDefault(id, 0.0) + score + + # Sort by hybrid score + result.hybridScores = scores + result.totalResults = scores.len + +proc newCrossModalQuery*(mode: QueryMode): CrossModalQuery = + CrossModalQuery( + mode: mode, + vectorK: 10, + vectorMetric: "cosine", + maxDepth: -1, + fuzzyMaxDist: 2, + docWeight: 1.0, + vecWeight: 1.0, + ftsWeight: 1.0, + graphWeight: 1.0, + ) + +# 2PC Cross-Modal Transaction +type + TPCParticipant* = ref object + name*: string + prepared*: bool + committed*: bool + aborted*: bool + writeLog*: seq[(string, seq[byte])] + + TPCTransaction* = ref object + id*: uint64 + participants*: seq[TPCParticipant] + state*: string # "active", "prepared", "committed", "aborted" + +proc newTPCTransaction*(id: uint64): TPCTransaction = + TPCTransaction(id: id, participants: @[], state: "active") + +proc addParticipant*(txn: TPCTransaction, name: string) = + txn.participants.add(TPCParticipant(name: name, prepared: false, + committed: false, aborted: false, + writeLog: @[])) + +proc prepare*(txn: TPCTransaction): bool = + if txn.state != "active": + return false + for p in txn.participants: + # In a real system, would send PREPARE to each participant + p.prepared = true + txn.state = "prepared" + return true + +proc commit*(txn: TPCTransaction): bool = + if txn.state != "prepared": + return false + for p in txn.participants: + p.committed = true + txn.state = "committed" + return true + +proc rollback*(txn: TPCTransaction): bool = + if txn.state == "active" or txn.state == "prepared": + for p in txn.participants: + p.aborted = true + txn.state = "aborted" + return true + return false + +proc participantCount*(txn: TPCTransaction): int = txn.participants.len +proc isPrepared*(txn: TPCTransaction): bool = txn.state == "prepared" +proc isCommitted*(txn: TPCTransaction): bool = txn.state == "committed" +proc isAborted*(txn: TPCTransaction): bool = txn.state == "aborted" diff --git a/src/barabadb/core/gossip.nim b/src/barabadb/core/gossip.nim new file mode 100644 index 0000000..4d8f8d4 --- /dev/null +++ b/src/barabadb/core/gossip.nim @@ -0,0 +1,170 @@ +## Gossip Protocol — membership and failure detection +import std/tables +import std/sets +import std/random +import std/monotimes + +type + NodeState* = enum + nsAlive + nsSuspect + nsDead + + GossipNode* = ref object + id*: string + host*: string + port*: int + state*: NodeState + incarnation*: uint64 + lastSeen*: int64 + metadata*: Table[string, string] + + GossipMessage* = object + senderId*: string + senderIncarnation*: uint64 + nodes*: seq[(string, NodeState, uint64)] # (id, state, incarnation) + + GossipProtocol* = ref object + self*: GossipNode + members*: Table[string, GossipNode] + suspectTimeout*: int64 # nanoseconds + deadTimeout*: int64 # nanoseconds + fanout*: int # number of nodes to gossip to per round + onJoin*: proc(node: GossipNode) {.gcsafe.} + onLeave*: proc(nodeId: string) {.gcsafe.} + onSuspect*: proc(nodeId: string) {.gcsafe.} + +proc newGossipNode*(id: string, host: string, port: int): GossipNode = + GossipNode( + id: id, host: host, port: port, + state: nsAlive, incarnation: 1, + lastSeen: getMonoTime().ticks(), + metadata: initTable[string, string](), + ) + +proc newGossipProtocol*(id: string, host: string, port: int, + fanout: int = 3): GossipProtocol = + let self = newGossipNode(id, host, port) + GossipProtocol( + self: self, + members: initTable[string, GossipNode](), + suspectTimeout: 5_000_000_000, # 5 seconds + deadTimeout: 15_000_000_000, # 15 seconds + fanout: fanout, + onJoin: nil, + onLeave: nil, + onSuspect: nil, + ) + +proc join*(gp: GossipProtocol, seedNode: GossipNode) = + gp.members[seedNode.id] = seedNode + if gp.onJoin != nil: + gp.onJoin(seedNode) + +proc addMember*(gp: GossipProtocol, node: GossipNode) = + if node.id == gp.self.id: + return + gp.members[node.id] = node + if gp.onJoin != nil: + gp.onJoin(node) + +proc removeMember*(gp: GossipProtocol, nodeId: string) = + gp.members.del(nodeId) + if gp.onLeave != nil: + gp.onLeave(nodeId) + +proc suspect*(gp: GossipProtocol, nodeId: string) = + if nodeId in gp.members: + gp.members[nodeId].state = nsSuspect + if gp.onSuspect != nil: + gp.onSuspect(nodeId) + +proc declareDead*(gp: GossipProtocol, nodeId: string) = + if nodeId in gp.members: + gp.members[nodeId].state = nsDead + if gp.onLeave != nil: + gp.onLeave(nodeId) + +proc createGossipMessage*(gp: GossipProtocol): GossipMessage = + result = GossipMessage( + senderId: gp.self.id, + senderIncarnation: gp.self.incarnation, + nodes: @[], + ) + for id, node in gp.members: + result.nodes.add((id, node.state, node.incarnation)) + +proc applyGossipMessage*(gp: GossipProtocol, msg: GossipMessage) = + for (nodeId, state, incarnation) in msg.nodes: + if nodeId == gp.self.id: + # Someone suspects us — increment incarnation to refute + if state == nsSuspect and incarnation >= gp.self.incarnation: + inc gp.self.incarnation + continue + + if nodeId in gp.members: + let existing = gp.members[nodeId] + if incarnation > existing.incarnation: + existing.state = state + existing.incarnation = incarnation + existing.lastSeen = getMonoTime().ticks() + if state == nsDead: + gp.removeMember(nodeId) + elif incarnation == existing.incarnation and state == nsDead: + existing.state = nsDead + gp.removeMember(nodeId) + else: + # New node + if state != nsDead: + let newNode = GossipNode( + id: nodeId, host: "", port: 0, + state: state, incarnation: incarnation, + lastSeen: getMonoTime().ticks(), + ) + gp.addMember(newNode) + +proc selectGossipTargets*(gp: GossipProtocol): seq[GossipNode] = + var alive: seq[GossipNode] = @[] + for id, node in gp.members: + if node.state == nsAlive: + alive.add(node) + result = @[] + let count = min(gp.fanout, alive.len) + for i in 0.. gp.suspectTimeout: + node.state = nsSuspect + if gp.onSuspect != nil: + gp.onSuspect(id) + elif node.state == nsSuspect and elapsed > gp.deadTimeout: + toRemove.add(id) + for id in toRemove: + gp.declareDead(id) + +proc aliveMembers*(gp: GossipProtocol): seq[GossipNode] = + result = @[] + for id, node in gp.members: + if node.state == nsAlive: + result.add(node) + +proc memberCount*(gp: GossipProtocol): int = gp.members.len +proc aliveCount*(gp: GossipProtocol): int = gp.aliveMembers().len + +proc memberIds*(gp: GossipProtocol): seq[string] = + result = @[] + for id in gp.members.keys: + result.add(id) + +proc isMember*(gp: GossipProtocol, nodeId: string): bool = + return nodeId in gp.members + +proc getMember*(gp: GossipProtocol, nodeId: string): GossipNode = + gp.members.getOrDefault(nodeId, nil) diff --git a/src/barabadb/fts/multilang.nim b/src/barabadb/fts/multilang.nim new file mode 100644 index 0000000..7dfd14d --- /dev/null +++ b/src/barabadb/fts/multilang.nim @@ -0,0 +1,209 @@ +## Multi-Language FTS — tokenizers for different languages +import std/tables +import std/unicode +import std/strutils +import std/sets + +type + Language* = enum + langEnglish = "en" + langSpanish = "es" + langFrench = "fr" + langGerman = "de" + langRussian = "ru" + langBulgarian = "bg" + langChinese = "zh" + langJapanese = "ja" + langArabic = "ar" + langAuto = "auto" + + Stemmer* = proc(word: string): string {.gcsafe.} + StopWords* = HashSet[string] + + LanguageConfig* = object + language*: Language + stemmer*: Stemmer + stopWords*: StopWords + tokenizer*: proc(text: string): seq[string] {.gcsafe.} + +const + stopWordsEn* = [ + "a", "an", "the", "is", "it", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "into", "through", "during", + "and", "or", "not", "but", "if", "then", "else", "when", + "be", "was", "were", "been", "being", "have", "has", "had", + "do", "does", "did", "will", "would", "could", "should", "may", + "this", "that", "these", "those", "i", "you", "he", "she", "we", "they", + ] + + stopWordsBg* = [ + "и", "в", "на", "за", "от", "да", "се", "е", "са", "по", + "не", "че", "с", "към", "но", "или", "ако", "при", "до", + "как", "какво", "кой", "коя", "кое", "кои", "този", "тази", + "това", "тези", "със", "между", "след", "преди", "без", + "още", "вече", "само", "може", "трябва", "има", "няма", + ] + + stopWordsDe* = [ + "der", "die", "das", "ein", "eine", "und", "oder", "aber", "nicht", + "ist", "sind", "war", "hat", "haben", "werden", "wird", "kann", + "mit", "von", "für", "auf", "in", "an", "zu", "bei", "nach", + "über", "unter", "vor", "zwischen", "durch", "gegen", "ohne", + ] + + stopWordsFr* = [ + "le", "la", "les", "un", "une", "des", "et", "ou", "mais", "pas", + "est", "sont", "était", "a", "ont", "sera", "peut", "avec", + "de", "du", "pour", "sur", "dans", "par", "entre", "sous", "chez", + "je", "tu", "il", "elle", "nous", "vous", "ils", "elles", + ] + + stopWordsRu* = [ + "и", "в", "на", "не", "что", "он", "я", "с", "а", "как", + "это", "по", "но", "они", "мы", "за", "от", "из", "у", "к", + "бы", "ты", "его", "её", "их", "её", "мой", "твой", "наш", + "ваш", "свой", "который", "этот", "тот", "такой", "каждый", + ] + +proc stemEnglish*(word: string): string = + if word.len <= 3: return word + if word.endsWith("ing"): return word[0..^4] + if word.endsWith("tion"): return word[0..^5] + if word.endsWith("ness"): return word[0..^5] + if word.endsWith("ment"): return word[0..^5] + if word.endsWith("able"): return word[0..^5] + if word.endsWith("ies"): return word[0..^4] & "y" + if word.endsWith("es") and word.len > 4: return word[0..^3] + if word.endsWith("ed") and word.len > 4: return word[0..^3] + if word.endsWith("ly") and word.len > 4: return word[0..^3] + if word.endsWith("s") and not word.endsWith("ss") and word.len > 3: return word[0..^2] + return word + +proc stemBulgarian*(word: string): string = + if word.len <= 3: return word + # Bulgarian suffixes + if word.endsWith("ища"): return word[0..^4] + if word.endsWith("ище"): return word[0..^4] + if word.endsWith("ция"): return word[0..^4] + if word.endsWith("ние"): return word[0..^4] + if word.endsWith("ост"): return word[0..^4] + if word.endsWith("ство"): return word[0..^5] + if word.endsWith("ски"): return word[0..^4] + if word.endsWith("ски"): return word[0..^4] + if word.endsWith("на"): return word[0..^3] + if word.endsWith("та"): return word[0..^3] + return word + +proc stemGerman*(word: string): string = + if word.len <= 3: return word + if word.endsWith("ung"): return word[0..^4] + if word.endsWith("heit"): return word[0..^5] + if word.endsWith("keit"): return word[0..^5] + if word.endsWith("lich"): return word[0..^5] + if word.endsWith("isch"): return word[0..^5] + if word.endsWith("chen"): return word[0..^5] + if word.endsWith("schaft"): return word[0..^7] + if word.endsWith("en"): return word[0..^3] + if word.endsWith("er"): return word[0..^3] + if word.endsWith("es"): return word[0..^3] + return word + +proc stemFrench*(word: string): string = + if word.len <= 3: return word + if word.endsWith("ement"): return word[0..^6] + if word.endsWith("ment"): return word[0..^5] + if word.endsWith("tion"): return word[0..^5] + if word.endsWith("eur"): return word[0..^4] + if word.endsWith("euse"): return word[0..^5] + if word.endsWith("ique"): return word[0..^5] + if word.endsWith("esse"): return word[0..^5] + if word.endsWith("eux"): return word[0..^4] + if word.endsWith("er"): return word[0..^3] + if word.endsWith("es"): return word[0..^3] + return word + +proc stemRussian*(word: string): string = + if word.len <= 3: return word + # Russian suffixes + if word.endsWith("ость"): return word[0..^5] + if word.endsWith("ение"): return word[0..^5] + if word.endsWith("ание"): return word[0..^5] + if word.endsWith("тель"): return word[0..^5] + if word.endsWith("ский"): return word[0..^5] + if word.endsWith("ция"): return word[0..^4] + if word.endsWith("ние"): return word[0..^4] + if word.endsWith("ать"): return word[0..^4] + if word.endsWith("ить"): return word[0..^4] + if word.endsWith("ыть"): return word[0..^4] + return word + +proc getLanguageConfig*(lang: Language): LanguageConfig = + case lang + of langEnglish: + LanguageConfig(language: lang, stemmer: stemEnglish, + stopWords: stopWordsEn.toHashSet()) + of langBulgarian: + LanguageConfig(language: lang, stemmer: stemBulgarian, + stopWords: stopWordsBg.toHashSet()) + of langGerman: + LanguageConfig(language: lang, stemmer: stemGerman, + stopWords: stopWordsDe.toHashSet()) + of langFrench: + LanguageConfig(language: lang, stemmer: stemFrench, + stopWords: stopWordsFr.toHashSet()) + of langRussian: + LanguageConfig(language: lang, stemmer: stemRussian, + stopWords: stopWordsRu.toHashSet()) + else: + LanguageConfig(language: lang, stemmer: stemEnglish, + stopWords: stopWordsEn.toHashSet()) + +proc tokenize*(text: string, config: LanguageConfig): seq[string] = + result = @[] + var word = "" + for ch in text: + let cp = ord(ch) + # Accept ASCII alphanumeric, Cyrillic, CJK, Arabic, and common separators + if (cp >= 0x30 and cp <= 0x39) or # digits + (cp >= 0x41 and cp <= 0x5A) or # A-Z + (cp >= 0x61 and cp <= 0x7A) or # a-z + (cp >= 0xC0 and cp <= 0xFF) or # Latin Extended + Cyrillic start + (cp >= 0x400 and cp <= 0x4FF) or # Cyrillic + ch == '_' or ch == '-': + word &= ch + else: + if word.len > 0: + var token = word.toLower() + if config.stemmer != nil: + token = config.stemmer(token) + if token.len >= 2 and token notin config.stopWords: + result.add(token) + word = "" + if word.len > 0: + var token = word.toLower() + if config.stemmer != nil: + token = config.stemmer(token) + if token.len >= 2 and token notin config.stopWords: + result.add(token) + +proc detectLanguage*(text: string): Language = + # Simple heuristic based on byte patterns + # UTF-8 Cyrillic: bytes 0xD0-0xD1 followed by 0x80-0xBF + var cyrillicCount = 0 + var latinCount = 0 + var i = 0 + let bytes = cast[seq[byte]](text) + while i < bytes.len: + let b = bytes[i] + if b >= 0xD0'u8 and b <= 0xD1'u8 and i + 1 < bytes.len and bytes[i+1] >= 0x80'u8: + inc cyrillicCount + i += 2 + elif b >= 0x41'u8 and b <= 0x7A'u8: + inc latinCount + i += 1 + else: + i += 1 + + if cyrillicCount > latinCount: + return langRussian # Could be BG or RU — default to RU + return langEnglish diff --git a/src/barabadb/protocol/tls.nim b/src/barabadb/protocol/tls.nim new file mode 100644 index 0000000..641f6eb --- /dev/null +++ b/src/barabadb/protocol/tls.nim @@ -0,0 +1,100 @@ +## TLS/SSL — transport layer security wrapper +import std/os +import std/strutils + +type + TLSVersion* = enum + tls12 = "TLSv1.2" + tls13 = "TLSv1.3" + + TLSConfig* = object + certFile*: string + keyFile*: string + caFile*: string + minVersion*: TLSVersion + verifyPeer*: bool + cipherSuites*: seq[string] + + TLSState* = enum + tsDisconnected + tsHandshaking + tsConnected + tsError + + TLSConnection* = ref object + config*: TLSConfig + state*: TLSState + host*: string + port*: int + +proc defaultTLSConfig*(): TLSConfig = + TLSConfig( + certFile: "", + keyFile: "", + caFile: "", + minVersion: tls12, + verifyPeer: false, + cipherSuites: @[ + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_GCM_SHA256", + ], + ) + +proc newTLSConfig*(certFile, keyFile: string, caFile: string = "", + minVersion: TLSVersion = tls12, + verifyPeer: bool = false): TLSConfig = + TLSConfig( + certFile: certFile, + keyFile: keyFile, + caFile: caFile, + minVersion: minVersion, + verifyPeer: verifyPeer, + cipherSuites: @[ + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_GCM_SHA256", + ], + ) + +proc validateConfig*(config: TLSConfig): seq[string] = + result = @[] + if config.certFile.len == 0: + result.add("Certificate file not specified") + elif not fileExists(config.certFile): + result.add("Certificate file not found: " & config.certFile) + if config.keyFile.len == 0: + result.add("Key file not specified") + elif not fileExists(config.keyFile): + result.add("Key file not found: " & config.keyFile) + if config.caFile.len > 0 and not fileExists(config.caFile): + result.add("CA file not found: " & config.caFile) + +proc isValid*(config: TLSConfig): bool = + return config.validateConfig().len == 0 + +proc newTLSConnection*(config: TLSConfig, host: string, port: int): TLSConnection = + TLSConnection(config: config, state: tsDisconnected, host: host, port: port) + +proc state*(conn: TLSConnection): TLSState = conn.state + +# Self-signed certificate generation helper +proc generateSelfSignedCert*(outputDir: string): (string, string) = + let certPath = outputDir / "server.crt" + let keyPath = outputDir / "server.key" + + createDir(outputDir) + # Use openssl to generate self-signed cert + let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & keyPath & + " -out " & certPath & " -days 365 -nodes -subj '/CN=localhost' 2>/dev/null" + discard execShellCmd(cmd) + + return (certPath, keyPath) + +proc certificateInfo*(certPath: string): Table[string, string] = + result = initTable[string, string]() + if not fileExists(certPath): + return + # Would use openssl to parse cert in production + result["path"] = certPath + result["exists"] = "true" diff --git a/tests/test_all.nim b/tests/test_all.nim index 12680e9..39f5811 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -22,6 +22,11 @@ import barabadb/query/ir as qir import barabadb/query/codegen import barabadb/query/udf import barabadb/vector/simd +import barabadb/core/crossmodal +import barabadb/core/gossip +import barabadb/client/client +import barabadb/client/fileops +import barabadb/fts/multilang as mlang import barabadb/vector/engine as vengine import barabadb/vector/quant as vquant import barabadb/graph/engine as gengine @@ -1254,3 +1259,226 @@ suite "Vector SIMD": let results = batchDistance(queries, corpus, "cosine") check results.len == 2 check results[0].len == 3 + +suite "Cross-Modal Engine": + test "Create engine": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal") + check engine != nil + + test "Document operations": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal2") + engine.put("key1", cast[seq[byte]]("value1")) + let (found, val) = engine.get("key1") + check found + check cast[string](val) == "value1" + + test "Vector operations": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal3") + engine.insertVector(1, @[1.0'f32, 0.0'f32, 0.0'f32], {"cat": "A"}.toTable) + engine.insertVector(2, @[0.0'f32, 1.0'f32, 0.0'f32], {"cat": "B"}.toTable) + let results = engine.searchVector(@[1.0'f32, 0.1'f32, 0.0'f32], 2) + check results.len == 2 + + test "Graph operations": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal4") + let n1 = engine.addNode("Person") + let n2 = engine.addNode("Person") + discard engine.addEdge(n1, n2, "knows") + let traversal = engine.traverseGraph(n1, "bfs") + check traversal.len >= 1 + + test "FTS operations": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal5") + engine.indexText(1, "Nim programming language") + engine.indexText(2, "Python data science") + let results = engine.searchText("programming") + check results.len >= 1 + + test "2PC transaction": + var txn = newTPCTransaction(1) + txn.addParticipant("storage") + txn.addParticipant("vector") + txn.addParticipant("graph") + check txn.participantCount == 3 + check txn.prepare() + check txn.isPrepared + check txn.commit() + check txn.isCommitted + + test "2PC rollback": + var txn = newTPCTransaction(2) + txn.addParticipant("storage") + txn.addParticipant("vector") + check txn.prepare() + check txn.rollback() + check txn.isAborted + + test "Hybrid query": + let engine = newCrossModalEngine("/tmp/baradb_test_crossmodal6") + engine.insertVector(1, @[1.0'f32, 0.0'f32], {"cat": "A"}.toTable) + engine.indexText(1, "fast database") + var query = newCrossModalQuery(qmHybrid) + query.vector = @[1.0'f32, 0.0'f32] + query.vectorK = 5 + query.searchQuery = "fast" + query.vecWeight = 1.0 + query.ftsWeight = 1.0 + let result = engine.hybridSearch(query) + check result.totalResults >= 0 + +suite "Gossip Protocol": + test "Create gossip node": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946) + check gp.self.id == "node1" + check gp.memberCount == 0 + + test "Add members": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946) + let node2 = newGossipNode("node2", "10.0.0.2", 7946) + let node3 = newGossipNode("node3", "10.0.0.3", 7946) + gp.addMember(node2) + gp.addMember(node3) + check gp.memberCount == 2 + check gp.aliveCount == 2 + + test "Suspect and declare dead": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946) + let node2 = newGossipNode("node2", "10.0.0.2", 7946) + gp.addMember(node2) + gp.suspect("node2") + check gp.getMember("node2").state == nsSuspect + gp.declareDead("node2") + check gp.memberCount == 1 + + test "Gossip message": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946) + let node2 = newGossipNode("node2", "10.0.0.2", 7946) + gp.addMember(node2) + let msg = gp.createGossipMessage() + check msg.senderId == "node1" + check msg.nodes.len == 1 + + test "Select gossip targets": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946, fanout = 2) + for i in 2..5: + gp.addMember(newGossipNode("node" & $i, "10.0.0." & $i, 7946)) + let targets = gp.selectGossipTargets() + check targets.len <= 2 + + test "Member IDs": + var gp = newGossipProtocol("node1", "10.0.0.1", 7946) + gp.addMember(newGossipNode("node2", "10.0.0.2", 7946)) + check gp.isMember("node2") + check not gp.isMember("node99") + +suite "Client Library": + test "Connection string parser": + let config = parseConnectionString("host=localhost port=5432 dbname=test user=admin") + check config.host == "localhost" + check config.port == 5432 + check config.database == "test" + check config.username == "admin" + + test "Client config defaults": + let config = defaultClientConfig() + check config.host == "127.0.0.1" + check config.port == 5432 + + test "Query builder": + let client = newBaraClient() + let qb = newQueryBuilder(client) + let sql = qb.select("name", "age").from("users") + .where("age > 18").orderBy("name", "ASC").limit(10).build() + check sql == "SELECT name, age FROM users WHERE age > 18 ORDER BY name ASC LIMIT 10" + + test "Query builder with JOIN": + let client = newBaraClient() + let qb = newQueryBuilder(client) + let sql = qb.select("u.name", "o.total") + .from("users u").join("orders o", "u.id = o.user_id") + .where("o.total > 100").build() + check "JOIN" in sql + check "WHERE" in sql + + test "Query builder with GROUP BY": + let client = newBaraClient() + let qb = newQueryBuilder(client) + let sql = qb.select("dept", "count(*)").from("employees") + .groupBy("dept").having("count(*) > 5").build() + check "GROUP BY" in sql + check "HAVING" in sql + +suite "Import/Export": + test "JSON export": + let columns = @["name", "age"] + let rows = @[@["Alice", "30"], @["Bob", "25"]] + let json = fileops.toJson(columns, rows) + check json.startsWith("[") + check "Alice" in json + + test "CSV export": + let columns = @["name", "age"] + let rows = @[@["Alice", "30"], @["Bob", "25"]] + let csv = fileops.toCsv(columns, rows) + check csv.startsWith("name,age") + check "Alice" in csv + + test "JSON import": + let json = """[{"name": "Alice", "age": "30"}, {"name": "Bob", "age": "25"}]""" + let (columns, rows) = fileops.parseJsonTable(json) + check columns.len == 2 + check rows.len == 2 + + test "CSV import": + let csv = "name,age\nAlice,30\nBob,25" + let (columns, rows) = fileops.parseCsvTable(csv) + check columns.len == 2 + check rows.len == 2 + check rows[0][0] == "Alice" + + test "NDJSON export/import": + let columns = @["name", "age"] + let rows = @[@["Alice", "30"]] + let ndjson = fileops.toNdjson(columns, rows) + check "Alice" in ndjson + + test "CSV with quoted fields": + let csv = "name,bio\nAlice,\"Software engineer, Nim\"\nBob,Data scientist" + let (columns, rows) = fileops.parseCsvTable(csv) + check rows.len == 2 + +suite "Multi-Language FTS": + test "English tokenizer": + let config = mlang.getLanguageConfig(mlang.langEnglish) + let tokens = mlang.tokenize("The quick brown fox jumps over the lazy dog", config) + check tokens.len > 0 + check "the" notin tokens # stop word + + test "Bulgarian tokenizer": + let config = mlang.getLanguageConfig(mlang.langBulgarian) + let tokens = mlang.tokenize("Бързата кафява лисица прескача мързеливото куче", config) + check tokens.len > 0 + + test "German tokenizer": + let config = mlang.getLanguageConfig(mlang.langGerman) + let tokens = mlang.tokenize("Der schnelle braune Fuchs springt über den faulen Hund", config) + check tokens.len > 0 + check "der" notin tokens + + test "Russian tokenizer": + let config = mlang.getLanguageConfig(mlang.langRussian) + let tokens = mlang.tokenize("Быстрая браун лиса прыгает через ленивую собаку", config) + check tokens.len > 0 + + test "Language detection": + check mlang.detectLanguage("Hello world how are you") == mlang.langEnglish + # Bulgarian text is also Cyrillic — detected as Russian by default + check mlang.detectLanguage("Здравей свят как си") == mlang.langRussian + + test "English stemming": + check mlang.stemEnglish("running") == "runn" + check mlang.stemEnglish("cats") == "cat" + check mlang.stemEnglish("programming") == "programm" + + test "Bulgarian stemming": + check mlang.stemBulgarian("красота") == "красот" # -та suffix