From 6aaabb518d3f250d2af7acb08e45461cbc963fa4 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Tue, 12 May 2026 23:02:54 +0300 Subject: [PATCH] =?UTF-8?q?fix:=20=D0=B2=D0=B8=D1=81=D0=BE=D0=BA=D0=BE=20?= =?UTF-8?q?=D0=B8=20=D1=81=D1=80=D0=B5=D0=B4=D0=BD=D0=BE=20=D0=BF=D1=80?= =?UTF-8?q?=D0=B8=D0=BE=D1=80=D0=B8=D1=82=D0=B5=D1=82=D0=BD=D0=B8=20=D0=B1?= =?UTF-8?q?=D1=8A=D0=B3=D0=BE=D0=B2=D0=B5=20=D0=B2=20query,=20protocol,=20?= =?UTF-8?q?storage,=20vector,=20fts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправени проблеми: - Wire protocol: bounds checking, max length limits, recursion depth limit - SQL injection: escape-ване на quotes в exprToSql string literals - ReDoS: escape-ване на regex metachars в LIKE/ILIKE - Stale BTree indexes: добавен BTree.remove() + изтриване при UPDATE/DELETE - SSL: quoteShell за всички shell команди с пътища - Boolean literals: parser вече разпознава tkTrue/tkFalse - Unary minus: lowerExpr map-ва ukNeg към irNeg (вместо irNot) - Non-aggregate UDFs: lowerExpr създава irekFuncCall вместо NULL literal - UTF-8 FTS: tokenize използва runes вместо байтове - HNSW: добавен Lock за thread-safe insert/search 292 теста, 0 failure-а. --- src/barabadb/fts/engine.nim | 7 ++-- src/barabadb/protocol/ssl.nim | 12 +++--- src/barabadb/protocol/wire.nim | 39 +++++++++++++++++-- src/barabadb/query/executor.nim | 69 ++++++++++++++++++++++++++------- src/barabadb/query/ir.nim | 2 +- src/barabadb/query/parser.nim | 4 +- src/barabadb/storage/btree.nim | 28 +++++++++++++ src/barabadb/vector/engine.nim | 12 +++++- 8 files changed, 144 insertions(+), 29 deletions(-) diff --git a/src/barabadb/fts/engine.nim b/src/barabadb/fts/engine.nim index 314dc75..003624f 100644 --- a/src/barabadb/fts/engine.nim +++ b/src/barabadb/fts/engine.nim @@ -85,9 +85,10 @@ proc simpleStem(word: string): string = proc tokenize*(text: string, config: TokenizerConfig = defaultTokenizerConfig()): seq[string] = result = @[] var word = "" - for ch in text: - if ch.isAlphaNumeric() or ch in {'_', '-'}: - word.add(ch) + for r in text.runes: + let rStr = $r + if r.isAlpha() or rStr == "_" or rStr == "-": + word.add(rStr) else: if word.len > 0: var token = word diff --git a/src/barabadb/protocol/ssl.nim b/src/barabadb/protocol/ssl.nim index d8d9329..2c9baf1 100644 --- a/src/barabadb/protocol/ssl.nim +++ b/src/barabadb/protocol/ssl.nim @@ -79,8 +79,8 @@ proc generateSelfSignedCert*(outputDir: string, commonName: string = "localhost" let certPath = outputDir / (commonName & ".crt") let keyPath = outputDir / (commonName & ".key") createDir(outputDir) - let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & keyPath & - " -out " & certPath & " -days 365 -nodes -subj '/CN=" & commonName & "' 2>/dev/null" + let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & quoteShell(keyPath) & + " -out " & quoteShell(certPath) & " -days 365 -nodes -subj " & quoteShell("/CN=" & commonName) & " 2>/dev/null" if execShellCmd(cmd) == 0 and fileExists(certPath): return (certPath, keyPath) return ("", "") @@ -88,7 +88,7 @@ proc generateSelfSignedCert*(outputDir: string, commonName: string = "localhost" proc certificateFingerprint*(certPath: string): string = if not fileExists(certPath): return "" - let cmd = "openssl x509 -in " & certPath & " -fingerprint -noout 2>/dev/null" + let cmd = "openssl x509 -in " & quoteShell(certPath) & " -fingerprint -noout 2>/dev/null" let (output, _) = execCmdEx(cmd) for line in output.splitLines(): if "Fingerprint=" in line: @@ -100,17 +100,17 @@ proc certificateFingerprint*(certPath: string): string = proc isExpired*(certPath: string): bool = if not fileExists(certPath): return true - let cmd = "openssl x509 -in " & certPath & " -checkend 0 2>/dev/null" + let cmd = "openssl x509 -in " & quoteShell(certPath) & " -checkend 0 2>/dev/null" return execShellCmd(cmd) != 0 proc daysUntilExpiry*(certPath: string): int = if not fileExists(certPath): return -1 # Check if expires within 1 day - let cmd1 = "openssl x509 -in " & certPath & " -checkend 86400 2>/dev/null" + let cmd1 = "openssl x509 -in " & quoteShell(certPath) & " -checkend 86400 2>/dev/null" if execShellCmd(cmd1) == 0: # Check if expires within 30 days - let cmd30 = "openssl x509 -in " & certPath & " -checkend 2592000 2>/dev/null" + let cmd30 = "openssl x509 -in " & quoteShell(certPath) & " -checkend 2592000 2>/dev/null" if execShellCmd(cmd30) == 0: return 365 return 30 diff --git a/src/barabadb/protocol/wire.nim b/src/barabadb/protocol/wire.nim index 5ce0aae..88963c7 100644 --- a/src/barabadb/protocol/wire.nim +++ b/src/barabadb/protocol/wire.nim @@ -4,6 +4,11 @@ import std/endians const ProtocolVersion* = 1'u32 Magic* = 0x42415241'u32 # "BARA" + MaxWireStringLen* = 64 * 1024 * 1024 # 64 MB + MaxWireArrayLen* = 1_000_000 + MaxWireObjectLen* = 1_000_000 + MaxWireVectorLen* = 1_000_000 + MaxWireDeserializeDepth* = 100 type MsgKind* = enum @@ -110,6 +115,8 @@ proc writeBytes*(buf: var seq[byte], data: openArray[byte]) = buf.add(b) proc readUint32*(buf: openArray[byte], pos: var int): uint32 = + if pos + 4 > buf.len: + raise newException(ValueError, "Wire protocol: truncated uint32") var bytes: array[4, byte] for i in 0..3: bytes[i] = buf[pos + i] @@ -117,6 +124,8 @@ proc readUint32*(buf: openArray[byte], pos: var int): uint32 = pos += 4 proc readUint64*(buf: openArray[byte], pos: var int): uint64 = + if pos + 8 > buf.len: + raise newException(ValueError, "Wire protocol: truncated uint64") var bytes: array[8, byte] for i in 0..7: bytes[i] = buf[pos + i] @@ -125,6 +134,10 @@ proc readUint64*(buf: openArray[byte], pos: var int): uint64 = proc readString*(buf: openArray[byte], pos: var int): string = let len = int(readUint32(buf, pos)) + if len > MaxWireStringLen: + raise newException(ValueError, "Wire protocol: string exceeds max length") + if pos + len > buf.len: + raise newException(ValueError, "Wire protocol: truncated string data") result = newString(len) for i in 0.. MaxWireStringLen: + raise newException(ValueError, "Wire protocol: bytes exceed max length") + if pos + len > buf.len: + raise newException(ValueError, "Wire protocol: truncated bytes data") result = newSeq[byte](len) for i in 0.. MaxWireDeserializeDepth: + raise newException(ValueError, "Wire protocol: max deserialization depth exceeded") + if pos >= buf.len: + raise newException(ValueError, "Wire protocol: unexpected end of buffer") let kind = FieldKind(buf[pos]) inc pos case kind @@ -222,20 +243,28 @@ proc deserializeValue*(buf: openArray[byte], pos: var int): WireValue = result = WireValue(kind: fkBytes, bytesVal: readBytes(buf, pos)) of fkArray: let count = int(readUint32(buf, pos)) + if count > MaxWireArrayLen: + raise newException(ValueError, "Wire protocol: array exceeds max length") var arr: seq[WireValue] = @[] for i in 0.. MaxWireObjectLen: + raise newException(ValueError, "Wire protocol: object exceeds max length") var obj: seq[(string, WireValue)] = @[] for i in 0.. MaxWireVectorLen: + raise newException(ValueError, "Wire protocol: vector exceeds max length") + if pos + count * 4 > buf.len: + raise newException(ValueError, "Wire protocol: truncated vector data") var vec: seq[float32] = @[] for i in 0..= payload.len: + raise newException(ValueError, "Wire protocol: missing format byte") discard payload[pos] # format byte pos += 1 let paramCount = int(readUint32(payload, pos)) + if paramCount > MaxWireArrayLen: + raise newException(ValueError, "Wire protocol: param count exceeds max") var params: seq[WireValue] = @[] for i in 0.. 0 and not isNull(oldIdxVal): + ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existingVal))) # Update FTS indexes for ftsKey, ftsIdx in ctx.ftsIndexes: if ftsKey.startsWith(table & "."): @@ -826,6 +859,9 @@ proc execUpdateRow*(ctx: ExecutionContext, table: string, key: string, sets: Tab newVals.add(parsed[c]) else: newVals.add("") + let oldIdxVal = oldVals.join("|") + if oldIdxVal.len > 0 and not isNull(oldIdxVal): + ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existing))) let newIdxVal = newVals.join("|") if newIdxVal.len > 0 and not isNull(newIdxVal): ctx.btrees[colName].insert(newIdxVal, IndexEntry(lsmKey: fullKey, rowValue: newVal)) @@ -1036,19 +1072,26 @@ proc lowerExpr*(node: Node): IRExpr = result.binRight = lowerExpr(node.binRight) of nkUnaryOp: result = IRExpr(kind: irekUnary) - result.unOp = if node.unOp == ukNot: irNot else: irNot + result.unOp = if node.unOp == ukNot: irNot else: irNeg result.unExpr = lowerExpr(node.unOperand) of nkFuncCall: - result = IRExpr(kind: irekAggregate) case node.funcName.toLower() - of "count": result.aggOp = irCount - of "sum": result.aggOp = irSum - of "avg": result.aggOp = irAvg - of "min": result.aggOp = irMin - of "max": result.aggOp = irMax - else: result = IRExpr(kind: irekLiteral, literal: IRLiteral(kind: vkNull)) - result.aggArgs = @[] - for arg in node.funcArgs: result.aggArgs.add(lowerExpr(arg)) + of "count", "sum", "avg", "min", "max": + result = IRExpr(kind: irekAggregate) + case node.funcName.toLower() + of "count": result.aggOp = irCount + of "sum": result.aggOp = irSum + of "avg": result.aggOp = irAvg + of "min": result.aggOp = irMin + of "max": result.aggOp = irMax + else: discard + result.aggArgs = @[] + for arg in node.funcArgs: result.aggArgs.add(lowerExpr(arg)) + else: + result = IRExpr(kind: irekFuncCall) + result.irFunc = node.funcName + result.irFuncArgs = @[] + for arg in node.funcArgs: result.irFuncArgs.add(lowerExpr(arg)) of nkIsExpr: result = IRExpr(kind: irekUnary) result.unOp = if node.isNegated: irIsNotNull else: irIsNull diff --git a/src/barabadb/query/ir.nim b/src/barabadb/query/ir.nim index 5512ca7..ea4273f 100644 --- a/src/barabadb/query/ir.nim +++ b/src/barabadb/query/ir.nim @@ -21,7 +21,7 @@ type IROperator* = enum irAdd, irSub, irMul, irDiv, irMod, irPow irEq, irNeq, irLt, irLte, irGt, irGte - irAnd, irOr, irNot + irAnd, irOr, irNot, irNeg irIn, irNotIn irLike, irILike irBetween diff --git a/src/barabadb/query/parser.nim b/src/barabadb/query/parser.nim index 79cb650..d371db3 100644 --- a/src/barabadb/query/parser.nim +++ b/src/barabadb/query/parser.nim @@ -51,9 +51,9 @@ proc parsePrimary(p: var Parser): Node = of tkStringLit: discard p.advance() Node(kind: nkStringLit, strVal: tok.value, line: tok.line, col: tok.col) - of tkBoolLit: + of tkBoolLit, tkTrue, tkFalse: discard p.advance() - Node(kind: nkBoolLit, boolVal: tok.value == "true", line: tok.line, col: tok.col) + Node(kind: nkBoolLit, boolVal: tok.value == "true" or tok.kind == tkTrue, line: tok.line, col: tok.col) of tkNull: discard p.advance() Node(kind: nkNullLit, line: tok.line, col: tok.col) diff --git a/src/barabadb/storage/btree.nim b/src/barabadb/storage/btree.nim index c1f0530..5eef249 100644 --- a/src/barabadb/storage/btree.nim +++ b/src/barabadb/storage/btree.nim @@ -119,3 +119,31 @@ proc scan*[K, V](btree: BTreeIndex[K, V], startKey, endKey: K): seq[(K, seq[V])] node = node.next proc len*[K, V](btree: BTreeIndex[K, V]): int = btree.size + +proc remove*[K, V](btree: var BTreeIndex[K, V], key: K, value: V) = + proc removeRec(node: BTreeNode[K, V]): bool = + var i = 0 + while i < node.keys.len and key > node.keys[i]: + inc i + if node.isLeaf: + if i < node.keys.len and key == node.keys[i]: + var vals = node.values[i] + var idx = -1 + for j in 0..= 0: + vals.del(idx) + if vals.len == 0: + node.keys.del(i) + node.values.del(i) + else: + node.values[i] = vals + return true + return false + else: + return removeRec(node.children[i]) + + if removeRec(btree.root): + dec btree.size diff --git a/src/barabadb/vector/engine.nim b/src/barabadb/vector/engine.nim index ed79d3c..42a310d 100644 --- a/src/barabadb/vector/engine.nim +++ b/src/barabadb/vector/engine.nim @@ -5,6 +5,7 @@ import std/random import std/tables import std/sets import std/monotimes +import std/locks type DistanceMetric* = enum @@ -35,6 +36,7 @@ type maxM*: int metric*: DistanceMetric dimensions*: int + lock*: Lock IVFCluster* = object centroid*: Vector @@ -92,7 +94,7 @@ proc distance*(a, b: Vector, metric: DistanceMetric): float64 = proc newHNSWIndex*(dimensions: int, m: int = 16, efConstruction: int = 200, metric: DistanceMetric = dmCosine): HNSWIndex = - HNSWIndex( + var idx = HNSWIndex( nodes: initTable[uint64, HNSWNode](), entryPoint: 0, maxLevel: 0, @@ -102,6 +104,8 @@ proc newHNSWIndex*(dimensions: int, m: int = 16, efConstruction: int = 200, metric: metric, dimensions: dimensions, ) + initLock(idx.lock) + return idx proc randomLevel(m: int): int = ## Geometric distribution: probability of level L is (1/m)^L @@ -192,6 +196,8 @@ proc addBidirectionalLink(idx: HNSWIndex, nodeId, neighborId: uint64, level: int proc insert*(idx: HNSWIndex, id: uint64, vector: Vector, metadata: Table[string, string] = initTable[string, string]()) = + acquire(idx.lock) + defer: release(idx.lock) let level = randomLevel(idx.m) let node = HNSWNode(id: id, vector: vector, metadata: metadata, neighbors: newSeq[seq[uint64]](level + 1)) @@ -228,6 +234,8 @@ proc insert*(idx: HNSWIndex, id: uint64, vector: Vector, proc search*(idx: HNSWIndex, query: Vector, k: int, metric: DistanceMetric = dmCosine): seq[(uint64, float64)] = + acquire(idx.lock) + defer: release(idx.lock) if idx.nodes.len == 0: return @[] @@ -251,6 +259,8 @@ proc search*(idx: HNSWIndex, query: Vector, k: int, proc searchWithFilter*(idx: HNSWIndex, query: Vector, k: int, filter: proc(metadata: Table[string, string]): bool {.gcsafe.}, metric: DistanceMetric = dmCosine): seq[(uint64, float64)] = + acquire(idx.lock) + defer: release(idx.lock) if idx.nodes.len == 0: return @[]