feat: compaction, page cache, WebSocket, rate limiter, TF-IDF, fuzzy search, regex, metadata filter — 73 tests
- SSTable compaction: size-tiered strategy, level-based scheduling - Page cache: LRU eviction, hit rate tracking, capacity management - WebSocket: full duplex streaming, frame encoding, ping/pong - Rate limiter: token bucket + sliding window algorithms - FTS: TF-IDF ranking, Levenshtein fuzzy matching, wildcard regex - Vector: metadata filtering on HNSW search - 16 new tests (73 total, all passing)
This commit is contained in:
@@ -21,6 +21,7 @@ type
|
||||
HNSWNode* = ref object
|
||||
id*: uint64
|
||||
vector*: Vector
|
||||
metadata*: Table[string, string]
|
||||
neighbors*: seq[seq[uint64]] # neighbors per level
|
||||
|
||||
HNSWIndex* = ref object
|
||||
@@ -102,8 +103,9 @@ proc randomLevel(maxLevel: int): int =
|
||||
r = rand(1.0)
|
||||
return level
|
||||
|
||||
proc insert*(idx: HNSWIndex, id: uint64, vector: Vector) =
|
||||
let node = HNSWNode(id: id, vector: vector, neighbors: @[])
|
||||
proc insert*(idx: HNSWIndex, id: uint64, vector: Vector,
|
||||
metadata: Table[string, string] = initTable[string, string]()) =
|
||||
let node = HNSWNode(id: id, vector: vector, metadata: metadata, neighbors: @[])
|
||||
let level = randomLevel(16)
|
||||
|
||||
for i in 0..level:
|
||||
@@ -137,6 +139,23 @@ proc search*(idx: HNSWIndex, query: Vector, k: int,
|
||||
|
||||
return candidates
|
||||
|
||||
proc searchWithFilter*(idx: HNSWIndex, query: Vector, k: int,
|
||||
filter: proc(metadata: Table[string, string]): bool {.gcsafe.},
|
||||
metric: DistanceMetric = dmCosine): seq[(uint64, float64)] =
|
||||
if idx.nodes.len == 0:
|
||||
return @[]
|
||||
|
||||
var candidates: seq[(uint64, float64)] = @[]
|
||||
for nodeId, node in idx.nodes:
|
||||
if filter(node.metadata):
|
||||
let dist = distance(query, node.vector, metric)
|
||||
candidates.add((nodeId, dist))
|
||||
|
||||
candidates.sort(proc(a, b: (uint64, float64)): int = cmp(a[1], b[1]))
|
||||
if candidates.len > k:
|
||||
candidates = candidates[0..<k]
|
||||
return candidates
|
||||
|
||||
proc newIVFPQIndex*(dimensions: int, nClusters: int = 100,
|
||||
nSubquantizers: int = 8, nBits: int = 8,
|
||||
metric: DistanceMetric = dmCosine): IVFPQIndex =
|
||||
|
||||
Reference in New Issue
Block a user