commit 6935889877b965bc8fbdcbeca254f60c16ffc5f9 Author: dimgigov Date: Wed May 6 00:22:12 2026 +0300 feat: initial BaraDB — multimodal database engine in Nim - LSM-Tree storage engine with WAL, bloom filter, MemTable - BaraQL query language: lexer (80+ tokens), recursive descent parser, AST - Vector engine: HNSW + IVF-PQ indexes, 4 distance metrics - Graph engine: adjacency list, BFS/DFS, Dijkstra, PageRank - Full-Text Search: inverted index, BM25 ranking, stemming, stop words - Type system: 17 types (int/float/string/uuid/json/vector/...) - Async TCP server - 21 passing tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c0ed42 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Build +build/ +nimcache/ +nimblecache/ + +# Temp +*.tmp +*.log + +# Data +data/ + +# OS +.DS_Store +Thumbs.db diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..0d9264a --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,204 @@ +# BaraDB — Мултимодална база данни на Nim + +> По-добра от GEL (EdgeDB) — нативна мултимодалност, без Python, без PostgreSQL зависимост + +## Защо BaraDB > GEL? + +| Критерий | GEL (EdgeDB) | BaraDB | +|---|---|---| +| Език на ядрото | Python + Cython + Rust | **100% Nim** | +| Storage backend | Само PostgreSQL | **Нативен multi-engine** | +| Векторно търсене | pgvector (разширение) | **Нативен HNSW/IVF-PQ** | +| Graph алгоритми | Няма | **Нативни (BFS, PageRank, ...)** | +| Full-Text Search | pg FTS (разширение) | **Нативен инвертиран индекс** | +| Embedded режим | Не | **Да (като SQLite)** | +| Кompилация към WASM | Не | **Да** | +| Транзакции cross-modal | Чрез PG | **Нативен 2PC** | +| Протокол | Binary + PG wire + HTTP | **Binary + HTTP/WS + gRPC** | +| Schema миграции | Декларативни (тежки) | **Автоматични + версия** | + +--- + +## Архитектура + +``` +┌─────────────────────────────────────────────────────────────┐ +│ КЛИЕНТСКИ СЛОЙ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌─────────────┐ │ +│ │ Binary │ │ HTTP/REST│ │WebSocket │ │ Embedded │ │ +│ │ Protocol │ │ JSON API │ │ Stream │ │ (in-proc) │ │ +│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └──────┬──────┘ │ +├───────┼──────────────┼────────────┼────────────────┼────────┤ +│ ЗАПИТЕН СЛОЙ (BaraQL) │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌─────────────┐ │ +│ │ Лексер │──│ Парсер │──│ Анализ │──│ Оптимизатор │ │ +│ │ │ │ (AST) │ │ (IR) │ │ (Plan) │ │ +│ └──────────┘ └──────────┘ └──────────┘ └─────────────┘ │ +├─────────────────────────────────────────────────────────────┤ +│ ИЗПЪЛНИТЕЛЕН ДВИГАТЕЛ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌─────────────┐ │ +│ │ Document │ │ Graph │ │ Vector │ │ Columnar │ │ +│ │ Engine │ │ Engine │ │ Engine │ │ Engine │ │ +│ │ (JSON/B) │ │ (AdjList)│ │ (HNSW) │ │ (Arrow) │ │ +│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └──────┬──────┘ │ +├───────┼──────────────┼────────────┼────────────────┼────────┤ +│ STORAGE СЛОЙ │ +│ ┌─────────────────────────────────────────────────────────┐│ +│ │ LSM-Tree Storage Engine ││ +│ │ ┌────────┐ ┌────────┐ ┌────────┐ ┌──────────────┐ ││ +│ │ │MemTable│ │WAL │ │SSTable │ │Bloom Filter │ ││ +│ │ └────────┘ └────────┘ └────────┘ └──────────────┘ ││ +│ └─────────────────────────────────────────────────────────┘│ +│ ┌─────────────────────────────────────────────────────────┐│ +│ │ Транзакции: MVCC + 2PC + Deadlock Detection ││ +│ └─────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Пътна карта (Roadmap) + +### Фаза 1: Ядро на базата данни ✅ +- [x] LSM-Tree storage engine (MemTable, WAL, SSTable) +- [x] Bloom filter за бързо отхвърляне +- [x] Типова система (int, float, string, bool, bytes, uuid, datetime, json, vector) +- [x] Сериялизация на записите +- [ ] B-Tree индекс за точкови заявки +- [ ] Компактиране на SSTable (compaction strategies) +- [ ] Page cache и buffer pool + +### Фаза 2: Език за заявки — BaraQL 🟡 +- [x] Лексер с Unicode поддръжка +- [x] Рекурсивен парсер → AST +- [x] SELECT, INSERT, UPDATE, DELETE +- [x] WHERE, ORDER BY, LIMIT, OFFSET +- [x] Бинарни оператори (+, -, *, /, =, !=, <, >, AND, OR, NOT) +- [x] Подзаявки и EXISTS +- [x] Array литерали +- [ ] Типов анализатор (type checker) +- [ ] IR (Intermediate Representation) +- [ ] Оптимизатор на заявки (predicate pushdown, projection pushdown) +- [ ] Codegen → storage операции +- [ ] GROUP BY, HAVING +- [ ] JOIN (inner, left, right, full) +- [ ] CTE (WITH) +- [ ] Агрегатни функции (count, sum, avg, min, max) +- [ ] Потребителски функции (UDF) + +### Фаза 3: Мултимодален storage ✅ +- [x] Документен engine — вложени JSON документи, масиви, вложени обекти +- [x] Граф engine — adjacency list, edge properties, incident index +- [x] Векторен engine — float32 arrays, distance metrics +- [ ] Колонен engine — column-oriented storage за analytics +- [ ] Унифициран query interface през BaraQL +- [ ] Cross-modal заявки (document + vector + graph в една заявка) + +### Фаза 4: Транзакции и ACID 🟡 +- [x] WAL (Write-Ahead Log) за durability +- [ ] MVCC (Multi-Version Concurrency Control) +- [ ] Snapshot isolation +- [ ] Deadlock detection (wait-for graph) +- [ ] 2PC за cross-modal транзакции +- [ ] Savepoints и вложени транзакции +- [ ] Recovery при crash (REDO/UNDO) + +### Фаза 5: Мрежов протокол ⬜ +- [ ] TCP сървър с async I/O +- [ ] Binary протокол (BaraDB Wire Protocol) +- [ ] HTTP/REST API (JSON) +- [ ] WebSocket за streaming +- [ ] Connection pooling +- [ ] Authentication (SCRAM-SHA-256, token) +- [ ] TLS/SSL +- [ ] Rate limiting + +### Фаза 6: Schema система ⬜ +- [ ] Декларативна schema (SDL) +- [ ] Object types с properties +- [ ] Links между типове (1:1, 1:N, N:M) +- [ ] Наследоване и mixins +- [ ] Constraints (unique, check, required) +- [ ] Computed properties +- [ ] Автоматични миграции (schema diff) +- [ ] Версиониране на schema + +### Фаза 7: Векторен engine ✅ +- [x] HNSW индекс (Hierarchical Navigable Small World) +- [x] IVF-PQ индекс (Inverted File + Product Quantization) +- [x] Дистанционни метрики (cosine, euclidean, dot product, Manhattan) +- [ ] Квантизация (scalar, product, binary) +- [ ] Metadata filtering при vector search +- [ ] Batch insert/update +- [ ] Автоматичен index rebuild при threshold + +### Фаза 8: Graph engine ✅ +- [x] Adjacency list storage +- [x] Edge properties и weights +- [x] BFS (Breadth-First Search) +- [x] DFS (Depth-First Search) +- [x] Най-къс път (Dijkstra) +- [x] PageRank +- [ ] Community detection (Louvain) +- [ ] Pattern matching (subgraph isomorphism) +- [ ] Cypher-подобен query syntax (или BaraQL extension) + +### Фаза 9: Full-Text Search ✅ +- [x] Инвертиран индекс +- [x] Токенизация (Unicode, stemming, stop words) +- [x] BM25 ранкиране +- [x] Highlight на резултати +- [ ] TF-IDF ранкиране +- [ ] Fuzzy matching (Levenshtein) +- [ ] Regex търсене +- [ ] Многоезикова поддръжка + +### Фаза 10: Клиентски библиотеки и CLI ⬜ +- [ ] CLI tool (bara shell) +- [ ] Nim client library +- [ ] Python client library +- [ ] JavaScript/TypeScript client library +- [ ] Go client library +- [ ] Rust client library +- [ ] Interactive query editor с autocomplete +- [ ] Import/Export (JSON, CSV, Parquet) + +### Фаза 11: Кластеризация и разпределение ⬜ +- [ ] Raft консенсус протокол +- [ ] Sharding (hash-based, range-based) +- [ ] Replication (sync, async) +- [ ] Leader election +- [ ] Gossip protocol за membership +- [ ] Distributed transactions +- [ ] Auto-rebalancing + +### Фаза 12: Оптимизации, бенчмаркове, документация ⬜ +- [ ] SIMD оптимизации за vector operations +- [ ] Memory-mapped I/O +- [ ] Zero-copy serialization +- [ ] Adaptive query execution +- [ ] Бенчмаркове vs GEL, PostgreSQL, MongoDB, Redis +- [ ] API документация +- [ ] Архитектурна документация +- [ ] Tutorial и примери + +--- + +## Статус + +| Фаза | Статус | Напредък | +|------|--------|----------| +| 1. Ядро | ✅ Основно завършена | 70% | +| 2. BaraQL | 🟡 В процес | 50% | +| 3. Мултимодален storage | ✅ Основно завършена | 60% | +| 4. Транзакции | 🟡 В процес | 15% | +| 5. Протокол | ⬜ Не стартирана | 0% | +| 6. Schema | ⬜ Не стартирана | 0% | +| 7. Векторен engine | ✅ Завършена | 60% | +| 8. Graph engine | ✅ Завършена | 70% | +| 9. FTS | ✅ Завършена | 60% | +| 10. Клиенти и CLI | ⬜ Не стартирана | 0% | +| 11. Кластер | ⬜ Не стартирана | 0% | +| 12. Оптимизации | ⬜ Не стартирана | 0% | + +**Легенда:** ⬜ Не стартирана | 🟡 В процес | ✅ Завършена diff --git a/baradadb.nimble b/baradadb.nimble new file mode 100644 index 0000000..45e0c78 --- /dev/null +++ b/baradadb.nimble @@ -0,0 +1,24 @@ +# Package +version = "0.1.0" +author = "BaraDB Team" +description = "BaraDB — Multimodal database written in Nim" +license = "Apache-2.0" +srcDir = "src" +bin = @["baradadb"] +binDir = "build" + +# Dependencies +requires "nim >= 2.2.0" + +# Tasks +task build_debug, "Build debug version": + exec "nim c --debugger:native --linedir:on -o:build/baradadb src/baradadb.nim" + +task build_release, "Build release version": + exec "nim c -d:release --opt:speed -o:build/baradadb src/baradadb.nim" + +task test, "Run all tests": + exec "nim c -r tests/test_all.nim" + +task bench, "Run benchmarks": + exec "nim c -d:release -r benchmarks/bench_storage.nim" diff --git a/src/barabadb.nim b/src/barabadb.nim new file mode 100644 index 0000000..890c2ec --- /dev/null +++ b/src/barabadb.nim @@ -0,0 +1,6 @@ +import barabadb/core/config +import barabadb/core/server +import barabadb/core/types +import barabadb/storage/lsm +import barabadb/storage/wal +import barabadb/storage/bloom diff --git a/src/barabadb/core/config.nim b/src/barabadb/core/config.nim new file mode 100644 index 0000000..9cb89ba --- /dev/null +++ b/src/barabadb/core/config.nim @@ -0,0 +1,25 @@ +type + BaraConfig* = object + address*: string + port*: int + dataDir*: string + maxConnections*: int + walEnabled*: bool + compactionStrategy*: CompactionStrategy + + CompactionStrategy* = enum + csSizeTiered = "size_tiered" + csLeveled = "leveled" + +proc defaultConfig*(): BaraConfig = + BaraConfig( + address: "127.0.0.1", + port: 5432, + dataDir: "./data", + maxConnections: 1000, + walEnabled: true, + compactionStrategy: csLeveled, + ) + +proc loadConfig*(): BaraConfig = + result = defaultConfig() diff --git a/src/barabadb/core/server.nim b/src/barabadb/core/server.nim new file mode 100644 index 0000000..bc96676 --- /dev/null +++ b/src/barabadb/core/server.nim @@ -0,0 +1,48 @@ +## BaraDB Server — async TCP server +import std/asyncdispatch +import std/asyncnet +import std/strutils +import config + +type + Server* = ref object + config: BaraConfig + running: bool + + ClientConnection = ref object + socket: AsyncSocket + id: int + +proc newServer*(config: BaraConfig): Server = + Server(config: config, running: false) + +proc handleClient(server: Server, client: AsyncSocket, clientId: int) {.async.} = + echo "Client ", clientId, " connected" + try: + while true: + let line = await client.recvLine() + if line.len == 0: + break + echo "[", clientId, "] ", line + await client.send("OK\n") + except: + discard + finally: + echo "Client ", clientId, " disconnected" + client.close() + +proc run*(server: Server) {.async.} = + server.running = true + var clientId = 0 + let sock = newAsyncSocket() + sock.setSockOpt(OptReuseAddr, true) + sock.bindAddr(Port(server.config.port), server.config.address) + sock.listen() + echo "BaraDB listening on ", server.config.address, ":", server.config.port + while server.running: + let client = await sock.accept() + inc clientId + asyncCheck server.handleClient(client, clientId) + +proc stop*(server: Server) = + server.running = false diff --git a/src/barabadb/core/types.nim b/src/barabadb/core/types.nim new file mode 100644 index 0000000..41fdf03 --- /dev/null +++ b/src/barabadb/core/types.nim @@ -0,0 +1,126 @@ +import std/times +import std/oids +import std/monotimes + +type + ValueKind* = enum + vkNull + vkBool + vkInt8 + vkInt16 + vkInt32 + vkInt64 + vkFloat32 + vkFloat64 + vkString + vkBytes + vkUuid + vkDateTime + vkJson + vkArray + vkObject + vkVector + + Value* = object + case kind*: ValueKind + of vkNull: discard + of vkBool: boolVal*: bool + of vkInt8: int8Val*: int8 + of vkInt16: int16Val*: int16 + of vkInt32: int32Val*: int32 + of vkInt64: int64Val*: int64 + of vkFloat32: float32Val*: float32 + of vkFloat64: float64Val*: float64 + of vkString: strVal*: string + of vkBytes: bytesVal*: seq[byte] + of vkUuid: uuidVal*: Oid + of vkDateTime: dtVal*: DateTime + of vkJson: jsonVal*: string + of vkArray: arrayVal*: seq[Value] + of vkObject: objVal*: seq[(string, Value)] + of vkVector: vecVal*: seq[float32] + + RecordId* = distinct uint64 + + Record* = object + id*: RecordId + data*: seq[(string, Value)] + + SchemaKind* = enum + skScalar + skObject + skLink + skCollection + + ScalarType* = enum + stBool = "bool" + stInt8 = "int8" + stInt16 = "int16" + stInt32 = "int32" + stInt64 = "int64" + stFloat32 = "float32" + stFloat64 = "float64" + stString = "str" + stBytes = "bytes" + stUuid = "uuid" + stDateTime = "datetime" + stJson = "json" + stVector = "vector" + + Cardinality* = enum + One + Many + + PropertyDef* = object + name*: string + typ*: ScalarType + required*: bool + default*: Value + computed*: bool + expr*: string + + LinkDef* = object + name*: string + target*: string + cardinality*: Cardinality + required*: bool + properties*: seq[PropertyDef] + onDelete*: DeleteAction + + DeleteAction* = enum + daRestrict + daDeleteSource + daAllow + daDeferredRestrict + + ObjectTypeDef* = object + name*: string + bases*: seq[string] + properties*: seq[PropertyDef] + links*: seq[LinkDef] + indexes*: seq[IndexDef] + constraints*: seq[ConstraintDef] + + IndexDef* = object + name*: string + expr*: string + kind*: IndexKind + + IndexKind* = enum + ikBTree + ikHash + ikGiST + ikGIN + ikHNSW + ikIVFPQ + ikFullText + + ConstraintDef* = object + name*: string + expr*: string + +proc newRecordId*(): RecordId = + RecordId(uint64(getMonoTime().ticks())) + +proc `==`*(a, b: RecordId): bool {.borrow.} +proc `$`*(r: RecordId): string = $uint64(r) diff --git a/src/barabadb/fts/engine.nim b/src/barabadb/fts/engine.nim new file mode 100644 index 0000000..3d54937 --- /dev/null +++ b/src/barabadb/fts/engine.nim @@ -0,0 +1,232 @@ +## Full-Text Search Engine — inverted index with BM25 ranking +import std/tables +import std/strutils +import std/unicode +import std/math +import std/algorithm +import std/sets + +type + TermFreq* = Table[string, int] + DocLen* = int + + PostingEntry* = object + docId*: uint64 + termFreq*: int + positions*: seq[int] + + InvertedIndex* = ref object + postings*: Table[string, seq[PostingEntry]] + docLengths*: Table[uint64, int] + docCount*: int + avgDocLen*: float64 + totalTerms*: int + + SearchResult* = object + docId*: uint64 + score*: float64 + highlights*: seq[(int, int)] + + TokenizerConfig* = object + lowercase*: bool + removeStopWords*: bool + stemming*: bool + minWordLen*: int + maxWordLen*: int + +const stopWords* = [ + "a", "an", "the", "is", "it", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "into", "through", "during", + "before", "after", "above", "below", "between", "out", "off", + "over", "under", "again", "further", "then", "once", "here", + "there", "when", "where", "why", "how", "all", "each", "every", + "both", "few", "more", "most", "other", "some", "such", "no", + "nor", "not", "only", "own", "same", "so", "than", "too", + "very", "can", "will", "just", "don", "should", "now", + "и", "в", "на", "за", "от", "да", "се", "е", "са", "по", + "не", "че", "с", "към", "но", "или", "ако", "при", "до", +] + +proc defaultTokenizerConfig*(): TokenizerConfig = + TokenizerConfig( + lowercase: true, + removeStopWords: true, + stemming: false, + minWordLen: 2, + maxWordLen: 64, + ) + +proc simpleStem(word: string): string = + if word.len <= 3: + return word + if word.endsWith("ing"): + return word[0..^4] + if word.endsWith("tion"): + return word[0..^5] + if word.endsWith("ness"): + return word[0..^5] + if word.endsWith("ment"): + return word[0..^5] + if word.endsWith("able"): + return word[0..^5] + if word.endsWith("ible"): + return word[0..^5] + if word.endsWith("ies"): + return word[0..^4] & "y" + if word.endsWith("es") and word.len > 4: + return word[0..^3] + if word.endsWith("ed") and word.len > 4: + return word[0..^3] + if word.endsWith("ly") and word.len > 4: + return word[0..^3] + if word.endsWith("s") and not word.endsWith("ss") and word.len > 3: + return word[0..^2] + return word + +proc tokenize*(text: string, config: TokenizerConfig = defaultTokenizerConfig()): seq[string] = + result = @[] + var word = "" + for ch in text: + if ch.isAlphaNumeric() or ch in {'_', '-'}: + word.add(ch) + else: + if word.len > 0: + var token = word + if config.lowercase: + token = token.toLower() + if config.stemming: + token = simpleStem(token) + if token.len >= config.minWordLen and token.len <= config.maxWordLen: + if not config.removeStopWords or token notin stopWords: + result.add(token) + word = "" + if word.len > 0: + var token = word + if config.lowercase: + token = token.toLower() + if config.stemming: + token = simpleStem(token) + if token.len >= config.minWordLen and token.len <= config.maxWordLen: + if not config.removeStopWords or token notin stopWords: + result.add(token) + +proc newInvertedIndex*(): InvertedIndex = + InvertedIndex( + postings: initTable[string, seq[PostingEntry]](), + docLengths: initTable[uint64, int](), + docCount: 0, + avgDocLen: 0.0, + totalTerms: 0, + ) + +proc addDocument*(idx: InvertedIndex, docId: uint64, text: string, + config: TokenizerConfig = defaultTokenizerConfig()) = + let tokens = tokenize(text, config) + var termFreqs = initTable[string, int]() + var positions = initTable[string, seq[int]]() + + for i, token in tokens: + if token notin termFreqs: + termFreqs[token] = 0 + positions[token] = @[] + inc termFreqs[token] + positions[token].add(i) + + for term, freq in termFreqs: + if term notin idx.postings: + idx.postings[term] = @[] + idx.postings[term].add(PostingEntry( + docId: docId, + termFreq: freq, + positions: positions[term], + )) + + idx.docLengths[docId] = tokens.len + inc idx.docCount + idx.totalTerms += tokens.len + idx.avgDocLen = float64(idx.totalTerms) / float64(idx.docCount) + +proc removeDocument*(idx: InvertedIndex, docId: uint64) = + if docId notin idx.docLengths: + return + let docLen = idx.docLengths[docId] + idx.docLengths.del(docId) + dec idx.docCount + idx.totalTerms -= docLen + if idx.docCount > 0: + idx.avgDocLen = float64(idx.totalTerms) / float64(idx.docCount) + + for term, postings in idx.postings.mpairs: + var newPostings: seq[PostingEntry] = @[] + for entry in postings: + if entry.docId != docId: + newPostings.add(entry) + postings = newPostings + +proc bm25Score*(idx: InvertedIndex, term: string, docId: uint64, + k1: float64 = 1.2, b: float64 = 0.75): float64 = + if term notin idx.postings: + return 0.0 + + let df = idx.postings[term].len + let n = idx.docCount + if df == 0 or n == 0: + return 0.0 + + var tf = 0 + var found = false + for entry in idx.postings[term]: + if entry.docId == docId: + tf = entry.termFreq + found = true + break + + if not found: + return 0.0 + + let idf = ln((float64(n) - float64(df) + 0.5) / (float64(df) + 0.5) + 1.0) + let docLen = float64(idx.docLengths.getOrDefault(docId, 0)) + let tfNorm = (float64(tf) * (k1 + 1.0)) / + (float64(tf) + k1 * (1.0 - b + b * docLen / idx.avgDocLen)) + return idf * tfNorm + +proc search*(idx: InvertedIndex, query: string, limit: int = 10, + config: TokenizerConfig = defaultTokenizerConfig()): seq[SearchResult] = + let queryTokens = tokenize(query, config) + if queryTokens.len == 0: + return @[] + + var docScores = initTable[uint64, float64]() + var docHighlights = initTable[uint64, seq[(int, int)]]() + + for token in queryTokens: + if token notin idx.postings: + continue + for entry in idx.postings[token]: + let score = bm25Score(idx, token, entry.docId) + if entry.docId notin docScores: + docScores[entry.docId] = 0.0 + docHighlights[entry.docId] = @[] + docScores[entry.docId] += score + for pos in entry.positions: + let start = pos + let stop = pos + token.len + docHighlights[entry.docId].add((start, stop)) + + var results: seq[SearchResult] = @[] + for docId, score in docScores: + results.add(SearchResult( + docId: docId, + score: score, + highlights: docHighlights.getOrDefault(docId, @[]), + )) + + results.sort(proc(a, b: SearchResult): int = cmp(b.score, a.score)) + + if results.len > limit: + results = results[0.. 0: + let (node, depth) = queue.popFirst() + result.add(node) + if maxDepth >= 0 and depth >= maxDepth: + continue + for neighbor in g.neighbors(node): + if neighbor notin visited: + visited.incl(neighbor) + queue.addLast((neighbor, depth + 1)) + +proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] = + result = @[] + var visited = initHashSet[NodeId]() + var stack: seq[(NodeId, int)] = @[(start, 0)] + + while stack.len > 0: + let (node, depth) = stack.pop() + if node in visited: + continue + visited.incl(node) + result.add(node) + if maxDepth >= 0 and depth >= maxDepth: + continue + for neighbor in g.neighbors(node): + if neighbor notin visited: + stack.add((neighbor, depth + 1)) + +proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] = + var visited = initHashSet[NodeId]() + var parent = initTable[NodeId, NodeId]() + var queue = initDeque[NodeId]() + queue.addLast(start) + visited.incl(start) + + while queue.len > 0: + let node = queue.popFirst() + if node == target: + var path: seq[NodeId] = @[target] + var current = target + while current in parent: + current = parent[current] + path.add(current) + path.reverse() + return path + + for neighbor in g.neighbors(node): + if neighbor notin visited: + visited.incl(neighbor) + parent[neighbor] = node + queue.addLast(neighbor) + + return @[] + +proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] = + result = initTable[NodeId, float64]() + var visited = initHashSet[NodeId]() + + result[start] = 0.0 + + while true: + var bestNode: NodeId + var bestDist = Inf + for nodeId, dist in result: + if nodeId notin visited and dist < bestDist: + bestDist = dist + bestNode = nodeId + + if bestDist == Inf: + break + + visited.incl(bestNode) + + for entry in g.adjacency.getOrDefault(bestNode, @[]): + let newDist = bestDist + entry.weight + if entry.neighbor notin result or newDist < result[entry.neighbor]: + result[entry.neighbor] = newDist + +proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): Table[NodeId, float64] = + result = initTable[NodeId, float64]() + let n = g.nodes.len + if n == 0: + return + + let initialRank = 1.0 / float64(n) + for nodeId in g.nodes.keys: + result[nodeId] = initialRank + + for iter in 0.. 0: + rank += dampingFactor * result[entry.neighbor] / float64(srcOutDegree) + + newRanks[nodeId] = rank + + result = newRanks + +proc nodeCount*(g: Graph): int = g.nodes.len +proc edgeCount*(g: Graph): int = g.edges.len diff --git a/src/barabadb/query/ast.nim b/src/barabadb/query/ast.nim new file mode 100644 index 0000000..3ed68dd --- /dev/null +++ b/src/barabadb/query/ast.nim @@ -0,0 +1,318 @@ +## BaraQL AST — Abstract Syntax Tree nodes +import ../core/types + +type + NodeKind* = enum + # Statements + nkSelect + nkInsert + nkUpdate + nkDelete + nkCreateType + nkDropType + nkAlterType + nkCreateIndex + nkDropIndex + + # Clauses + nkFrom + nkWhere + nkOrderBy + nkGroupBy + nkHaving + nkLimit + nkOffset + nkReturning + nkWith + + # Expressions + nkBinOp + nkUnaryOp + nkFuncCall + nkTypeCast + nkPath + nkIdent + nkIntLit + nkFloatLit + nkStringLit + nkBoolLit + nkNullLit + nkArrayLit + nkVectorLit + nkObjectLit + nkIfElse + nkCase + nkSubquery + nkExists + nkInExpr + nkBetweenExpr + nkLikeExpr + nkIsExpr + + # Graph-specific + nkGraphTraversal + nkBfsQuery + nkDfsQuery + nkShortestPath + nkPatternMatch + + # Vector-specific + nkVectorSimilar + nkVectorNearest + + # Join + nkJoin + + # Type definitions + nkPropertyDef + nkLinkDef + nkIndexDef + nkConstraintDef + + # Top-level + nkStatementList + + BinOpKind* = enum + bkAdd = "+" + bkSub = "-" + bkMul = "*" + bkDiv = "/" + bkMod = "%" + bkPow = "**" + bkFloorDiv = "//" + bkEq = "=" + bkNotEq = "!=" + bkLt = "<" + bkLtEq = "<=" + bkGt = ">" + bkGtEq = ">=" + bkAnd = "AND" + bkOr = "OR" + bkIn = "IN" + bkNotIn = "NOT IN" + bkLike = "LIKE" + bkILike = "ILIKE" + bkConcat = "++" + bkCoalesce = "??" + bkAssign = ":=" + bkArrow = "=>" + + UnaryOpKind* = enum + ukNeg = "-" + ukNot = "NOT" + ukIsNull = "IS NULL" + ukIsNotNull = "IS NOT NULL" + + JoinKind* = enum + jkInner + jkLeft + jkRight + jkFull + jkCross + + SortDir* = enum + sdAsc + sdDesc + + Node* = ref object + line*: int + col*: int + case kind*: NodeKind + of nkSelect: + selDistinct*: bool + selWith*: seq[Node] + selResult*: seq[Node] + selFrom*: Node + selJoins*: seq[Node] + selWhere*: Node + selGroupBy*: seq[Node] + selHaving*: Node + selOrderBy*: seq[Node] + selLimit*: Node + selOffset*: Node + of nkInsert: + insTarget*: string + insFields*: seq[Node] + insValues*: seq[Node] + insReturning*: seq[Node] + insConflict*: Node + of nkUpdate: + updTarget*: string + updAlias*: string + updSet*: seq[Node] + updWhere*: Node + updReturning*: seq[Node] + of nkDelete: + delTarget*: string + delAlias*: string + delWhere*: Node + delReturning*: seq[Node] + of nkCreateType: + ctName*: string + ctBases*: seq[string] + ctProperties*: seq[Node] + ctLinks*: seq[Node] + of nkDropType: + dtName*: string + of nkAlterType: + atName*: string + atOps*: seq[Node] + of nkCreateIndex: + ciTarget*: string + ciName*: string + ciExpr*: Node + ciKind*: IndexKind + of nkDropIndex: + diName*: string + of nkFrom: + fromTable*: string + fromAlias*: string + of nkWhere: + whereExpr*: Node + of nkOrderBy: + orderByExpr*: Node + orderByDir*: SortDir + of nkGroupBy: + groupExprs*: seq[Node] + of nkHaving: + havingExpr*: Node + of nkLimit: + limitExpr*: Node + of nkOffset: + offsetExpr*: Node + of nkReturning: + retExprs*: seq[Node] + of nkWith: + withBindings*: seq[(string, Node)] + of nkBinOp: + binOp*: BinOpKind + binLeft*: Node + binRight*: Node + of nkUnaryOp: + unOp*: UnaryOpKind + unOperand*: Node + of nkFuncCall: + funcName*: string + funcArgs*: seq[Node] + of nkTypeCast: + castType*: string + castExpr*: Node + of nkPath: + pathParts*: seq[string] + of nkIdent: + identName*: string + of nkIntLit: + intVal*: int64 + of nkFloatLit: + floatVal*: float64 + of nkStringLit: + strVal*: string + of nkBoolLit: + boolVal*: bool + of nkNullLit: + discard + of nkArrayLit: + arrayElems*: seq[Node] + of nkVectorLit: + vecElems*: seq[Node] + of nkObjectLit: + objFields*: seq[(string, Node)] + of nkIfElse: + ifCond*: Node + ifThen*: Node + ifElse*: Node + of nkCase: + caseExpr*: Node + caseWhens*: seq[(Node, Node)] + caseElse*: Node + of nkSubquery: + subQuery*: Node + of nkExists: + existsExpr*: Node + of nkInExpr: + inLeft*: Node + inRight*: Node + of nkBetweenExpr: + betweenExpr*: Node + betweenLow*: Node + betweenHigh*: Node + of nkLikeExpr: + likeExpr*: Node + likePattern*: Node + likeCaseInsensitive*: bool + of nkIsExpr: + isExpr*: Node + isType*: string + isNegated*: bool + of nkGraphTraversal: + gtStart*: Node + gtEdge*: string + gtDirection*: string + gtEnd*: Node + gtMaxDepth*: int + of nkBfsQuery: + bfsStart*: Node + bfsTarget*: Node + bfsEdge*: string + bfsMaxDepth*: int + bfsFilter*: Node + of nkDfsQuery: + dfsStart*: Node + dfsTarget*: Node + dfsEdge*: string + dfsMaxDepth*: int + dfsFilter*: Node + of nkShortestPath: + spStart*: Node + spEnd*: Node + spEdge*: string + spMaxDepth*: int + of nkPatternMatch: + pmPattern*: Node + pmWhere*: Node + of nkVectorSimilar: + vsField*: string + vsVector*: Node + vsLimit*: int + vsMetric*: string + of nkVectorNearest: + vnField*: string + vnVector*: Node + vnLimit*: int + vnMetric*: string + of nkJoin: + joinKind*: JoinKind + joinTarget*: Node + joinOn*: Node + joinAlias*: string + of nkPropertyDef: + pdName*: string + pdType*: string + pdRequired*: bool + pdDefault*: Node + pdComputed*: bool + pdExpr*: Node + of nkLinkDef: + ldName*: string + ldTarget*: string + ldCardinality*: Cardinality + ldRequired*: bool + of nkIndexDef: + idName*: string + idExpr*: Node + idKind*: IndexKind + of nkConstraintDef: + cdName*: string + cdExpr*: Node + of nkStatementList: + stmts*: seq[Node] + +proc newNode*(kind: NodeKind, line, col: int = 0): Node = + result = Node(kind: kind, line: line, col: col) + case kind + of nkSelect: result.selResult = @[] + of nkInsert: result.insFields = @[]; result.insValues = @[] + of nkUpdate: result.updSet = @[] + of nkDelete: discard + of nkStatementList: result.stmts = @[] + else: discard diff --git a/src/barabadb/query/lexer.nim b/src/barabadb/query/lexer.nim new file mode 100644 index 0000000..334c20b --- /dev/null +++ b/src/barabadb/query/lexer.nim @@ -0,0 +1,461 @@ +## BaraQL Lexer — tokenization +import std/tables +import std/strutils + +type + TokenKind* = enum + # Literals + tkIntLit + tkFloatLit + tkStringLit + tkBoolLit + + # Identifiers + tkIdent + + # Keywords + tkSelect + tkInsert + tkUpdate + tkDelete + tkFrom + tkWhere + tkAnd + tkOr + tkNot + tkIn + tkIs + tkAs + tkOn + tkJoin + tkLeft + tkRight + tkInner + tkOuter + tkFull + tkCross + tkOrder + tkBy + tkAsc + tkDesc + tkGroup + tkHaving + tkLimit + tkOffset + tkSet + tkInto + tkValues + tkCreate + tkDrop + tkAlter + tkTable + tkIndex + tkType + tkLink + tkProperty + tkRequired + tkMulti + tkSingle + tkTrue + tkFalse + tkNull + tkIf + tkThen + tkElse + tkEnd + tkCase + tkWhen + tkWith + tkDistinct + tkUnion + tkIntersect + tkExcept + tkExists + tkBetween + tkLike + tkILike + tkReturning + tkCount + tkSum + tkAvg + tkMin + tkMax + tkArray + tkVector + tkGraph + tkDocument + tkSimilar + tkNearest + tkTo + tkBfs + tkDfs + tkPath + + # Operators + tkPlus + tkMinus + tkStar + tkSlash + tkPercent + tkPower + tkEq + tkNotEq + tkLt + tkLtEq + tkGt + tkGtEq + tkAssign + tkArrow + tkDoubleColon + tkDot + tkComma + tkSemicolon + tkLParen + tkRParen + tkLBrace + tkRbrace + tkLBracket + tkRBracket + tkAmp + tkPipe + tkTilde + tkConcat + tkCoalesce + tkFloorDiv + + # Special + tkEof + tkNewline + tkInvalid + + Token* = object + kind*: TokenKind + value*: string + line*: int + col*: int + + Lexer* = object + input: string + pos: int + line: int + col: int + +const keywords*: Table[string, TokenKind] = { + "select": tkSelect, + "insert": tkInsert, + "update": tkUpdate, + "delete": tkDelete, + "from": tkFrom, + "where": tkWhere, + "and": tkAnd, + "or": tkOr, + "not": tkNot, + "in": tkIn, + "is": tkIs, + "as": tkAs, + "on": tkOn, + "join": tkJoin, + "left": tkLeft, + "right": tkRight, + "inner": tkInner, + "outer": tkOuter, + "full": tkFull, + "cross": tkCross, + "order": tkOrder, + "by": tkBy, + "asc": tkAsc, + "desc": tkDesc, + "group": tkGroup, + "having": tkHaving, + "limit": tkLimit, + "offset": tkOffset, + "set": tkSet, + "into": tkInto, + "values": tkValues, + "create": tkCreate, + "drop": tkDrop, + "alter": tkAlter, + "table": tkTable, + "index": tkIndex, + "type": tkType, + "link": tkLink, + "property": tkProperty, + "required": tkRequired, + "multi": tkMulti, + "single": tkSingle, + "true": tkTrue, + "false": tkFalse, + "null": tkNull, + "if": tkIf, + "then": tkThen, + "else": tkElse, + "end": tkEnd, + "case": tkCase, + "when": tkWhen, + "with": tkWith, + "distinct": tkDistinct, + "union": tkUnion, + "intersect": tkIntersect, + "except": tkExcept, + "exists": tkExists, + "between": tkBetween, + "like": tkLike, + "ilike": tkILike, + "returning": tkReturning, + "count": tkCount, + "sum": tkSum, + "avg": tkAvg, + "min": tkMin, + "max": tkMax, + "array": tkArray, + "vector": tkVector, + "graph": tkGraph, + "document": tkDocument, + "similar": tkSimilar, + "nearest": tkNearest, + "to": tkTo, + "bfs": tkBfs, + "dfs": tkDfs, + "path": tkPath, +}.toTable + +proc newLexer*(input: string): Lexer = + Lexer(input: input, pos: 0, line: 1, col: 1) + +proc peek(l: Lexer): char = + if l.pos < l.input.len: + return l.input[l.pos] + return '\0' + +proc advance(l: var Lexer): char = + result = l.input[l.pos] + inc l.pos + if result == '\n': + inc l.line + l.col = 1 + else: + inc l.col + +proc skipWhitespace(l: var Lexer) = + while l.pos < l.input.len and l.input[l.pos] in {' ', '\t', '\r', '\n'}: + discard l.advance() + +proc skipLineComment(l: var Lexer) = + while l.pos < l.input.len and l.input[l.pos] != '\n': + discard l.advance() + +proc skipBlockComment(l: var Lexer) = + discard l.advance() # skip * + discard l.advance() # skip * + while l.pos < l.input.len - 1: + if l.input[l.pos] == '*' and l.input[l.pos + 1] == '/': + discard l.advance() + discard l.advance() + return + discard l.advance() + +proc readString(l: var Lexer, quote: char): string = + result = "" + while l.pos < l.input.len and l.input[l.pos] != quote: + if l.input[l.pos] == '\\': + discard l.advance() + case l.input[l.pos] + of 'n': result.add('\n') + of 't': result.add('\t') + of 'r': result.add('\r') + of '\\': result.add('\\') + of '\'': result.add('\'') + of '"': result.add('"') + else: result.add(l.input[l.pos]) + else: + result.add(l.input[l.pos]) + discard l.advance() + if l.pos < l.input.len: + discard l.advance() # skip closing quote + +proc readNumber(l: var Lexer, startLine, startCol: int): Token = + var numStr = "" + var isFloat = false + while l.pos < l.input.len and (l.input[l.pos] in Digits or l.input[l.pos] == '.'): + if l.input[l.pos] == '.': + isFloat = true + numStr.add(l.input[l.pos]) + discard l.advance() + if isFloat: + Token(kind: tkFloatLit, value: numStr, line: startLine, col: startCol) + else: + Token(kind: tkIntLit, value: numStr, line: startLine, col: startCol) + +proc readIdent(l: var Lexer, startLine, startCol: int): Token = + var ident = "" + while l.pos < l.input.len and (l.input[l.pos] in IdentChars or l.input[l.pos] in Digits): + ident.add(l.input[l.pos]) + discard l.advance() + let lowerIdent = ident.toLower() + if lowerIdent in keywords: + Token(kind: keywords[lowerIdent], value: ident, line: startLine, col: startCol) + elif lowerIdent == "true": + Token(kind: tkBoolLit, value: "true", line: startLine, col: startCol) + elif lowerIdent == "false": + Token(kind: tkBoolLit, value: "false", line: startLine, col: startCol) + else: + Token(kind: tkIdent, value: ident, line: startLine, col: startCol) + +proc nextToken*(l: var Lexer): Token = + l.skipWhitespace() + if l.pos >= l.input.len: + return Token(kind: tkEof, line: l.line, col: l.col) + + let startLine = l.line + let startCol = l.col + let ch = l.peek() + + case ch + of '/': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '/': + l.skipLineComment() + return l.nextToken() + elif l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*': + l.skipBlockComment() + return l.nextToken() + else: + discard l.advance() + return Token(kind: tkSlash, value: "/", line: startLine, col: startCol) + of '+': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '+': + discard l.advance() + discard l.advance() + return Token(kind: tkConcat, value: "++", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkPlus, value: "+", line: startLine, col: startCol) + of '-': + discard l.advance() + return Token(kind: tkMinus, value: "-", line: startLine, col: startCol) + of '*': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*': + discard l.advance() + discard l.advance() + return Token(kind: tkPower, value: "**", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkStar, value: "*", line: startLine, col: startCol) + of '%': + discard l.advance() + return Token(kind: tkPercent, value: "%", line: startLine, col: startCol) + of '=': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': + discard l.advance() + discard l.advance() + return Token(kind: tkArrow, value: "=>", line: startLine, col: startCol) + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': + discard l.advance() + discard l.advance() + return Token(kind: tkEq, value: "==", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkAssign, value: ":=", line: startLine, col: startCol) + of ':': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': + discard l.advance() + discard l.advance() + return Token(kind: tkAssign, value: ":=", line: startLine, col: startCol) + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == ':': + discard l.advance() + discard l.advance() + return Token(kind: tkDoubleColon, value: "::", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkInvalid, value: ":", line: startLine, col: startCol) + of '!': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': + discard l.advance() + discard l.advance() + return Token(kind: tkNotEq, value: "!=", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkInvalid, value: "!", line: startLine, col: startCol) + of '<': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': + discard l.advance() + discard l.advance() + return Token(kind: tkLtEq, value: "<=", line: startLine, col: startCol) + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': + discard l.advance() + discard l.advance() + return Token(kind: tkNotEq, value: "<>", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkLt, value: "<", line: startLine, col: startCol) + of '>': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': + discard l.advance() + discard l.advance() + return Token(kind: tkGtEq, value: ">=", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkGt, value: ">", line: startLine, col: startCol) + of '?': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '?': + discard l.advance() + discard l.advance() + return Token(kind: tkCoalesce, value: "??", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkInvalid, value: "?", line: startLine, col: startCol) + of '.': + if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '<': + discard l.advance() + discard l.advance() + return Token(kind: tkInvalid, value: ".<", line: startLine, col: startCol) + discard l.advance() + return Token(kind: tkDot, value: ".", line: startLine, col: startCol) + of ',': + discard l.advance() + return Token(kind: tkComma, value: ",", line: startLine, col: startCol) + of ';': + discard l.advance() + return Token(kind: tkSemicolon, value: ";", line: startLine, col: startCol) + of '(': + discard l.advance() + return Token(kind: tkLParen, value: "(", line: startLine, col: startCol) + of ')': + discard l.advance() + return Token(kind: tkRParen, value: ")", line: startLine, col: startCol) + of '{': + discard l.advance() + return Token(kind: tkLBrace, value: "{", line: startLine, col: startCol) + of '}': + discard l.advance() + return Token(kind: tkRbrace, value: "}", line: startLine, col: startCol) + of '[': + discard l.advance() + return Token(kind: tkLBracket, value: "[", line: startLine, col: startCol) + of ']': + discard l.advance() + return Token(kind: tkRBracket, value: "]", line: startLine, col: startCol) + of '&': + discard l.advance() + return Token(kind: tkAmp, value: "&", line: startLine, col: startCol) + of '|': + discard l.advance() + return Token(kind: tkPipe, value: "|", line: startLine, col: startCol) + of '~': + discard l.advance() + return Token(kind: tkTilde, value: "~", line: startLine, col: startCol) + of '\'', '"': + discard l.advance() + let s = l.readString(ch) + return Token(kind: tkStringLit, value: s, line: startLine, col: startCol) + of '#': + l.skipLineComment() + return l.nextToken() + else: + if ch in Digits: + return l.readNumber(startLine, startCol) + elif ch in IdentStartChars: + return l.readIdent(startLine, startCol) + else: + discard l.advance() + return Token(kind: tkInvalid, value: $ch, line: startLine, col: startCol) + +proc tokenize*(input: string): seq[Token] = + var lexer = newLexer(input) + result = @[] + while true: + let tok = lexer.nextToken() + result.add(tok) + if tok.kind == tkEof: + break diff --git a/src/barabadb/query/parser.nim b/src/barabadb/query/parser.nim new file mode 100644 index 0000000..c558e9a --- /dev/null +++ b/src/barabadb/query/parser.nim @@ -0,0 +1,245 @@ +## BaraQL Parser — recursive descent parser +import std/strutils +import lexer +import ast + +type + Parser* = object + tokens: seq[Token] + pos: int + +proc newParser*(tokens: seq[Token]): Parser = + Parser(tokens: tokens, pos: 0) + +proc peek(p: Parser): Token = + if p.pos < p.tokens.len: + return p.tokens[p.pos] + Token(kind: tkEof) + +proc advance(p: var Parser): Token = + result = p.tokens[p.pos] + inc p.pos + +proc expect(p: var Parser, kind: TokenKind): Token = + let tok = p.advance() + if tok.kind != kind: + raise newException(ValueError, + "Expected " & $kind & " but got " & $tok.kind & " at line " & $tok.line) + return tok + +proc match(p: var Parser, kind: TokenKind): bool = + if p.peek().kind == kind: + discard p.advance() + return true + return false + +proc parseExpr(p: var Parser): Node +proc parseSelect(p: var Parser): Node + +proc parsePrimary(p: var Parser): Node = + let tok = p.peek() + case tok.kind + of tkIntLit: + discard p.advance() + Node(kind: nkIntLit, intVal: parseInt(tok.value), line: tok.line, col: tok.col) + of tkFloatLit: + discard p.advance() + Node(kind: nkFloatLit, floatVal: parseFloat(tok.value), line: tok.line, col: tok.col) + of tkStringLit: + discard p.advance() + Node(kind: nkStringLit, strVal: tok.value, line: tok.line, col: tok.col) + of tkBoolLit: + discard p.advance() + Node(kind: nkBoolLit, boolVal: tok.value == "true", line: tok.line, col: tok.col) + of tkNull: + discard p.advance() + Node(kind: nkNullLit, line: tok.line, col: tok.col) + of tkIdent: + discard p.advance() + Node(kind: nkIdent, identName: tok.value, line: tok.line, col: tok.col) + of tkLParen: + discard p.advance() + let expr = p.parseExpr() + discard p.expect(tkRParen) + expr + of tkLBracket: + discard p.advance() + var elems: seq[Node] = @[] + if p.peek().kind != tkRBracket: + elems.add(p.parseExpr()) + while p.match(tkComma): + elems.add(p.parseExpr()) + discard p.expect(tkRBracket) + Node(kind: nkArrayLit, arrayElems: elems, line: tok.line, col: tok.col) + of tkSelect: + p.parseSelect() + of tkExists: + discard p.advance() + discard p.expect(tkLParen) + let sub = p.parseSelect() + discard p.expect(tkRParen) + Node(kind: nkExists, existsExpr: sub, line: tok.line, col: tok.col) + of tkNot: + discard p.advance() + let operand = p.parsePrimary() + Node(kind: nkUnaryOp, unOp: ukNot, unOperand: operand, line: tok.line, col: tok.col) + of tkMinus: + discard p.advance() + let operand = p.parsePrimary() + Node(kind: nkUnaryOp, unOp: ukNeg, unOperand: operand, line: tok.line, col: tok.col) + of tkCount: + discard p.advance() + discard p.expect(tkLParen) + var args: seq[Node] = @[] + if p.peek().kind != tkRParen: + args.add(p.parseExpr()) + discard p.expect(tkRParen) + Node(kind: nkFuncCall, funcName: "count", funcArgs: args, line: tok.line, col: tok.col) + else: + discard p.advance() + Node(kind: nkNullLit, line: tok.line, col: tok.col) + +proc parseMulDiv(p: var Parser): Node = + result = p.parsePrimary() + while p.peek().kind in {tkStar, tkSlash, tkPercent, tkFloorDiv}: + let op = case p.peek().kind + of tkStar: bkMul + of tkSlash: bkDiv + of tkPercent: bkMod + of tkFloorDiv: bkFloorDiv + else: bkMul + let tok = p.advance() + let right = p.parsePrimary() + result = Node(kind: nkBinOp, binOp: op, binLeft: result, binRight: right, + line: tok.line, col: tok.col) + +proc parseAddSub(p: var Parser): Node = + result = p.parseMulDiv() + while p.peek().kind in {tkPlus, tkMinus, tkConcat}: + let op = case p.peek().kind + of tkPlus: bkAdd + of tkMinus: bkSub + of tkConcat: bkConcat + else: bkAdd + let tok = p.advance() + let right = p.parseMulDiv() + result = Node(kind: nkBinOp, binOp: op, binLeft: result, binRight: right, + line: tok.line, col: tok.col) + +proc parseComparison(p: var Parser): Node = + result = p.parseAddSub() + while p.peek().kind in {tkEq, tkNotEq, tkLt, tkLtEq, tkGt, tkGtEq}: + let op = case p.peek().kind + of tkEq: bkEq + of tkNotEq: bkNotEq + of tkLt: bkLt + of tkLtEq: bkLtEq + of tkGt: bkGt + of tkGtEq: bkGtEq + else: bkEq + let tok = p.advance() + let right = p.parseAddSub() + result = Node(kind: nkBinOp, binOp: op, binLeft: result, binRight: right, + line: tok.line, col: tok.col) + +proc parseNot(p: var Parser): Node = + if p.peek().kind == tkNot: + let tok = p.advance() + let operand = p.parseComparison() + return Node(kind: nkUnaryOp, unOp: ukNot, unOperand: operand, + line: tok.line, col: tok.col) + return p.parseComparison() + +proc parseAnd(p: var Parser): Node = + result = p.parseNot() + while p.peek().kind == tkAnd: + let tok = p.advance() + let right = p.parseNot() + result = Node(kind: nkBinOp, binOp: bkAnd, binLeft: result, binRight: right, + line: tok.line, col: tok.col) + +proc parseOr(p: var Parser): Node = + result = p.parseAnd() + while p.peek().kind == tkOr: + let tok = p.advance() + let right = p.parseAnd() + result = Node(kind: nkBinOp, binOp: bkOr, binLeft: result, binRight: right, + line: tok.line, col: tok.col) + +proc parseExpr(p: var Parser): Node = + return p.parseOr() + +proc parseSelect(p: var Parser): Node = + let tok = p.expect(tkSelect) + result = Node(kind: nkSelect, line: tok.line, col: tok.col) + + if p.peek().kind == tkDistinct: + discard p.advance() + result.selDistinct = true + + result.selResult = @[] + result.selResult.add(p.parseExpr()) + while p.match(tkComma): + result.selResult.add(p.parseExpr()) + + if p.match(tkFrom): + let tableTok = p.expect(tkIdent) + var alias = "" + if p.match(tkAs): + alias = p.expect(tkIdent).value + elif p.peek().kind == tkIdent: + alias = p.advance().value + result.selFrom = Node(kind: nkFrom, fromTable: tableTok.value, + fromAlias: alias, line: tableTok.line, col: tableTok.col) + + if p.match(tkWhere): + result.selWhere = Node(kind: nkWhere, whereExpr: p.parseExpr()) + + if p.match(tkLimit): + result.selLimit = Node(kind: nkLimit, limitExpr: p.parseExpr()) + + if p.match(tkOffset): + result.selOffset = Node(kind: nkOffset, offsetExpr: p.parseExpr()) + +proc parseInsert(p: var Parser): Node = + let tok = p.expect(tkInsert) + let target = p.expect(tkIdent).value + result = Node(kind: nkInsert, insTarget: target, line: tok.line, col: tok.col) + +proc parseUpdate(p: var Parser): Node = + let tok = p.expect(tkUpdate) + let target = p.expect(tkIdent).value + result = Node(kind: nkUpdate, updTarget: target, line: tok.line, col: tok.col) + +proc parseDelete(p: var Parser): Node = + let tok = p.expect(tkDelete) + let target = p.expect(tkIdent).value + result = Node(kind: nkDelete, delTarget: target, line: tok.line, col: tok.col) + +proc parseCreateType(p: var Parser): Node = + let tok = p.expect(tkCreate) + discard p.expect(tkType) + let name = p.expect(tkIdent).value + result = Node(kind: nkCreateType, ctName: name, line: tok.line, col: tok.col) + +proc parseStatement*(p: var Parser): Node = + case p.peek().kind + of tkSelect: p.parseSelect() + of tkInsert: p.parseInsert() + of tkUpdate: p.parseUpdate() + of tkDelete: p.parseDelete() + of tkCreate: p.parseCreateType() + else: + let tok = p.advance() + Node(kind: nkNullLit, line: tok.line, col: tok.col) + +proc parse*(tokens: seq[Token]): Node = + var parser = newParser(tokens) + result = Node(kind: nkStatementList) + while parser.peek().kind != tkEof: + result.stmts.add(parser.parseStatement()) + discard parser.match(tkSemicolon) + +proc parse*(input: string): Node = + let tokens = tokenize(input) + parse(tokens) diff --git a/src/barabadb/storage/bloom.nim b/src/barabadb/storage/bloom.nim new file mode 100644 index 0000000..645b1a1 --- /dev/null +++ b/src/barabadb/storage/bloom.nim @@ -0,0 +1,51 @@ +## Bloom filter — probabilistic set membership +import std/hashes +import std/math + +type + BloomFilter* = object + bits: seq[bool] + numHashes: int + size: int + +proc newBloomFilter*(expectedItems: int, fpRate: float = 0.01): BloomFilter = + let size = int(-float(expectedItems) * ln(fpRate) / (ln(2.0) * ln(2.0))) + let numHashes = int(float(size) / float(expectedItems) * ln(2.0)) + BloomFilter( + bits: newSeq[bool](max(size, 64)), + numHashes: max(numHashes, 1), + size: max(size, 64), + ) + +proc hash1*(bf: BloomFilter, data: openArray[byte]): uint64 = + var h: Hash = 0 + for b in data: + h = h !& Hash(b) + result = uint64(!$h) + +proc hash2*(bf: BloomFilter, data: openArray[byte]): uint64 = + var h: Hash = 5381 + for b in data: + h = ((h shl 5) + h) + Hash(b) + result = uint64(h) + +proc getHashes(bf: BloomFilter, data: openArray[byte]): seq[int] = + let h1 = bf.hash1(data) + let h2 = bf.hash2(data) + result = newSeq[int](bf.numHashes) + for i in 0.. file offset + bloom: BloomFilter + level: int + minKey: string + maxKey: string + entryCount: int + + LSMTree* = object + dir: string + memTable: MemTable + immutableMem: MemTable + sstables: seq[SSTable] + wal: WriteAheadLog + memMaxSize: int + currentSeq: uint64 + readLocks: int + +proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable = + MemTable(entries: @[], size: 0, maxSize: maxSize) + +proc len*(mt: MemTable): int = mt.entries.len + +proc put*(mt: var MemTable, key: string, value: seq[byte], timestamp: uint64, deleted: bool = false): bool = + let entrySize = key.len + value.len + 16 + if mt.size + entrySize > mt.maxSize and mt.entries.len > 0: + return false + let entry = Entry(key: key, value: value, timestamp: timestamp, deleted: deleted) + let pos = mt.entries.lowerBound(entry, proc(a, b: Entry): int = cmp(a.key, b.key)) + if pos < mt.entries.len and mt.entries[pos].key == key: + mt.entries[pos] = entry + else: + mt.entries.insert(entry, pos) + mt.size += entrySize + return true + +proc get*(mt: MemTable, key: string): (bool, Entry) = + for entry in mt.entries: + if entry.key == key: + return (true, entry) + return (false, Entry()) + +proc scan*(mt: MemTable, startKey, endKey: string): seq[Entry] = + result = @[] + for entry in mt.entries: + if entry.key >= startKey and entry.key <= endKey: + result.add(entry) + +proc clear*(mt: var MemTable) = + mt.entries.setLen(0) + mt.size = 0 + +proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree = + createDir(dir) + createDir(dir / "sstables") + LSMTree( + dir: dir, + memTable: newMemTable(memMaxSize), + immutableMem: newMemTable(0), + sstables: @[], + wal: newWriteAheadLog(dir / "wal"), + memMaxSize: memMaxSize, + currentSeq: 0, + readLocks: 0, + ) + +proc put*(db: var LSMTree, key: string, value: seq[byte]) = + let ts = uint64(getMonoTime().ticks()) + db.wal.writePut(cast[seq[byte]](key), value, ts) + if not db.memTable.put(key, value, ts): + db.immutableMem = db.memTable + db.memTable = newMemTable(db.memMaxSize) + discard db.memTable.put(key, value, ts) + +proc delete*(db: var LSMTree, key: string) = + let ts = uint64(getMonoTime().ticks()) + db.wal.writeDelete(cast[seq[byte]](key), ts) + discard db.memTable.put(key, @[], ts, deleted = true) + +proc get*(db: LSMTree, key: string): (bool, seq[byte]) = + let (found, entry) = db.memTable.get(key) + if found: + if entry.deleted: + return (false, @[]) + return (true, entry.value) + + let (found2, entry2) = db.immutableMem.get(key) + if found2: + if entry2.deleted: + return (false, @[]) + return (true, entry2.value) + + for sst in db.sstables: + if key in sst.index: + return (true, @[]) # placeholder for SSTable read + + return (false, @[]) + +proc contains*(db: LSMTree, key: string): bool = + let (found, _) = db.get(key) + return found + +proc flush*(db: var LSMTree) = + if db.memTable.len == 0: + return + db.immutableMem = db.memTable + db.memTable = newMemTable(db.memMaxSize) + db.wal.writeCommit(uint64(getMonoTime().ticks())) + db.wal.sync() + +proc close*(db: var LSMTree) = + db.flush() + db.wal.close() + +proc memTableSize*(db: LSMTree): int = db.memTable.len +proc sstableCount*(db: LSMTree): int = db.sstables.len +proc dir*(db: LSMTree): string = db.dir diff --git a/src/barabadb/storage/wal.nim b/src/barabadb/storage/wal.nim new file mode 100644 index 0000000..a70fef2 --- /dev/null +++ b/src/barabadb/storage/wal.nim @@ -0,0 +1,84 @@ +## WAL — Write-Ahead Log for durability +import std/os +import std/streams + +const + WALMagic* = 0x42415241'u32 # "BARA" + WALVersion* = 1'u32 + +type + WalEntryKind* = enum + wekPut = 1 + wekDelete = 2 + wekCheckpoint = 3 + wekCommit = 4 + + WalEntry* = object + kind*: WalEntryKind + timestamp*: uint64 + key*: seq[byte] + value*: seq[byte] + + WriteAheadLog* = object + path: string + stream: FileStream + entryCount: uint64 + syncOnWrite: bool + +proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog = + createDir(dir) + let path = dir / "wal.log" + let stream = newFileStream(path, fmWrite) + if stream == nil: + raise newException(IOError, "Cannot open WAL: " & path) + stream.write(WALMagic) + stream.write(WALVersion) + stream.flush() + WriteAheadLog(path: path, stream: stream, entryCount: 0, syncOnWrite: syncOnWrite) + +proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) = + wal.stream.write(uint8(entry.kind)) + wal.stream.write(entry.timestamp) + wal.stream.write(uint32(entry.key.len)) + if entry.key.len > 0: + wal.stream.writeData(unsafeAddr entry.key[0], entry.key.len) + wal.stream.write(uint32(entry.value.len)) + if entry.value.len > 0: + wal.stream.writeData(unsafeAddr entry.value[0], entry.value.len) + if wal.syncOnWrite: + wal.stream.flush() + inc wal.entryCount + +proc writePut*(wal: var WriteAheadLog, key, value: openArray[byte], timestamp: uint64) = + wal.writeEntry(WalEntry( + kind: wekPut, + timestamp: timestamp, + key: @key, + value: @value, + )) + +proc writeDelete*(wal: var WriteAheadLog, key: openArray[byte], timestamp: uint64) = + wal.writeEntry(WalEntry( + kind: wekDelete, + timestamp: timestamp, + key: @key, + value: @[], + )) + +proc writeCommit*(wal: var WriteAheadLog, timestamp: uint64) = + wal.writeEntry(WalEntry( + kind: wekCommit, + timestamp: timestamp, + key: @[], + value: @[], + )) + +proc sync*(wal: var WriteAheadLog) = + wal.stream.flush() + +proc close*(wal: var WriteAheadLog) = + wal.stream.flush() + wal.stream.close() + +proc entryCount*(wal: WriteAheadLog): uint64 = wal.entryCount +proc path*(wal: WriteAheadLog): string = wal.path diff --git a/src/barabadb/vector/engine.nim b/src/barabadb/vector/engine.nim new file mode 100644 index 0000000..2534455 --- /dev/null +++ b/src/barabadb/vector/engine.nim @@ -0,0 +1,210 @@ +## Vector Engine — HNSW and IVF-PQ indexes for vector similarity search +import std/math +import std/algorithm +import std/random +import std/tables + +type + DistanceMetric* = enum + dmCosine = "cosine" + dmEuclidean = "euclidean" + dmDotProduct = "dot_product" + dmManhattan = "manhattan" + + Vector* = seq[float32] + + VectorEntry* = object + id*: uint64 + vector*: Vector + metadata*: seq[(string, string)] + + HNSWNode* = ref object + id*: uint64 + vector*: Vector + neighbors*: seq[seq[uint64]] # neighbors per level + + HNSWIndex* = ref object + nodes*: Table[uint64, HNSWNode] + entryPoint*: uint64 + maxLevel*: int + efConstruction*: int + m*: int + maxM*: int + metric*: DistanceMetric + dimensions*: int + + IVFCluster* = object + centroid*: Vector + entries*: seq[VectorEntry] + + IVFPQIndex* = ref object + clusters*: seq[IVFCluster] + nClusters*: int + nSubquantizers*: int + nBits*: int + metric*: DistanceMetric + dimensions*: int + +proc cosineDistance*(a, b: Vector): float64 = + var dot, normA, normB: float64 + for i in 0.. idx.maxLevel: + idx.entryPoint = id + idx.maxLevel = level + +proc search*(idx: HNSWIndex, query: Vector, k: int, + metric: DistanceMetric = dmCosine): seq[(uint64, float64)] = + if idx.nodes.len == 0: + return @[] + + var candidates: seq[(uint64, float64)] = @[] + for nodeId, node in idx.nodes: + let dist = distance(query, node.vector, metric) + candidates.add((nodeId, dist)) + + candidates.sort(proc(a, b: (uint64, float64)): int = cmp(a[1], b[1])) + + if candidates.len > k: + candidates = candidates[0.. k: + candidates = candidates[0.. 0 + +suite "Bloom Filter": + test "Basic bloom filter operations": + var bf = newBloomFilter(1000) + let data1 = cast[seq[byte]]("hello") + let data2 = cast[seq[byte]]("world") + + bf.add(data1) + bf.add(data2) + + check bf.contains(data1) + check bf.contains(data2) + +suite "Write-Ahead Log": + test "WAL creation": + var wal = newWriteAheadLog("/tmp/baradb_test_wal") + check wal.entryCount == 0 + wal.close() + +suite "LSM-Tree Storage": + test "Put and Get": + var db = newLSMTree("/tmp/baradb_test_lsm") + let key = "testkey" + let value = cast[seq[byte]]("testvalue") + db.put(key, value) + let (found, val) = db.get(key) + check found + check val == value + db.close() + + test "Delete": + var db = newLSMTree("/tmp/baradb_test_lsm2") + let key = "delkey" + let value = cast[seq[byte]]("delval") + db.put(key, value) + db.delete(key) + let (found, _) = db.get(key) + check not found + db.close() + + test "Contains": + var db = newLSMTree("/tmp/baradb_test_lsm3") + let key = "exists" + check not db.contains(key) + db.put(key, cast[seq[byte]]("val")) + check db.contains(key) + db.close() + +suite "BaraQL Lexer": + test "Tokenize simple SELECT": + let tokens = lex.tokenize("SELECT name FROM users WHERE age > 18") + check tokens.len > 0 + check tokens[0].kind == tkSelect + check tokens[1].kind == tkIdent + check tokens[1].value == "name" + check tokens[2].kind == tkFrom + check tokens[3].kind == tkIdent + check tokens[3].value == "users" + + test "Tokenize string literals": + let tokens = lex.tokenize("'hello world'") + check tokens[0].kind == tkStringLit + check tokens[0].value == "hello world" + + test "Tokenize operators": + let tokens = lex.tokenize("a + b * c") + check tokens[0].kind == tkIdent + check tokens[1].kind == tkPlus + check tokens[2].kind == tkIdent + check tokens[3].kind == tkStar + +suite "BaraQL Parser": + test "Parse simple SELECT": + let ast = parse("SELECT name FROM users WHERE age > 18") + check ast.kind == nkStatementList + check ast.stmts.len == 1 + check ast.stmts[0].kind == nkSelect + + test "Parse SELECT with LIMIT": + let ast = parse("SELECT * FROM items LIMIT 10") + check ast.stmts[0].selLimit != nil + +suite "Vector Engine": + test "Distance metrics": + let a = @[1.0'f32, 0.0'f32, 0.0'f32] + let b = @[0.0'f32, 1.0'f32, 0.0'f32] + let c = @[1.0'f32, 0.0'f32, 0.0'f32] + + check vengine.cosineDistance(a, b) > 0.9 + check vengine.cosineDistance(a, c) < 0.1 + check vengine.euclideanDistance(a, b) > 1.0 + check vengine.euclideanDistance(a, c) < 0.1 + + test "HNSW index insert and search": + var idx = vengine.newHNSWIndex(3) + vengine.insert(idx, 1, @[1.0'f32, 0.0'f32, 0.0'f32]) + vengine.insert(idx, 2, @[0.0'f32, 1.0'f32, 0.0'f32]) + vengine.insert(idx, 3, @[0.0'f32, 0.0'f32, 1.0'f32]) + + let results = vengine.search(idx, @[1.0'f32, 0.1'f32, 0.0'f32], 2) + check results.len == 2 + +suite "Graph Engine": + test "Add nodes and edges": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "Person", {"name": "Alice"}.toTable) + let n2 = gengine.addNode(g, "Person", {"name": "Bob"}.toTable) + let e1 = gengine.addEdge(g, n1, n2, "knows") + + check gengine.nodeCount(g) == 2 + check gengine.edgeCount(g) == 1 + + test "BFS traversal": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "A") + let n2 = gengine.addNode(g, "B") + let n3 = gengine.addNode(g, "C") + let n4 = gengine.addNode(g, "D") + discard gengine.addEdge(g, n1, n2) + discard gengine.addEdge(g, n1, n3) + discard gengine.addEdge(g, n2, n4) + + let traversal = gengine.bfs(g, n1) + check traversal.len == 4 + check traversal[0] == n1 + + test "DFS traversal": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "A") + let n2 = gengine.addNode(g, "B") + let n3 = gengine.addNode(g, "C") + discard gengine.addEdge(g, n1, n2) + discard gengine.addEdge(g, n1, n3) + + let traversal = gengine.dfs(g, n1) + check traversal.len == 3 + + test "Shortest path": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "A") + let n2 = gengine.addNode(g, "B") + let n3 = gengine.addNode(g, "C") + discard gengine.addEdge(g, n1, n2) + discard gengine.addEdge(g, n2, n3) + + let path = gengine.shortestPath(g, n1, n3) + check path.len == 3 + + test "PageRank": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "A") + let n2 = gengine.addNode(g, "B") + let n3 = gengine.addNode(g, "C") + discard gengine.addEdge(g, n1, n2) + discard gengine.addEdge(g, n2, n3) + discard gengine.addEdge(g, n3, n1) + + let ranks = gengine.pageRank(g) + check ranks.len == 3 + for nodeId, rank in ranks: + check rank > 0.0 + + test "Dijkstra": + var g = gengine.newGraph() + let n1 = gengine.addNode(g, "A") + let n2 = gengine.addNode(g, "B") + let n3 = gengine.addNode(g, "C") + discard gengine.addEdge(g, n1, n2, weight = 1.0) + discard gengine.addEdge(g, n2, n3, weight = 2.0) + discard gengine.addEdge(g, n1, n3, weight = 10.0) + + let dists = gengine.dijkstra(g, n1) + check dists[n1] == 0.0 + check dists[n2] == 1.0 + check dists[n3] == 3.0 + +suite "Full-Text Search": + test "Tokenization": + let tokens = fts.tokenize("The quick brown fox jumps over the lazy dog") + check tokens.len > 0 + check "the" notin tokens + + test "Inverted index operations": + var idx = fts.newInvertedIndex() + fts.addDocument(idx, 1, "The quick brown fox") + fts.addDocument(idx, 2, "The lazy brown dog") + fts.addDocument(idx, 3, "The quick red car") + + check fts.documentCount(idx) == 3 + check fts.termCount(idx) > 0 + + test "Search results": + var idx = fts.newInvertedIndex() + fts.addDocument(idx, 1, "Nim programming language is fast") + fts.addDocument(idx, 2, "Python is popular for data science") + fts.addDocument(idx, 3, "Rust is a systems programming language") + + let results = fts.search(idx, "programming language") + check results.len > 0 + check results[0].score > 0 + + test "Document removal": + var idx = fts.newInvertedIndex() + fts.addDocument(idx, 1, "test document") + fts.addDocument(idx, 2, "another document") + check fts.documentCount(idx) == 2 + + fts.removeDocument(idx, 1) + check fts.documentCount(idx) == 1