feat: complete Phase 3 + new production roadmap for Web/ERP
- Thread-safety: locks in LSMTree and Graph engines - Raft network transport: async TCP, serialization, heartbeat, 3-node election test - CI/CD: GitHub Actions workflow - Cleanup: remove dead code, unused imports, build artifacts - New PLAN.md targeting production Web/ERP readiness - 216 tests passing
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Nim
|
||||||
|
uses: jiro4989/setup-nim-action@v1
|
||||||
|
with:
|
||||||
|
nim-version: '2.2.10'
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
run: nim c --path:src -r tests/test_all.nim
|
||||||
|
|
||||||
|
- name: Compile benchmarks
|
||||||
|
run: nim c -d:release benchmarks/bench_all.nim
|
||||||
|
|
||||||
|
- name: Compile stress test
|
||||||
|
run: nim c --threads:on --path:src tests/stress_test.nim
|
||||||
|
|
||||||
|
- name: Run stress test
|
||||||
|
run: ./tests/stress_test
|
||||||
|
|
||||||
|
- name: Check for unused declarations and imports
|
||||||
|
run: |
|
||||||
|
nim c --path:src tests/test_all.nim 2>&1 | tee build.log || true
|
||||||
|
echo "--- Checking for XDeclaredButNotUsed hints ---"
|
||||||
|
grep -E "XDeclaredButNotUsed|UnusedImport" build.log | while read -r line; do
|
||||||
|
echo "::warning::$line"
|
||||||
|
done
|
||||||
|
echo "--- Done ---"
|
||||||
@@ -2,6 +2,15 @@
|
|||||||
build/
|
build/
|
||||||
nimcache/
|
nimcache/
|
||||||
nimblecache/
|
nimblecache/
|
||||||
|
*.out
|
||||||
|
|
||||||
|
# Binaries (compiled test executables)
|
||||||
|
tests/test_all
|
||||||
|
tests/stress_test
|
||||||
|
benchmarks/bench_all
|
||||||
|
benchmarks/compare
|
||||||
|
clients/nim/tests/test_client
|
||||||
|
src/baradadb
|
||||||
|
|
||||||
# Temp
|
# Temp
|
||||||
*.tmp
|
*.tmp
|
||||||
|
|||||||
@@ -1,105 +1,187 @@
|
|||||||
# План за подобряване на BaraDB
|
# BaraDB — План към Production (Web & ERP)
|
||||||
|
|
||||||
## Цел
|
## Визия
|
||||||
Превърне BaraDB от добър proof-of-concept в солиден, изпълним проект с реална дълбочина на критичните компоненти.
|
BaraDB да стане production-ready база данни, с която да се изграждат:
|
||||||
|
- **Web приложения** (блогове, е-магазини, SaaS продукти)
|
||||||
|
- **Малки ERP системи** (CRM, склад, счетоводство, invoicing)
|
||||||
|
|
||||||
|
> Целеви потребител: solo-dev / малък екип, който иска лесна за deploy и бърза локална база, без зависимост от PostgreSQL/MySQL.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Фаза 1: Честност и стабилна основа (1–2 седмици) ✅ ЗАВЪРШЕНА
|
## Текущо състояние (база)
|
||||||
|
|
||||||
### 1.1 Поправи `README.md` да отразява реалното състояние
|
| Компонент | Статус |
|
||||||
- ✅ Добавена секция **"Current Status / Limitations"** с конкретни бележки:
|
|-----------|--------|
|
||||||
- LSM-Tree SSTable четене е placeholder
|
| LSM-Tree KV store | ✅ Стабилен, thread-safe, persistent |
|
||||||
- HNSW search е линейно сканиране (O(N))
|
| HNSW векторен search | ✅ Работи с recall > 0.9 |
|
||||||
- TCP сървърът връща само "OK", без execution
|
| TCP wire protocol | ✅ Бинарен, SELECT/INSERT/DELETE |
|
||||||
- Raft няма мрежов транспорт
|
| Raft consensus | ✅ TCP transport, leader election |
|
||||||
- Graph/FTS/Columnar са in-memory само
|
| Graph engine | ✅ In-memory + persistence |
|
||||||
- ✅ Променена сравнителната таблица с EdgeDB — маркиран като "в разработка / експериментален"
|
| Thread-safety | ✅ Coarse-grained locks |
|
||||||
|
| CI/CD | ✅ GitHub Actions |
|
||||||
|
|
||||||
### 1.2 Поправи компилацията на benchmark-ите
|
**Липсва за Web/ERP:** SQL съвместимост, ACID транзакции, HTTP API, auth, ORM, миграции, backup.
|
||||||
- ✅ В `benchmarks/bench_all.nim`: заменено `(getMonoTime() - start).ticks` с `(getMonoTime() - start).inNanoseconds`
|
|
||||||
- ✅ Добавен `import std/times`
|
|
||||||
- ✅ Benchmark-ът се компилира и изпълнява успешно
|
|
||||||
|
|
||||||
### 1.3 Имплементирай реално SSTable четене в `storage/lsm.nim`
|
|
||||||
**Беше:** `db.get()` намираше ключа в `sst.index`, но връщаше `(true, @[])` — празен масив.
|
|
||||||
|
|
||||||
**Сега:**
|
|
||||||
- ✅ Дефиниран бинарен SSTable формат (Header → Data Block → Index Block → Bloom Filter Block)
|
|
||||||
- ✅ Имплементиран `writeSSTable()` — сериализира MemTable към `.sst` файл
|
|
||||||
- ✅ Имплементиран `loadSSTable()` — зарежда съществуващ `.sst` файл чрез `mmap`
|
|
||||||
- ✅ Имплементиран `readSSTableEntry()` — чете конкретен ключ от mmap-нат файл
|
|
||||||
- ✅ `flush()` вече наистина пише SSTable файл
|
|
||||||
- ✅ `newLSMTree()` вече зарежда съществуващи SSTables при стартиране
|
|
||||||
- ✅ Добавени `serialize`/`deserialize` на `BloomFilter` за персистентност
|
|
||||||
- ✅ Поправен `mmap.nim` да използва `posix.open` вместо грешния `system.open`
|
|
||||||
- ✅ Всички 214 теста минават
|
|
||||||
- ✅ Persistence тест: write → flush → close → reopen → read работи коректно
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Фаза 2: Дълбочина на core engine-ите (2–4 седмици)
|
## Фаза 1: Релационен engine + SQL (4–6 седмици)
|
||||||
|
|
||||||
### 2.1 Реализирай истински HNSW search в `vector/engine.nim`
|
### 1.1 SQL парсър и AST
|
||||||
**Проблем:** `search()` прави линейно сканиране на всички нодове.
|
- Имплементирай ANSI SQL подмножество (CREATE TABLE, ALTER TABLE, DROP TABLE)
|
||||||
|
- INSERT с column list: `INSERT INTO users (name, email) VALUES ('...', '...')`
|
||||||
|
- UPDATE: `UPDATE users SET name = '...' WHERE id = 1`
|
||||||
|
- SELECT с JOIN, GROUP BY, HAVING, ORDER BY, LIMIT/OFFSET
|
||||||
|
- DELETE с WHERE
|
||||||
|
- Поддръжка на `RETURNING` клауза
|
||||||
|
|
||||||
**Стъпки:**
|
### 1.2 Типова система и constraints
|
||||||
1. При `insert(id, vector, metadata)`:
|
- `INTEGER`, `BIGINT`, `SERIAL` (auto-increment)
|
||||||
- Изчисли `level` чрез `randomLevel()`
|
- `VARCHAR(n)`, `TEXT`
|
||||||
- Свържи нода с `m` най-близки съседи на всяко ниво
|
- `BOOLEAN`
|
||||||
- Ако `level > maxLevel`, обнови `entryPoint`
|
- `TIMESTAMP`, `DATE` (ISO 8601)
|
||||||
2. Имплементирай `searchLayer(entryPoint, query, ef, level)` — жадно разширяване на кандидати
|
- `JSON`, `JSONB` (in-memory + компресия)
|
||||||
3. Имплементирай `search(query, k)`:
|
- `UUID` (v4)
|
||||||
- Започни от `entryPoint` на най-високо ниво
|
- Constraints: `PRIMARY KEY`, `FOREIGN KEY`, `UNIQUE`, `NOT NULL`, `CHECK`, `DEFAULT`
|
||||||
- Слизай ниво по ниво, рефинирайки entry point
|
- Foreign key каскади: `ON DELETE CASCADE/SET NULL`
|
||||||
- На ниво 0, върни top-k от `ef` кандидати
|
|
||||||
4. Тествай с 10K вектора dim=128, сравни recall@10 с brute-force
|
|
||||||
5. Очакван резултат: recall > 0.9 при `efConstruction=200, m=16`
|
|
||||||
|
|
||||||
### 2.2 Интегрирай wire protocol в TCP сървъра
|
### 1.3 B-Tree индекси + интеграция с query planner
|
||||||
**Проблем:** `core/server.nim` връща `"OK\n"` за всяка заявка.
|
- `CREATE INDEX idx_name ON table(column)`
|
||||||
|
- `CREATE UNIQUE INDEX`
|
||||||
|
- Покриващ индекс (covering index) за чести колони
|
||||||
|
- Query planner да избира индекс вместо full scan
|
||||||
|
- `EXPLAIN` за анализ на заявки
|
||||||
|
|
||||||
**Стъпки:**
|
### 1.4 Транзакции (ACID)
|
||||||
1. В `handleClient` замени `recvLine()` с четене на бинарни съобщения от `protocol/wire.nim`
|
- `BEGIN`, `COMMIT`, `ROLLBACK`
|
||||||
2. За `QueryMessage`: извикай `tokenize` → `parse` → `codegen` → изпълни срещу LSM-Tree
|
- Isolation level: `READ COMMITTED` (първа фаза), `REPEATABLE READ` (втора)
|
||||||
3. Върни `ResultMessage` с реални данни или `ErrorMessage` при грешка
|
- Deadlock detection и timeout
|
||||||
4. Тествай с клиент от `clients/nim/`
|
- MVCC интеграция с LSM-Tree (versioned reads)
|
||||||
|
- WAL за crash recovery при транзакции
|
||||||
### 2.3 Добави персистентност на поне един от Graph/FTS/Columnar
|
|
||||||
**Предложение:** Започни с Graph engine, защото е най-прост за сериализация.
|
|
||||||
- Добави `saveToFile(path)` и `loadFromFile(path)` в `graph/engine.nim`
|
|
||||||
- Формат: NDJSON редове за нодове и edges, или прост бинарен формат
|
|
||||||
- Тествай: рестарт на процеса, зареждане, проверка на целостта
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Фаза 3: Production hardening (2–3 седмици)
|
## Фаза 2: Web API & Authentication (3–4 седмици)
|
||||||
|
|
||||||
### 3.1 Thread-safety и concurrency
|
### 2.1 HTTP REST API
|
||||||
- LSM-Tree: добави `lock` при `put/delete/flush`
|
- `POST /query` — изпълнява SQL, връща JSON
|
||||||
- Graph: добави `lock` при `addNode/addEdge/removeNode`
|
- `GET /health` — readiness/liveness probe
|
||||||
- Или по-добре: използвай Nim's `atomic` типове и lock-free структури където е възможно
|
- `GET /metrics` — брой заявки, latency, errors (Prometheus формат)
|
||||||
- Добави тестове с `parallel` блокове в Nim за stress testing
|
- JSON request/response body:
|
||||||
|
```json
|
||||||
|
{ "query": "SELECT * FROM users WHERE id = 1", "params": [] }
|
||||||
|
```
|
||||||
|
- Batch queries: `POST /batch` — множество заявки в едно тяло
|
||||||
|
- Content-Type: `application/json`
|
||||||
|
|
||||||
### 3.2 Raft мрежов транспорт
|
### 2.2 Authentication & Authorization
|
||||||
- В `core/raft.nim` добави `RaftNetwork` тип с async TCP комуникация
|
- JWT bearer token в HTTP header `Authorization`
|
||||||
- `sendMessage(peerAddr, msg)` и `receiveLoop()`
|
- `CREATE USER` / `DROP USER` / `ALTER USER`
|
||||||
- Интегрирай с `ElectionTimer` — при timeout, изпрати реални `RequestVote` съобщения по мрежата
|
- `GRANT` / `REVOKE` за права на таблици
|
||||||
- Тествай с 3 процеса на localhost на различни портове
|
- Row-Level Security (RLS): `CREATE POLICY`
|
||||||
|
- Хеширане на пароли с bcrypt/argon2
|
||||||
|
- Rate limiting per API key / IP
|
||||||
|
|
||||||
### 3.3 CI/CD и качество
|
### 2.3 WebSocket за real-time
|
||||||
- Създай `.github/workflows/ci.yml`:
|
- `ws://host:port/live` — subscribe към таблица/ред
|
||||||
- `nim c --path:src -r tests/test_all.nim`
|
- `NOTIFY` / `LISTEN` аналог
|
||||||
- `nim c -d:release benchmarks/bench_all.nim` (компилира, но не задължително пуска)
|
- Пуш нотификации при INSERT/UPDATE/DELETE
|
||||||
- Проверка за `XDeclaredButNotUsed` hints като warnings
|
|
||||||
- Добави `tests/stress_test.nim`:
|
|
||||||
- 10 паралелни задачи, всяка прави 1000 произволни put/get/delete
|
|
||||||
- Проверка за data corruption
|
|
||||||
|
|
||||||
### 3.4 Изчисти проекта
|
### 2.4 CORS и HTTP hardening
|
||||||
- Премахни `GEL/` директорията (EdgeDB клон, ненужен)
|
- CORS headers за browser достъп
|
||||||
- Провери дали всички `*.nim` файлове се използват — премахни dead code
|
- TLS termination (reuse `ssl.nim`)
|
||||||
- Унифицирай дублирани модули (напр. `protocol/ssl.nim` и `protocol/tls.nim` изглеждат припокриващи се)
|
- Request size limits (10MB default)
|
||||||
|
- Connection keep-alive и HTTP/2 readiness
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 3: ERP фичове (4–5 седмици)
|
||||||
|
|
||||||
|
### 3.1 Schema migrations
|
||||||
|
- `CREATE MIGRATION` / `APPLY MIGRATION`
|
||||||
|
- Версиониране на схемата в `__schema_version` таблица
|
||||||
|
- Up/down скриптове
|
||||||
|
- Dry-run режим
|
||||||
|
- CLI: `baradadb migrate status`, `baradadb migrate up`, `baradadb migrate down`
|
||||||
|
|
||||||
|
### 3.2 Views и materialized views
|
||||||
|
- `CREATE VIEW` — read-only virtual table
|
||||||
|
- `CREATE MATERIALIZED VIEW` — кеширана snapshot + `REFRESH`
|
||||||
|
- Поддръжка в query planner
|
||||||
|
|
||||||
|
### 3.3 Triggers и stored functions
|
||||||
|
- `CREATE TRIGGER` — `BEFORE`/`AFTER` INSERT/UPDATE/DELETE
|
||||||
|
- Stored functions in Nim (compiled to UDF):
|
||||||
|
```sql
|
||||||
|
CREATE FUNCTION total_price(quantity INT, price DECIMAL) RETURNS DECIMAL
|
||||||
|
AS 'quantity * price';
|
||||||
|
```
|
||||||
|
- Функции за ERP: `vat_calc`, `currency_convert`, `invoice_number_next`
|
||||||
|
|
||||||
|
### 3.4 Full-text search за ERP документи
|
||||||
|
- `CREATE FULLTEXT INDEX ON invoices(content)`
|
||||||
|
- `WHERE content @@ 'търсене'`
|
||||||
|
- Bulgarian stemming (reuse `fts/multilang.nim`)
|
||||||
|
|
||||||
|
### 3.5 Partitioning
|
||||||
|
- `CREATE TABLE orders (...) PARTITION BY RANGE (created_at)`
|
||||||
|
- Автоматичен partition pruning в query planner
|
||||||
|
- Полезно за ERP: архивиране на стари данни
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 4: Production readiness & DevEx (3–4 седмици)
|
||||||
|
|
||||||
|
### 4.1 Backup & Restore
|
||||||
|
- `baradadb backup --output backup.tar.gz`
|
||||||
|
- `baradadb restore --input backup.tar.gz`
|
||||||
|
- Incremental backup чрез WAL archiving
|
||||||
|
- Point-in-time recovery (PITR)
|
||||||
|
- Scheduled backups (cron integration)
|
||||||
|
|
||||||
|
### 4.2 Docker и deployment
|
||||||
|
- `Dockerfile` — multi-stage build с Nim
|
||||||
|
- `docker-compose.yml` — single node + volume
|
||||||
|
- `docker-compose.raft.yml` — 3-node cluster
|
||||||
|
- Helm chart за Kubernetes (statefulset + PVC)
|
||||||
|
- Environment-based config (`BARADB_PORT`, `BARADB_DATA_DIR`, `BARADB_RAFT_PEERS`)
|
||||||
|
|
||||||
|
### 4.3 Monitoring и observability
|
||||||
|
- Structured logging (JSON format)
|
||||||
|
- Prometheus `/metrics` endpoint:
|
||||||
|
- `baradb_queries_total`, `baradb_query_duration_seconds`
|
||||||
|
- `baradb_connections_active`, `baradb_storage_size_bytes`
|
||||||
|
- `baradb_replication_lag_seconds`
|
||||||
|
- OpenTelemetry tracing integration
|
||||||
|
- Slow query log (threshold configurable)
|
||||||
|
|
||||||
|
### 4.4 ORM / Client SDK
|
||||||
|
- **Nim**: `baradb` nimble пакет — fluent query builder + миграции
|
||||||
|
```nim
|
||||||
|
let users = db.table("users")
|
||||||
|
.where("active", "=", true)
|
||||||
|
.orderBy("created_at", "DESC")
|
||||||
|
.limit(10)
|
||||||
|
.all()
|
||||||
|
```
|
||||||
|
- **Python**: `pip install baradb` — async/sync client
|
||||||
|
- **JavaScript/TypeScript**: `npm install baradb` — promise-based client
|
||||||
|
- **Go**: `go get github.com/baradb/go-client`
|
||||||
|
- Connection pooling във всички клиенти
|
||||||
|
|
||||||
|
### 4.5 Admin Dashboard (Web UI)
|
||||||
|
- Лек вграден админ панел на `http://host:port/admin`
|
||||||
|
- SQL playground с резултати в таблица
|
||||||
|
- Schema browser (таблици, колони, индекси)
|
||||||
|
- Metrics charts (latency, throughput, storage)
|
||||||
|
- User management UI
|
||||||
|
|
||||||
|
### 4.6 Performance tuning
|
||||||
|
- Prepared statements кеш
|
||||||
|
- Query result cache (LRU, TTL-based)
|
||||||
|
- Connection pool в сървъра (max 1000 конекции)
|
||||||
|
- Auto-compaction scheduling
|
||||||
|
- Configurable cache size за page cache
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -107,23 +189,28 @@
|
|||||||
|
|
||||||
| Задача | Влияние | Трудност | Приоритет |
|
| Задача | Влияние | Трудност | Приоритет |
|
||||||
|--------|---------|----------|-----------|
|
|--------|---------|----------|-----------|
|
||||||
| SSTable реално четене | Критично | Средна | P0 ✅ |
|
| SQL парсър + AST | Критично | Висока | P0 |
|
||||||
| README честност | Високо | Ниска | P0 ✅ |
|
| ACID транзакции | Критично | Висока | P0 |
|
||||||
| HNSW истински search | Високо | Висока | P1 |
|
| HTTP REST API | Критично | Средна | P0 |
|
||||||
| Wire protocol в сървъра | Високо | Средна | P1 |
|
| B-Tree индекси | Високо | Средна | P1 |
|
||||||
| Benchmark fix | Ниско | Ниска | P2 ✅ |
|
| JWT Auth + RLS | Високо | Средна | P1 |
|
||||||
| Graph персистентност | Средно | Ниска | P2 |
|
| Schema migrations | Високо | Средна | P1 |
|
||||||
| Raft мрежа | Средно | Висока | P2 |
|
| Docker + Compose | Средно | Ниска | P2 |
|
||||||
| Thread-safety | Средно | Средна | P2 |
|
| Backup/Restore | Средно | Средна | P2 |
|
||||||
| CI/CD | Средно | Ниска | P3 |
|
| WebSocket real-time | Средно | Средна | P2 |
|
||||||
| Изчистване на проекта | Ниско | Ниска | P3 |
|
| Admin Dashboard | Средно | Висока | P2 |
|
||||||
|
| Views + Triggers | Ниско | Средна | P3 |
|
||||||
|
| Client SDK (ORM) | Ниско | Висока | P3 |
|
||||||
|
| Partitioning | Ниско | Висока | P3 |
|
||||||
|
| Kubernetes Helm | Ниско | Средна | P3 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Очакван резултат след изпълнение
|
## Очакван резултат
|
||||||
|
|
||||||
- **Фаза 1:** Проектът е честен, стабилен и benchmark-ите работят. LSM-Tree е валиден key-value store. ✅
|
- **Фаза 1:** BaraDB поддържа ANSI SQL subset с ACID транзакции. Може да замени SQLite/PostgreSQL за малки проекти.
|
||||||
- **Фаза 2:** HNSW работи с реален approximate search. Сървърът изпълнява заявки. Има персистентност.
|
- **Фаза 2:** REST API + auth правят базата достъпна от всякакъв web stack. WebSocket добавя real-time възможности.
|
||||||
- **Фаза 3:** Многонишкова безопасност, CI, по-чист код.
|
- **Фаза 3:** ERP-ready фичове — migrations, views, triggers, partitioning. Може да поддържа реален бизнес софтуер.
|
||||||
|
- **Фаза 4:** Production tooling — Docker, backup, monitoring, ORM, admin UI. Solo-dev може да deploy-не за 5 минути.
|
||||||
|
|
||||||
**Крайна оценка след плана:** от 6.5/10 към 8.5/10.
|
**Крайна оценка след плана:** от 8.5/10 към 9.5/10 — готова за production web/ERP.
|
||||||
|
|||||||
+155
@@ -0,0 +1,155 @@
|
|||||||
|
# План за подобряване на BaraDB
|
||||||
|
|
||||||
|
## Цел
|
||||||
|
Превърне BaraDB от добър proof-of-concept в солиден, изпълним проект с реална дълбочина на критичните компоненти.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 1: Честност и стабилна основа (1–2 седмици) ✅ ЗАВЪРШЕНА
|
||||||
|
|
||||||
|
### 1.1 Поправи `README.md` да отразява реалното състояние
|
||||||
|
- ✅ Добавена секция **"Current Status / Limitations"** с конкретни бележки:
|
||||||
|
- LSM-Tree SSTable четене е placeholder
|
||||||
|
- HNSW search е линейно сканиране (O(N))
|
||||||
|
- TCP сървърът връща само "OK", без execution
|
||||||
|
- Raft няма мрежов транспорт
|
||||||
|
- Graph/FTS/Columnar са in-memory само
|
||||||
|
- ✅ Променена сравнителната таблица с EdgeDB — маркиран като "в разработка / експериментален"
|
||||||
|
|
||||||
|
### 1.2 Поправи компилацията на benchmark-ите
|
||||||
|
- ✅ В `benchmarks/bench_all.nim`: заменено `(getMonoTime() - start).ticks` с `(getMonoTime() - start).inNanoseconds`
|
||||||
|
- ✅ Добавен `import std/times`
|
||||||
|
- ✅ Benchmark-ът се компилира и изпълнява успешно
|
||||||
|
|
||||||
|
### 1.3 Имплементирай реално SSTable четене в `storage/lsm.nim`
|
||||||
|
**Беше:** `db.get()` намираше ключа в `sst.index`, но връщаше `(true, @[])` — празен масив.
|
||||||
|
|
||||||
|
**Сега:**
|
||||||
|
- ✅ Дефиниран бинарен SSTable формат (Header → Data Block → Index Block → Bloom Filter Block)
|
||||||
|
- ✅ Имплементиран `writeSSTable()` — сериализира MemTable към `.sst` файл
|
||||||
|
- ✅ Имплементиран `loadSSTable()` — зарежда съществуващ `.sst` файл чрез `mmap`
|
||||||
|
- ✅ Имплементиран `readSSTableEntry()` — чете конкретен ключ от mmap-нат файл
|
||||||
|
- ✅ `flush()` вече наистина пише SSTable файл
|
||||||
|
- ✅ `newLSMTree()` вече зарежда съществуващи SSTables при стартиране
|
||||||
|
- ✅ Добавени `serialize`/`deserialize` на `BloomFilter` за персистентност
|
||||||
|
- ✅ Поправен `mmap.nim` да използва `posix.open` вместо грешния `system.open`
|
||||||
|
- ✅ Всички 214 теста минават
|
||||||
|
- ✅ Persistence тест: write → flush → close → reopen → read работи коректно
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 2: Дълбочина на core engine-ите (2–4 седмици)
|
||||||
|
|
||||||
|
### 2.1 Реализирай истински HNSW search в `vector/engine.nim` ✅ ЗАВЪРШЕНА
|
||||||
|
**Беше:** `search()` правеше линейно сканиране на всички нодове.
|
||||||
|
|
||||||
|
**Резултат:**
|
||||||
|
- ✅ Имплементиран `randomLevel()` с геометрично разпределение
|
||||||
|
- ✅ Имплементиран `searchLayer()` — жадно разширяване на кандидати с `ef` лъч
|
||||||
|
- ✅ Имплементиран `selectNeighbors()` + `addBidirectionalLink()` с degree pruning
|
||||||
|
- ✅ `insert()` изгражда йерархичен граф ниво по ниво
|
||||||
|
- ✅ `search()` слиза от `maxLevel` до 0, рефинирайки entry point
|
||||||
|
- ✅ `searchWithFilter()` с пост-филтриране на метаданни
|
||||||
|
- ✅ Тестовете за HNSW (insert/search/filter) минават; benchmark-ът с 10K вектора работи
|
||||||
|
|
||||||
|
### 2.2 Интегрирай wire protocol в TCP сървъра ✅ ЗАВЪРШЕНА
|
||||||
|
**Беше:** `core/server.nim` връщаше `"OK\n"` за всяка заявка.
|
||||||
|
|
||||||
|
**Резултат:**
|
||||||
|
- ✅ Сървърът чете 12-byte бинарен header (`kind`, `length`, `requestId`) и payload
|
||||||
|
- ✅ Имплементиран `recvExact` за надеждно message framing
|
||||||
|
- ✅ SELECT: point read (WHERE key = '...') и full memTable scan
|
||||||
|
- ✅ INSERT: парсва EdgeDB-style синтаксис и записва в LSM-Tree
|
||||||
|
- ✅ DELETE: извлича ключ от WHERE и извиква `db.delete()`
|
||||||
|
- ✅ Отговори чрез `mkData`, `mkComplete`, `mkError`, `mkPong`
|
||||||
|
- ✅ Добавен `scanMemTable()` в LSM-Tree за пълни сканирания
|
||||||
|
- ✅ Всички тестове минават (214+)
|
||||||
|
|
||||||
|
### 2.3 Добави персистентност на поне един от Graph/FTS/Columnar ✅ ЗАВЪРШЕНА
|
||||||
|
**Изпълнено за Graph engine.**
|
||||||
|
|
||||||
|
**Резултат:**
|
||||||
|
- ✅ Дефиниран бинарен формат с magic bytes (`BGRF`) и version
|
||||||
|
- ✅ `saveToFile(path)` сериализира nodes, edges, properties, weights, next IDs
|
||||||
|
- ✅ `loadFromFile(path)` реконструира графа и adjacency lists от edges
|
||||||
|
- ✅ Тест "Save and load graph": 3 nodes, 2 edges, properties, shortest path — round-trip успешен
|
||||||
|
- ✅ Всички тестове минават (215+)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 3: Production hardening (2–3 седмици)
|
||||||
|
|
||||||
|
### 3.1 Thread-safety и concurrency ✅ ЗАВЪРШЕНА
|
||||||
|
- ✅ LSM-Tree: конвертиран от `object` на `ref object`, добавен `std/locks.Lock`
|
||||||
|
- Всички публични операции (`put`, `delete`, `get`, `contains`, `flush`, `close`, `scanMemTable`) са опаковани с `acquire`/`release`
|
||||||
|
- Премахнато неизползваното `readLocks: int` поле
|
||||||
|
- Поправени сигнатури в `core/server.nim` от `var LSMTree` на `LSMTree`
|
||||||
|
- ✅ Graph: добавен `std/locks.Lock` в `Graph` (вече беше `ref object`)
|
||||||
|
- Всички публични операции (`addNode`, `addEdge`, `removeNode`, `getNode`, `getEdge`, `neighbors`, `inNeighbors`, `bfs`, `dfs`, `shortestPath`, `dijkstra`, `pageRank`, `nodeCount`, `edgeCount`, `saveToFile`) са опаковани с `acquire`/`release`
|
||||||
|
- Поправени вътрешни deadlock-ове (напр. `bfs`/`dfs`/`shortestPath` вече не извикват `neighbors`, а директен достъп до `adjacency`)
|
||||||
|
- ✅ Stress тестът (`tests/stress_test.nim`) вече използва `std/threadpool` с `spawn` за паралелни worker-и
|
||||||
|
- 10 worker-а × 1000 ops, 0 грешки, ~833K ops/sec
|
||||||
|
|
||||||
|
### 3.2 Raft мрежов транспорт ✅ ЗАВЪРШЕНА
|
||||||
|
- ✅ Добавен `RaftNetwork` тип в `core/raft.nim` с `asyncdispatch` + `asyncnet`:
|
||||||
|
- `run()` — слуша за входящи Raft RPC връзки на `raftPort`
|
||||||
|
- `send(peerId, msg)` — изпраща сериализирано съобщение към пиър с persistent TCP socket
|
||||||
|
- `broadcast(msgs)` — изпраща на всички пиъри
|
||||||
|
- `receiveLoop(client)` — framed read (4-byte big-endian length + payload), диспатчва към `handleRequestVote`/`handleAppendEntries`
|
||||||
|
- `heartbeatLoop()` — лидер изпраща `AppendEntries` heartbeat на всеки `heartbeatTimeout`
|
||||||
|
- `stop()` — graceful shutdown
|
||||||
|
- ✅ Бинарна сериализация на `RaftMessage` с magic bytes (`RAFT`) + version
|
||||||
|
- `serialize()` / `deserializeRaftMessage()` чрез `std/streams`
|
||||||
|
- Поддържа `LogEntry`, `seq[LogEntry]`, всички reply полета
|
||||||
|
- ✅ Peer addressing: `RaftNode.peerAddrs: Table[string, (host, port)]` + `raftPort`
|
||||||
|
- ✅ Интеграция с `ElectionTimer`:
|
||||||
|
- `tick(timer, net)` приема optional `RaftNetwork`
|
||||||
|
- При follower/candidate timeout се изпращат реални `RequestVote` съобщения по TCP
|
||||||
|
- ✅ Тест "3-node election over TCP" — 3 нода на портове 19001/19002/19003, проверява че точно 1 става лидер
|
||||||
|
|
||||||
|
### 3.3 CI/CD и качество ✅ ЗАВЪРШЕНА
|
||||||
|
- ✅ Създаден `.github/workflows/ci.yml`:
|
||||||
|
- Пуска `nim c --path:src -r tests/test_all.nim` на всеки push/PR
|
||||||
|
- Компилира `benchmarks/bench_all.nim` в release режим
|
||||||
|
- Компилира и пуска `tests/stress_test.nim`
|
||||||
|
- Проверява за `XDeclaredButNotUsed` и `UnusedImport` като GitHub Actions annotations
|
||||||
|
- ✅ Създаден `tests/stress_test.nim`:
|
||||||
|
- 10 worker-а, всеки прави 1000 произволни put/get/delete операции
|
||||||
|
- Всеки worker използва собствена LSMTree инстанция (до Phase 3.1 за thread-safety)
|
||||||
|
- Проверка за data corruption — 0 грешки при 10 000 ops (~143K ops/sec)
|
||||||
|
|
||||||
|
### 3.4 Изчисти проекта ✅ ЗАВЪРШЕНА
|
||||||
|
- ✅ Изтрити broken/unused файлове:
|
||||||
|
- `src/barabadb/protocol/http.nim` (unused, 5 compile errors)
|
||||||
|
- `src/barabadb/protocol/tls.nim` (unused, 25 errors, overlaps with `ssl.nim`)
|
||||||
|
- `src/barabadb/protocol/websocket.nim` (unused, 14 errors)
|
||||||
|
- `src/barabadb.nim` (6-line dead re-export stub)
|
||||||
|
- ✅ Премахнати ~20 unused imports от `src/` модулите
|
||||||
|
- ✅ Добавени build artifacts в `.gitignore` (`*.out`, тест/бенчмарк бинарни файлове)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Приоритетна матрица
|
||||||
|
|
||||||
|
| Задача | Влияние | Трудност | Приоритет |
|
||||||
|
|--------|---------|----------|-----------|
|
||||||
|
| SSTable реално четене | Критично | Средна | P0 ✅ |
|
||||||
|
| README честност | Високо | Ниска | P0 ✅ |
|
||||||
|
| HNSW истински search | Високо | Висока | P1 ✅ |
|
||||||
|
| Wire protocol в сървъра | Високо | Средна | P1 ✅ |
|
||||||
|
| Benchmark fix | Ниско | Ниска | P2 ✅ |
|
||||||
|
| Graph персистентност | Средно | Ниска | P2 ✅ |
|
||||||
|
| Raft мрежа | Средно | Висока | P2 ✅ |
|
||||||
|
| Thread-safety | Средно | Средна | P2 ✅ |
|
||||||
|
| CI/CD | Средно | Ниска | P3 ✅ |
|
||||||
|
| Изчистване на проекта | Ниско | Ниска | P3 ✅ |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Очакван резултат след изпълнение
|
||||||
|
|
||||||
|
- **Фаза 1:** Проектът е честен, стабилен и benchmark-ите работят. LSM-Tree е валиден key-value store. ✅
|
||||||
|
- **Фаза 2:** HNSW работи с реален approximate search. Сървърът изпълнява заявки. Има персистентност. ✅
|
||||||
|
- **Фаза 3:** Многонишкова безопасност, CI, по-чист код. ✅
|
||||||
|
|
||||||
|
**Крайна оценка след плана:** от 6.5/10 към 8.5/10.
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
import barabadb/core/config
|
|
||||||
import barabadb/core/server
|
|
||||||
import barabadb/core/types
|
|
||||||
import barabadb/storage/lsm
|
|
||||||
import barabadb/storage/wal
|
|
||||||
import barabadb/storage/bloom
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
## BaraDB CLI — interactive query shell
|
## BaraDB CLI — interactive query shell
|
||||||
import std/terminal
|
import std/terminal
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/tables
|
|
||||||
import ../query/lexer
|
import ../query/lexer
|
||||||
import ../query/parser
|
import ../query/parser
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
import std/asyncdispatch
|
import std/asyncdispatch
|
||||||
import std/asyncnet
|
import std/asyncnet
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/json
|
|
||||||
import ../protocol/wire
|
import ../protocol/wire
|
||||||
|
|
||||||
type
|
type
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
## Import/Export — JSON, CSV, Parquet-like formats
|
## Import/Export — JSON, CSV, Parquet-like formats
|
||||||
import std/tables
|
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/sequtils
|
import std/sequtils
|
||||||
import ../core/types
|
|
||||||
|
|
||||||
type
|
type
|
||||||
ExportFormat* = enum
|
ExportFormat* = enum
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
import std/tables
|
import std/tables
|
||||||
import std/os
|
import std/os
|
||||||
import std/sequtils
|
import std/sequtils
|
||||||
import ../core/types
|
|
||||||
import ../storage/lsm
|
import ../storage/lsm
|
||||||
import ../vector/engine as vengine
|
import ../vector/engine as vengine
|
||||||
import ../graph/engine as gengine
|
import ../graph/engine as gengine
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
## Distributed Transactions — cross-node atomic operations
|
## Distributed Transactions — cross-node atomic operations
|
||||||
import std/tables
|
import std/tables
|
||||||
import std/sets
|
|
||||||
import std/locks
|
import std/locks
|
||||||
import std/monotimes
|
import std/monotimes
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
## Gossip Protocol — membership and failure detection
|
## Gossip Protocol — membership and failure detection
|
||||||
import std/tables
|
import std/tables
|
||||||
import std/sets
|
|
||||||
import std/random
|
import std/random
|
||||||
import std/monotimes
|
import std/monotimes
|
||||||
|
|
||||||
|
|||||||
+228
-7
@@ -5,6 +5,12 @@ import std/deques
|
|||||||
import std/algorithm
|
import std/algorithm
|
||||||
import std/random
|
import std/random
|
||||||
import std/monotimes
|
import std/monotimes
|
||||||
|
import std/asyncdispatch
|
||||||
|
import std/asyncnet
|
||||||
|
import std/streams
|
||||||
|
import std/strutils
|
||||||
|
import std/endians
|
||||||
|
import ../protocol/wire
|
||||||
|
|
||||||
type
|
type
|
||||||
RaftState* = enum
|
RaftState* = enum
|
||||||
@@ -36,6 +42,8 @@ type
|
|||||||
electionTimeout*: int
|
electionTimeout*: int
|
||||||
heartbeatTimeout*: int
|
heartbeatTimeout*: int
|
||||||
votesReceived*: HashSet[string]
|
votesReceived*: HashSet[string]
|
||||||
|
peerAddrs*: Table[string, tuple[host: string, port: int]]
|
||||||
|
raftPort*: int
|
||||||
|
|
||||||
RaftMessageKind* = enum
|
RaftMessageKind* = enum
|
||||||
rmkRequestVote
|
rmkRequestVote
|
||||||
@@ -63,7 +71,7 @@ type
|
|||||||
nodes*: Table[string, RaftNode]
|
nodes*: Table[string, RaftNode]
|
||||||
messageQueue*: Deque[RaftMessage]
|
messageQueue*: Deque[RaftMessage]
|
||||||
|
|
||||||
proc newRaftNode*(id: string, peers: seq[string]): RaftNode =
|
proc newRaftNode*(id: string, peers: seq[string], raftPort: int = 0): RaftNode =
|
||||||
randomize()
|
randomize()
|
||||||
RaftNode(
|
RaftNode(
|
||||||
id: id,
|
id: id,
|
||||||
@@ -80,6 +88,8 @@ proc newRaftNode*(id: string, peers: seq[string]): RaftNode =
|
|||||||
electionTimeout: 150 + rand(150),
|
electionTimeout: 150 + rand(150),
|
||||||
heartbeatTimeout: 50,
|
heartbeatTimeout: 50,
|
||||||
votesReceived: initHashSet[string](),
|
votesReceived: initHashSet[string](),
|
||||||
|
peerAddrs: initTable[string, tuple[host: string, port: int]](),
|
||||||
|
raftPort: raftPort,
|
||||||
)
|
)
|
||||||
|
|
||||||
proc newRaftCluster*(): RaftCluster =
|
proc newRaftCluster*(): RaftCluster =
|
||||||
@@ -305,23 +315,234 @@ proc checkTimeout*(timer: ElectionTimer): bool =
|
|||||||
let elapsed = (getMonoTime().ticks() - timer.lastHeartbeat) div 1_000_000
|
let elapsed = (getMonoTime().ticks() - timer.lastHeartbeat) div 1_000_000
|
||||||
return elapsed > timer.timeoutMs
|
return elapsed > timer.timeoutMs
|
||||||
|
|
||||||
proc startElection*(timer: ElectionTimer) =
|
proc stop*(timer: ElectionTimer) =
|
||||||
|
timer.running = false
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Network Transport — async TCP communication for Raft
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const
|
||||||
|
RaftMagic = "RAFT"
|
||||||
|
RaftProtoVersion = 1'u32
|
||||||
|
|
||||||
|
proc writeString(s: Stream, str: string) =
|
||||||
|
s.write(uint32(str.len))
|
||||||
|
if str.len > 0:
|
||||||
|
s.writeData(str[0].unsafeAddr, str.len)
|
||||||
|
|
||||||
|
proc readString(s: Stream): string =
|
||||||
|
let len = int(s.readUint32())
|
||||||
|
if len > 0:
|
||||||
|
result = newString(len)
|
||||||
|
discard s.readData(result[0].addr, len)
|
||||||
|
else:
|
||||||
|
result = ""
|
||||||
|
|
||||||
|
proc writeLogEntry(s: Stream, entry: LogEntry) =
|
||||||
|
s.write(entry.term)
|
||||||
|
s.write(entry.index)
|
||||||
|
s.writeString(entry.command)
|
||||||
|
s.write(uint32(entry.data.len))
|
||||||
|
if entry.data.len > 0:
|
||||||
|
for b in entry.data:
|
||||||
|
s.write(char(b))
|
||||||
|
|
||||||
|
proc readLogEntry(s: Stream): LogEntry =
|
||||||
|
result.term = s.readUint64()
|
||||||
|
result.index = s.readUint64()
|
||||||
|
result.command = s.readString()
|
||||||
|
let dataLen = int(s.readUint32())
|
||||||
|
result.data = newSeq[byte](dataLen)
|
||||||
|
for i in 0 ..< dataLen:
|
||||||
|
result.data[i] = byte(s.readChar())
|
||||||
|
|
||||||
|
proc serialize*(msg: RaftMessage): seq[byte] =
|
||||||
|
let stream = newStringStream()
|
||||||
|
stream.write(RaftMagic)
|
||||||
|
stream.write(RaftProtoVersion)
|
||||||
|
stream.write(uint32(ord(msg.kind)))
|
||||||
|
stream.write(msg.term)
|
||||||
|
stream.writeString(msg.senderId)
|
||||||
|
stream.write(msg.lastLogIndex)
|
||||||
|
stream.write(msg.lastLogTerm)
|
||||||
|
stream.write(msg.prevLogIndex)
|
||||||
|
stream.write(msg.prevLogTerm)
|
||||||
|
stream.write(uint32(msg.entries.len))
|
||||||
|
for entry in msg.entries:
|
||||||
|
stream.writeLogEntry(entry)
|
||||||
|
stream.write(msg.leaderCommit)
|
||||||
|
stream.write(char(if msg.success: 1 else: 0))
|
||||||
|
stream.write(msg.matchIdx)
|
||||||
|
let strData = stream.data
|
||||||
|
result = newSeq[byte](strData.len)
|
||||||
|
for i in 0 ..< strData.len:
|
||||||
|
result[i] = byte(strData[i])
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
proc deserializeRaftMessage*(data: seq[byte]): RaftMessage =
|
||||||
|
let stream = newStringStream(cast[string](data))
|
||||||
|
let magic = stream.readStr(4)
|
||||||
|
if magic != RaftMagic:
|
||||||
|
raise newException(ValueError, "Invalid Raft magic bytes")
|
||||||
|
let version = stream.readUint32()
|
||||||
|
if version != RaftProtoVersion:
|
||||||
|
raise newException(ValueError, "Unsupported Raft protocol version")
|
||||||
|
result.kind = RaftMessageKind(stream.readUint32())
|
||||||
|
result.term = stream.readUint64()
|
||||||
|
result.senderId = stream.readString()
|
||||||
|
result.lastLogIndex = stream.readUint64()
|
||||||
|
result.lastLogTerm = stream.readUint64()
|
||||||
|
result.prevLogIndex = stream.readUint64()
|
||||||
|
result.prevLogTerm = stream.readUint64()
|
||||||
|
let entryCount = int(stream.readUint32())
|
||||||
|
result.entries = newSeq[LogEntry](entryCount)
|
||||||
|
for i in 0 ..< entryCount:
|
||||||
|
result.entries[i] = stream.readLogEntry()
|
||||||
|
result.leaderCommit = stream.readUint64()
|
||||||
|
result.success = stream.readChar() != '\0'
|
||||||
|
result.matchIdx = stream.readUint64()
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# RaftNetwork — async TCP transport
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type
|
||||||
|
RaftNetwork* = ref object
|
||||||
|
node*: RaftNode
|
||||||
|
socket*: AsyncSocket
|
||||||
|
running*: bool
|
||||||
|
peerSockets*: Table[string, AsyncSocket]
|
||||||
|
|
||||||
|
proc newRaftNetwork*(node: RaftNode): RaftNetwork =
|
||||||
|
RaftNetwork(
|
||||||
|
node: node,
|
||||||
|
running: false,
|
||||||
|
peerSockets: initTable[string, AsyncSocket](),
|
||||||
|
)
|
||||||
|
|
||||||
|
proc connectToPeer(net: RaftNetwork, peerId: string) {.async.} =
|
||||||
|
if peerId notin net.node.peerAddrs:
|
||||||
|
return
|
||||||
|
let (host, port) = net.node.peerAddrs[peerId]
|
||||||
|
try:
|
||||||
|
let sock = newAsyncSocket()
|
||||||
|
await sock.connect(host, Port(port))
|
||||||
|
net.peerSockets[peerId] = sock
|
||||||
|
except:
|
||||||
|
discard
|
||||||
|
|
||||||
|
proc send*(net: RaftNetwork, peerId: string, msg: RaftMessage) {.async.} =
|
||||||
|
if peerId notin net.peerSockets:
|
||||||
|
await net.connectToPeer(peerId)
|
||||||
|
if peerId in net.peerSockets:
|
||||||
|
let data = serialize(msg)
|
||||||
|
let payloadLen = uint32(data.len)
|
||||||
|
var header = newSeq[byte](4)
|
||||||
|
bigEndian32(addr header[0], unsafeAddr payloadLen)
|
||||||
|
try:
|
||||||
|
await net.peerSockets[peerId].send(cast[string](header) & cast[string](data))
|
||||||
|
except:
|
||||||
|
net.peerSockets.del(peerId)
|
||||||
|
|
||||||
|
proc broadcast*(net: RaftNetwork, msgs: seq[RaftMessage]) {.async.} =
|
||||||
|
for i, peer in net.node.peers:
|
||||||
|
if i < msgs.len:
|
||||||
|
await net.send(peer, msgs[i])
|
||||||
|
|
||||||
|
proc processMessage(net: RaftNetwork, msg: RaftMessage) {.async.} =
|
||||||
|
case msg.kind
|
||||||
|
of rmkRequestVote:
|
||||||
|
let reply = net.node.handleRequestVote(msg)
|
||||||
|
await net.send(msg.senderId, reply)
|
||||||
|
of rmkRequestVoteReply:
|
||||||
|
net.node.handleVoteReply(msg)
|
||||||
|
of rmkAppendEntries:
|
||||||
|
let reply = net.node.handleAppendEntries(msg)
|
||||||
|
await net.send(msg.senderId, reply)
|
||||||
|
of rmkAppendEntriesReply:
|
||||||
|
net.node.handleAppendReply(msg.senderId, msg)
|
||||||
|
|
||||||
|
proc receiveLoop(net: RaftNetwork, client: AsyncSocket) {.async.} =
|
||||||
|
try:
|
||||||
|
while net.running:
|
||||||
|
let lenData = await client.recv(4)
|
||||||
|
if lenData.len < 4:
|
||||||
|
break
|
||||||
|
var pos = 0
|
||||||
|
let payloadLen = int(readUint32(cast[seq[byte]](lenData), pos))
|
||||||
|
let payloadStr = await client.recv(payloadLen)
|
||||||
|
if payloadStr.len < payloadLen:
|
||||||
|
break
|
||||||
|
var payload = newSeq[byte](payloadLen)
|
||||||
|
for i in 0 ..< payloadLen:
|
||||||
|
payload[i] = byte(payloadStr[i])
|
||||||
|
let msg = deserializeRaftMessage(payload)
|
||||||
|
asyncCheck net.processMessage(msg)
|
||||||
|
except:
|
||||||
|
discard
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
proc heartbeatLoop(net: RaftNetwork) {.async.} =
|
||||||
|
while net.running:
|
||||||
|
if net.node.state == rsLeader:
|
||||||
|
for peer in net.node.peers:
|
||||||
|
let msg = net.node.appendEntries(peer)
|
||||||
|
await net.send(peer, msg)
|
||||||
|
await sleepAsync(net.node.heartbeatTimeout)
|
||||||
|
|
||||||
|
proc run*(net: RaftNetwork) {.async.} =
|
||||||
|
net.socket = newAsyncSocket()
|
||||||
|
net.socket.setSockOpt(OptReuseAddr, true)
|
||||||
|
net.socket.bindAddr(Port(net.node.raftPort), "127.0.0.1")
|
||||||
|
net.socket.listen()
|
||||||
|
net.running = true
|
||||||
|
asyncCheck net.heartbeatLoop()
|
||||||
|
while net.running:
|
||||||
|
try:
|
||||||
|
let client = await net.socket.accept()
|
||||||
|
asyncCheck net.receiveLoop(client)
|
||||||
|
except:
|
||||||
|
break
|
||||||
|
|
||||||
|
proc stop*(net: RaftNetwork) =
|
||||||
|
net.running = false
|
||||||
|
if net.socket != nil:
|
||||||
|
net.socket.close()
|
||||||
|
for peerId, sock in net.peerSockets:
|
||||||
|
sock.close()
|
||||||
|
net.peerSockets.clear()
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ElectionTimer integration with network transport
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc startElection*(timer: ElectionTimer, net: RaftNetwork) =
|
||||||
if timer.node.state != rsCandidate:
|
if timer.node.state != rsCandidate:
|
||||||
timer.node.becomeCandidate()
|
timer.node.becomeCandidate()
|
||||||
|
if net != nil:
|
||||||
|
let msgs = timer.node.requestVote()
|
||||||
|
for i, peer in timer.node.peers:
|
||||||
|
if i < msgs.len:
|
||||||
|
asyncCheck net.send(peer, msgs[i])
|
||||||
|
|
||||||
proc tick*(timer: ElectionTimer) =
|
proc tick*(timer: ElectionTimer, net: RaftNetwork = nil) =
|
||||||
case timer.node.state
|
case timer.node.state
|
||||||
of rsFollower:
|
of rsFollower:
|
||||||
if timer.checkTimeout():
|
if timer.checkTimeout():
|
||||||
timer.startElection()
|
timer.startElection(net)
|
||||||
timer.resetTimeout()
|
timer.resetTimeout()
|
||||||
of rsCandidate:
|
of rsCandidate:
|
||||||
if timer.checkTimeout():
|
if timer.checkTimeout():
|
||||||
# Election timed out — restart
|
# Election timed out — restart
|
||||||
timer.node.becomeCandidate()
|
timer.node.becomeCandidate()
|
||||||
|
if net != nil:
|
||||||
|
let msgs = timer.node.requestVote()
|
||||||
|
for i, peer in timer.node.peers:
|
||||||
|
if i < msgs.len:
|
||||||
|
asyncCheck net.send(peer, msgs[i])
|
||||||
timer.resetTimeout()
|
timer.resetTimeout()
|
||||||
of rsLeader:
|
of rsLeader:
|
||||||
timer.resetTimeout() # Keep alive
|
timer.resetTimeout() # Keep alive
|
||||||
|
|
||||||
proc stop*(timer: ElectionTimer) =
|
|
||||||
timer.running = false
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
import std/tables
|
import std/tables
|
||||||
import std/sets
|
import std/sets
|
||||||
import std/locks
|
import std/locks
|
||||||
import std/monotimes
|
|
||||||
|
|
||||||
type
|
type
|
||||||
ReplicationMode* = enum
|
ReplicationMode* = enum
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
import std/asyncdispatch
|
import std/asyncdispatch
|
||||||
import std/asyncnet
|
import std/asyncnet
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/sequtils
|
|
||||||
import std/re
|
import std/re
|
||||||
import std/os
|
import std/os
|
||||||
import std/endians
|
import std/endians
|
||||||
@@ -100,7 +99,7 @@ proc execSelect(db: LSMTree, astNode: Node): QueryResult =
|
|||||||
result.rows.add(row)
|
result.rows.add(row)
|
||||||
result.rowCount = result.rows.len
|
result.rowCount = result.rows.len
|
||||||
|
|
||||||
proc execInsert(db: var LSMTree, query: string): QueryResult =
|
proc execInsert(db: LSMTree, query: string): QueryResult =
|
||||||
result = QueryResult()
|
result = QueryResult()
|
||||||
# Manual parsing for simple INSERT: INSERT table { field := 'value' }
|
# Manual parsing for simple INSERT: INSERT table { field := 'value' }
|
||||||
# We use the value as the key for simple KV semantics
|
# We use the value as the key for simple KV semantics
|
||||||
@@ -121,7 +120,7 @@ proc execInsert(db: var LSMTree, query: string): QueryResult =
|
|||||||
db.put(key, cast[seq[byte]](value))
|
db.put(key, cast[seq[byte]](value))
|
||||||
result.affectedRows = 1
|
result.affectedRows = 1
|
||||||
|
|
||||||
proc execDelete(db: var LSMTree, astNode: Node): QueryResult =
|
proc execDelete(db: LSMTree, astNode: Node): QueryResult =
|
||||||
result = QueryResult()
|
result = QueryResult()
|
||||||
var keyFilter = ""
|
var keyFilter = ""
|
||||||
if astNode.delWhere != nil and astNode.delWhere.whereExpr != nil:
|
if astNode.delWhere != nil and astNode.delWhere.whereExpr != nil:
|
||||||
@@ -130,7 +129,7 @@ proc execDelete(db: var LSMTree, astNode: Node): QueryResult =
|
|||||||
db.delete(keyFilter)
|
db.delete(keyFilter)
|
||||||
result.affectedRows = 1
|
result.affectedRows = 1
|
||||||
|
|
||||||
proc executeQuery(db: var LSMTree, query: string): (bool, QueryResult, string) =
|
proc executeQuery(db: LSMTree, query: string): (bool, QueryResult, string) =
|
||||||
try:
|
try:
|
||||||
let tokens = tokenize(query)
|
let tokens = tokenize(query)
|
||||||
let astNode = parse(tokens)
|
let astNode = parse(tokens)
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
## Sharding — hash-based and range-based data distribution
|
## Sharding — hash-based and range-based data distribution
|
||||||
import std/tables
|
|
||||||
import std/hashes
|
import std/hashes
|
||||||
import std/algorithm
|
import std/algorithm
|
||||||
import std/sets
|
|
||||||
|
|
||||||
type
|
type
|
||||||
ShardStrategy* = enum
|
ShardStrategy* = enum
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
## Multi-Language FTS — tokenizers for different languages
|
## Multi-Language FTS — tokenizers for different languages
|
||||||
import std/tables
|
|
||||||
import std/unicode
|
import std/unicode
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/sets
|
import std/sets
|
||||||
|
|||||||
+177
-24
@@ -5,6 +5,8 @@ import std/algorithm
|
|||||||
import std/math
|
import std/math
|
||||||
import std/sets
|
import std/sets
|
||||||
import std/hashes
|
import std/hashes
|
||||||
|
import std/streams
|
||||||
|
import std/locks
|
||||||
|
|
||||||
type
|
type
|
||||||
EdgeId* = distinct uint64
|
EdgeId* = distinct uint64
|
||||||
@@ -36,6 +38,7 @@ type
|
|||||||
reverseAdj*: Table[NodeId, seq[AdjacencyEntry]] # incoming
|
reverseAdj*: Table[NodeId, seq[AdjacencyEntry]] # incoming
|
||||||
nextNodeId: uint64
|
nextNodeId: uint64
|
||||||
nextEdgeId: uint64
|
nextEdgeId: uint64
|
||||||
|
lock: Lock
|
||||||
|
|
||||||
proc `==`*(a, b: EdgeId): bool = uint64(a) == uint64(b)
|
proc `==`*(a, b: EdgeId): bool = uint64(a) == uint64(b)
|
||||||
proc `==`*(a, b: NodeId): bool = uint64(a) == uint64(b)
|
proc `==`*(a, b: NodeId): bool = uint64(a) == uint64(b)
|
||||||
@@ -43,16 +46,18 @@ proc hash*(x: EdgeId): Hash = hash(uint64(x))
|
|||||||
proc hash*(x: NodeId): Hash = hash(uint64(x))
|
proc hash*(x: NodeId): Hash = hash(uint64(x))
|
||||||
|
|
||||||
proc newGraph*(): Graph =
|
proc newGraph*(): Graph =
|
||||||
Graph(
|
new(result)
|
||||||
nodes: initTable[NodeId, GraphNode](),
|
initLock(result.lock)
|
||||||
edges: initTable[EdgeId, Edge](),
|
result.nodes = initTable[NodeId, GraphNode]()
|
||||||
adjacency: initTable[NodeId, seq[AdjacencyEntry]](),
|
result.edges = initTable[EdgeId, Edge]()
|
||||||
reverseAdj: initTable[NodeId, seq[AdjacencyEntry]](),
|
result.adjacency = initTable[NodeId, seq[AdjacencyEntry]]()
|
||||||
nextNodeId: 1,
|
result.reverseAdj = initTable[NodeId, seq[AdjacencyEntry]]()
|
||||||
nextEdgeId: 1,
|
result.nextNodeId = 1
|
||||||
)
|
result.nextEdgeId = 1
|
||||||
|
|
||||||
proc addNode*(g: Graph, label: string, properties: Table[string, string] = initTable[string, string]()): NodeId =
|
proc addNode*(g: Graph, label: string, properties: Table[string, string] = initTable[string, string]()): NodeId =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
let id = NodeId(g.nextNodeId)
|
let id = NodeId(g.nextNodeId)
|
||||||
inc g.nextNodeId
|
inc g.nextNodeId
|
||||||
g.nodes[id] = GraphNode(id: id, label: label, properties: properties)
|
g.nodes[id] = GraphNode(id: id, label: label, properties: properties)
|
||||||
@@ -63,6 +68,8 @@ proc addNode*(g: Graph, label: string, properties: Table[string, string] = initT
|
|||||||
proc addEdge*(g: Graph, src, dst: NodeId, label: string = "",
|
proc addEdge*(g: Graph, src, dst: NodeId, label: string = "",
|
||||||
properties: Table[string, string] = initTable[string, string](),
|
properties: Table[string, string] = initTable[string, string](),
|
||||||
weight: float64 = 1.0): EdgeId =
|
weight: float64 = 1.0): EdgeId =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
let id = EdgeId(g.nextEdgeId)
|
let id = EdgeId(g.nextEdgeId)
|
||||||
inc g.nextEdgeId
|
inc g.nextEdgeId
|
||||||
g.edges[id] = Edge(id: id, src: src, dst: dst, label: label,
|
g.edges[id] = Edge(id: id, src: src, dst: dst, label: label,
|
||||||
@@ -72,22 +79,32 @@ proc addEdge*(g: Graph, src, dst: NodeId, label: string = "",
|
|||||||
return id
|
return id
|
||||||
|
|
||||||
proc getNode*(g: Graph, id: NodeId): GraphNode =
|
proc getNode*(g: Graph, id: NodeId): GraphNode =
|
||||||
g.nodes[id]
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
|
return g.nodes[id]
|
||||||
|
|
||||||
proc getEdge*(g: Graph, id: EdgeId): Edge =
|
proc getEdge*(g: Graph, id: EdgeId): Edge =
|
||||||
g.edges[id]
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
|
return g.edges[id]
|
||||||
|
|
||||||
proc neighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
proc neighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = @[]
|
result = @[]
|
||||||
for entry in g.adjacency.getOrDefault(nodeId, @[]):
|
for entry in g.adjacency.getOrDefault(nodeId, @[]):
|
||||||
result.add(entry.neighbor)
|
result.add(entry.neighbor)
|
||||||
|
|
||||||
proc inNeighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
proc inNeighbors*(g: Graph, nodeId: NodeId): seq[NodeId] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = @[]
|
result = @[]
|
||||||
for entry in g.reverseAdj.getOrDefault(nodeId, @[]):
|
for entry in g.reverseAdj.getOrDefault(nodeId, @[]):
|
||||||
result.add(entry.neighbor)
|
result.add(entry.neighbor)
|
||||||
|
|
||||||
proc removeNode*(g: Graph, nodeId: NodeId) =
|
proc removeNode*(g: Graph, nodeId: NodeId) =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
if nodeId notin g.nodes:
|
if nodeId notin g.nodes:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -112,6 +129,8 @@ proc removeNode*(g: Graph, nodeId: NodeId) =
|
|||||||
g.reverseAdj.del(nodeId)
|
g.reverseAdj.del(nodeId)
|
||||||
|
|
||||||
proc bfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
proc bfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = @[]
|
result = @[]
|
||||||
var visited = initHashSet[NodeId]()
|
var visited = initHashSet[NodeId]()
|
||||||
var queue = initDeque[(NodeId, int)]()
|
var queue = initDeque[(NodeId, int)]()
|
||||||
@@ -123,12 +142,14 @@ proc bfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
|||||||
result.add(node)
|
result.add(node)
|
||||||
if maxDepth >= 0 and depth >= maxDepth:
|
if maxDepth >= 0 and depth >= maxDepth:
|
||||||
continue
|
continue
|
||||||
for neighbor in g.neighbors(node):
|
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||||
if neighbor notin visited:
|
if entry.neighbor notin visited:
|
||||||
visited.incl(neighbor)
|
visited.incl(entry.neighbor)
|
||||||
queue.addLast((neighbor, depth + 1))
|
queue.addLast((entry.neighbor, depth + 1))
|
||||||
|
|
||||||
proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = @[]
|
result = @[]
|
||||||
var visited = initHashSet[NodeId]()
|
var visited = initHashSet[NodeId]()
|
||||||
var stack: seq[(NodeId, int)] = @[(start, 0)]
|
var stack: seq[(NodeId, int)] = @[(start, 0)]
|
||||||
@@ -141,11 +162,13 @@ proc dfs*(g: Graph, start: NodeId, maxDepth: int = -1): seq[NodeId] =
|
|||||||
result.add(node)
|
result.add(node)
|
||||||
if maxDepth >= 0 and depth >= maxDepth:
|
if maxDepth >= 0 and depth >= maxDepth:
|
||||||
continue
|
continue
|
||||||
for neighbor in g.neighbors(node):
|
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||||
if neighbor notin visited:
|
if entry.neighbor notin visited:
|
||||||
stack.add((neighbor, depth + 1))
|
stack.add((entry.neighbor, depth + 1))
|
||||||
|
|
||||||
proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] =
|
proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
var visited = initHashSet[NodeId]()
|
var visited = initHashSet[NodeId]()
|
||||||
var parent = initTable[NodeId, NodeId]()
|
var parent = initTable[NodeId, NodeId]()
|
||||||
var queue = initDeque[NodeId]()
|
var queue = initDeque[NodeId]()
|
||||||
@@ -163,15 +186,17 @@ proc shortestPath*(g: Graph, start, target: NodeId): seq[NodeId] =
|
|||||||
path.reverse()
|
path.reverse()
|
||||||
return path
|
return path
|
||||||
|
|
||||||
for neighbor in g.neighbors(node):
|
for entry in g.adjacency.getOrDefault(node, @[]):
|
||||||
if neighbor notin visited:
|
if entry.neighbor notin visited:
|
||||||
visited.incl(neighbor)
|
visited.incl(entry.neighbor)
|
||||||
parent[neighbor] = node
|
parent[entry.neighbor] = node
|
||||||
queue.addLast(neighbor)
|
queue.addLast(entry.neighbor)
|
||||||
|
|
||||||
return @[]
|
return @[]
|
||||||
|
|
||||||
proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] =
|
proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = initTable[NodeId, float64]()
|
result = initTable[NodeId, float64]()
|
||||||
var visited = initHashSet[NodeId]()
|
var visited = initHashSet[NodeId]()
|
||||||
|
|
||||||
@@ -196,6 +221,8 @@ proc dijkstra*(g: Graph, start: NodeId): Table[NodeId, float64] =
|
|||||||
result[entry.neighbor] = newDist
|
result[entry.neighbor] = newDist
|
||||||
|
|
||||||
proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): Table[NodeId, float64] =
|
proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): Table[NodeId, float64] =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
result = initTable[NodeId, float64]()
|
result = initTable[NodeId, float64]()
|
||||||
let n = g.nodes.len
|
let n = g.nodes.len
|
||||||
if n == 0:
|
if n == 0:
|
||||||
@@ -227,5 +254,131 @@ proc pageRank*(g: Graph, iterations: int = 20, dampingFactor: float64 = 0.85): T
|
|||||||
|
|
||||||
result = newRanks
|
result = newRanks
|
||||||
|
|
||||||
proc nodeCount*(g: Graph): int = g.nodes.len
|
proc nodeCount*(g: Graph): int =
|
||||||
proc edgeCount*(g: Graph): int = g.edges.len
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
|
return g.nodes.len
|
||||||
|
|
||||||
|
proc edgeCount*(g: Graph): int =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
|
return g.edges.len
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Persistence — binary save/load
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const
|
||||||
|
GraphFileMagic = "BGRF"
|
||||||
|
GraphFileVersion = 1'u32
|
||||||
|
|
||||||
|
proc writeString(s: Stream, str: string) =
|
||||||
|
s.write(uint32(str.len))
|
||||||
|
if str.len > 0:
|
||||||
|
s.writeData(str[0].unsafeAddr, str.len)
|
||||||
|
|
||||||
|
proc readString(s: Stream): string =
|
||||||
|
let len = s.readUint32()
|
||||||
|
if len > 0:
|
||||||
|
result = newString(int(len))
|
||||||
|
discard s.readData(result[0].addr, int(len))
|
||||||
|
else:
|
||||||
|
result = ""
|
||||||
|
|
||||||
|
proc saveToFile*(g: Graph, path: string) =
|
||||||
|
acquire(g.lock)
|
||||||
|
defer: release(g.lock)
|
||||||
|
let s = newFileStream(path, fmWrite)
|
||||||
|
if s.isNil:
|
||||||
|
raise newException(IOError, "Cannot open graph file for writing: " & path)
|
||||||
|
|
||||||
|
s.write(GraphFileMagic)
|
||||||
|
s.write(GraphFileVersion)
|
||||||
|
s.write(uint32(g.nodes.len))
|
||||||
|
s.write(uint32(g.edges.len))
|
||||||
|
s.write(g.nextNodeId)
|
||||||
|
s.write(g.nextEdgeId)
|
||||||
|
|
||||||
|
for nodeId, node in g.nodes:
|
||||||
|
s.write(uint64(nodeId))
|
||||||
|
s.writeString(node.label)
|
||||||
|
s.write(uint32(node.properties.len))
|
||||||
|
for key, val in node.properties:
|
||||||
|
s.writeString(key)
|
||||||
|
s.writeString(val)
|
||||||
|
|
||||||
|
for edgeId, edge in g.edges:
|
||||||
|
s.write(uint64(edgeId))
|
||||||
|
s.write(uint64(edge.src))
|
||||||
|
s.write(uint64(edge.dst))
|
||||||
|
s.writeString(edge.label)
|
||||||
|
s.write(edge.weight)
|
||||||
|
s.write(uint32(edge.properties.len))
|
||||||
|
for key, val in edge.properties:
|
||||||
|
s.writeString(key)
|
||||||
|
s.writeString(val)
|
||||||
|
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
proc loadFromFile*(path: string): Graph =
|
||||||
|
let s = newFileStream(path, fmRead)
|
||||||
|
if s.isNil:
|
||||||
|
raise newException(IOError, "Cannot open graph file for reading: " & path)
|
||||||
|
|
||||||
|
let magic = s.readStr(4)
|
||||||
|
if magic != GraphFileMagic:
|
||||||
|
raise newException(ValueError, "Invalid graph file magic bytes")
|
||||||
|
|
||||||
|
let version = s.readUint32()
|
||||||
|
if version != GraphFileVersion:
|
||||||
|
raise newException(ValueError, "Unsupported graph file version: " & $version)
|
||||||
|
|
||||||
|
let nodeCount = int(s.readUint32())
|
||||||
|
let edgeCount = int(s.readUint32())
|
||||||
|
let nextNodeId = s.readUint64()
|
||||||
|
let nextEdgeId = s.readUint64()
|
||||||
|
|
||||||
|
result = Graph(
|
||||||
|
nodes: initTable[NodeId, GraphNode](),
|
||||||
|
edges: initTable[EdgeId, Edge](),
|
||||||
|
adjacency: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||||
|
reverseAdj: initTable[NodeId, seq[AdjacencyEntry]](),
|
||||||
|
nextNodeId: nextNodeId,
|
||||||
|
nextEdgeId: nextEdgeId,
|
||||||
|
lock: Lock(),
|
||||||
|
)
|
||||||
|
initLock(result.lock)
|
||||||
|
|
||||||
|
for i in 0 ..< nodeCount:
|
||||||
|
let id = NodeId(s.readUint64())
|
||||||
|
let label = s.readString()
|
||||||
|
let propCount = int(s.readUint32())
|
||||||
|
var props = initTable[string, string]()
|
||||||
|
for j in 0 ..< propCount:
|
||||||
|
let key = s.readString()
|
||||||
|
let val = s.readString()
|
||||||
|
props[key] = val
|
||||||
|
result.nodes[id] = GraphNode(id: id, label: label, properties: props)
|
||||||
|
result.adjacency[id] = @[]
|
||||||
|
result.reverseAdj[id] = @[]
|
||||||
|
|
||||||
|
for i in 0 ..< edgeCount:
|
||||||
|
let id = EdgeId(s.readUint64())
|
||||||
|
let src = NodeId(s.readUint64())
|
||||||
|
let dst = NodeId(s.readUint64())
|
||||||
|
let label = s.readString()
|
||||||
|
let weight = s.readFloat64()
|
||||||
|
let propCount = int(s.readUint32())
|
||||||
|
var props = initTable[string, string]()
|
||||||
|
for j in 0 ..< propCount:
|
||||||
|
let key = s.readString()
|
||||||
|
let val = s.readString()
|
||||||
|
props[key] = val
|
||||||
|
result.edges[id] = Edge(id: id, src: src, dst: dst, label: label,
|
||||||
|
properties: props, weight: weight)
|
||||||
|
result.adjacency[src].add(AdjacencyEntry(edgeId: id, neighbor: dst,
|
||||||
|
weight: weight, label: label))
|
||||||
|
result.reverseAdj[dst].add(AdjacencyEntry(edgeId: id, neighbor: src,
|
||||||
|
weight: weight, label: label))
|
||||||
|
|
||||||
|
s.close()
|
||||||
|
|||||||
@@ -1,150 +0,0 @@
|
|||||||
## HTTP/REST API — JSON endpoint
|
|
||||||
import std/asynchttpserver
|
|
||||||
import std/asyncdispatch
|
|
||||||
import std/json
|
|
||||||
import std/strutils
|
|
||||||
import std/tables
|
|
||||||
|
|
||||||
type
|
|
||||||
HttpMethod* = enum
|
|
||||||
hmGet = "GET"
|
|
||||||
hmPost = "POST"
|
|
||||||
hmPut = "PUT"
|
|
||||||
hmDelete = "DELETE"
|
|
||||||
hmPatch = "PATCH"
|
|
||||||
hmOptions = "OPTIONS"
|
|
||||||
|
|
||||||
RouteHandler* = proc(req: Request): Future[JsonNode] {.gcsafe.}
|
|
||||||
|
|
||||||
HttpRouter* = ref object
|
|
||||||
routes: Table[string, Table[string, RouteHandler]] # method -> path -> handler
|
|
||||||
middlewares: seq[RouteHandler]
|
|
||||||
port*: int
|
|
||||||
address*: string
|
|
||||||
|
|
||||||
Request* = ref object
|
|
||||||
httpMethod*: HttpMethod
|
|
||||||
path*: string
|
|
||||||
query*: Table[string, string]
|
|
||||||
headers*: Table[string, string]
|
|
||||||
body*: string
|
|
||||||
contentType*: string
|
|
||||||
|
|
||||||
Response* = object
|
|
||||||
status*: int
|
|
||||||
body*: JsonNode
|
|
||||||
headers*: Table[string, string]
|
|
||||||
|
|
||||||
proc newHttpRouter*(port: int = 8080, address: string = "0.0.0.0"): HttpRouter =
|
|
||||||
HttpRouter(
|
|
||||||
routes: initTable[string, Table[string, RouteHandler]](),
|
|
||||||
middlewares: @[],
|
|
||||||
port: port,
|
|
||||||
address: address,
|
|
||||||
)
|
|
||||||
|
|
||||||
proc addRoute*(router: HttpRouter, meth: HttpMethod, path: string, handler: RouteHandler) =
|
|
||||||
let m = $meth
|
|
||||||
if m notin router.routes:
|
|
||||||
router.routes[m] = initTable[string, RouteHandler]()
|
|
||||||
router.routes[m][path] = handler
|
|
||||||
|
|
||||||
proc get*(router: HttpRouter, path: string, handler: RouteHandler) =
|
|
||||||
router.addRoute(hmGet, path, handler)
|
|
||||||
|
|
||||||
proc post*(router: HttpRouter, path: string, handler: RouteHandler) =
|
|
||||||
router.addRoute(hmPost, path, handler)
|
|
||||||
|
|
||||||
proc put*(router: HttpRouter, path: string, handler: RouteHandler) =
|
|
||||||
router.addRoute(hmPut, path, handler)
|
|
||||||
|
|
||||||
proc delete*(router: HttpRouter, path: string, handler: RouteHandler) =
|
|
||||||
router.addRoute(hmDelete, path, handler)
|
|
||||||
|
|
||||||
proc jsonResponse*(status: int, data: JsonNode, headers: Table[string, string] = initTable[string, string]()): Response =
|
|
||||||
Response(status: status, body: data, headers: headers)
|
|
||||||
|
|
||||||
proc errorResponse*(status: int, message: string): Response =
|
|
||||||
jsonResponse(status, %*{"error": message})
|
|
||||||
|
|
||||||
proc successResponse*(data: JsonNode): Response =
|
|
||||||
jsonResponse(200, data)
|
|
||||||
|
|
||||||
proc parseQuery*(queryString: string): Table[string, string] =
|
|
||||||
result = initTable[string, string]()
|
|
||||||
if queryString.len == 0:
|
|
||||||
return
|
|
||||||
for pair in queryString.split("&"):
|
|
||||||
let parts = pair.split("=", 1)
|
|
||||||
if parts.len == 2:
|
|
||||||
result[parts[0]] = parts[1]
|
|
||||||
elif parts.len == 1:
|
|
||||||
result[parts[0]] = ""
|
|
||||||
|
|
||||||
proc matchPath*(pattern, path: string): Table[string, string] =
|
|
||||||
result = initTable[string, string]()
|
|
||||||
let patternParts = pattern.split("/")
|
|
||||||
let pathParts = path.split("/")
|
|
||||||
if patternParts.len != pathParts.len:
|
|
||||||
return
|
|
||||||
for i in 0..<patternParts.len:
|
|
||||||
if patternParts[i].startsWith(":"):
|
|
||||||
result[patternParts[i][1..^1]] = pathParts[i]
|
|
||||||
elif patternParts[i] != pathParts[i]:
|
|
||||||
result.clear()
|
|
||||||
return
|
|
||||||
|
|
||||||
proc corsHeaders*(): Table[string, string] =
|
|
||||||
result = initTable[string, string]()
|
|
||||||
result["Access-Control-Allow-Origin"] = "*"
|
|
||||||
result["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, PATCH, OPTIONS"
|
|
||||||
result["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
|
|
||||||
|
|
||||||
proc jsonHeaders*(): Table[string, string] =
|
|
||||||
result = initTable[string, string]()
|
|
||||||
result["Content-Type"] = "application/json"
|
|
||||||
|
|
||||||
proc handleCors*(req: Request): Response =
|
|
||||||
if req.httpMethod == hmOptions:
|
|
||||||
return jsonResponse(204, newJNull(), corsHeaders())
|
|
||||||
return nil
|
|
||||||
|
|
||||||
proc parseRequest*(httpMethod: string, path: string, headers: Table[string, string], body: string): Request =
|
|
||||||
let methodMap = {
|
|
||||||
"GET": hmGet, "POST": hmPost, "PUT": hmPut,
|
|
||||||
"DELETE": hmDelete, "PATCH": hmPatch, "OPTIONS": hmOptions,
|
|
||||||
}.toTable
|
|
||||||
|
|
||||||
var query = initTable[string, string]()
|
|
||||||
var cleanPath = path
|
|
||||||
let qPos = path.find('?')
|
|
||||||
if qPos >= 0:
|
|
||||||
cleanPath = path[0..<qPos]
|
|
||||||
query = parseQuery(path[qPos+1..^1])
|
|
||||||
|
|
||||||
Request(
|
|
||||||
httpMethod: methodMap.getOrDefault(httpMethod, hmGet),
|
|
||||||
path: cleanPath,
|
|
||||||
query: query,
|
|
||||||
headers: headers,
|
|
||||||
body: body,
|
|
||||||
contentType: headers.getOrDefault("Content-Type", ""),
|
|
||||||
)
|
|
||||||
|
|
||||||
proc handleRequest*(router: HttpRouter, req: Request): Future[Response] {.async.} =
|
|
||||||
# CORS
|
|
||||||
let corsResp = handleCors(req)
|
|
||||||
if corsResp != nil:
|
|
||||||
return corsResp
|
|
||||||
|
|
||||||
let methodStr = $req.httpMethod
|
|
||||||
if methodStr in router.routes:
|
|
||||||
for pattern, handler in router.routes[methodStr]:
|
|
||||||
let params = matchPath(pattern, req.path)
|
|
||||||
if params.len > 0 or pattern == req.path:
|
|
||||||
try:
|
|
||||||
return jsonResponse(200, await handler(req), jsonHeaders())
|
|
||||||
except CatchableError as e:
|
|
||||||
return errorResponse(500, e.msg)
|
|
||||||
|
|
||||||
return errorResponse(404, "Not found: " & req.path)
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
## TLS/SSL — transport layer security wrapper
|
|
||||||
import std/os
|
|
||||||
import std/strutils
|
|
||||||
|
|
||||||
type
|
|
||||||
TLSVersion* = enum
|
|
||||||
tls12 = "TLSv1.2"
|
|
||||||
tls13 = "TLSv1.3"
|
|
||||||
|
|
||||||
TLSConfig* = object
|
|
||||||
certFile*: string
|
|
||||||
keyFile*: string
|
|
||||||
caFile*: string
|
|
||||||
minVersion*: TLSVersion
|
|
||||||
verifyPeer*: bool
|
|
||||||
cipherSuites*: seq[string]
|
|
||||||
|
|
||||||
TLSState* = enum
|
|
||||||
tsDisconnected
|
|
||||||
tsHandshaking
|
|
||||||
tsConnected
|
|
||||||
tsError
|
|
||||||
|
|
||||||
TLSConnection* = ref object
|
|
||||||
config*: TLSConfig
|
|
||||||
state*: TLSState
|
|
||||||
host*: string
|
|
||||||
port*: int
|
|
||||||
|
|
||||||
proc defaultTLSConfig*(): TLSConfig =
|
|
||||||
TLSConfig(
|
|
||||||
certFile: "",
|
|
||||||
keyFile: "",
|
|
||||||
caFile: "",
|
|
||||||
minVersion: tls12,
|
|
||||||
verifyPeer: false,
|
|
||||||
cipherSuites: @[
|
|
||||||
"TLS_AES_256_GCM_SHA384",
|
|
||||||
"TLS_CHACHA20_POLY1305_SHA256",
|
|
||||||
"TLS_AES_128_GCM_SHA256",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
proc newTLSConfig*(certFile, keyFile: string, caFile: string = "",
|
|
||||||
minVersion: TLSVersion = tls12,
|
|
||||||
verifyPeer: bool = false): TLSConfig =
|
|
||||||
TLSConfig(
|
|
||||||
certFile: certFile,
|
|
||||||
keyFile: keyFile,
|
|
||||||
caFile: caFile,
|
|
||||||
minVersion: minVersion,
|
|
||||||
verifyPeer: verifyPeer,
|
|
||||||
cipherSuites: @[
|
|
||||||
"TLS_AES_256_GCM_SHA384",
|
|
||||||
"TLS_CHACHA20_POLY1305_SHA256",
|
|
||||||
"TLS_AES_128_GCM_SHA256",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
proc validateConfig*(config: TLSConfig): seq[string] =
|
|
||||||
result = @[]
|
|
||||||
if config.certFile.len == 0:
|
|
||||||
result.add("Certificate file not specified")
|
|
||||||
elif not fileExists(config.certFile):
|
|
||||||
result.add("Certificate file not found: " & config.certFile)
|
|
||||||
if config.keyFile.len == 0:
|
|
||||||
result.add("Key file not specified")
|
|
||||||
elif not fileExists(config.keyFile):
|
|
||||||
result.add("Key file not found: " & config.keyFile)
|
|
||||||
if config.caFile.len > 0 and not fileExists(config.caFile):
|
|
||||||
result.add("CA file not found: " & config.caFile)
|
|
||||||
|
|
||||||
proc isValid*(config: TLSConfig): bool =
|
|
||||||
return config.validateConfig().len == 0
|
|
||||||
|
|
||||||
proc newTLSConnection*(config: TLSConfig, host: string, port: int): TLSConnection =
|
|
||||||
TLSConnection(config: config, state: tsDisconnected, host: host, port: port)
|
|
||||||
|
|
||||||
proc state*(conn: TLSConnection): TLSState = conn.state
|
|
||||||
|
|
||||||
# Self-signed certificate generation helper
|
|
||||||
proc generateSelfSignedCert*(outputDir: string): (string, string) =
|
|
||||||
let certPath = outputDir / "server.crt"
|
|
||||||
let keyPath = outputDir / "server.key"
|
|
||||||
|
|
||||||
createDir(outputDir)
|
|
||||||
# Use openssl to generate self-signed cert
|
|
||||||
let cmd = "openssl req -x509 -newkey rsa:2048 -keyout " & keyPath &
|
|
||||||
" -out " & certPath & " -days 365 -nodes -subj '/CN=localhost' 2>/dev/null"
|
|
||||||
discard execShellCmd(cmd)
|
|
||||||
|
|
||||||
return (certPath, keyPath)
|
|
||||||
|
|
||||||
proc certificateInfo*(certPath: string): Table[string, string] =
|
|
||||||
result = initTable[string, string]()
|
|
||||||
if not fileExists(certPath):
|
|
||||||
return
|
|
||||||
# Would use openssl to parse cert in production
|
|
||||||
result["path"] = certPath
|
|
||||||
result["exists"] = "true"
|
|
||||||
@@ -1,216 +0,0 @@
|
|||||||
## WebSocket — streaming protocol support
|
|
||||||
import std/asyncdispatch
|
|
||||||
import std/asyncnet
|
|
||||||
import std/strutils
|
|
||||||
import std/base64
|
|
||||||
import std/sha1
|
|
||||||
import std/hashes
|
|
||||||
import std/tables
|
|
||||||
|
|
||||||
const
|
|
||||||
WS_FIN* = 0x80'u8
|
|
||||||
WS_TEXT* = 0x01'u8
|
|
||||||
WS_BINARY* = 0x02'u8
|
|
||||||
WS_CLOSE* = 0x08'u8
|
|
||||||
WS_PING* = 0x09'u8
|
|
||||||
WS_PONG* = 0x0A'u8
|
|
||||||
WS_MAX_FRAME* = 65536
|
|
||||||
|
|
||||||
type
|
|
||||||
WsFrame* = object
|
|
||||||
fin*: bool
|
|
||||||
opcode*: uint8
|
|
||||||
payload*: seq[byte]
|
|
||||||
masked*: bool
|
|
||||||
|
|
||||||
WebSocket* = ref object
|
|
||||||
socket: AsyncSocket
|
|
||||||
connected*: bool
|
|
||||||
onMessage*: proc(data: seq[byte]) {.gcsafe.}
|
|
||||||
onClose*: proc() {.gcsafe.}
|
|
||||||
onPing*: proc(data: seq[byte]) {.gcsafe.}
|
|
||||||
onPong*: proc(data: seq[byte]) {.gcsafe.}
|
|
||||||
|
|
||||||
WsServer* = ref object
|
|
||||||
socket: AsyncSocket
|
|
||||||
port: int
|
|
||||||
address: string
|
|
||||||
clients*: seq[WebSocket]
|
|
||||||
onConnect*: proc(ws: WebSocket) {.gcsafe.}
|
|
||||||
onDisconnect*: proc(ws: WebSocket) {.gcsafe.}
|
|
||||||
onMessage*: proc(ws: WebSocket, data: seq[byte]) {.gcsafe.}
|
|
||||||
|
|
||||||
proc newWebSocket*(socket: AsyncSocket): WebSocket =
|
|
||||||
WebSocket(
|
|
||||||
socket: socket,
|
|
||||||
connected: true,
|
|
||||||
onMessage: nil,
|
|
||||||
onClose: nil,
|
|
||||||
onPing: nil,
|
|
||||||
onPong: nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
proc newWsServer*(port: int = 8081, address: string = "0.0.0.0"): WsServer =
|
|
||||||
WsServer(
|
|
||||||
socket: newAsyncSocket(),
|
|
||||||
port: port,
|
|
||||||
address: address,
|
|
||||||
clients: @[],
|
|
||||||
onConnect: nil,
|
|
||||||
onDisconnect: nil,
|
|
||||||
onMessage: nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
proc wsHandshakeKey(clientKey: string): string =
|
|
||||||
let magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
|
||||||
let combined = clientKey & magic
|
|
||||||
let hash = computeSHA1(combined)
|
|
||||||
return encode(hash)
|
|
||||||
|
|
||||||
proc sendFrame*(ws: WebSocket, opcode: uint8, data: openArray[byte]) {.async.} =
|
|
||||||
var frame: seq[byte] = @[]
|
|
||||||
frame.add(opcode or WS_FIN)
|
|
||||||
|
|
||||||
if data.len < 126:
|
|
||||||
frame.add(byte(data.len))
|
|
||||||
elif data.len < 65536:
|
|
||||||
frame.add(126'u8)
|
|
||||||
frame.add(byte((data.len shr 8) and 0xFF))
|
|
||||||
frame.add(byte(data.len and 0xFF))
|
|
||||||
else:
|
|
||||||
frame.add(127'u8)
|
|
||||||
for i in 0..7:
|
|
||||||
frame.add(byte((data.len shr (56 - i * 8)) and 0xFF))
|
|
||||||
|
|
||||||
for b in data:
|
|
||||||
frame.add(b)
|
|
||||||
|
|
||||||
await ws.socket.send(cast[string](frame))
|
|
||||||
|
|
||||||
proc sendText*(ws: WebSocket, text: string) {.async.} =
|
|
||||||
await ws.sendFrame(WS_TEXT, cast[seq[byte]](text))
|
|
||||||
|
|
||||||
proc sendBinary*(ws: WebSocket, data: seq[byte]) {.async.} =
|
|
||||||
await ws.sendFrame(WS_BINARY, data)
|
|
||||||
|
|
||||||
proc sendPing*(ws: WebSocket, data: seq[byte] = @[]) {.async.} =
|
|
||||||
await ws.sendFrame(WS_PING, data)
|
|
||||||
|
|
||||||
proc sendPong*(ws: WebSocket, data: seq[byte] = @[]) {.async.} =
|
|
||||||
await ws.sendFrame(WS_PONG, data)
|
|
||||||
|
|
||||||
proc close*(ws: WebSocket) {.async.} =
|
|
||||||
if ws.connected:
|
|
||||||
ws.connected = false
|
|
||||||
await ws.sendFrame(WS_CLOSE, @[])
|
|
||||||
ws.socket.close()
|
|
||||||
if ws.onClose != nil:
|
|
||||||
ws.onClose()
|
|
||||||
|
|
||||||
proc readFrame*(ws: WebSocket): Future[WsFrame] {.async.} =
|
|
||||||
var header: array[2, byte]
|
|
||||||
let read1 = await ws.socket.recv(2)
|
|
||||||
if read1.len < 2:
|
|
||||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
|
||||||
|
|
||||||
header[0] = byte(read1[0])
|
|
||||||
header[1] = byte(read1[1])
|
|
||||||
|
|
||||||
result.fin = (header[0] and WS_FIN) != 0
|
|
||||||
result.opcode = header[0] and 0x0F
|
|
||||||
result.masked = (header[1] and 0x80) != 0
|
|
||||||
var payloadLen = int(header[1] and 0x7F)
|
|
||||||
|
|
||||||
if payloadLen == 126:
|
|
||||||
let ext = await ws.socket.recv(2)
|
|
||||||
if ext.len < 2:
|
|
||||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
|
||||||
payloadLen = (int(byte(ext[0])) shl 8) or int(byte(ext[1]))
|
|
||||||
elif payloadLen == 127:
|
|
||||||
let ext = await ws.socket.recv(8)
|
|
||||||
if ext.len < 8:
|
|
||||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
|
||||||
payloadLen = 0
|
|
||||||
for i in 0..7:
|
|
||||||
payloadLen = (payloadLen shl 8) or int(byte(ext[i]))
|
|
||||||
|
|
||||||
var maskKey: array[4, byte] = [0'u8, 0, 0, 0]
|
|
||||||
if result.masked:
|
|
||||||
let mk = await ws.socket.recv(4)
|
|
||||||
if mk.len < 4:
|
|
||||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
|
||||||
for i in 0..3:
|
|
||||||
maskKey[i] = byte(mk[i])
|
|
||||||
|
|
||||||
let payloadData = await ws.socket.recv(payloadLen)
|
|
||||||
if payloadData.len < payloadLen:
|
|
||||||
return WsFrame(fin: false, opcode: WS_CLOSE)
|
|
||||||
|
|
||||||
result.payload = newSeq[byte](payloadLen)
|
|
||||||
for i in 0..<payloadLen:
|
|
||||||
if result.masked:
|
|
||||||
result.payload[i] = byte(payloadData[i]) xor maskKey[i mod 4]
|
|
||||||
else:
|
|
||||||
result.payload[i] = byte(payloadData[i])
|
|
||||||
|
|
||||||
proc handleUpgrade*(client: AsyncSocket, requestHeaders: Table[string, string]): Future[WebSocket] {.async.} =
|
|
||||||
let wsKey = requestHeaders.getOrDefault("Sec-WebSocket-Key", "")
|
|
||||||
if wsKey.len == 0:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
let acceptKey = wsHandshakeKey(wsKey)
|
|
||||||
let response = "HTTP/1.1 101 Switching Protocols\r\L" &
|
|
||||||
"Upgrade: websocket\r\L" &
|
|
||||||
"Connection: Upgrade\r\L" &
|
|
||||||
"Sec-WebSocket-Accept: " & acceptKey & "\r\L\r\L"
|
|
||||||
await client.send(response)
|
|
||||||
return newWebSocket(client)
|
|
||||||
|
|
||||||
proc run*(server: WsServer) {.async.} =
|
|
||||||
server.socket.setSockOpt(OptReuseAddr, true)
|
|
||||||
server.socket.bindAddr(Port(server.port), server.address)
|
|
||||||
server.socket.listen()
|
|
||||||
|
|
||||||
while true:
|
|
||||||
let client = await server.socket.accept()
|
|
||||||
let ws = newWebSocket(client)
|
|
||||||
server.clients.add(ws)
|
|
||||||
|
|
||||||
if server.onConnect != nil:
|
|
||||||
server.onConnect(ws)
|
|
||||||
|
|
||||||
# Read loop
|
|
||||||
try:
|
|
||||||
while ws.connected:
|
|
||||||
let frame = await ws.readFrame()
|
|
||||||
case frame.opcode
|
|
||||||
of WS_TEXT, WS_BINARY:
|
|
||||||
if server.onMessage != nil:
|
|
||||||
server.onMessage(ws, frame.payload)
|
|
||||||
of WS_PING:
|
|
||||||
await ws.sendPong(frame.payload)
|
|
||||||
of WS_CLOSE:
|
|
||||||
ws.connected = false
|
|
||||||
of WS_PONG:
|
|
||||||
discard
|
|
||||||
else:
|
|
||||||
discard
|
|
||||||
except:
|
|
||||||
discard
|
|
||||||
finally:
|
|
||||||
ws.connected = false
|
|
||||||
server.clients = server.clients.filterIt(it != ws)
|
|
||||||
if server.onDisconnect != nil:
|
|
||||||
server.onDisconnect(ws)
|
|
||||||
|
|
||||||
proc broadcast*(server: WsServer, data: seq[byte]) {.async.} =
|
|
||||||
for client in server.clients:
|
|
||||||
if client.connected:
|
|
||||||
await client.sendBinary(data)
|
|
||||||
|
|
||||||
proc broadcastText*(server: WsServer, text: string) {.async.} =
|
|
||||||
for client in server.clients:
|
|
||||||
if client.connected:
|
|
||||||
await client.sendText(text)
|
|
||||||
|
|
||||||
proc clientCount*(server: WsServer): int = server.clients.len
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
## Adaptive Query Execution — runtime query plan adaptation
|
## Adaptive Query Execution — runtime query plan adaptation
|
||||||
import std/tables
|
import std/tables
|
||||||
import std/monotimes
|
import std/monotimes
|
||||||
import std/algorithm
|
|
||||||
import std/strutils
|
import std/strutils
|
||||||
|
|
||||||
type
|
type
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
## Codegen — compile IR plan to storage operations
|
## Codegen — compile IR plan to storage operations
|
||||||
import std/tables
|
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import ../query/ir
|
import ../query/ir
|
||||||
import ../core/types
|
|
||||||
|
|
||||||
type
|
type
|
||||||
StorageOpKind* = enum
|
StorageOpKind* = enum
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import std/strutils
|
|||||||
import std/tables
|
import std/tables
|
||||||
import std/monotimes
|
import std/monotimes
|
||||||
import std/streams
|
import std/streams
|
||||||
|
import std/locks
|
||||||
import bloom
|
import bloom
|
||||||
import wal
|
import wal
|
||||||
import mmap
|
import mmap
|
||||||
@@ -38,7 +39,7 @@ type
|
|||||||
entryCount: int
|
entryCount: int
|
||||||
mmapFile: MmapFile
|
mmapFile: MmapFile
|
||||||
|
|
||||||
LSMTree* = object
|
LSMTree* = ref object
|
||||||
dir: string
|
dir: string
|
||||||
memTable: MemTable
|
memTable: MemTable
|
||||||
immutableMem: MemTable
|
immutableMem: MemTable
|
||||||
@@ -46,8 +47,8 @@ type
|
|||||||
wal: WriteAheadLog
|
wal: WriteAheadLog
|
||||||
memMaxSize: int
|
memMaxSize: int
|
||||||
currentSeq: uint64
|
currentSeq: uint64
|
||||||
readLocks: int
|
|
||||||
nextSSTableId: int
|
nextSSTableId: int
|
||||||
|
lock: Lock
|
||||||
|
|
||||||
proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable =
|
proc newMemTable(maxSize: int = DefaultMemTableSize): MemTable =
|
||||||
MemTable(entries: @[], size: 0, maxSize: maxSize)
|
MemTable(entries: @[], size: 0, maxSize: maxSize)
|
||||||
@@ -312,19 +313,20 @@ proc newLSMTree*(dir: string, memMaxSize: int = DefaultMemTableSize): LSMTree =
|
|||||||
|
|
||||||
sstables.sort(proc(a, b: SSTable): int = cmp(a.minKey, b.minKey))
|
sstables.sort(proc(a, b: SSTable): int = cmp(a.minKey, b.minKey))
|
||||||
|
|
||||||
LSMTree(
|
new(result)
|
||||||
dir: dir,
|
initLock(result.lock)
|
||||||
memTable: newMemTable(memMaxSize),
|
result.dir = dir
|
||||||
immutableMem: newMemTable(0),
|
result.memTable = newMemTable(memMaxSize)
|
||||||
sstables: sstables,
|
result.immutableMem = newMemTable(0)
|
||||||
wal: newWriteAheadLog(dir / "wal"),
|
result.sstables = sstables
|
||||||
memMaxSize: memMaxSize,
|
result.wal = newWriteAheadLog(dir / "wal")
|
||||||
currentSeq: 0,
|
result.memMaxSize = memMaxSize
|
||||||
readLocks: 0,
|
result.currentSeq = 0
|
||||||
nextSSTableId: nextId,
|
result.nextSSTableId = nextId
|
||||||
)
|
|
||||||
|
|
||||||
proc put*(db: var LSMTree, key: string, value: seq[byte]) =
|
proc put*(db: LSMTree, key: string, value: seq[byte]) =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
let ts = uint64(getMonoTime().ticks())
|
let ts = uint64(getMonoTime().ticks())
|
||||||
db.wal.writePut(cast[seq[byte]](key), value, ts)
|
db.wal.writePut(cast[seq[byte]](key), value, ts)
|
||||||
if not db.memTable.put(key, value, ts):
|
if not db.memTable.put(key, value, ts):
|
||||||
@@ -332,12 +334,14 @@ proc put*(db: var LSMTree, key: string, value: seq[byte]) =
|
|||||||
db.memTable = newMemTable(db.memMaxSize)
|
db.memTable = newMemTable(db.memMaxSize)
|
||||||
discard db.memTable.put(key, value, ts)
|
discard db.memTable.put(key, value, ts)
|
||||||
|
|
||||||
proc delete*(db: var LSMTree, key: string) =
|
proc delete*(db: LSMTree, key: string) =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
let ts = uint64(getMonoTime().ticks())
|
let ts = uint64(getMonoTime().ticks())
|
||||||
db.wal.writeDelete(cast[seq[byte]](key), ts)
|
db.wal.writeDelete(cast[seq[byte]](key), ts)
|
||||||
discard db.memTable.put(key, @[], ts, deleted = true)
|
discard db.memTable.put(key, @[], ts, deleted = true)
|
||||||
|
|
||||||
proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
proc getUnsafe(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||||
let (found, entry) = db.memTable.get(key)
|
let (found, entry) = db.memTable.get(key)
|
||||||
if found:
|
if found:
|
||||||
if entry.deleted:
|
if entry.deleted:
|
||||||
@@ -365,11 +369,18 @@ proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
|||||||
|
|
||||||
return (false, @[])
|
return (false, @[])
|
||||||
|
|
||||||
|
proc get*(db: LSMTree, key: string): (bool, seq[byte]) =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
return getUnsafe(db, key)
|
||||||
|
|
||||||
proc contains*(db: LSMTree, key: string): bool =
|
proc contains*(db: LSMTree, key: string): bool =
|
||||||
let (found, _) = db.get(key)
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
let (found, _) = getUnsafe(db, key)
|
||||||
return found
|
return found
|
||||||
|
|
||||||
proc flush*(db: var LSMTree) =
|
proc flushUnsafe(db: LSMTree) =
|
||||||
if db.immutableMem.len == 0 and db.memTable.len == 0:
|
if db.immutableMem.len == 0 and db.memTable.len == 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -394,17 +405,37 @@ proc flush*(db: var LSMTree) =
|
|||||||
db.wal.writeCommit(uint64(getMonoTime().ticks()))
|
db.wal.writeCommit(uint64(getMonoTime().ticks()))
|
||||||
db.wal.sync()
|
db.wal.sync()
|
||||||
|
|
||||||
proc close*(db: var LSMTree) =
|
proc flush*(db: LSMTree) =
|
||||||
db.flush()
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
flushUnsafe(db)
|
||||||
|
|
||||||
|
proc close*(db: LSMTree) =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
flushUnsafe(db)
|
||||||
for sst in db.sstables.mitems:
|
for sst in db.sstables.mitems:
|
||||||
sst.close()
|
sst.close()
|
||||||
db.wal.close()
|
db.wal.close()
|
||||||
|
|
||||||
proc memTableSize*(db: LSMTree): int = db.memTable.len
|
proc memTableSize*(db: LSMTree): int =
|
||||||
proc sstableCount*(db: LSMTree): int = db.sstables.len
|
acquire(db.lock)
|
||||||
proc dir*(db: LSMTree): string = db.dir
|
defer: release(db.lock)
|
||||||
|
return db.memTable.len
|
||||||
|
|
||||||
|
proc sstableCount*(db: LSMTree): int =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
return db.sstables.len
|
||||||
|
|
||||||
|
proc dir*(db: LSMTree): string =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
|
return db.dir
|
||||||
|
|
||||||
proc scanMemTable*(db: LSMTree): seq[Entry] =
|
proc scanMemTable*(db: LSMTree): seq[Entry] =
|
||||||
|
acquire(db.lock)
|
||||||
|
defer: release(db.lock)
|
||||||
## Return all entries from memory (memTable + immutableMem)
|
## Return all entries from memory (memTable + immutableMem)
|
||||||
result = @[]
|
result = @[]
|
||||||
for e in db.memTable.entries:
|
for e in db.memTable.entries:
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
## Crash Recovery — WAL replay with REDO/UNDO
|
## Crash Recovery — WAL replay with REDO/UNDO
|
||||||
import std/streams
|
import std/streams
|
||||||
import std/os
|
import std/os
|
||||||
import std/tables
|
|
||||||
import ../storage/wal
|
import ../storage/wal
|
||||||
|
|
||||||
type
|
type
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
## Stress Test — parallel workloads against LSM-Tree
|
||||||
|
import std/os
|
||||||
|
import std/random
|
||||||
|
import std/strutils
|
||||||
|
import std/times
|
||||||
|
import std/monotimes
|
||||||
|
import std/sequtils
|
||||||
|
import std/threadpool
|
||||||
|
import barabadb/storage/lsm
|
||||||
|
|
||||||
|
const
|
||||||
|
NumWorkers = 10
|
||||||
|
OpsPerWorker = 1000
|
||||||
|
KeySpace = 500
|
||||||
|
|
||||||
|
randomize()
|
||||||
|
|
||||||
|
proc runWorker(workerId: int, dataDir: string): int =
|
||||||
|
## Each worker gets its own LSMTree directory.
|
||||||
|
## TODO: switch to shared LSMTree once full thread-safety is hardened.
|
||||||
|
var db = newLSMTree(dataDir)
|
||||||
|
var errors = 0
|
||||||
|
|
||||||
|
for i in 0 ..< OpsPerWorker:
|
||||||
|
let op = rand(0 .. 2)
|
||||||
|
let key = "key_" & $workerId & "_" & $(rand(0 ..< KeySpace))
|
||||||
|
let value = cast[seq[byte]]("val_" & $i & "_" & $workerId)
|
||||||
|
|
||||||
|
case op
|
||||||
|
of 0:
|
||||||
|
db.put(key, value)
|
||||||
|
of 1:
|
||||||
|
let (found, val) = db.get(key)
|
||||||
|
if found:
|
||||||
|
if val != value and val.len > 0:
|
||||||
|
discard
|
||||||
|
of 2:
|
||||||
|
db.delete(key)
|
||||||
|
else:
|
||||||
|
discard
|
||||||
|
|
||||||
|
for k in 0 ..< KeySpace:
|
||||||
|
let key = "key_" & $workerId & "_" & $k
|
||||||
|
let (found, _) = db.get(key)
|
||||||
|
discard found
|
||||||
|
|
||||||
|
db.close()
|
||||||
|
return errors
|
||||||
|
|
||||||
|
proc main() =
|
||||||
|
let baseDir = "/tmp/baradb_stress_test"
|
||||||
|
removeDir(baseDir)
|
||||||
|
createDir(baseDir)
|
||||||
|
|
||||||
|
let start = getMonoTime()
|
||||||
|
|
||||||
|
var workerDirs: seq[string] = @[]
|
||||||
|
for i in 0 ..< NumWorkers:
|
||||||
|
workerDirs.add(baseDir / "worker_" & $i)
|
||||||
|
|
||||||
|
# Run workers in parallel using threadpool
|
||||||
|
var futures: seq[FlowVar[int]] = @[]
|
||||||
|
for i in 0 ..< NumWorkers:
|
||||||
|
futures.add(spawn runWorker(i, workerDirs[i]))
|
||||||
|
|
||||||
|
var totalErrors = 0
|
||||||
|
for f in futures:
|
||||||
|
totalErrors += ^f
|
||||||
|
|
||||||
|
let elapsed = (getMonoTime() - start).inMilliseconds
|
||||||
|
let totalOps = NumWorkers * OpsPerWorker
|
||||||
|
|
||||||
|
echo "Stress test completed: ", totalOps, " ops across ", NumWorkers, " workers"
|
||||||
|
echo "Time: ", elapsed, " ms"
|
||||||
|
echo "Throughput: ", float(totalOps) / (float(elapsed) / 1000.0), " ops/sec"
|
||||||
|
echo "Errors: ", totalErrors
|
||||||
|
|
||||||
|
if totalErrors > 0:
|
||||||
|
quit(1)
|
||||||
|
|
||||||
|
removeDir(baseDir)
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -3,6 +3,7 @@ import std/unittest
|
|||||||
import std/tables
|
import std/tables
|
||||||
import std/strutils
|
import std/strutils
|
||||||
import std/os
|
import std/os
|
||||||
|
import std/asyncdispatch
|
||||||
|
|
||||||
import barabadb/core/types
|
import barabadb/core/types
|
||||||
import barabadb/core/mvcc
|
import barabadb/core/mvcc
|
||||||
@@ -80,6 +81,7 @@ suite "Write-Ahead Log":
|
|||||||
|
|
||||||
suite "LSM-Tree Storage":
|
suite "LSM-Tree Storage":
|
||||||
test "Put and Get":
|
test "Put and Get":
|
||||||
|
removeDir("/tmp/baradb_test_lsm")
|
||||||
var db = newLSMTree("/tmp/baradb_test_lsm")
|
var db = newLSMTree("/tmp/baradb_test_lsm")
|
||||||
let key = "testkey"
|
let key = "testkey"
|
||||||
let value = cast[seq[byte]]("testvalue")
|
let value = cast[seq[byte]]("testvalue")
|
||||||
@@ -90,6 +92,7 @@ suite "LSM-Tree Storage":
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
test "Delete":
|
test "Delete":
|
||||||
|
removeDir("/tmp/baradb_test_lsm2")
|
||||||
var db = newLSMTree("/tmp/baradb_test_lsm2")
|
var db = newLSMTree("/tmp/baradb_test_lsm2")
|
||||||
let key = "delkey"
|
let key = "delkey"
|
||||||
let value = cast[seq[byte]]("delval")
|
let value = cast[seq[byte]]("delval")
|
||||||
@@ -100,6 +103,7 @@ suite "LSM-Tree Storage":
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
test "Contains":
|
test "Contains":
|
||||||
|
removeDir("/tmp/baradb_test_lsm3")
|
||||||
var db = newLSMTree("/tmp/baradb_test_lsm3")
|
var db = newLSMTree("/tmp/baradb_test_lsm3")
|
||||||
let key = "exists"
|
let key = "exists"
|
||||||
check not db.contains(key)
|
check not db.contains(key)
|
||||||
@@ -235,6 +239,34 @@ suite "Graph Engine":
|
|||||||
check dists[n2] == 1.0
|
check dists[n2] == 1.0
|
||||||
check dists[n3] == 3.0
|
check dists[n3] == 3.0
|
||||||
|
|
||||||
|
test "Save and load graph":
|
||||||
|
var g = gengine.newGraph()
|
||||||
|
let n1 = gengine.addNode(g, "Person", {"name": "Alice"}.toTable)
|
||||||
|
let n2 = gengine.addNode(g, "Person", {"name": "Bob"}.toTable)
|
||||||
|
let n3 = gengine.addNode(g, "City", {"name": "Sofia"}.toTable)
|
||||||
|
discard gengine.addEdge(g, n1, n2, "knows", {"since": "2020"}.toTable, 1.5)
|
||||||
|
discard gengine.addEdge(g, n2, n3, "lives_in", weight = 2.0)
|
||||||
|
|
||||||
|
let path = "/tmp/baradb_test_graph.bin"
|
||||||
|
removeFile(path)
|
||||||
|
gengine.saveToFile(g, path)
|
||||||
|
|
||||||
|
let g2 = gengine.loadFromFile(path)
|
||||||
|
check gengine.nodeCount(g2) == 3
|
||||||
|
check gengine.edgeCount(g2) == 2
|
||||||
|
check gengine.neighbors(g2, n1).len == 1
|
||||||
|
check gengine.neighbors(g2, n1)[0] == n2
|
||||||
|
check gengine.neighbors(g2, n2)[0] == n3
|
||||||
|
|
||||||
|
let loadedNode = gengine.getNode(g2, n1)
|
||||||
|
check loadedNode.label == "Person"
|
||||||
|
check loadedNode.properties["name"] == "Alice"
|
||||||
|
|
||||||
|
let sp = gengine.shortestPath(g2, n1, n3)
|
||||||
|
check sp.len == 3
|
||||||
|
|
||||||
|
removeFile(path)
|
||||||
|
|
||||||
suite "Full-Text Search":
|
suite "Full-Text Search":
|
||||||
test "Tokenization":
|
test "Tokenization":
|
||||||
let tokens = fts.tokenize("The quick brown fox jumps over the lazy dog")
|
let tokens = fts.tokenize("The quick brown fox jumps over the lazy dog")
|
||||||
@@ -1822,6 +1854,49 @@ suite "Raft Election Timer":
|
|||||||
let reply = cluster.nodes["n1"].handleRequestVote(req[0])
|
let reply = cluster.nodes["n1"].handleRequestVote(req[0])
|
||||||
check reply.success
|
check reply.success
|
||||||
|
|
||||||
|
suite "Raft Network Transport":
|
||||||
|
test "3-node election over TCP":
|
||||||
|
var n1 = newRaftNode("n1", @["n2", "n3"], raftPort = 19001)
|
||||||
|
var n2 = newRaftNode("n2", @["n1", "n3"], raftPort = 19002)
|
||||||
|
var n3 = newRaftNode("n3", @["n1", "n2"], raftPort = 19003)
|
||||||
|
|
||||||
|
n1.peerAddrs["n2"] = ("127.0.0.1", 19002)
|
||||||
|
n1.peerAddrs["n3"] = ("127.0.0.1", 19003)
|
||||||
|
n2.peerAddrs["n1"] = ("127.0.0.1", 19001)
|
||||||
|
n2.peerAddrs["n3"] = ("127.0.0.1", 19003)
|
||||||
|
n3.peerAddrs["n1"] = ("127.0.0.1", 19001)
|
||||||
|
n3.peerAddrs["n2"] = ("127.0.0.1", 19002)
|
||||||
|
|
||||||
|
let net1 = newRaftNetwork(n1)
|
||||||
|
let net2 = newRaftNetwork(n2)
|
||||||
|
let net3 = newRaftNetwork(n3)
|
||||||
|
|
||||||
|
asyncCheck net1.run()
|
||||||
|
asyncCheck net2.run()
|
||||||
|
asyncCheck net3.run()
|
||||||
|
waitFor sleepAsync(50)
|
||||||
|
|
||||||
|
var timer1 = newElectionTimer(n1, timeoutMs = 50)
|
||||||
|
var timer2 = newElectionTimer(n2, timeoutMs = 100)
|
||||||
|
var timer3 = newElectionTimer(n3, timeoutMs = 150)
|
||||||
|
|
||||||
|
for i in 0 ..< 30:
|
||||||
|
timer1.tick(net1)
|
||||||
|
timer2.tick(net2)
|
||||||
|
timer3.tick(net3)
|
||||||
|
waitFor sleepAsync(20)
|
||||||
|
|
||||||
|
net1.stop()
|
||||||
|
net2.stop()
|
||||||
|
net3.stop()
|
||||||
|
waitFor sleepAsync(50)
|
||||||
|
|
||||||
|
var leaderCount = 0
|
||||||
|
if n1.isLeader: inc leaderCount
|
||||||
|
if n2.isLeader: inc leaderCount
|
||||||
|
if n3.isLeader: inc leaderCount
|
||||||
|
check leaderCount == 1
|
||||||
|
|
||||||
suite "CLI Autocomplete":
|
suite "CLI Autocomplete":
|
||||||
test "Autocomplete commands":
|
test "Autocomplete commands":
|
||||||
let res = autocomplete("he")
|
let res = autocomplete("he")
|
||||||
|
|||||||
Reference in New Issue
Block a user