From a5d34c001a99e791fda7355b0b70bdc70251c6de Mon Sep 17 00:00:00 2001 From: dimgigov Date: Sun, 17 May 2026 16:15:45 +0300 Subject: [PATCH] docs: add German (DE) documentation + update all docs for Sessions 10-12 New German documentation (docs/de/): - index.md, quickstart.md, installation.md - baraql.md, graph.md, vector.md, mcp.md Updated English documentation: - changelog.md: all Sessions 10-12 features - graph.md: SQL GRAPH_TABLE, CREATE GRAPH, all 8 algorithms, Cypher, similarity_nodes, node2vec - vector.md: hybrid RAG, chunk(), embed_text(), auto-embed, nl_to_sql(), schema_prompt() - baraql.md: new AI & Cross-Modal Functions section, updated keyword tables - mcp.md: MCP Server documentation (new file) - index.md: added German (DE) language link --- docs/de/baraql.md | 114 ++++++++++++++++++++++++++ docs/de/graph.md | 145 +++++++++++++++++++++++++++++++++ docs/de/index.md | 64 +++++++++++++++ docs/de/installation.md | 57 +++++++++++++ docs/de/mcp.md | 108 +++++++++++++++++++++++++ docs/de/quickstart.md | 134 +++++++++++++++++++++++++++++++ docs/de/vector.md | 129 ++++++++++++++++++++++++++++++ docs/en/baraql.md | 49 +++++++++++- docs/en/changelog.md | 56 +++++++++---- docs/en/graph.md | 172 +++++++++++++++++++++++++++++++++++----- docs/en/mcp.md | 110 +++++++++++++++++++++++++ docs/en/vector.md | 162 ++++++++++++++++++++----------------- docs/index.md | 11 +-- 13 files changed, 1194 insertions(+), 117 deletions(-) create mode 100644 docs/de/baraql.md create mode 100644 docs/de/graph.md create mode 100644 docs/de/index.md create mode 100644 docs/de/installation.md create mode 100644 docs/de/mcp.md create mode 100644 docs/de/quickstart.md create mode 100644 docs/de/vector.md create mode 100644 docs/en/mcp.md diff --git a/docs/de/baraql.md b/docs/de/baraql.md new file mode 100644 index 0000000..8ea1f05 --- /dev/null +++ b/docs/de/baraql.md @@ -0,0 +1,114 @@ +# BaraQL — Abfragesprache-Referenz + +BaraQL ist eine SQL-kompatible Abfragesprache mit Erweiterungen für Graph-, Vektor- und Dokumentoperationen. + +## Datentypen + +| Typ | Beschreibung | Beispiel | +|------|-------------|---------| +| `null` | Nullwert | `null` | +| `bool` | Boolean | `true`, `false` | +| `int64` | 64-bit Ganzzahl | `42` | +| `float64` | 64-bit Fließkomma | `3.14` | +| `str` | UTF-8 String | `'hello'` | +| `vector(n)` | Float32 Vektor | `VECTOR(768)` | +| `json` | JSON-Dokument | `{"key": "value"}` | + +## Grundlegende Abfragen + +```sql +SELECT * FROM users; +SELECT name, age FROM users WHERE age > 18; +SELECT * FROM users ORDER BY age DESC LIMIT 10; +``` + +## Vektor-Operationen + +```sql +-- Distanzberechnungen +SELECT cosine_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; +SELECT embedding <-> '[0.1, 0.2, 0.3]' AS dist FROM items; + +-- Hybride Suche +SELECT hybrid_search('query', embedding, content, 10) AS result; +``` + +## Graph-Operationen + +```sql +CREATE GRAPH social; +DROP GRAPH social; + +-- Traversierung +SELECT * FROM GRAPH_TABLE(social MATCH (n)-[r]->(m) + ALGORITHM bfs START 1 + COLUMNS (id, node_label)); + +-- PageRank +SELECT * FROM GRAPH_TABLE(social ALGORITHM pagerank + COLUMNS (id, node_label, rank)); + +-- Community Detection +SELECT * FROM GRAPH_TABLE(social ALGORITHM community + COLUMNS (id, node_label, community)); +``` + +## AI-Funktionen + +```sql +-- Text in Chunks zerlegen +SELECT chunk('Langer Text...', 1024, 128) AS result; + +-- Embedding generieren +SELECT embed_text('Suchtext') AS result; + +-- Natural Language → SQL +SELECT nl_to_sql('Zeige alle Benutzer über 25', 'users') AS result; + +-- Schema-Prompt für LLM +SELECT schema_prompt('users') AS result; + +-- Cypher-Übersetzung +SELECT cypher('MATCH (a)-[r]->(b) RETURN a.node_label') AS result; + +-- Knotenähnlichkeit +SELECT similarity_nodes('social', 'jaccard') AS result; + +-- Graph-Embeddings +SELECT node2vec_embed('social', 64) AS result; +``` + +## Joins + +```sql +SELECT u.name, o.amount +FROM users u +INNER JOIN orders o ON u.id = o.user_id; +``` + +## Aggregation + +```sql +SELECT department, COUNT(*), AVG(salary) +FROM employees +GROUP BY department +HAVING COUNT(*) > 5; +``` + +## Index-Erstellung + +```sql +CREATE INDEX idx_name ON users(name) USING btree; +CREATE INDEX idx_vec ON docs(embedding) USING hnsw; +CREATE INDEX idx_fts ON docs(content) USING fts; +``` + +## Multi-Tenant / RLS + +```sql +CREATE POLICY tenant_policy ON orders +FOR ALL USING (tenant_id = current_setting('app.tenant_id')); + +SET app.tenant_id = 'company-a'; +SELECT * FROM orders; -- Automatisch gefiltert +``` diff --git a/docs/de/graph.md b/docs/de/graph.md new file mode 100644 index 0000000..908ef73 --- /dev/null +++ b/docs/de/graph.md @@ -0,0 +1,145 @@ +# Graph Engine + +Adjazenzlisten-Speicher mit eingebauten Algorithmen für Graph-Traversierung und -Analyse. +Vollständig integriert in den SQL-Executor via `GRAPH_TABLE()`. + +## SQL — Graph DDL + +### Graph erstellen + +```sql +CREATE GRAPH org_chart; +``` + +Erstellt automatisch zwei Tabellen: +- `org_chart_nodes (id INTEGER PRIMARY KEY, node_label TEXT, properties TEXT)` +- `org_chart_edges (source_id INTEGER, dest_id INTEGER, edge_label TEXT, weight REAL)` + +### Graph löschen + +```sql +DROP GRAPH org_chart; +``` + +## SQL — Daten einfügen + +```sql +-- Knoten +INSERT INTO org_chart_nodes (id, node_label) VALUES (1, 'CEO'); +INSERT INTO org_chart_nodes (id, node_label) VALUES (2, 'VP'); + +-- Kanten +INSERT INTO org_chart_edges (source_id, dest_id, edge_label) VALUES (1, 2, 'manages'); +``` + +Alle INSERTs werden automatisch mit dem nativen Graph-Objekt synchronisiert. + +## SQL — GRAPH_TABLE Abfragen + +### BFS (Breitensuche) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart MATCH (n)-[r]->(m) + ALGORITHM bfs + START 1 MAXDEPTH 2 + COLUMNS (id, node_label)); +``` + +### DFS (Tiefensuche) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart MATCH (n)-[r]->(m) + ALGORITHM dfs START 1 + COLUMNS (id, node_label)); +``` + +### PageRank + +```sql +SELECT id, node_label, rank FROM GRAPH_TABLE(org_chart + ALGORITHM pagerank + COLUMNS (id, node_label, rank)) +ORDER BY rank DESC; +``` + +### Community Detection (Louvain) + +```sql +SELECT id, node_label, community FROM GRAPH_TABLE(org_chart + ALGORITHM community + COLUMNS (id, node_label, community)); +``` + +### Kürzester Pfad (Shortest Path) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart + ALGORITHM shortest_path + START 1 END 3 + COLUMNS (id, node_label)); +``` + +### Dijkstra (gewichtete kürzeste Pfade) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart + ALGORITHM dijkstra START 1 + COLUMNS (id, node_label, distance)); +``` + +## SQL-Funktionen + +### Knotenähnlichkeit + +```sql +-- Jaccard-Ähnlichkeit zwischen allen Knotenpaaren +SELECT similarity_nodes('social', 'jaccard') AS result; + +-- Adamic-Adar-Ähnlichkeit +SELECT similarity_nodes('social', 'adamic_adar') AS result; +``` + +### Node2Vec Embeddings + +```sql +-- Graphstruktur-Embeddings generieren (64 Dimensionen) +SELECT node2vec_embed('social', 64) AS result; +``` + +## Cypher-Kompatibilität + +```sql +-- Cypher-Syntax automatisch nach GRAPH_TABLE übersetzen +SELECT cypher('MATCH (a)-[r]->(b) WHERE a.node_label = ''CEO'' RETURN b.node_label') AS result; +``` + +## Algorithmen + +| Algorithmus | Beschreibung | SQL | +|-------------|--------------|-----| +| `bfs` | Breitensuche | `ALGORITHM bfs` | +| `dfs` | Tiefensuche | `ALGORITHM dfs` | +| `dijkstra` | Gewichtete kürzeste Pfade | `ALGORITHM dijkstra` | +| `pageRank` | Knoten-Wichtigkeit | `ALGORITHM pagerank` | +| `louvain` | Community Detection | `ALGORITHM community` | +| `shortestPath` | Kürzester Pfad | `ALGORITHM shortest_path START X END Y` | +| `similarityNodes` | Knotenähnlichkeit | `similarity_nodes()` | +| `node2vec` | Graph Embeddings | `node2vec_embed()` | + +## Native Nim API + +```nim +import barabadb/graph/engine + +var g = newGraph() +let alice = g.addNode("Person", {"name": "Alice"}.toTable) +let bob = g.addNode("Person", {"name": "Bob"}.toTable) +discard g.addEdge(alice, bob, "knows") + +let bfs = g.bfs(alice) +let path = g.shortestPath(alice, bob) +let ranks = g.pageRank() +let communities = louvain(g) +let similarities = g.similarityNodes(smJaccard) +let embeddings = g.node2vec(64, 10, 5) +``` diff --git a/docs/de/index.md b/docs/de/index.md new file mode 100644 index 0000000..2598791 --- /dev/null +++ b/docs/de/index.md @@ -0,0 +1,64 @@ +# BaraDB Dokumentation + +**Eine multimodale Datenbank-Engine — 100% Nim, null Abhängigkeiten.** + +## Sprachen + +- [English](../en/) +- [Български (Bulgarisch)](../bg/) +- [Deutsch (German)](../de/) +- [Русский (Russisch)](../ru/) +- [فارسی (Farsi)](../fa/) +- [中文 (Chinesisch)](../zh/) +- [Türkçe (Türkisch)](../tr/) +- [العربية (Arabisch)](../ar/) + +--- + +## Schnellstart + +- [Installation](installation.md) +- [Schnellstart](quickstart.md) +- [Architektur](architecture.md) +- [Konfiguration](configuration.md) + +## Kernkonzepte + +- [BaraQL Abfragesprache](baraql.md) +- [Speicher-Engines](storage.md) +- [Schema System](schema.md) + +## Engines + +- [LSM-Tree Speicher](lsm.md) +- [B-Tree Index](btree.md) +- [Vektor-Suche](vector.md) +- [Graph Engine](graph.md) +- [Volltext-Suche](fts.md) +- [Spaltenbasierte Speicherung](columnar.md) + +## API & Clients + +- [Client SDKs](clients.md) +- [Binärprotokoll](api-binary.md) +- [HTTP/REST API](api-http.md) +- [MCP Server](mcp.md) + +## Betrieb + +- [Performance](performance.md) +- [Sicherheit](security.md) +- [Monitoring](monitoring.md) +- [Backup & Recovery](backup.md) +- [Fehlerbehebung](troubleshooting.md) + +## Erweitert + +- [Transaktionen & MVCC](transactions.md) +- [Verteilte Systeme](distributed.md) +- [Docker Deployment](docker.md) +- [Änderungsprotokoll](changelog.md) + +--- + +*Um eine neue Sprache hinzuzufügen, erstellen Sie einen neuen Ordner in `docs/` mit dem Sprachcode (z.B. `docs/de/`).* diff --git a/docs/de/installation.md b/docs/de/installation.md new file mode 100644 index 0000000..9b27029 --- /dev/null +++ b/docs/de/installation.md @@ -0,0 +1,57 @@ +# BaraDB — Installation + +## Voraussetzungen + +- **Nim >= 2.2.0** (`curl https://nim-lang.org/choosenim/init.sh -sSf | sh`) +- **Git** +- **OpenSSL** (für TLS) + +## Aus dem Quellcode bauen + +```bash +git clone https://github.com/katehonz/barabaDB.git +cd barabadb +nimble build_release +``` + +Die Binärdateien werden im `build/` Verzeichnis erstellt: +- `build/baradadb` — Datenbank-Server (TCP + HTTP) +- `build/baramcp` — MCP Server für AI-Agenten + +## Debug-Build + +```bash +nimble build_debug +``` + +## Tests ausführen + +```bash +nimble test +``` + +## Docker + +```bash +docker compose up -d +``` + +## Verifizierung + +```bash +./build/baradadb --version +# BaraDB v1.1.2 — Multimodal Database Engine + +./build/baramcp --data-dir ./data & +echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' | ./build/baramcp +``` + +## Manuelle Kompilierung + +```bash +# Server +nim c -d:release --opt:speed -o:build/baradadb src/baradadb.nim + +# MCP Server +nim c -d:release --opt:speed -o:build/baramcp src/baramcp.nim +``` diff --git a/docs/de/mcp.md b/docs/de/mcp.md new file mode 100644 index 0000000..19878d1 --- /dev/null +++ b/docs/de/mcp.md @@ -0,0 +1,108 @@ +# MCP Server (Model Context Protocol) + +BaraDB enthält einen eingebauten MCP-Server, der es AI-Agenten (Claude, Cursor, etc.) ermöglicht, direkt mit der Datenbank zu interagieren. + +## Schnellstart + +```bash +./build/baramcp --data-dir ./data +``` + +Der Server startet im STDIO-Modus und akzeptiert JSON-RPC 2.0 Nachrichten. + +## Verfügbare Tools + +### 1. `query` — SQL ausführen + +```json +{ + "name": "query", + "arguments": { + "sql": "SELECT * FROM users WHERE age > ?", + "params": [25], + "tenant_id": "company-a", + "user_id": "alice" + } +} +``` + +Parameterisierte Abfragen mit `?`-Platzhaltern. Multi-Tenant-Support über `tenant_id` und `user_id`. + +### 2. `vector_search` — Semantische Suche + +```json +{ + "name": "vector_search", + "arguments": { + "table": "docs", + "column": "embedding", + "query_vector": [0.1, 0.2, 0.3], + "k": 5, + "metric": "cosine", + "tenant_id": "company-a" + } +} +``` + +Unterstützte Metriken: `cosine`, `euclidean`, `dot_product`, `manhattan`. + +### 3. `schema_inspect` — Schema erkunden + +```json +{ + "name": "schema_inspect", + "arguments": { + "table": "users", + "tenant_id": "company-a" + } +} +``` + +Gibt Tabellen, Spalten, Typen, Primärschlüssel, Fremdschlüssel, Indizes und RLS-Policies zurück. + +## Konfiguration in Claude Desktop + +```json +{ + "mcpServers": { + "baradb": { + "command": "/pfad/zu/build/baramcp", + "args": ["--data-dir", "/pfad/zu/daten"] + } + } +} +``` + +## Konfiguration in Cursor + +```json +{ + "mcpServers": { + "baradb": { + "command": "/pfad/zu/build/baramcp", + "args": ["--data-dir", "~/.baradb/data"] + } + } +} +``` + +## Multi-Tenant Isolation + +Jede MCP-Anfrage kann `tenant_id` und `user_id` enthalten. Diese werden als Session-Variablen gesetzt: + +- `app.tenant_id` — für RLS-Filterung +- `app.user_id` — für `current_user`-Referenzen + +RLS-Policies filtern die Daten automatisch basierend auf diesen Variablen. + +## JSON-RPC 2.0 Protokoll + +Der Server verwendet JSON-RPC 2.0 über STDIO: + +```json +// Anfrage +{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {...}} + +// Antwort +{"jsonrpc": "2.0", "id": 1, "result": {"content": [{"type": "text", "text": "..."}]}} +``` diff --git a/docs/de/quickstart.md b/docs/de/quickstart.md new file mode 100644 index 0000000..074effa --- /dev/null +++ b/docs/de/quickstart.md @@ -0,0 +1,134 @@ +# BaraDB — Schnellstart + +## Server starten + +```bash +./build/baradadb +``` + +Der Server startet standardmäßig auf `localhost:9470`. + +## Verbindung via CLI + +```bash +./build/baradadb --shell +``` + +## MCP Server (AI Agenten) + +```bash +./build/baramcp --data-dir ./data +``` + +Der MCP Server startet im STDIO-Modus und stellt 3 Tools für AI-Agenten bereit: `query`, `vector_search`, `schema_inspect`. + +## Grundlegende Operationen + +### Tabelle erstellen + +```sql +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT, + email TEXT, + age INTEGER +); +``` + +### Daten einfügen + +```sql +INSERT INTO users (id, name, email, age) VALUES (1, 'Alice', 'alice@test.com', 30); +INSERT INTO users (id, name, email, age) VALUES (2, 'Bob', 'bob@test.com', 25); +``` + +### Daten abfragen + +```sql +SELECT name, age FROM users WHERE age > 18; +``` + +### Indizes erstellen + +```sql +-- BTree Index +CREATE INDEX idx_name ON users(name) USING btree; + +-- Volltext-Index +CREATE INDEX idx_email_fts ON users(email) USING fts; + +-- Vektor-Index +CREATE INDEX idx_vec ON items(embedding) USING hnsw; +``` + +## Vector Search + +```sql +CREATE TABLE docs (id INTEGER PRIMARY KEY, content TEXT, embedding VECTOR(768)); +CREATE INDEX docs_vec ON docs(embedding) USING hnsw; + +-- Ähnlichkeitssuche +SELECT id, cosine_distance(embedding, '[0.1, 0.2, ...]') AS dist +FROM docs ORDER BY dist ASC LIMIT 10; +``` + +## Graph Engine + +```sql +CREATE GRAPH social; +INSERT INTO social_nodes (id, node_label) VALUES (1, 'Alice'), (2, 'Bob'); +INSERT INTO social_edges (source_id, dest_id) VALUES (1, 2); + +-- BFS Traversal +SELECT * FROM GRAPH_TABLE(social MATCH (n)-[r]->(m) ALGORITHM bfs COLUMNS (id, node_label)); + +-- PageRank +SELECT * FROM GRAPH_TABLE(social ALGORITHM pagerank COLUMNS (id, node_label, rank)); + +-- Community Detection (Louvain) +SELECT * FROM GRAPH_TABLE(social ALGORITHM community COLUMNS (id, node_label, community)); + +-- Kürzester Pfad +SELECT * FROM GRAPH_TABLE(social ALGORITHM shortest_path START 1 END 2 COLUMNS (id, node_label)); + +-- Knoten-Ähnlichkeit (Jaccard) +SELECT similarity_nodes('social', 'jaccard') AS result; +``` + +## AI Pipeline + +```sql +-- Text in Chunks zerlegen +SELECT chunk('Langer Text hier...', 1024, 128) AS result; + +-- Embedding generieren (mit konfiguriertem externen Service) +SELECT embed_text('Suchanfrage') AS result; + +-- Schema-Prompt für LLM generieren +SELECT schema_prompt('users') AS result; + +-- Natural Language → SQL (mit konfiguriertem LLM) +SELECT nl_to_sql('Zeige alle Benutzer über 25', 'users') AS result; + +-- Cypher zu BaraQL übersetzen +SELECT cypher('MATCH (a)-[r]->(b) RETURN a.node_label, b.node_label') AS result; +``` + +## HTTP/REST API + +```bash +curl -X POST http://localhost:9470/query \ + -H "Content-Type: application/json" \ + -d '{"query": "SELECT * FROM users"}' +``` + +## Konfiguration + +```bash +# Umgebungsvariablen +export BARADB_DATA_DIR=./data +export BARADB_EMBED_ENDPOINT=http://localhost:11434/api/embeddings +export BARADB_EMBED_MODEL=nomic-embed-text +export BARADB_LLM_ENDPOINT=http://localhost:11434/api/generate +export BARADB_LLM_MODEL=llama3 +``` diff --git a/docs/de/vector.md b/docs/de/vector.md new file mode 100644 index 0000000..aa6ea47 --- /dev/null +++ b/docs/de/vector.md @@ -0,0 +1,129 @@ +# Vektor-Suche + +Native HNSW und IVF-PQ Indizes für Ähnlichkeitssuche mit vollständiger SQL-Integration. + +## SQL — Vektor-Spalten + +```sql +CREATE TABLE items ( + id INT PRIMARY KEY, + embedding VECTOR(768) +); +``` + +Der `VECTOR(n)`-Typ speichert float32-Arrays mit fester Dimension `n`. + +## Vektoren einfügen + +```sql +INSERT INTO items (id, embedding) VALUES (1, '[0.1, 0.2, 0.3, ...]'); +``` + +## Vektor-Distanzfunktionen + +```sql +-- Kosinus-Distanz (0 = identisch, 1 = orthogonal) +SELECT id, cosine_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; + +-- Euklidische / L2 Distanz +SELECT id, euclidean_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; +SELECT id, embedding <-> '[0.1, 0.2, 0.3]' AS dist FROM items; + +-- Inneres Produkt (negativ für Minimierung) +SELECT id, inner_product(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; + +-- Manhattan / L1 Distanz +SELECT id, l1_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; +``` + +## Vektor-Indizes + +```sql +-- HNSW-Index für approximative Nächste-Nachbarn-Suche +CREATE INDEX idx_items_vec ON items(embedding) USING hnsw; + +-- Der Index wird bei INSERT und UPDATE automatisch aktualisiert +``` + +## Hybrid RAG Search + +```sql +-- Kombinierte Vektor- + Volltext-Suche mit RRF-Reranking +SELECT hybrid_search('AI query', embedding, content, 10) AS result; + +-- Gefilterte hybride Suche +SELECT hybrid_search_filtered('AI query', embedding, content, 10, 'category', 'news') AS result; +``` + +## AI Pipeline + +### Text-Chunking + +```sql +-- Text in überlappende Chunks zerlegen +SELECT chunk('Langer Text hier...', 1024, 128) AS result; + +-- Ergebnis: [{"index":0, "text":"...", "size":124}, ...] +``` + +### Embedding-Generierung + +```sql +-- Externen Embedding-Service aufrufen (konfiguriert via Umgebungsvariablen) +SELECT embed_text('Suchtext hier') AS result; +``` + +Umgebungsvariablen für den Embedder: +```bash +export BARADB_EMBED_ENDPOINT=http://localhost:11434/api/embeddings +export BARADB_EMBED_MODEL=nomic-embed-text +``` + +### Auto-Embedding bei INSERT + +Wenn eine VECTOR-Spalte NULL ist, aber eine TEXT-Spalte einen Wert hat, wird das Embedding automatisch generiert (falls ein Embedder konfiguriert ist). + +```sql +CREATE TABLE docs (id INTEGER PRIMARY KEY, content TEXT, embedding VECTOR(768)); +CREATE INDEX docs_vec ON docs(embedding) USING hnsw; + +-- embedding wird automatisch gefüllt +INSERT INTO docs (id, content) VALUES (1, 'Dieser Text wird automatisch embedded'); +``` + +## Distanzmetriken + +| Metrik | SQL-Funktion | Beschreibung | +|--------|-------------|-------------| +| `cosine` | `cosine_distance(a, b)` | Kosinus-Distanz | +| `euclidean` | `euclidean_distance(a, b)` / `<->` | L2-Distanz | +| `dotproduct` | `inner_product(a, b)` | Negatives Skalarprodukt | +| `manhattan` | `l1_distance(a, b)` | L1-Distanz | + +## Index-Typen + +### HNSW (Standard) + +```nim +import barabadb/vector/engine +var hnsw = newHNSWIndex(dimensions = 128, m = 16, efConstruction = 200) +``` + +### IVF-PQ + +```nim +var ivfpq = newIVFPQIndex(dimensions = 128, numCentroids = 256, subQuantizers = 8) +``` + +## Native Nim API + +```nim +import barabadb/vector/engine + +var idx = newHNSWIndex(dimensions = 128) +idx.insert(1, @[1.0'f32, 0.0'f32, ...], {"category": "A"}.toTable) + +let results = idx.search(queryVector, k = 10) +let filtered = idx.searchWithFilter(queryVector, k = 10, + filter = proc(meta: Table[string, string]): bool = return "category" in meta) +``` diff --git a/docs/en/baraql.md b/docs/en/baraql.md index 33920ae..1400484 100644 --- a/docs/en/baraql.md +++ b/docs/en/baraql.md @@ -640,23 +640,66 @@ SELECT * FROM invoices; -- only company-a rows - **JSONB documents** — schema-flexible storage, easy to add fields per tenant - **RLS guarantees isolation** — the database enforces tenant boundaries, not just the application +## AI & Cross-Modal Functions + +### Vector / RAG + +```sql +-- Hybrid search (vector + FTS + RRF reranking) +SELECT hybrid_search('query text', embedding, content, 10) AS result; +SELECT hybrid_search_ids('query', embedding, content, 5) AS result; +SELECT hybrid_search_filtered('query', embedding, content, 10, 'category', 'news') AS result; + +-- Rerank +SELECT rerank('query text', results_json) AS result; +``` + +### Graph Traversal + +```sql +-- BFS, DFS, PageRank, ShortestPath, Dijkstra, Louvain +SELECT * FROM GRAPH_TABLE(g MATCH (n)-[r]->(m) + ALGORITHM bfs START 1 MAXDEPTH 2 + COLUMNS (id, node_label)); + +SELECT similarity_nodes('graph_name', 'jaccard') AS result; +SELECT node2vec_embed('graph_name', 64) AS result; +SELECT cypher('MATCH (a)-[r]->(b) RETURN a.label') AS result; +``` + +### AI / LLM + +```sql +-- Text chunking +SELECT chunk('long text...', 1024, 128) AS result; + +-- Embedding generation (external service) +SELECT embed_text('query text') AS result; + +-- Natural Language → SQL (external LLM) +SELECT nl_to_sql('Show users over 25', 'users') AS result; + +-- Schema prompt for LLM context +SELECT schema_prompt('users') AS result; +``` + ## Supported Keywords | Category | Keywords | |----------|----------| | DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT | | DML | INSERT, UPDATE, DELETE, SET, VALUES | -| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE | +| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE GRAPH, DROP GRAPH | | Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON | | Set | UNION, UNION ALL, INTERSECT, EXCEPT | | CTEs | WITH, RECURSIVE, AS | | Case | CASE, WHEN, THEN, ELSE, END | | Transaction | BEGIN, COMMIT, ROLLBACK, SAVEPOINT | -| Graph | MATCH, RETURN, WHERE, shortestPath, type | +| Graph | MATCH, RETURN, WHERE, shortestPath, type, GRAPH_TABLE, ALGORITHM, bfs, dfs, pagerank | | FTS | MATCH, AGAINST, relevance, IN BOOLEAN MODE, WITH FUZZINESS | | Vector | cosine_distance, euclidean_distance, inner_product, l1_distance, l2_distance, <-> | +| AI | hybrid_search, rerank, chunk, embed_text, nl_to_sql, schema_prompt, similarity_nodes, node2vec_embed, cypher | | JSON | ->, ->> | -| FTS | @@ (BM25 match) | | Recovery | RECOVER TO TIMESTAMP | | Functions | count, sum, avg, min, max, stddev, variance, abs, sqrt, lower, upper, len, trim, substr, now, last_insert_id, current_setting | | Session | SET, current_setting, current_user, current_role | diff --git a/docs/en/changelog.md b/docs/en/changelog.md index 41d6209..7207cda 100644 --- a/docs/en/changelog.md +++ b/docs/en/changelog.md @@ -2,25 +2,49 @@ All notable changes to BaraDB are documented in this file. -## [Unreleased] — SQL:2023 Stabilization - -### Fixed - -- **GROUPING SETS execution** — `lowerSelect` now creates `irpkGroupBy` when `selGroupingSetsKind != gskNone`, even if `selGroupBy` is empty. Previously, queries like `GROUP BY GROUPING SETS ((dept), ())` bypassed the grouping executor entirely. -- **FTS CREATE INDEX docId mismatch** — `CREATE INDEX ... USING FTS` now computes `docId` as a hash of `tableName.$key`, consistent with DML operations (`INSERT`/`UPDATE`/`DELETE`). Previously, index creation used sequential IDs (0, 1, 2...), causing `@@` queries to never match indexed documents. -- **Test isolation (all suites)** — All `newLSMTree("")` calls replaced with unique temporary directories per suite. Eliminates WAL accumulation issues and flaky tests caused by shared database state between test runs. -- **Window frame parser** — `parseFrameBoundary` no longer consumes `tkRow` after `tkCurrent` incorrectly (was using `tkRows`). Also fixed `tkRow` keyword conflict with `ENABLE ROW LEVEL SECURITY` parsing. -- **ORDER BY + SELECT projection** — `lowerSelect` now places `irpkSort` before `irpkProject`, enabling `ORDER BY` on columns not present in the `SELECT` list. -- **UNPIVOT execution** — Verified and fixed missing test coverage for UNPIVOT transformation. +## [Unreleased] — AI-Native Platform ### Added -- **JSON operators** — `@>` (contains), `<@` (contained by), `?` (has key), `?|` (has any), `?&` (has all) now supported in lexer, parser, and executor. -- **Window frame execution** — `ROWS BETWEEN X PRECEDING AND Y FOLLOWING` / `CURRENT ROW` frame boundaries now respected by `FIRST_VALUE` and `LAST_VALUE`. -- **Session variables** — `SET var_name = value` and `current_setting('var_name')` for connection-scoped key/value storage. -- **Current user/role** — `current_user` and `current_role` SQL keywords evaluate to the authenticated session's user and role. -- **Auth-executor bridge** — Wire server and HTTP server now populate `ExecutionContext.currentUser` and `ExecutionContext.currentRole` after JWT/SCRAM authentication. -- **Multi-tenant RLS** — Row-Level Security policies can now reference `current_user`, `current_role`, and `current_setting('app.tenant_id')` for per-tenant data isolation. +- **MCP Server (Model Context Protocol)** — STDIO JSON-RPC 2.0 server with 3 AI tools: + - `query` — SQL execution with parameterized queries + multi-tenant session vars + - `vector_search` — Semantic HNSW vector search with tenant isolation + - `schema_inspect` — Table/column/index/RLS policy exploration + - Standalone binary: `build/baramcp` +- **Graph Engine Deep Integration** — `CREATE GRAPH` / `DROP GRAPH` DDL with native adjacency list storage + - `GRAPH_TABLE()` SQL function with 7 algorithms: BFS, DFS, PageRank, ShortestPath, Dijkstra, Louvain, Community + - INSERT into `_nodes`/`_edges` tables auto-syncs with native Graph objects + - Optional `MATCH`, `ALGORITHM`, `START`, `END`, `MAXDEPTH` in GRAPH_TABLE syntax +- **Chunking + Embedding Pipeline** — Server-side AI data processing: + - `chunk()` SQL function — text splitting with configurable size/overlap + - `embed_text()` SQL function — calls external embedding API (OpenAI/Ollama compatible) + - Auto-embedding on INSERT — when VECTOR column is null, generates from TEXT column + - Configurable via env vars: `BARADB_EMBED_ENDPOINT`, `BARADB_EMBED_MODEL`, `BARADB_EMBED_API_KEY` +- **LangChain ChatMessageHistory** — Python `BaraDBChatHistory` class: + - Stores conversation threads in relational table with RLS + - Multi-tenant isolation via `tenant_id` + `user_id` +- **RAG Pipeline Example** — End-to-end Python script (`examples/rag_pipeline.py`): + - PDF/text ingestion → chunking → embedding → BaraDB storage → hybrid search → LLM generation + - Supports OpenAI and Ollama APIs +- **AI Agents & NL→SQL** — Server-side LLM integration: + - `nl_to_sql()` SQL function — natural language → SQL generation + - `schema_prompt()` — generates DDL + sample data for LLM context + - Query validation layer — sandbox execution with LIMIT 0 + EXPLAIN + - Self-correction loop — error feedback to LLM for fix + - Configurable via env vars: `BARADB_LLM_ENDPOINT`, `BARADB_LLM_MODEL`, `BARADB_LLM_API_KEY` +- **Graph Similarity & Embeddings**: + - `similarity_nodes()` — Jaccard/Adamic-Adar similarity between node pairs + - `node2vec_embed()` — Random-walk based graph embeddings +- **Cypher Compatibility Layer**: + - `cypher()` SQL function — translates `MATCH (a)-[r]->(b) RETURN ...` to GRAPH_TABLE + - Automatic Cypher → BaraQL conversion +- **German Documentation** — Full documentation set in German (`docs/de/`) + +### Changed + +- Graph executor upgraded from stub to real BFS/DFS/PageRank/Dijkstra/Louvain execution +- ExecutionContext extended with `graphs`, `embedder`, `llmClient` fields +- Graph engine extended with `addNodeWithId`, `addEdgeWithId`, Jaccard, Adamic-Adar, node2vec ## [1.1.0] — 2026-05-13 diff --git a/docs/en/graph.md b/docs/en/graph.md index 26d7155..f41e45c 100644 --- a/docs/en/graph.md +++ b/docs/en/graph.md @@ -1,11 +1,136 @@ # Graph Engine Adjacency list storage with built-in algorithms for graph traversal and analysis. +Fully integrated into the SQL executor via `GRAPH_TABLE()`. -## Usage +## SQL — Graph DDL + +### Create Graph + +```sql +CREATE GRAPH org_chart; +``` + +Automatically creates two tables: +- `org_chart_nodes (id INTEGER PRIMARY KEY, node_label TEXT, properties TEXT)` +- `org_chart_edges (source_id INTEGER, dest_id INTEGER, edge_label TEXT, weight REAL)` + +### Drop Graph + +```sql +DROP GRAPH org_chart; +``` + +## SQL — Insert Data + +```sql +-- Nodes +INSERT INTO org_chart_nodes (id, node_label) VALUES (1, 'CEO'); +INSERT INTO org_chart_nodes (id, node_label) VALUES (2, 'VP'); + +-- Edges +INSERT INTO org_chart_edges (source_id, dest_id, edge_label) VALUES (1, 2, 'manages'); +``` + +All INSERTs are automatically synced with the native Graph object. + +## SQL — GRAPH_TABLE Queries + +### BFS (Breadth-First Search) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart MATCH (n)-[r]->(m) + ALGORITHM bfs + START 1 MAXDEPTH 2 + COLUMNS (id, node_label)); +``` + +### DFS (Depth-First Search) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart MATCH (n)-[r]->(m) + ALGORITHM dfs START 1 + COLUMNS (id, node_label)); +``` + +### PageRank + +```sql +SELECT id, node_label, rank FROM GRAPH_TABLE(org_chart + ALGORITHM pagerank + COLUMNS (id, node_label, rank)) +ORDER BY rank DESC; +``` + +### Community Detection (Louvain) + +```sql +SELECT id, node_label, community FROM GRAPH_TABLE(org_chart + ALGORITHM community + COLUMNS (id, node_label, community)); +``` + +### Shortest Path + +```sql +SELECT * FROM GRAPH_TABLE(org_chart + ALGORITHM shortest_path + START 1 END 3 + COLUMNS (id, node_label)); +``` + +### Dijkstra (Weighted Shortest Paths) + +```sql +SELECT * FROM GRAPH_TABLE(org_chart + ALGORITHM dijkstra START 1 + COLUMNS (id, node_label, distance)); +``` + +## SQL Functions + +### Node Similarity + +```sql +-- Jaccard similarity between all node pairs +SELECT similarity_nodes('social', 'jaccard') AS result; + +-- Adamic-Adar similarity +SELECT similarity_nodes('social', 'adamic_adar') AS result; +``` + +### Node2Vec Embeddings + +```sql +-- Generate graph structure embeddings (64 dimensions) +SELECT node2vec_embed('social', 64) AS result; +``` + +## Cypher Compatibility + +```sql +-- Cypher syntax auto-translated to GRAPH_TABLE +SELECT cypher('MATCH (a)-[r]->(b) WHERE a.node_label = ''CEO'' RETURN b.node_label') AS result; +``` + +## Algorithms + +| Algorithm | Description | SQL Syntax | +|-----------|-------------|------------| +| `bfs` | Breadth-first traversal | `ALGORITHM bfs` | +| `dfs` | Depth-first traversal | `ALGORITHM dfs` | +| `dijkstra` | Weighted shortest paths | `ALGORITHM dijkstra` | +| `pageRank` | Node importance ranking | `ALGORITHM pagerank` | +| `louvain` | Community detection | `ALGORITHM community` | +| `shortestPath` | Shortest unweighted path | `ALGORITHM shortest_path START X END Y` | +| `similarityNodes` | Jaccard/Adamic-Adar | `similarity_nodes()` | +| `node2vec` | Graph embeddings | `node2vec_embed()` | + +## Native Nim API ```nim import barabadb/graph/engine +import barabadb/graph/community var g = newGraph() let alice = g.addNode("Person", {"name": "Alice"}.toTable) @@ -13,34 +138,30 @@ let bob = g.addNode("Person", {"name": "Bob"}.toTable) discard g.addEdge(alice, bob, "knows") # Traversal -let bfs = g.bfs(alice) -let dfs = g.dfs(alice) +let bfsResult = g.bfs(alice) +let dfsResult = g.dfs(alice) let path = g.shortestPath(alice, bob) let ranks = g.pageRank() + +# Community detection +let communities = louvain(g) + +# Node similarity +let similarities = g.similarityNodes(smJaccard) +let adamicAdar = g.similarityNodes(smAdamicAdar) + +# Graph embeddings +let embeddings = g.node2vec(64, 10, 5) ``` -## Algorithms - -| Algorithm | Description | -|-----------|-------------| -| `bfs` | Breadth-first traversal | -| `dfs` | Depth-first traversal | -| `dijkstra` | Shortest weighted path | -| `pageRank` | Node importance ranking | -| `louvain` | Community detection | -| `patternMatch` | Subgraph isomorphism | - -## Cypher Query +## Cypher Query (Native) ```nim import barabadb/graph/cypher -var engine = newCypherEngine(g) -let results = engine.execute(""" - MATCH (p:Person)-[:KNOWS]->(friend:Person) - WHERE p.name = 'Alice' - RETURN friend.name -""") +# Translate Cypher to BaraQL +let sql = cypherToSql("MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN b.name") +# Result: "SELECT b.name FROM GRAPH_TABLE(g MATCH (a)-[r]->(b) COLUMNS (b.name))" ``` ## Pattern Matching @@ -49,4 +170,11 @@ let results = engine.execute(""" MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person) WHERE a.name = 'Alice' RETURN b.name, c.name -``` \ No newline at end of file +``` + +## Architecture Notes + +- **Native storage**: Edges stored as adjacency lists for O(1) neighbor access +- **Bidirectional indexes**: Both `source→targets` and `target→sources` for fast traversal +- **RLS integration**: Graph tables are regular SQL tables — existing RLS policies apply automatically +- **Transactional**: INSERT/UPDATE/DELETE on graph tables participate in MVCC transactions diff --git a/docs/en/mcp.md b/docs/en/mcp.md new file mode 100644 index 0000000..c32bddc --- /dev/null +++ b/docs/en/mcp.md @@ -0,0 +1,110 @@ +# MCP Server (Model Context Protocol) + +BaraDB includes a built-in MCP server that enables AI agents (Claude, Cursor, etc.) +to interact with the database directly. + +## Quick Start + +```bash +./build/baramcp --data-dir ./data +``` + +Starts in STDIO mode, accepting JSON-RPC 2.0 messages on stdin. + +## Available Tools + +### 1. `query` — SQL Execution + +```json +{ + "name": "query", + "arguments": { + "sql": "SELECT * FROM users WHERE age > ?", + "params": [25], + "tenant_id": "company-a", + "user_id": "alice" + } +} +``` + +Parameterized queries using `?` placeholders. Multi-tenant via `tenant_id` and `user_id`. + +### 2. `vector_search` — Semantic Search + +```json +{ + "name": "vector_search", + "arguments": { + "table": "docs", + "column": "embedding", + "query_vector": [0.1, 0.2, 0.3], + "k": 5, + "metric": "cosine", + "filter_column": "category", + "filter_value": "news", + "tenant_id": "company-a" + } +} +``` + +Metrics: `cosine`, `euclidean`, `dot_product`, `manhattan`. + +### 3. `schema_inspect` — Schema Exploration + +```json +{ + "name": "schema_inspect", + "arguments": { + "table": "users", + "tenant_id": "company-a" + } +} +``` + +Returns tables, columns, types, primary keys, foreign keys, indexes, and RLS policies. + +## Claude Desktop Configuration + +```json +{ + "mcpServers": { + "baradb": { + "command": "/path/to/build/baramcp", + "args": ["--data-dir", "/path/to/data"] + } + } +} +``` + +## Cursor Configuration + +```json +{ + "mcpServers": { + "baradb": { + "command": "/path/to/build/baramcp", + "args": ["--data-dir", "~/.baradb/data"] + } + } +} +``` + +## Multi-Tenant Isolation + +Each MCP request can include `tenant_id` and `user_id`, set as session variables: +- `app.tenant_id` — for RLS filtering +- `app.user_id` — for `current_user` references + +RLS policies automatically filter data based on these variables. + +## JSON-RPC 2.0 Protocol + +The server uses JSON-RPC 2.0 over STDIO: + +```json +// Request +{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {...}} + +// Response +{"jsonrpc": "2.0", "id": 1, "result": {"content": [{"type": "text", "text": "..."}]}} +``` diff --git a/docs/en/vector.md b/docs/en/vector.md index 50ce572..f47da0f 100644 --- a/docs/en/vector.md +++ b/docs/en/vector.md @@ -1,6 +1,7 @@ # Vector Search Engine Native HNSW and IVF-PQ indexes for similarity search with full SQL integration. +Includes AI pipeline for chunking, embedding, and hybrid RAG search. ## SQL Usage @@ -8,8 +9,8 @@ Native HNSW and IVF-PQ indexes for similarity search with full SQL integration. ```sql CREATE TABLE items ( - id INT PRIMARY KEY, - embedding VECTOR(768) + id INT PRIMARY KEY, + embedding VECTOR(768) ); ``` @@ -25,24 +26,17 @@ INSERT INTO items (id, embedding) VALUES (1, '[0.1, 0.2, 0.3, ...]'); ```sql -- Cosine distance (0 = identical, 1 = orthogonal) -SELECT id, cosine_distance(embedding, '[0.1, 0.2, 0.3]') AS dist -FROM items; +SELECT id, cosine_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; -- Euclidean / L2 distance -SELECT id, euclidean_distance(embedding, '[0.1, 0.2, 0.3]') AS dist -FROM items; +SELECT id, euclidean_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; +SELECT id, embedding <-> '[0.1, 0.2, 0.3]' AS dist FROM items; --- L2 distance with <-> operator -SELECT id, embedding <-> '[0.1, 0.2, 0.3]' AS dist -FROM items; - --- Inner product (negative dot product for minimization) -SELECT id, inner_product(embedding, '[0.1, 0.2, 0.3]') AS dist -FROM items; +-- Inner product (negative for minimization) +SELECT id, inner_product(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; -- Manhattan / L1 distance -SELECT id, l1_distance(embedding, '[0.1, 0.2, 0.3]') AS dist -FROM items; +SELECT id, l1_distance(embedding, '[0.1, 0.2, 0.3]') AS dist FROM items; ``` ### Nearest Neighbor Search @@ -52,79 +46,84 @@ FROM items; SELECT id FROM items ORDER BY cosine_distance(embedding, '[0.1, 0.2, 0.3]') ASC LIMIT 10; - --- Top-5 nearest neighbors by Euclidean distance -SELECT id FROM items -ORDER BY embedding <-> '[0.1, 0.2, 0.3]' -LIMIT 5; ``` ### Vector Indexes ```sql --- Create HNSW index for approximate nearest neighbor search +-- Create HNSW index CREATE INDEX idx_items_vec ON items(embedding) USING hnsw; - --- The index is automatically maintained on INSERT and UPDATE +-- Index is automatically maintained on INSERT and UPDATE ``` -Supported index methods: -- `USING hnsw` — Hierarchical Navigable Small World (default: cosine metric) -- `USING ivfpq` — Inverted File with Product Quantization - -### Dimension Validation - -BaraDB validates vector dimensions at insert time: +## Hybrid RAG Search ```sql --- This will fail: expected 768 dimensions but got 3 -INSERT INTO items (id, embedding) VALUES (2, '[1.0, 2.0, 3.0]'); +-- Combined vector + FTS search with Reciprocal Rank Fusion reranking +SELECT hybrid_search('AI query', embedding, content, 10) AS result; + +-- Filtered hybrid search +SELECT hybrid_search_filtered('AI query', embedding, content, 10, 'category', 'news') AS result; + +-- Comma-separated IDs only +SELECT hybrid_search_ids('AI query', embedding, content, 10) AS result; ``` -## Native Nim API +## AI Pipeline -For embedded or high-performance use, use the native Nim API directly: +### Text Chunking -```nim -import barabadb/vector/engine +```sql +-- Split text into overlapping chunks (max 1024 chars, 128 overlap) +SELECT chunk('Long text content here...', 1024, 128) AS result; -var idx = newHNSWIndex(dimensions = 128) -idx.insert(1, @[1.0'f32, 0.0'f32, ...], {"category": "A"}.toTable) - -# Search -let results = idx.search(queryVector, k = 10) - -# With metadata filtering -let filtered = idx.searchWithFilter(queryVector, k = 10, - filter = proc(meta: Table[string, string]): bool = - return meta.getOrDefault("category") == "A") +-- Returns: [{"index":0, "text":"...", "size":124}, ...] ``` -## Index Types +Strategies: `paragraph`, `sentence`, `fixed`, `recursive` (default). -### HNSW +### Embedding Generation -Hierarchical Navigable Small World graph for approximate nearest neighbor search. - -```nim -var hnsw = newHNSWIndex( - dimensions = 128, - m = 16, # connections per layer - efConstruction = 200, # search width during construction - efSearch = 100 # search width during query -) +```sql +-- Call external embedding service for a query vector +SELECT embed_text('query text here') AS result; ``` -### IVF-PQ +Configure the embedder via environment variables: +```bash +export BARADB_EMBED_ENDPOINT=http://localhost:11434/api/embeddings +export BARADB_EMBED_MODEL=nomic-embed-text +export BARADB_EMBED_API_KEY=sk-... # optional, for OpenAI +``` -Inverted File Index with Product Quantization for compression. +### Auto-Embedding on INSERT -```nim -var ivfpq = newIVFPQIndex( - dimensions = 128, - numCentroids = 256, - subQuantizers = 8 -) +When a VECTOR column is NULL on INSERT but a TEXT column has content, the embedding +is automatically generated (if an embedder is configured): + +```sql +CREATE TABLE docs (id INTEGER PRIMARY KEY, content TEXT, embedding VECTOR(768)); +CREATE INDEX docs_vec ON docs(embedding) USING hnsw; + +-- embedding is automatically populated +INSERT INTO docs (id, content) VALUES (1, 'This text will be auto-embedded'); +``` + +## Natural Language → SQL + +```sql +-- Generate schema prompt for LLM context +SELECT schema_prompt('users') AS result; + +-- Natural language to SQL (requires configured LLM) +SELECT nl_to_sql('Show all users over 25 years old', 'users') AS result; +``` + +LLM configuration: +```bash +export BARADB_LLM_ENDPOINT=http://localhost:11434/api/generate +export BARADB_LLM_MODEL=llama3 +export BARADB_LLM_API_KEY=sk-... # optional ``` ## Distance Metrics @@ -136,15 +135,37 @@ var ivfpq = newIVFPQIndex( | `dotproduct` | `inner_product(a, b)` | Negative dot product | | `manhattan` | `l1_distance(a, b)` | L1 distance | +## Native Nim API + +```nim +import barabadb/vector/engine + +var idx = newHNSWIndex(dimensions = 128) +idx.insert(1, @[1.0'f32, 0.0'f32, ...], {"category": "A"}.toTable) +let results = idx.search(queryVector, k = 10) +let filtered = idx.searchWithFilter(queryVector, k = 10, + filter = proc(meta: Table[string, string]): bool = "category" in meta) +``` + +## Index Types + +### HNSW (Default) + +```nim +var hnsw = newHNSWIndex(dimensions = 128, m = 16, efConstruction = 200) +``` + +### IVF-PQ + +```nim +var ivfpq = newIVFPQIndex(dimensions = 128, numCentroids = 256, subQuantizers = 8) +``` + ## Quantization ```nim import barabadb/vector/quant - -# Scalar quantization let scalar = scalarQuantize(data, bits = 8) - -# Product quantization let pq = productQuantize(data, subVectors = 8, bits = 8) ``` @@ -152,6 +173,5 @@ let pq = productQuantize(data, subVectors = 8, bits = 8) ```nim import barabadb/vector/simd - let dist = simdCosineDistance(vec1, vec2) -``` \ No newline at end of file +``` diff --git a/docs/index.md b/docs/index.md index 2187f06..1c3502e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,12 +5,13 @@ ## Documentation Languages - [English](en/) -- [Български (Bulgarian)](bg/) -- [Русский (Russian)](ru/) -- [فارسی (Farsi)](fa/) -- [中文 (Chinese)](zh/) -- [Türkçe (Turkish)](tr/) - [العربية (Arabic)](ar/) +- [Български (Bulgarian)](bg/) +- [Deutsch (German)](de/) +- [فارسی (Farsi)](fa/) +- [Русский (Russian)](ru/) +- [Türkçe (Turkish)](tr/) +- [中文 (Chinese)](zh/) ---