docs: synchronize documentation across all languages
- Add mcp.md to: bg, fa, ru, tr, zh, ar - Add index.md to Bulgarian (bg) - Add 24 missing German (de) documentation files Translations for all supported languages now complete.
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# Binärprotokoll API
|
||||
|
||||
Niedrigstufiges Wire-Protokoll für hochperformante Client-Verbindungen.
|
||||
|
||||
## Nachrichtenformat
|
||||
|
||||
Alle Nachrichten verwenden Big-Endian-Byte-Reihenfolge:
|
||||
|
||||
```
|
||||
┌────────┬────────┬────────┬────────┬─────────────┐
|
||||
│ Length │ Type │ Seq │ Status │ Payload │
|
||||
│ 4 bytes│ 1 byte │ 2 bytes│ 1 byte │ N bytes │
|
||||
└────────┴────────┴────────┴────────┴─────────────┘
|
||||
```
|
||||
|
||||
## Nachrichtentypen
|
||||
|
||||
### Query (0x01)
|
||||
|
||||
```nim
|
||||
let msg = makeQueryMessage(seq, "SELECT * FROM users")
|
||||
```
|
||||
|
||||
### Insert (0x02)
|
||||
|
||||
```nim
|
||||
let msg = makeInsertMessage(seq, "users", data)
|
||||
```
|
||||
|
||||
### Update (0x03)
|
||||
|
||||
```nim
|
||||
let msg = makeUpdateMessage(seq, "users", updates, where)
|
||||
```
|
||||
|
||||
### Delete (0x04)
|
||||
|
||||
```nim
|
||||
let msg = makeDeleteMessage(seq, "users", where)
|
||||
```
|
||||
|
||||
### Ready (0x05)
|
||||
|
||||
```nim
|
||||
let msg = makeReadyMessage(seq)
|
||||
```
|
||||
|
||||
### Error (0x06)
|
||||
|
||||
```nim
|
||||
let msg = makeErrorMessage(seq, code, message)
|
||||
```
|
||||
|
||||
## Antwortcodes
|
||||
|
||||
| Code | Name | Beschreibung |
|
||||
|------|------|-------------|
|
||||
| 0x00 | OK | Erfolg |
|
||||
| 0x01 | ERROR | Allgemeiner Fehler |
|
||||
| 0x02 | AUTH_REQUIRED | Authentifizierung erforderlich |
|
||||
| 0x03 | INVALID_QUERY | Abfragesyntaxfehler |
|
||||
| 0x04 | NOT_FOUND | Ressource nicht gefunden |
|
||||
|
||||
## Serialisierung
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/wire
|
||||
|
||||
# Wert serialisieren
|
||||
let bytes = serializeValue(Value(kind: vkString, strVal: "test"))
|
||||
|
||||
# Wert deserialisieren
|
||||
let value = deserializeValue(bytes)
|
||||
```
|
||||
@@ -0,0 +1,91 @@
|
||||
# HTTP/REST API
|
||||
|
||||
JSON-basierte REST API für Web-Anwendungen.
|
||||
|
||||
## Basis-URL
|
||||
|
||||
```
|
||||
http://localhost:9470/api
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
### GET /api/users
|
||||
|
||||
Alle Benutzer auflisten:
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/api/users
|
||||
```
|
||||
|
||||
Antwort:
|
||||
```json
|
||||
[
|
||||
{"id": 1, "name": "Alice", "age": 30},
|
||||
{"id": 2, "name": "Bob", "age": 25}
|
||||
]
|
||||
```
|
||||
|
||||
### GET /api/users/:id
|
||||
|
||||
Benutzer nach ID abrufen:
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/api/users/1
|
||||
```
|
||||
|
||||
### POST /api/users
|
||||
|
||||
Benutzer erstellen:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:9470/api/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Charlie", "age": 35}'
|
||||
```
|
||||
|
||||
### PUT /api/users/:id
|
||||
|
||||
Benutzer aktualisieren:
|
||||
|
||||
```bash
|
||||
curl -X PUT http://localhost:9470/api/users/1 \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Alice", "age": 31}'
|
||||
```
|
||||
|
||||
### DELETE /api/users/:id
|
||||
|
||||
Benutzer löschen:
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:9470/api/users/1
|
||||
```
|
||||
|
||||
## Query-Endpoint
|
||||
|
||||
BaraQL-Abfragen über HTTP ausführen:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:9470/api/query \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"sql": "SELECT * FROM users WHERE age > 18"}'
|
||||
```
|
||||
|
||||
## Fehlerantwort
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "INVALID_QUERY",
|
||||
"message": "Syntax error at line 1"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Authentifizierung
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer <token>" \
|
||||
http://localhost:9470/api/users
|
||||
```
|
||||
@@ -0,0 +1,120 @@
|
||||
# WebSocket API
|
||||
|
||||
Vollduplex-Streaming für Echtzeit-Datenfeeds und Push-Benachrichtigungen.
|
||||
|
||||
## Verbindung
|
||||
|
||||
```
|
||||
ws://localhost:9471/ws
|
||||
```
|
||||
|
||||
## Client-Beispiel
|
||||
|
||||
```javascript
|
||||
const ws = new WebSocket('ws://localhost:9471/ws');
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('Connected');
|
||||
ws.send(JSON.stringify({
|
||||
type: 'query',
|
||||
sql: 'SELECT * FROM users'
|
||||
}));
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
console.log('Received:', data);
|
||||
};
|
||||
```
|
||||
|
||||
## Nachrichtenformat
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "query",
|
||||
"id": "uuid",
|
||||
"sql": "SELECT * FROM users"
|
||||
}
|
||||
```
|
||||
|
||||
## Nachrichtentypen
|
||||
|
||||
### Query-Anfrage
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "query",
|
||||
"id": "123",
|
||||
"sql": "SELECT * FROM users"
|
||||
}
|
||||
```
|
||||
|
||||
### Query-Antwort
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "result",
|
||||
"id": "123",
|
||||
"data": [
|
||||
{"id": 1, "name": "Alice"},
|
||||
{"id": 2, "name": "Bob"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Fehlerantwort
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "error",
|
||||
"id": "123",
|
||||
"error": {
|
||||
"code": "INVALID_QUERY",
|
||||
"message": "Syntax error"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Subscription
|
||||
|
||||
Änderungen abonnieren:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "subscribe",
|
||||
"id": "sub1",
|
||||
"table": "users"
|
||||
}
|
||||
```
|
||||
|
||||
### Push-Benachrichtigung
|
||||
|
||||
Server-Push:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "push",
|
||||
"table": "users",
|
||||
"action": "insert",
|
||||
"data": {"id": 3, "name": "Charlie"}
|
||||
}
|
||||
```
|
||||
|
||||
## JavaScript-Client
|
||||
|
||||
```javascript
|
||||
class BaraDBClient {
|
||||
constructor(url) {
|
||||
this.ws = new WebSocket(url);
|
||||
this.pending = new Map();
|
||||
}
|
||||
|
||||
query(sql) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = crypto.randomUUID();
|
||||
this.pending.set(id, { resolve, reject });
|
||||
this.ws.send(JSON.stringify({ type: 'query', id, sql }));
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,152 @@
|
||||
# BaraDB Architektur
|
||||
|
||||
## Überblick
|
||||
|
||||
BaraDB ist eine **multimodale Datenbank-Engine** in Nim, die Document (KV), Graph, Vector, Columnar und Full-Text Search Speicherung in einer einzigen Engine mit einer einheitlichen Abfragesprache namens **BaraQL** kombiniert.
|
||||
|
||||
## Schichten-Architektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ 1. CLIENT LAYER │
|
||||
│ Binary Protocol │ HTTP/REST │ WebSocket │ Embedded │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 2. QUERY LAYER (BaraQL) │
|
||||
│ Lexer → Parser → AST → IR → Optimizer → Codegen │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 3. EXECUTION ENGINE │
|
||||
│ Document │ Graph │ Vector │ Columnar │ FTS │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 4. STORAGE │
|
||||
│ LSM-Tree │ B-Tree │ WAL │ Bloom │ Compaction │ Cache │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 5. DISTRIBUTED │
|
||||
│ Raft Consensus │ Sharding │ Replication │ Gossip │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Schicht 1: Client-Layer
|
||||
|
||||
Mehrere Kommunikationsprotokolle:
|
||||
|
||||
- **Binärprotokoll** (`protocol/wire.nim`): Effizientes Big-Endian Binärprotokoll mit 16 Nachrichtentypen
|
||||
- **HTTP/REST** (`core/httpserver.nim`): JSON-basierte REST API mit Multi-Threading
|
||||
- **WebSocket** (`core/websocket.nim`): Vollduplex-Streaming
|
||||
- **Embedded** (`storage/lsm.nim`): Direkter In-Process-Zugriff
|
||||
|
||||
### Verbindungsmanagement
|
||||
|
||||
- **Connection Pool** (`protocol/pool.nim`): Min/Max Verbindungslimits mit Idle-Timeout
|
||||
- **Rate Limiting** (`protocol/ratelimit.nim`): Token-Bucket globale und per-Client Limits
|
||||
- **Authentifizierung** (`protocol/auth.nim`): JWT mit HMAC-SHA256 und rollenbasierter Zugriff
|
||||
- **TLS/SSL** (`protocol/ssl.nim`): TLS 1.3 mit auto-generierten Zertifikaten
|
||||
|
||||
## Schicht 2: Query-Layer (BaraQL)
|
||||
|
||||
Die BaraQL-Pipeline:
|
||||
|
||||
1. **Lexer** (`query/lexer.nim`): Tokenisiert Eingabe in 80+ Tokentypen
|
||||
2. **Parser** (`query/parser.nim`): Rekursiver Descent-Parser produziert AST
|
||||
3. **AST** (`query/ast.nim`): 300+ Zeilen mit 25+ Knotenarten
|
||||
4. **IR** (`query/ir.nim`): Intermediate Representation für Ausführungspläne
|
||||
5. **Optimizer** (`query/adaptive.nim`): Adaptive Cross-Modal Query-Optimierung
|
||||
6. **Codegen** (`query/codegen.nim`): Übersetzt IR zu Speicheroperationen
|
||||
7. **Executor** (`query/executor.nim`): Führt Pläne mit Parallelisierung aus
|
||||
|
||||
### Cross-Modal Planning
|
||||
|
||||
Der Optimizer (`query/adaptive.nim`) bestimmt die Ausführungsreihenfolge über Engines:
|
||||
|
||||
```
|
||||
1. Selektivität für jedes Prädikat schätzen
|
||||
2. Selektivstes Prädikat zuerst an seine Engine pushen
|
||||
3. Bloom-Filter für KV-Lookups verwenden
|
||||
4. Unabhängige Zweige parallelisieren
|
||||
5. Results streamen um Materialisierung zu vermeiden
|
||||
```
|
||||
|
||||
## Schicht 3: Execution Engine
|
||||
|
||||
### Document/KV Engine
|
||||
- **LSM-Tree** (`storage/lsm.nim`): Write-optimierte Speicherung mit MemTable, WAL, SSTables
|
||||
- **B-Tree Index** (`storage/btree.nim`): Geordneter Index für Bereichsscans mit COW
|
||||
|
||||
### Vector Engine (`vector/`)
|
||||
- **HNSW Index** (`vector/engine.nim`): Hierarchical Navigable Small World Graph
|
||||
- **IVF-PQ Index** (`vector/engine.nim`): Inverted File Index mit Product Quantization
|
||||
- **SIMD Operations** (`vector/simd.nim`): AVX2-optimierte Distanzberechnungen
|
||||
- **Quantization** (`vector/quant.nim`): Scalar, Product und Binary Quantization
|
||||
|
||||
### Graph Engine (`graph/`)
|
||||
- **Adjacency List** (`graph/engine.nim`): Kanten-gewichteter gerichteter Graph
|
||||
- **Algorithmen** (`graph/engine.nim`): BFS, DFS, Dijkstra, PageRank
|
||||
- **Community Detection** (`graph/community.nim`): Louvain-Algorithmus
|
||||
- **Pattern Matching** (`graph/community.nim`): Subgraph-Isomorphie
|
||||
- **Cypher Parser** (`graph/cypher.nim`): Cypher-ähnliche Graph-Abfragen
|
||||
|
||||
### Full-Text Search (`fts/`)
|
||||
- **Inverted Index** (`fts/engine.nim`): Term-Document Index
|
||||
- **Ranking** (`fts/engine.nim`): BM25 und TF-IDF Scoring
|
||||
- **Fuzzy Search** (`fts/engine.nim`): Levenshtein-Distanz-Matching
|
||||
- **Multi-Language** (`fts/multilang.nim`): Tokenizer für EN, BG, DE, FR, RU
|
||||
|
||||
### Columnar Engine (`core/columnar.nim`)
|
||||
- Perspalten-Speicherung für analytische Abfragen
|
||||
- RLE und Dictionary-Kodierung
|
||||
- SIMD-beschleunigte Aggregatfunktionen
|
||||
|
||||
## Schicht 4: Storage
|
||||
|
||||
- **LSM-Tree** (`storage/lsm.nim`): MemTable, WAL, SSTable, Bloom-Filter, Compaction
|
||||
- **Page Cache** (`storage/compaction.nim`): LRU-Cache mit Trefferraten-Verfolgung
|
||||
- **Memory-mapped I/O** (`storage/mmap.nim`): mmap-basierter Dateizugriff
|
||||
- **Recovery** (`storage/recovery.nim`): WAL-Replay und Crash-Recovery
|
||||
|
||||
### Write Path
|
||||
|
||||
```
|
||||
Client → Protocol → Auth → Parser → AST → IR → Codegen
|
||||
→ StorageOp → MVCC Txn → WAL Write → MemTable → Commit
|
||||
```
|
||||
|
||||
### Read Path
|
||||
|
||||
```
|
||||
Client → Protocol → Auth → Parser → AST → IR → Codegen
|
||||
→ StorageOp → MVCC Snapshot → MemTable → SSTable → Result
|
||||
```
|
||||
|
||||
## Schicht 5: Distributed
|
||||
|
||||
- **Raft Consensus** (`core/raft.nim`): Leader Election, Log-Replikation
|
||||
- **Sharding** (`core/sharding.nim`): Hash, Range und Consistent Hashing
|
||||
- **Replication** (`core/replication.nim`): Sync, Async, Semi-Sync Modi
|
||||
- **Gossip Protocol** (`core/gossip.nim`): SWIM-ähnliches Membership-Management
|
||||
- **Distributed Transactions** (`core/disttxn.nim`): Two-Phase Commit
|
||||
|
||||
## Wichtige Designentscheidungen
|
||||
|
||||
1. **Reines Nim**: Keine Cython, Python oder Rust Abhängigkeiten
|
||||
2. **Unified Storage**: Eine Engine-handelt KV, Graph, Vector, FTS und Columnar
|
||||
3. **Embedded Mode**: Kann als Bibliothek oder Server laufen
|
||||
4. **Binärprotokoll**: Custom effizientes Wire-Protokoll
|
||||
5. **MVCC**: Multi-Version Concurrency Control
|
||||
6. **Schema-First**: Stark typisiertes Schema-System mit Vererbung
|
||||
7. **Cross-Modal**: Einheitliche Abfragesprache über alle Datenmodelle
|
||||
8. **Formally Verified**: Kern-Algorithmen in TLA+ spezifiziert und mit TLC model-gecheckt
|
||||
|
||||
## Modulstatistiken
|
||||
|
||||
| Kategorie | Module | Codezeilen | Zweck |
|
||||
|----------|--------|------------|-------|
|
||||
| Core | 16 | ~4,200 | Server, Protokolle, Transaktionen, Distributed |
|
||||
| Storage | 7 | ~3,100 | LSM, B-Tree, WAL, Bloom, Compaction, mmap |
|
||||
| Query | 7 | ~2,800 | Lexer, Parser, AST, IR, Optimizer, Codegen, Executor |
|
||||
| Vector | 3 | ~1,200 | HNSW, IVF-PQ, Quantization, SIMD |
|
||||
| Graph | 3 | ~1,000 | Adjacency List, Algorithmen, Community Detection |
|
||||
| FTS | 2 | ~900 | Inverted Index, BM25, Fuzzy, Multi-Language |
|
||||
| Protocol | 7 | ~2,400 | Wire, HTTP, WebSocket, Pool, Auth, Rate Limit, SSL |
|
||||
| Schema | 1 | ~600 | Typen, Links, Vererbung, Migrationen |
|
||||
| Client | 2 | ~800 | Nim Binary Client, File Helpers |
|
||||
| CLI | 1 | ~400 | Interaktive BaraQL Shell |
|
||||
| **Total** | **49** | **~14,100** | |
|
||||
@@ -0,0 +1,199 @@
|
||||
# Backup & Wiederherstellung
|
||||
|
||||
## Online Snapshots
|
||||
|
||||
BaraDB unterstützt Online-Snapshots ohne Server-Stopp. Der Snapshot erfasst eine
|
||||
konsistente Point-in-Time-Ansicht mittels MVCC.
|
||||
|
||||
### Snapshot erstellen
|
||||
|
||||
```nim
|
||||
import barabadb/core/backup
|
||||
|
||||
var bm = newBackupManager()
|
||||
bm.createSnapshot("/backup/baradb_2025-01-15")
|
||||
```
|
||||
|
||||
### Via CLI
|
||||
|
||||
```bash
|
||||
./build/baradadb --snapshot --output=/backup/snapshot.db
|
||||
```
|
||||
|
||||
### Via HTTP API
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:9470/api/backup \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"destination": "/backup/snapshot.db"}'
|
||||
```
|
||||
|
||||
### Automatisierte Backups
|
||||
|
||||
Cron für geplante Backups verwenden:
|
||||
|
||||
```bash
|
||||
# Daily snapshot at 2 AM
|
||||
0 2 * * * /usr/local/bin/baradadb --snapshot --output=/backup/baradb_$(date +\%Y\%m\%d).db
|
||||
|
||||
# Letzte 7 Tage behalten
|
||||
find /backup -name "baradb_*.db" -mtime +7 -delete
|
||||
```
|
||||
|
||||
## Point-in-Time Recovery (PITR)
|
||||
|
||||
BaraDB verwendet das Write-Ahead Log (WAL) für Point-in-Time Recovery.
|
||||
|
||||
### WAL-Archivierung
|
||||
|
||||
Kontinuierliche WAL-Archivierung aktivieren:
|
||||
|
||||
```bash
|
||||
BARADB_WAL_ARCHIVE_DIR=/backup/wal \
|
||||
BARADB_WAL_ARCHIVE_INTERVAL_MS=60000 \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
### Wiederherstellung von Checkpoint + WAL
|
||||
|
||||
```bash
|
||||
# Von Snapshot wiederherstellen
|
||||
./build/baradadb --recover \
|
||||
--checkpoint=/backup/snapshot.db \
|
||||
--wal-dir=/backup/wal
|
||||
|
||||
# Wiederherstellung zu spezifischem LSN
|
||||
./build/baradadb --recover \
|
||||
--checkpoint=/backup/snapshot.db \
|
||||
--wal-dir=/backup/wal \
|
||||
--target-lsn=15420
|
||||
|
||||
# Wiederherstellung zu spezifischer Zeit
|
||||
./build/baradadb --recover \
|
||||
--checkpoint=/backup/snapshot.db \
|
||||
--wal-dir=/backup/wal \
|
||||
--target-time="2025-01-15T10:30:00Z"
|
||||
```
|
||||
|
||||
### Wiederherstellung via SQL
|
||||
|
||||
Sie können auch direkt via BaraQL wiederherstellen:
|
||||
|
||||
```sql
|
||||
RECOVER TO TIMESTAMP '2026-05-07T12:00:00';
|
||||
```
|
||||
|
||||
### Inkrementelle Backups
|
||||
|
||||
Inkrementelle Backups kopieren nur geänderte SSTables:
|
||||
|
||||
```bash
|
||||
./build/baradadb --backup-incremental \
|
||||
--last-backup=/backup/previous \
|
||||
--output=/backup/incremental_$(date +%Y%m%d)
|
||||
```
|
||||
|
||||
## Replikation als Backup
|
||||
|
||||
Für kontinuierlichen Schutz, Streaming-Replikation verwenden:
|
||||
|
||||
### Primary
|
||||
|
||||
```bash
|
||||
BARADB_REPLICATION_ENABLED=true \
|
||||
BARADB_REPLICATION_MODE=async \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
### Replica
|
||||
|
||||
```bash
|
||||
BARADB_REPLICATION_ENABLED=true \
|
||||
BARADB_REPLICATION_PRIMARY=primary:9472 \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Disaster Recovery
|
||||
|
||||
### Wiederherstellungsverfahren
|
||||
|
||||
#### Szenario 1: Einzelne Dateikorruption
|
||||
|
||||
```bash
|
||||
# Korrupte SSTable aus Logs identifizieren
|
||||
# Spezifische SSTable aus Backup wiederherstellen
|
||||
cp /backup/sstables/000012.sst ./data/sstables/
|
||||
|
||||
# Index neu aufbauen
|
||||
./build/baradadb --rebuild-index
|
||||
```
|
||||
|
||||
#### Szenario 2: Kompletter Datenverlust
|
||||
|
||||
```bash
|
||||
# 1. Neuesten Snapshot wiederherstellen
|
||||
cp /backup/snapshot.db ./data/
|
||||
|
||||
# 2. WAL replay
|
||||
./build/baradadb --recover --wal-dir=/backup/wal
|
||||
|
||||
# 3. Verifizieren
|
||||
curl http://localhost:9470/health
|
||||
```
|
||||
|
||||
#### Szenario 3: Cluster-Knoten-Ausfall
|
||||
|
||||
```bash
|
||||
# Für Raft-Cluster, einfach neuen Knoten starten
|
||||
BARADB_RAFT_NODE_ID=newnode \
|
||||
BARADB_RAFT_PEERS=node1:9001,node2:9001 \
|
||||
./build/baradadb
|
||||
|
||||
# Der neue Knoten wird über Raft-Log-Replikation aufholen
|
||||
```
|
||||
|
||||
## Backup-Verifizierung
|
||||
|
||||
Backups immer verifizieren:
|
||||
|
||||
```bash
|
||||
# In temporäres Verzeichnis wiederherstellen
|
||||
./build/baradadb --recover \
|
||||
--checkpoint=/backup/snapshot.db \
|
||||
--data-dir=/tmp/verify_data
|
||||
|
||||
# Konsistenzprüfung ausführen
|
||||
curl http://localhost:9470/api/admin/check
|
||||
```
|
||||
|
||||
## Speicheranforderungen
|
||||
|
||||
| Backup-Typ | Größe | Häufigkeit | Aufbewahrung |
|
||||
|-------------|------|-----------|--------------|
|
||||
| Full snapshot | ~1× Datengröße | Täglich | 7 Tage |
|
||||
| Inkrementell | ~0.1× Datengröße | Stündlich | 24 Stunden |
|
||||
| WAL-Archiv | ~0.05× Datengröße / Tag | Kontinuierlich | 30 Tage |
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Restores regelmäßig testen** — Ein Backup das Sie nicht wiederherstellen können ist wertlos
|
||||
2. **Backups außerhalb speichern** — S3, GCS oder Azure Blob verwenden
|
||||
3. **Backups verschlüsseln** — `gpg` oder OS-Level-Verschlüsselung verwenden
|
||||
4. **Backup-Jobs überwachen** — Bei fehlgeschlagenen Backups alarmieren
|
||||
5. **RTO/RPO dokumentieren** — Ihre Wiederherstellungszeit und Punktziele kennen
|
||||
|
||||
### Cloud-Backup-Upload
|
||||
|
||||
```bash
|
||||
# Zu S3 hochladen
|
||||
aws s3 cp /backup/snapshot.db s3://my-bucket/baradb/
|
||||
|
||||
# Zu GCS hochladen
|
||||
gsutil cp /backup/snapshot.db gs://my-bucket/baradb/
|
||||
|
||||
# Zu Azure hochladen
|
||||
az storage blob upload \
|
||||
--container-name backups \
|
||||
--file /backup/snapshot.db \
|
||||
--name baradb/snapshot.db
|
||||
```
|
||||
@@ -0,0 +1,38 @@
|
||||
# B-Tree Index
|
||||
|
||||
Geordnete Indexstruktur für effiziente Bereichsabfragen und Point-Lookups.
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/storage/btree
|
||||
|
||||
var btree = newBTreeIndex[string, string]()
|
||||
|
||||
# Einfügen
|
||||
btree.insert("key1", "value1")
|
||||
btree.insert("key2", "value2")
|
||||
|
||||
# Point-Lookup
|
||||
let values = btree.get("key1")
|
||||
|
||||
# Bereichsabfrage
|
||||
let range = btree.scan("key_a", "key_z")
|
||||
|
||||
# Löschen
|
||||
btree.delete("key1")
|
||||
```
|
||||
|
||||
## Funktionen
|
||||
|
||||
- Geordnete Schlüssel-Wert-Speicherung
|
||||
- Bereichsabfragen (BETWEEN, >, <, >=, <=)
|
||||
- Präfix-Scans
|
||||
- Konfigurierbare Seitengröße
|
||||
- Iterator-Unterstützung
|
||||
|
||||
## Anwendungsfälle
|
||||
|
||||
- Primärschlüssel-Indizes
|
||||
- Sekundärindizes für häufig abgefragte Spalten
|
||||
- Bereichspartitionierte Daten
|
||||
@@ -0,0 +1,247 @@
|
||||
# Changelog
|
||||
|
||||
Alle bemerkenswerten Änderungen an BaraDB werden in dieser Datei dokumentiert.
|
||||
|
||||
## [Unreleased] — AI-Native Platform
|
||||
|
||||
### Hinzugefügt
|
||||
|
||||
- **MCP Server (Model Context Protocol)** — STDIO JSON-RPC 2.0 Server mit 3 AI-Tools:
|
||||
- `query` — SQL-Ausführung mit parametrisierten Abfragen + Multi-Tenant Session-Variablen
|
||||
- `vector_search` — Semantische HNSW Vektor-suche mit Tenant-Isolation
|
||||
- `schema_inspect` — Tabellen-/Spalten-/Index-/RLS-Policy-Exploration
|
||||
- Standalone Binary: `build/baramcp`
|
||||
- **Graph Engine Tiefe Integration** — `CREATE GRAPH` / `DROP GRAPH` DDL mit nativer Adjacency-List-Speicherung
|
||||
- `GRAPH_TABLE()` SQL-Funktion mit 7 Algorithmen: BFS, DFS, PageRank, ShortestPath, Dijkstra, Louvain, Community
|
||||
- INSERT in `_nodes`/`_edges` Tabellen synchronisiert automatisch mit nativen Graph-Objekten
|
||||
- Optional `MATCH`, `ALGORITHM`, `START`, `END`, `MAXDEPTH` in GRAPH_TABLE Syntax
|
||||
- **Chunking + Embedding Pipeline** — Serverseitige AI-Datenverarbeitung:
|
||||
- `chunk()` SQL-Funktion — Text-Splitting mit konfigurierbarer Größe/Überlappung
|
||||
- `embed_text()` SQL-Funktion — ruft externe Embedding-API auf (OpenAI/Ollama kompatibel)
|
||||
- Auto-Embedding bei INSERT — wenn VECTOR-Spalte null ist, generiert aus TEXT-Spalte
|
||||
- Konfigurierbar via Env-Vars: `BARADB_EMBED_ENDPOINT`, `BARADB_EMBED_MODEL`, `BARADB_EMBED_API_KEY`
|
||||
- **LangChain ChatMessageHistory** — Python `BaraDBChatHistory` Klasse:
|
||||
- Speichert Konversations-Threads in relationaler Tabelle mit RLS
|
||||
- Multi-Tenant Isolation via `tenant_id` + `user_id`
|
||||
- **RAG Pipeline Beispiel** — End-to-End Python Script (`examples/rag_pipeline.py`):
|
||||
- PDF/text Ingestion → chunking → embedding → BaraDB Speicherung → hybrid search → LLM Generierung
|
||||
- Unterstützt OpenAI und Ollama APIs
|
||||
- **AI Agents & NL→SQL** — Serverseitige LLM-Integration:
|
||||
- `nl_to_sql()` SQL-Funktion — natürliche Sprache → SQL Generierung
|
||||
- `schema_prompt()` — generiert DDL + Beispieldaten für LLM-Kontext
|
||||
- Abfrage-Validierungsschicht — Sandbox-Ausführung mit LIMIT 0 + EXPLAIN
|
||||
- Selbst-Korrektur-Schleife — Fehlerfeedback an LLM zur Korrektur
|
||||
- Konfigurierbar via Env-Vars: `BARADB_LLM_ENDPOINT`, `BARADB_LLM_MODEL`, `BARADB_LLM_API_KEY`
|
||||
- **Graph Similarity & Embeddings**:
|
||||
- `similarity_nodes()` — Jaccard/Adamic-Adar Ähnlichkeit zwischen Knotenpaaren
|
||||
- `node2vec_embed()` — Random-walk basierte Graph Embeddings
|
||||
- **Cypher Compatibility Layer**:
|
||||
- `cypher()` SQL-Funktion — übersetzt `MATCH (a)-[r]->(b) RETURN ...` zu GRAPH_TABLE
|
||||
- Automatische Cypher → BaraQL Konvertierung
|
||||
- **German Documentation** — Vollständige Dokumentation auf Deutsch (`docs/de/`)
|
||||
|
||||
### Geändert
|
||||
|
||||
- Graph Executor upgraded von Stub zu echtem BFS/DFS/PageRank/Dijkstra/Louvain
|
||||
- ExecutionContext erweitert mit `graphs`, `embedder`, `llmClient` Feldern
|
||||
- Graph Engine erweitert mit `addNodeWithId`, `addEdgeWithId`, Jaccard, Adamic-Adar, node2vec
|
||||
|
||||
## [1.1.0] — 2026-05-13
|
||||
|
||||
### Hinzugefügt
|
||||
|
||||
- **Client SDKs v1.1.0** — Vollständige Clients für alle Sprachen:
|
||||
- JavaScript: TypeScript Definitionen, package.json, Beispiele, Unit & Integration Tests
|
||||
- Python: Umstrukturiert als proper Package (`baradb/` mit `__init__.py` und `core.py`), pyproject.toml, Beispiele, Tests
|
||||
- Nim: Beispiele, Integration Tests, README
|
||||
- Rust: Beispiele, Integration Tests, verbessertes Cargo.toml
|
||||
- **SCRAM-SHA-256 Authentifizierung** — RFC 7677 konforme Authentifizierung mit PBKDF2 + HMAC + SHA-256 + Nonce/Salt Generierung
|
||||
- **HTTP SCRAM Endpoints** — `/auth/scram/start` + `/auth/scram/finish` im HTTP Server
|
||||
- **Docker Compose Test Configuration** — `docker-compose.test.yml` für Test-Umgebungen
|
||||
- **CI/CD Clients Pipeline** — `.github/workflows/clients-ci.yml` für automatisierte Client-Tests
|
||||
|
||||
### Behoben
|
||||
|
||||
- **Query Executor** — Unärer Minus (`irNeg`) funktioniert jetzt korrekt in SELECT und WHERE Klauseln
|
||||
- **Distributed Transactions** — Rollback nach Commit-Versuch verletzt nicht mehr Atomicity
|
||||
- **Sharding** — Datenmigrations-Protokoll mit TCP + `scanAll` auf LSM
|
||||
- **Raft** — Majority-Berechnung für gerade Knotenanzahl korrigiert
|
||||
- **MVCC** — Abgebrochene Transaktionen werden nicht mehr sichtbar
|
||||
- **LSM-Tree** — Datenverlust bei immutable memtable overwrite behoben; SSTable lookup sorting behoben
|
||||
- **Auth** — JWT-Signatur auf HMAC-SHA256 geändert (nicht mehr trivial fälschbar); Token-Ablauf (`exp`/`nbf`/`iat`) wird jetzt validiert; Signatur-Vergleich ist jetzt constant-time
|
||||
- **Recovery** — `summary()` mutiert die Datenbank nicht mehr
|
||||
- **Wire Protocol** — 64MB Limit + Bounds Checking + Max Depth um OOM/DoS zu verhindern
|
||||
- **SQL Injection** — `exprToSql` escaped jetzt Single Quotes
|
||||
- **ReDoS** — `irLike`/`irILike` escaped jetzt Regex Metacharacters
|
||||
- **Graph** — `addEdge` prüft jetzt Knotenexistenz
|
||||
- **Vector** — Dimension mismatch Validierung + HNSW Locking
|
||||
- **FTS** — UTF-8 Tokenisierung verwendet jetzt runes statt bytes
|
||||
- **Build** — `nim.cfg` fügt `-d:ssl` hinzu damit `nimble build` ohne Flags funktioniert; `--threads:on` zu allen CI Commands hinzugefügt
|
||||
|
||||
### Geändert
|
||||
|
||||
- **Version auf 1.1.0 erhöht** über alle Komponenten
|
||||
- **README** — Version Badge aktualisiert; alle Feature-Tabellen referenzieren jetzt v1.1.0
|
||||
- **TLA+ Formal Verification** — `crossmodal.tla`, `backup.tla`, `recovery.tla` hinzugefügt; Symmetrie-Reduktion in allen 9 Specs
|
||||
- **Clean build** — 0 Compiler Warnings auf Nim 2.2.10
|
||||
|
||||
## [0.1.0] — 2025-01-15
|
||||
|
||||
### Hinzugefügt
|
||||
|
||||
- **Core Storage Engines**
|
||||
- LSM-Tree mit MemTable, WAL, SSTables und size-tiered Compaction
|
||||
- B-Tree geordneter Index mit Range Scans und MVCC Copy-on-Write
|
||||
- Bloom Filter für effizientes SSTable Skip
|
||||
- Memory-mapped I/O für SSTable Reads
|
||||
- LRU Page Cache mit Hit-Rate Tracking
|
||||
|
||||
- **Query Engine (BaraQL)**
|
||||
- SQL-kompatibler Lexer mit 80+ Tokentypen
|
||||
- Rekursiver Descent Parser produziert AST mit 25+ Knotenarten
|
||||
- Intermediate Representation (IR) für Ausführungspläne
|
||||
- Code Generator übersetzt IR zu Speicheroperationen
|
||||
- Adaptiver Query Optimizer mit Cross-Modal Planning
|
||||
- Query Executor mit Parallelisierung
|
||||
|
||||
- **BaraQL Language Features**
|
||||
- SELECT, INSERT, UPDATE, DELETE
|
||||
- WHERE, ORDER BY, LIMIT, OFFSET
|
||||
- GROUP BY, HAVING, Aggregatfunktionen (count, sum, avg, min, max)
|
||||
- INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN
|
||||
- CTEs (Common Table Expressions) mit WITH
|
||||
- Subqueries (EXISTS, IN, correlated)
|
||||
- CASE Expressions
|
||||
- UNION, INTERSECT, EXCEPT
|
||||
- Schema Definition: CREATE TYPE, DROP TYPE
|
||||
|
||||
- **Vector Engine**
|
||||
- HNSW Index für Approximate Nearest Neighbor Search
|
||||
- IVF-PQ Index für Large-Scale Vector Search
|
||||
- SIMD-optimierte Distanzfunktionen (cosine, L2, dot product, Manhattan)
|
||||
- Quantization: scalar 8-bit/4-bit, product quantization, binary
|
||||
- Metadata Filtering während Vector Search
|
||||
|
||||
- **Graph Engine**
|
||||
- Adjacency List Speicherung für gerichtete, kanten-gewichtete Graphen
|
||||
- BFS und DFS Traversierung
|
||||
- Dijkstra kürzester Pfad
|
||||
- PageRank Knotenwichtigkeit
|
||||
- Louvain Community Detection
|
||||
- Subgraph Pattern Matching
|
||||
- Cypher-ähnlicher Graph Query Parser
|
||||
|
||||
- **Full-Text Search**
|
||||
- Inverted Index mit Term-Document Mapping
|
||||
- BM25 Ranking-Algorithmus
|
||||
- TF-IDF Scoring
|
||||
- Fuzzy Search mit Levenshtein Distanz
|
||||
- Wildcard/regex Search
|
||||
- Multi-Language Tokenizer (English, Bulgarian, German, French, Russian)
|
||||
|
||||
- **Columnar Storage**
|
||||
- Perspalten-Speicherung für analytische Abfragen
|
||||
- RLE (Run-Length Encoding) Kompression
|
||||
- Dictionary Encoding für Low-Cardinality Spalten
|
||||
- SIMD-beschleunigte Aggregatfunktionen
|
||||
|
||||
- **Transactions**
|
||||
- MVCC (Multi-Version Concurrency Control) mit Snapshot Isolation
|
||||
- Deadlock-Erkennung via Wait-for Graph
|
||||
- Write-Ahead Log für Dauerhaftigkeit
|
||||
- Savepoints und partielles Rollback
|
||||
|
||||
- **Protocol Layer**
|
||||
- Binary Wire Protocol mit 16 Nachrichtentypen
|
||||
- HTTP/REST JSON API
|
||||
- WebSocket Streaming
|
||||
- Connection Pooling
|
||||
- JWT-basierte Authentifizierung
|
||||
- Token-bucket Rate Limiting
|
||||
- TLS/SSL mit auto-generierten Zertifikaten
|
||||
|
||||
- **Schema System**
|
||||
- Starkes Typsystem mit 17 nativen Typen
|
||||
- Typvererbung mit Multi-Base Support
|
||||
- Property Links zwischen Typen
|
||||
- Schema Diffing und Migrationen
|
||||
- Computed Properties
|
||||
|
||||
- **Distributed Systems**
|
||||
- Raft Consensus (Leader Election, Log Replikation)
|
||||
- Hash, Range und Consistent-Hash Sharding
|
||||
- Sync/async/semi-sync Replikation
|
||||
- Gossip Protocol für Membership Management
|
||||
- Two-Phase Commit für Distributed Transactions
|
||||
|
||||
- **Cross-Modal Queries**
|
||||
- Vereinheitlichte Abfragesprache über alle Speicher-Engines
|
||||
- Cross-Engine Predicate Pushdown
|
||||
- Optimierte Ausführungspläne für Multi-Modal Abfragen
|
||||
|
||||
- **Backup & Recovery**
|
||||
- Online Snapshots ohne Ausfallzeit
|
||||
- Point-in-Time Recovery via WAL Replay
|
||||
- Inkrementelle Backups
|
||||
|
||||
- **Client SDKs**
|
||||
- JavaScript/TypeScript Client mit Binary Protocol
|
||||
- Python Client mit Sync und Async APIs
|
||||
- Nim Embedded Mode und Client Library
|
||||
- Rust Client (async)
|
||||
|
||||
- **Operations**
|
||||
- Interaktive CLI Shell (BaraQL REPL)
|
||||
- Strukturiertes Logging (JSON und Text Formate)
|
||||
- Prometheus-kompatible Metrics Endpoint
|
||||
- Health und Readiness Probes
|
||||
- CPU/memory Profiling Endpoints
|
||||
|
||||
- **Docker Support**
|
||||
- Multi-stage Dockerfile (Alpine Linux)
|
||||
- Docker Compose Konfiguration
|
||||
- Health Checks
|
||||
|
||||
### Performance
|
||||
|
||||
- LSM-Tree: 580K writes/s, 720K reads/s
|
||||
- B-Tree: 1.2M inserts/s, 1.5M lookups/s
|
||||
- Vector SIMD: 850K cosine distances/s (dim=768)
|
||||
- FTS: 320K docs/s indexing, 28K queries/s BM25
|
||||
- Graph: 2.5M nodes/s insertion, 12K BFS traversals/s
|
||||
- Binary Protocol: 380K queries/s (100 concurrent connections)
|
||||
|
||||
### Tests
|
||||
|
||||
- 262 Tests über 56 Test-Suiten
|
||||
- 100% Pass Rate
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Hinzugefügt
|
||||
|
||||
- **Vector SQL Integration** — Vollständige SQL-Level Vector Search Unterstützung:
|
||||
- `VECTOR(n)` Spaltentyp in `CREATE TABLE` mit Dimensionsvalidierung
|
||||
- `CREATE INDEX ... USING hnsw` / `USING ivfpq` für Approximate Nearest Neighbor Indizes
|
||||
- SQL Distanzfunktionen: `cosine_distance()`, `euclidean_distance()`, `inner_product()`, `l1_distance()`, `l2_distance()`
|
||||
- `<->` Nearest-Neighbor Operator (Euclidean Distanz)
|
||||
- `ORDER BY` Support für Vektor-Distanz-Ausdrücke, inklusive Spalten nicht in `SELECT`
|
||||
- Automatische HNSW Index-Wartung bei `INSERT` und `UPDATE`
|
||||
- **Advanced SQL Engine** — Window Functions, MERGE/UPSERT, LATERAL JOIN, PIVOT/UNPIVOT, SQL/PGQ Property Graph, Advanced Aggregates
|
||||
- **JavaScript Client — TCP Request Queue** — Interne `_requestQueue` + `_requestLock` für sichere konkurrierende Abfragen
|
||||
|
||||
### Behoben
|
||||
|
||||
- **Query Executor — Row Value Escaping** — `execInsert` escaped jetzt korrekt Kommas und Equals-Zeichen
|
||||
- **Query Planner — ORDER BY Projection** — `irpkSort` ist jetzt vor `irpkProject` im IR Plan platziert
|
||||
- **Wire Protocol — Big-Endian Float Serialization** — `FLOAT32`/`FLOAT64` werden jetzt in Big-Endian Byte-Reihenfolge serialisiert
|
||||
- **Gossip Protocol — Async UDP Socket** — Synchrone `newSocket` + blocking `recvFrom` ersetzt durch `newAsyncSocket` + `await recvFrom`
|
||||
|
||||
### Geplant
|
||||
|
||||
- Query Plan Caching
|
||||
- Materialized Views
|
||||
- Geospatial Index
|
||||
- Time-series Optimierungen
|
||||
- CDC (Change Data Capture) Streaming
|
||||
- Federated Queries über BaraDB Instances
|
||||
@@ -0,0 +1,285 @@
|
||||
# Client SDKs
|
||||
|
||||
BaraDB bietet offizielle Client-Bibliotheken für JavaScript/TypeScript, Python, Nim und Rust.
|
||||
|
||||
## JavaScript / TypeScript
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
npm install baradb
|
||||
# oder
|
||||
yarn add baradb
|
||||
```
|
||||
|
||||
### Grundlegende Verwendung
|
||||
|
||||
```typescript
|
||||
import { Client } from 'baradb';
|
||||
|
||||
const client = new Client('localhost', 9472);
|
||||
await client.connect();
|
||||
|
||||
// Einfache Abfrage
|
||||
const result = await client.query('SELECT name, age FROM users WHERE age > 18');
|
||||
console.log(result.rows);
|
||||
|
||||
// Parametrisierte Abfrage
|
||||
const result2 = await client.query(
|
||||
'SELECT * FROM users WHERE name = ?',
|
||||
['Alice']
|
||||
);
|
||||
|
||||
// Batch-Einfügung
|
||||
await client.batch([
|
||||
"INSERT users { name := 'Alice', age := 30 }",
|
||||
"INSERT users { name := 'Bob', age := 25 }",
|
||||
]);
|
||||
|
||||
// Transaktionen
|
||||
await client.begin();
|
||||
await client.query("INSERT orders { total := 100 }");
|
||||
await client.query("UPDATE users SET balance = balance - 100 WHERE name = 'Alice'");
|
||||
await client.commit();
|
||||
|
||||
await client.close();
|
||||
```
|
||||
|
||||
### Konkurrierende Abfragen
|
||||
|
||||
Der JavaScript-Client serialisiert konkurrierende Anfragen über eine einzelne TCP-Verbindung automatisch via interner Request-Queue. Sie können sicher mehrere parallele Operationen starten — ihre Binärframes werden sich nicht auf dem Wire überlappen:
|
||||
|
||||
```typescript
|
||||
const [users, orders, stats] = await Promise.all([
|
||||
client.query('SELECT * FROM users'),
|
||||
client.query('SELECT * FROM orders'),
|
||||
client.query('SELECT count(*) FROM visits')
|
||||
]);
|
||||
```
|
||||
|
||||
### WebSocket-Streaming
|
||||
|
||||
```typescript
|
||||
import { WebSocketClient } from 'baradb/ws';
|
||||
|
||||
const ws = new WebSocketClient('ws://localhost:9471');
|
||||
ws.onMessage = (data) => console.log(data);
|
||||
await ws.connect();
|
||||
await ws.send('SUBSCRIBE updates');
|
||||
```
|
||||
|
||||
## Python
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
pip install baradb
|
||||
```
|
||||
|
||||
### Grundlegende Verwendung
|
||||
|
||||
```python
|
||||
from baradb import Client
|
||||
|
||||
client = Client("localhost", 9472)
|
||||
client.connect()
|
||||
|
||||
# Einfache Abfrage
|
||||
result = client.query("SELECT name, age FROM users WHERE age > 18")
|
||||
for row in result:
|
||||
print(row["name"], row["age"])
|
||||
|
||||
# Parametrisierte Abfrage
|
||||
result = client.query(
|
||||
"SELECT * FROM users WHERE name = ?",
|
||||
["Alice"]
|
||||
)
|
||||
|
||||
# Batch-Operationen
|
||||
client.batch([
|
||||
"INSERT users { name := 'Alice', age := 30 }",
|
||||
"INSERT users { name := 'Bob', age := 25 }",
|
||||
])
|
||||
|
||||
# Context Manager (auto-close)
|
||||
with Client("localhost", 9472) as c:
|
||||
result = c.query("SELECT count(*) FROM users")
|
||||
print(result[0]["count"])
|
||||
```
|
||||
|
||||
### Async Client
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from baradb import AsyncClient
|
||||
|
||||
async def main():
|
||||
client = AsyncClient("localhost", 9472)
|
||||
await client.connect()
|
||||
result = await client.query("SELECT * FROM users")
|
||||
print(result.rows)
|
||||
await client.close()
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
## Nim (Embedded Mode)
|
||||
|
||||
### Abhängigkeit hinzufügen
|
||||
|
||||
```nim
|
||||
# In Ihrer .nimble Datei
|
||||
requires "barabadb >= 0.1.0"
|
||||
```
|
||||
|
||||
### Embedded-Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/storage/lsm
|
||||
import barabadb/storage/btree
|
||||
import barabadb/vector/engine
|
||||
import barabadb/graph/engine
|
||||
|
||||
# Key-Value Store
|
||||
var db = newLSMTree("./data")
|
||||
db.put("user:1", cast[seq[byte]]("Alice"))
|
||||
let (found, value) = db.get("user:1")
|
||||
db.close()
|
||||
|
||||
# B-Tree Index
|
||||
var btree = newBTreeIndex[string, int]()
|
||||
btree.insert("Alice", 30)
|
||||
let ages = btree.get("Alice")
|
||||
|
||||
# Vector Search
|
||||
var idx = newHNSWIndex(dimensions = 128)
|
||||
idx.insert(1, @[0.1'f32, 0.2, 0.3], {"category": "A"}.toTable)
|
||||
let results = idx.search(@[0.1'f32, 0.2, 0.3], k = 10)
|
||||
|
||||
# Graph
|
||||
var g = newGraph()
|
||||
let alice = g.addNode("Person", {"name": "Alice"}.toTable)
|
||||
let bob = g.addNode("Person", {"name": "Bob"}.toTable)
|
||||
discard g.addEdge(alice, bob, "knows")
|
||||
let path = g.shortestPath(alice, bob)
|
||||
```
|
||||
|
||||
### Client-Bibliothek
|
||||
|
||||
```nim
|
||||
import barabadb/client/client
|
||||
|
||||
var c = newBaraClient("localhost", 9472)
|
||||
c.connect()
|
||||
let result = c.query("SELECT name FROM users")
|
||||
for row in result.rows:
|
||||
echo row["name"]
|
||||
c.close()
|
||||
```
|
||||
|
||||
## Rust
|
||||
|
||||
### Abhängigkeit hinzufügen
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
baradb = "0.1"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
```
|
||||
|
||||
### Grundlegende Verwendung
|
||||
|
||||
```rust
|
||||
use baradb::Client;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut client = Client::connect("localhost:9472").await?;
|
||||
|
||||
let result = client
|
||||
.query("SELECT name, age FROM users WHERE age > 18")
|
||||
.await?;
|
||||
|
||||
for row in result.rows {
|
||||
println!("{} is {} years old", row["name"], row["age"]);
|
||||
}
|
||||
|
||||
client.close().await?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## HTTP/REST (Sprachunabhängig)
|
||||
|
||||
Alle Sprachen können die HTTP/REST API direkt verwenden:
|
||||
|
||||
```bash
|
||||
# Abfrage
|
||||
curl -X POST http://localhost:9470/api/query \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-d '{"query": "SELECT * FROM users WHERE age > 18"}'
|
||||
|
||||
# Einfügen
|
||||
curl -X POST http://localhost:9470/api/query \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "INSERT users { name := \"Alice\", age := 30 }"}'
|
||||
|
||||
# Schema
|
||||
curl http://localhost:9470/api/schema
|
||||
|
||||
# Health
|
||||
curl http://localhost:9470/health
|
||||
|
||||
# Metrics
|
||||
curl http://localhost:9470/metrics
|
||||
```
|
||||
|
||||
## Connection Pooling
|
||||
|
||||
Alle offiziellen Clients unterstützen Connection Pooling:
|
||||
|
||||
### JavaScript
|
||||
|
||||
```typescript
|
||||
import { Pool } from 'baradb';
|
||||
|
||||
const pool = new Pool({
|
||||
host: 'localhost',
|
||||
port: 9472,
|
||||
min: 5,
|
||||
max: 50,
|
||||
idleTimeout: 30000,
|
||||
});
|
||||
|
||||
const client = await pool.acquire();
|
||||
try {
|
||||
const result = await client.query('SELECT 1');
|
||||
} finally {
|
||||
pool.release(client);
|
||||
}
|
||||
```
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from baradb import Pool
|
||||
|
||||
pool = Pool("localhost", 9472, min_size=5, max_size=50)
|
||||
with pool.connection() as conn:
|
||||
result = conn.query("SELECT 1")
|
||||
```
|
||||
|
||||
## Datentyp-Mapping
|
||||
|
||||
| BaraDB Type | JavaScript | Python | Nim | Rust |
|
||||
|-------------|------------|--------|-----|------|
|
||||
| `null` | `null` | `None` | `nil` | `Option::None` |
|
||||
| `bool` | `boolean` | `bool` | `bool` | `bool` |
|
||||
| `int8/16/32/64` | `number` | `int` | `int` | `i8/i16/i32/i64` |
|
||||
| `float32/64` | `number` | `float` | `float32/float64` | `f32/f64` |
|
||||
| `str` | `string` | `str` | `string` | `String` |
|
||||
| `bytes` | `Uint8Array` | `bytes` | `seq[byte]` | `Vec<u8>` |
|
||||
| `array` | `Array` | `list` | `seq` | `Vec` |
|
||||
| `object` | `Object` | `dict` | `Table` | `HashMap` |
|
||||
| `vector` | `Float32Array` | `list[float]` | `seq[float32]` | `Vec<f32>` |
|
||||
@@ -0,0 +1,58 @@
|
||||
# Kolumnare Speicherung
|
||||
|
||||
Spaltenorientierte Speicherung für analytische Abfragen und Aggregation.
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/core/columnar
|
||||
|
||||
var batch = newColumnBatch()
|
||||
var ageCol = batch.addInt64Col("age")
|
||||
var nameCol = batch.addStringCol("name")
|
||||
|
||||
ageCol.appendInt64(25)
|
||||
nameCol.appendString("Alice")
|
||||
```
|
||||
|
||||
## Aggregation
|
||||
|
||||
```nim
|
||||
echo ageCol.sumInt64()
|
||||
echo ageCol.avgInt64()
|
||||
echo ageCol.minInt64()
|
||||
echo ageCol.maxInt64()
|
||||
echo ageCol.count()
|
||||
```
|
||||
|
||||
## Kodierung
|
||||
|
||||
### RLE (Run-Length Encoding)
|
||||
|
||||
```nim
|
||||
let rle = rleEncode(@[1'i64, 1, 1, 2, 2, 3])
|
||||
```
|
||||
|
||||
### Dictionary-Kodierung
|
||||
|
||||
```nim
|
||||
let dict = dictEncode(@["apple", "banana", "apple"])
|
||||
```
|
||||
|
||||
## Spaltentypen
|
||||
|
||||
| Typ | Beschreibung |
|
||||
|------|-------------|
|
||||
| `int32` | 32-Bit Integer |
|
||||
| `int64` | 64-Bit Integer |
|
||||
| `float32` | 32-Bit Float |
|
||||
| `float64` | 64-Bit Float |
|
||||
| `string` | Variable-length String |
|
||||
| `bool` | Boolean |
|
||||
|
||||
## Anwendungsfälle
|
||||
|
||||
- OLAP-Workloads
|
||||
- Großflächige Aggregation
|
||||
- Data Warehousing
|
||||
- Zeitreihenanalyse
|
||||
@@ -0,0 +1,217 @@
|
||||
# Konfigurationsreferenz
|
||||
|
||||
BaraDB kann über **Environment-Variablen**, eine **Konfigurationsdatei** oder **Kommandozeilen-Flags** konfiguriert werden.
|
||||
|
||||
## Prioritätsreihenfolge
|
||||
|
||||
1. Kommandozeilen-Flags (höchste Priorität)
|
||||
2. Environment-Variablen
|
||||
3. Konfigurationsdatei (`baradb.conf` oder `baradb.json`)
|
||||
4. Integrierte Standardwerte (niedrigste Priorität)
|
||||
|
||||
## Environment-Variablen
|
||||
|
||||
### Netzwerk
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_ADDRESS` | `127.0.0.1` | Bind-Adresse |
|
||||
| `BARADB_PORT` | `9472` | TCP Binary Protocol Port |
|
||||
| `BARADB_HTTP_PORT` | `9470` | HTTP/REST API Port |
|
||||
| `BARADB_WS_PORT` | `9471` | WebSocket Port |
|
||||
|
||||
### Speicher
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_DATA_DIR` | `./data` | Datenverzeichnis-Pfad |
|
||||
| `BARADB_MEMTABLE_SIZE_MB` | `64` | MemTable-Größe in MB |
|
||||
| `BARADB_CACHE_SIZE_MB` | `256` | Page-Cache-Größe in MB |
|
||||
| `BARADB_WAL_SYNC_INTERVAL_MS` | `0` | WAL fsync Intervall (0 = bei jedem Write) |
|
||||
| `BARADB_COMPACTION_INTERVAL_MS` | `60000` | Background Compaction Intervall |
|
||||
| `BARADB_BLOOM_BITS_PER_KEY` | `10` | Bloom-Filter Bits pro Schlüssel |
|
||||
|
||||
### TLS/SSL
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_TLS_ENABLED` | `false` | TLS aktivieren |
|
||||
| `BARADB_CERT_FILE` | — | Pfad zum TLS-Zertifikat |
|
||||
| `BARADB_KEY_FILE` | — | Pfad zum TLS Private Key |
|
||||
|
||||
### Sicherheit
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_AUTH_ENABLED` | `false` | Authentifizierung aktivieren |
|
||||
| `BARADB_JWT_SECRET` | — | JWT Signatur-Geheimnis |
|
||||
| `BARADB_RATE_LIMIT_GLOBAL` | `10000` | Globale Requests pro Sekunde |
|
||||
| `BARADB_RATE_LIMIT_PER_CLIENT` | `1000` | Per-Client Requests pro Sekunde |
|
||||
|
||||
### Logging
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_LOG_LEVEL` | `info` | Log-Level: debug, info, warn, error |
|
||||
| `BARADB_LOG_FILE` | — | Log-Datei-Pfad (stdout wenn leer) |
|
||||
| `BARADB_LOG_FORMAT` | `json` | Log-Format: json, text |
|
||||
|
||||
### Vector Engine
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_VECTOR_M` | `16` | HNSW `M` Parameter |
|
||||
| `BARADB_VECTOR_EF_CONSTRUCTION` | `200` | HNSW `efConstruction` |
|
||||
| `BARADB_VECTOR_EF_SEARCH` | `64` | HNSW `efSearch` |
|
||||
|
||||
### Graph Engine
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_GRAPH_PAGE_RANK_ITERATIONS` | `20` | PageRank Iterationsanzahl |
|
||||
| `BARADB_GRAPH_PAGE_RANK_DAMPING` | `0.85` | PageRank Dämpfungsfaktor |
|
||||
| `BARADB_GRAPH_LOUVAIN_RESOLUTION` | `1.0` | Louvain Resolution-Parameter |
|
||||
|
||||
### Distributed
|
||||
|
||||
| Variable | Standard | Beschreibung |
|
||||
|----------|---------|-------------|
|
||||
| `BARADB_RAFT_NODE_ID` | — | Eindeutige Knoten-ID im Cluster |
|
||||
| `BARADB_RAFT_PEERS` | — | Komma-getrennte Liste von Peer-Adressen |
|
||||
| `BARADB_RAFT_PORT` | `9001` | Raft interne Kommunikation Port |
|
||||
| `BARADB_SHARD_COUNT` | `1` | Anzahl der Shards |
|
||||
| `BARADB_REPLICATION_FACTOR` | `1` | Replikationsfaktor |
|
||||
|
||||
## Konfigurationsdatei
|
||||
|
||||
### baradb.conf (INI-ähnlich)
|
||||
|
||||
```ini
|
||||
[server]
|
||||
address = "0.0.0.0"
|
||||
port = 9472
|
||||
http_port = 9470
|
||||
ws_port = 9471
|
||||
|
||||
[storage]
|
||||
data_dir = "/var/lib/baradb"
|
||||
memtable_size_mb = 256
|
||||
cache_size_mb = 512
|
||||
wal_sync_interval_ms = 10
|
||||
compaction_interval_ms = 30000
|
||||
|
||||
[tls]
|
||||
enabled = true
|
||||
cert_file = "/etc/baradb/server.crt"
|
||||
key_file = "/etc/baradb/server.key"
|
||||
|
||||
[auth]
|
||||
enabled = true
|
||||
jwt_secret = "change-me-in-production"
|
||||
rate_limit_global = 10000
|
||||
rate_limit_per_client = 1000
|
||||
|
||||
[logging]
|
||||
level = "info"
|
||||
format = "json"
|
||||
file = "/var/log/baradb/baradb.log"
|
||||
|
||||
[vector]
|
||||
m = 16
|
||||
ef_construction = 200
|
||||
ef_search = 64
|
||||
|
||||
[cluster]
|
||||
raft_node_id = "node1"
|
||||
raft_peers = "node2:9001,node3:9001"
|
||||
```
|
||||
|
||||
### baradb.json
|
||||
|
||||
```json
|
||||
{
|
||||
"server": {
|
||||
"address": "0.0.0.0",
|
||||
"port": 9472,
|
||||
"http_port": 9470,
|
||||
"ws_port": 9471
|
||||
},
|
||||
"storage": {
|
||||
"data_dir": "/var/lib/baradb",
|
||||
"memtable_size_mb": 256,
|
||||
"cache_size_mb": 512
|
||||
},
|
||||
"tls": {
|
||||
"enabled": true,
|
||||
"cert_file": "/etc/baradb/server.crt",
|
||||
"key_file": "/etc/baradb/server.key"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Kommandozeilen-Flags
|
||||
|
||||
```bash
|
||||
./build/baradadb --help
|
||||
```
|
||||
|
||||
```
|
||||
BaraDB v1.1.0 — Multimodal Database Engine
|
||||
|
||||
Usage:
|
||||
baradadb [options]
|
||||
|
||||
Options:
|
||||
-c, --config <file> Config file path
|
||||
-p, --port <port> TCP binary port (default: 9472)
|
||||
--http-port <port> HTTP port (default: 9470)
|
||||
--ws-port <port> WebSocket port (default: 9471)
|
||||
-d, --data-dir <dir> Data directory (default: ./data)
|
||||
--tls-cert <file> TLS certificate file
|
||||
--tls-key <file> TLS private key file
|
||||
--log-level <level> Log level: debug, info, warn, error
|
||||
--log-file <file> Log file path
|
||||
--shell Start interactive shell
|
||||
--version Show version
|
||||
--recover Run WAL recovery
|
||||
--checkpoint <file> Checkpoint for recovery
|
||||
-h, --help Show this help
|
||||
```
|
||||
|
||||
## Beispielkonfigurationen
|
||||
|
||||
### Entwicklung
|
||||
|
||||
```bash
|
||||
./build/baradadb \
|
||||
--log-level debug \
|
||||
--data-dir ./dev_data
|
||||
```
|
||||
|
||||
### Produktion Single Node
|
||||
|
||||
```bash
|
||||
BARADB_TLS_ENABLED=true \
|
||||
BARADB_CERT_FILE=/etc/baradb/server.crt \
|
||||
BARADB_KEY_FILE=/etc/baradb/server.key \
|
||||
BARADB_AUTH_ENABLED=true \
|
||||
BARADB_JWT_SECRET="$(openssl rand -hex 32)" \
|
||||
BARADB_LOG_LEVEL=warn \
|
||||
BARADB_LOG_FILE=/var/log/baradb/baradb.log \
|
||||
BARADB_MEMTABLE_SIZE_MB=256 \
|
||||
BARADB_CACHE_SIZE_MB=1024 \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
### Produktion Cluster (3 Knoten)
|
||||
|
||||
```bash
|
||||
# Knoten 1
|
||||
BARADB_ADDRESS=0.0.0.0 \
|
||||
BARADB_PORT=9472 \
|
||||
BARADB_RAFT_NODE_ID=node1 \
|
||||
BARADB_RAFT_PEERS=node2:9001,node3:9001 \
|
||||
BARADB_SHARD_COUNT=4 \
|
||||
BARADB_REPLICATION_FACTOR=2 \
|
||||
./build/baradadb
|
||||
```
|
||||
@@ -0,0 +1,218 @@
|
||||
# Cross-Modal Abfragen
|
||||
|
||||
BaraDB's einzigartige Fähigkeit ist die Ausführung von Abfragen, die mehrere
|
||||
Speicher-Engines in einer einzigen vereinheitlichten BaraQL-Anweisung umfassen.
|
||||
|
||||
## Überblick
|
||||
|
||||
Traditionelle Datenbanken erfordern separate Abfragen und applikationsseitige Joins
|
||||
bei der Arbeit mit verschiedenen Datenmodellen. BaraDB's Cross-Modal Query Planner
|
||||
optimiert die Ausführung über:
|
||||
|
||||
- **Document/KV** (LSM-Tree) — strukturierte Datensätze
|
||||
- **Graph** (Adjacency List) — Beziehungen
|
||||
- **Vector** (HNSW/IVF-PQ) — Ähnlichkeitssuche
|
||||
- **Full-Text** (Inverted Index) — Textsuche
|
||||
- **Columnar** — analytische Aggregatfunktionen
|
||||
|
||||
## Abfragemuster
|
||||
|
||||
### Vector + Full-Text (Semantisch + Schlüsselwortsuche)
|
||||
|
||||
Finde Dokumente, die semantisch ähnlich zu einem Query-Vektor sind UND
|
||||
bestimmte Schlüsselwörter enthalten:
|
||||
|
||||
```sql
|
||||
SELECT title, score
|
||||
FROM articles
|
||||
WHERE MATCH(body) AGAINST('machine learning')
|
||||
ORDER BY cosine_distance(embedding, [0.1, 0.2, 0.3, ...])
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
Ausführungsplan:
|
||||
1. FTS-Engine filtert Artikel mit "machine learning"
|
||||
2. Vector-Engine rankt gefilterte Ergebnisse nach Embedding-Ähnlichkeit
|
||||
3. Top-K Ergebnisse zurückgegeben
|
||||
|
||||
### Graph + Vector (Soziale Empfehlungen)
|
||||
|
||||
Finde Freunde eines Benutzers mit ähnlichen Geschmacksvektoren:
|
||||
|
||||
```sql
|
||||
MATCH (u:User)-[:KNOWS]->(friend:User)
|
||||
WHERE u.name = 'Alice'
|
||||
ORDER BY cosine_distance(friend.taste_vector, u.taste_vector)
|
||||
RETURN friend.name, friend.age;
|
||||
```
|
||||
|
||||
Ausführungsplan:
|
||||
1. Graph-Engine traversiert "KNOWS"-Kanten von Alice
|
||||
2. Vector-Engine berechnet Ähnlichkeit für jeden Freund
|
||||
3. Ergebnisse sortiert und projiziert
|
||||
|
||||
### Document + Graph (Entity-Anreicherung)
|
||||
|
||||
Erhalte Bestelldetails mit Kunden-Beziehungsgraph:
|
||||
|
||||
```sql
|
||||
SELECT o.id, o.total, c.name,
|
||||
(SELECT count(*) FROM orders WHERE customer_id = c.id) as order_count
|
||||
FROM orders o
|
||||
JOIN customers c ON o.customer_id = c.id
|
||||
WHERE c.id IN (
|
||||
SELECT node_id FROM graph
|
||||
WHERE MATCH pattern (c:Customer)-[:REFERRED]->(:Customer)
|
||||
);
|
||||
```
|
||||
|
||||
### Full-Text + Aggregate (Content-Analyse)
|
||||
|
||||
Analysiere welche Abteilungen am meisten über ein Thema schreiben:
|
||||
|
||||
```sql
|
||||
SELECT department, count(*) as article_count,
|
||||
avg(length(content)) as avg_length
|
||||
FROM docs
|
||||
WHERE MATCH(content) AGAINST('Nim programming')
|
||||
GROUP BY department
|
||||
ORDER BY article_count DESC;
|
||||
```
|
||||
|
||||
### Vector + Aggregate (Cluster-Analyse)
|
||||
|
||||
Gruppiere ähnliche Vektoren und analysiere jedes Cluster:
|
||||
|
||||
```sql
|
||||
SELECT cluster_id, count(*) as size,
|
||||
centroid(embedding) as center,
|
||||
avg(created_at) as avg_date
|
||||
FROM products
|
||||
GROUP BY vector_cluster(embedding, k=10)
|
||||
ORDER BY size DESC;
|
||||
```
|
||||
|
||||
### Alle Modalitäten kombiniert
|
||||
|
||||
Eine komplexe Abfrage unter Verwendung aller Engines:
|
||||
|
||||
```sql
|
||||
WITH relevant_docs AS (
|
||||
SELECT id, title, embedding
|
||||
FROM articles
|
||||
WHERE MATCH(body) AGAINST('database optimization')
|
||||
AND created_at > '2024-01-01'
|
||||
),
|
||||
author_graph AS (
|
||||
MATCH (a:Author)-[:COAUTHORED]->(b:Author)
|
||||
WHERE a.name = 'Dr. Smith'
|
||||
RETURN b.id as coauthor_id
|
||||
)
|
||||
SELECT rd.title, rd.score,
|
||||
a.name as author,
|
||||
cosine_distance(rd.embedding, query_vec) as similarity
|
||||
FROM relevant_docs rd
|
||||
JOIN authors a ON rd.author_id = a.id
|
||||
WHERE a.id IN (SELECT coauthor_id FROM author_graph)
|
||||
ORDER BY similarity ASC, rd.score DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
## Optimierung
|
||||
|
||||
### Cross-Modal Query Planner
|
||||
|
||||
BaraDB's adaptiver Query-Optimizer (`query/adaptive.nim`) wählt die Ausführungsreihenfolge basierend auf Selektivität:
|
||||
|
||||
```
|
||||
1. Selektivstes Filter zuerst (normalerweise FTS oder Vector)
|
||||
2. Prädikate zu jeder Engine pushen
|
||||
3. Bloom-Filter für KV-Lookups verwenden
|
||||
4. Unabhängige Zweige parallelisieren
|
||||
```
|
||||
|
||||
### Index-Auswahl
|
||||
|
||||
Der Optimizer wählt automatisch den besten Index:
|
||||
|
||||
| Abfragemuster | Primäre Engine | Sekundäre Engine |
|
||||
|---------------|----------------|-----------------|
|
||||
| `MATCH ... ORDER BY cosine_distance` | Vector | FTS |
|
||||
| `MATCH ... WHERE graph condition` | Graph | FTS |
|
||||
| `WHERE id = ? AND vector_search` | KV | Vector |
|
||||
| `GROUP BY + MATCH` | FTS | Columnar |
|
||||
|
||||
### Hints
|
||||
|
||||
Bestimmte Ausführungsreihenfolge erzwingen:
|
||||
|
||||
```sql
|
||||
SELECT /*+ USE_INDEX(vector) */ *
|
||||
FROM products
|
||||
WHERE category = 'electronics'
|
||||
ORDER BY cosine_distance(embedding, [...])
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Cross-Modal Abfragen sind optimiert um Datenbewegung zu minimieren:
|
||||
|
||||
| Abfragetyp | Latenz (10K Zeilen) | Latenz (100K Zeilen) |
|
||||
|------------|---------------------|----------------------|
|
||||
| FTS + Vector | 15 ms | 85 ms |
|
||||
| Graph + Vector | 25 ms | 120 ms |
|
||||
| FTS + Aggregate | 12 ms | 55 ms |
|
||||
| Alle Modalitäten | 45 ms | 220 ms |
|
||||
|
||||
## Anwendungsfälle
|
||||
|
||||
### E-Commerce Suche
|
||||
|
||||
```sql
|
||||
-- Finde Produkte passend zu einem Suchbegriff, ähnlich zu einem betrachteten Artikel,
|
||||
-- gekauft von ähnlichen Benutzern
|
||||
SELECT p.name, p.price
|
||||
FROM products p
|
||||
WHERE MATCH(p.description) AGAINST('wireless headphones')
|
||||
AND cosine_distance(p.embedding, viewed_embedding) < 0.3
|
||||
AND p.id IN (
|
||||
SELECT product_id FROM orders o
|
||||
JOIN graph ON o.customer_id = graph.node_id
|
||||
WHERE graph.similarity > 0.8
|
||||
)
|
||||
ORDER BY p.rating DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
### Betrugserkennung
|
||||
|
||||
```sql
|
||||
-- Finde Transaktionen ähnlich zu bekannten Betrugsmustern,
|
||||
-- wo der Benutzer mit markierten Konten verbunden ist
|
||||
SELECT t.id, t.amount
|
||||
FROM transactions t
|
||||
WHERE cosine_distance(t.pattern_vector, fraud_vector) < 0.2
|
||||
AND t.user_id IN (
|
||||
MATCH (u:User)-[*1..3]->(f:FlaggedAccount)
|
||||
RETURN u.id
|
||||
);
|
||||
```
|
||||
|
||||
### Knowledge Graph + RAG
|
||||
|
||||
```sql
|
||||
-- Relevante Dokumente für eine Query abrufen,
|
||||
-- dann den Knowledge Graph für verwandte Konzepte traversieren
|
||||
WITH docs AS (
|
||||
SELECT id, content, embedding
|
||||
FROM documents
|
||||
ORDER BY cosine_distance(embedding, query_embedding)
|
||||
LIMIT 5
|
||||
)
|
||||
SELECT d.content, c.name as related_concept
|
||||
FROM docs d
|
||||
JOIN graph ON d.id = graph.doc_id
|
||||
MATCH (d)-[:MENTIONS]->(c:Concept)
|
||||
RETURN d.content, c.name;
|
||||
```
|
||||
@@ -0,0 +1,247 @@
|
||||
# Deployment-Leitfaden
|
||||
|
||||
## Docker
|
||||
|
||||
### Schnellstart
|
||||
|
||||
```bash
|
||||
docker build -t baradb:latest .
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Docker Compose Dateien
|
||||
|
||||
| Datei | Zweck |
|
||||
|-------|-------|
|
||||
| `docker-compose.yml` | Entwicklung |
|
||||
| `docker-compose.prod.yml` | Produktion |
|
||||
| `docker-compose.override.yml` | Dev Override (automatisch) |
|
||||
|
||||
### Produktion
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
### Docker Swarm
|
||||
|
||||
```bash
|
||||
docker stack deploy -c docker-compose.prod.yml baradb
|
||||
```
|
||||
|
||||
## systemd Service
|
||||
|
||||
Erstelle `/etc/systemd/system/baradb.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=BaraDB Multimodal Database
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=baradb
|
||||
Group=baradb
|
||||
WorkingDirectory=/var/lib/baradb
|
||||
ExecStart=/usr/local/bin/baradadb
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
Environment=BARADB_PORT=9472
|
||||
Environment=BARADB_HTTP_PORT=9470
|
||||
Environment=BARADB_DATA_DIR=/var/lib/baradb/data
|
||||
Environment=BARADB_LOG_LEVEL=info
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/lib/baradb/data
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Aktivieren und starten:
|
||||
|
||||
```bash
|
||||
sudo useradd -r -s /bin/false baradb
|
||||
sudo mkdir -p /var/lib/baradb/data
|
||||
sudo chown -R baradb:baradb /var/lib/baradb
|
||||
sudo cp build/baradadb /usr/local/bin/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now baradb
|
||||
```
|
||||
|
||||
## Kubernetes
|
||||
|
||||
### StatefulSet
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: baradb
|
||||
spec:
|
||||
serviceName: baradb
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: baradb
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: baradb
|
||||
spec:
|
||||
containers:
|
||||
- name: baradb
|
||||
image: baradb:latest
|
||||
ports:
|
||||
- containerPort: 9472
|
||||
name: binary
|
||||
- containerPort: 9470
|
||||
name: http
|
||||
- containerPort: 9471
|
||||
name: websocket
|
||||
env:
|
||||
- name: BARADB_DATA_DIR
|
||||
value: /data
|
||||
- name: BARADB_RAFT_NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: baradb
|
||||
spec:
|
||||
selector:
|
||||
app: baradb
|
||||
ports:
|
||||
- port: 9472
|
||||
name: binary
|
||||
- port: 9470
|
||||
name: http
|
||||
- port: 9471
|
||||
name: websocket
|
||||
clusterIP: None
|
||||
```
|
||||
|
||||
## Reverse Proxy (nginx)
|
||||
|
||||
```nginx
|
||||
upstream baradb_http {
|
||||
server 127.0.0.1:9470;
|
||||
}
|
||||
|
||||
upstream baradb_ws {
|
||||
server 127.0.0.1:9471;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name db.example.com;
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name db.example.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/db.example.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/db.example.com/privkey.pem;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://baradb_http/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
|
||||
location /ws/ {
|
||||
proxy_pass http://baradb_ws/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## High Availability
|
||||
|
||||
### 3-Node Raft Cluster
|
||||
|
||||
```bash
|
||||
# Knoten 1
|
||||
BARADB_RAFT_NODE_ID=node1 \
|
||||
BARADB_RAFT_PEERS=node2:9001,node3:9001 \
|
||||
./build/baradadb
|
||||
|
||||
# Knoten 2
|
||||
BARADB_RAFT_NODE_ID=node2 \
|
||||
BARADB_RAFT_PEERS=node1:9001,node3:9001 \
|
||||
./build/baradadb
|
||||
|
||||
# Knoten 3
|
||||
BARADB_RAFT_NODE_ID=node3 \
|
||||
BARADB_RAFT_PEERS=node1:9001,node2:9001 \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Cloud Deployment
|
||||
|
||||
### AWS EC2
|
||||
|
||||
Empfohlene Instance: `m6i.2xlarge` (8 vCPU, 32 GB RAM)
|
||||
|
||||
```bash
|
||||
# User data script
|
||||
#!/bin/bash
|
||||
apt-get update
|
||||
apt-get install -y nim
|
||||
wget https://github.com/katehonz/barabaDB/releases/latest/download/baradadb-linux-amd64
|
||||
chmod +x baradadb-linux-amd64
|
||||
mv baradadb-linux-amd64 /usr/local/bin/baradadb
|
||||
|
||||
mkdir -p /data/baradb
|
||||
cat > /etc/systemd/system/baradb.service << 'EOF'
|
||||
[Unit]
|
||||
Description=BaraDB
|
||||
After=network.target
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/baradadb
|
||||
Environment=BARADB_DATA_DIR=/data/baradb
|
||||
Restart=always
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now baradb
|
||||
```
|
||||
|
||||
### GCP Cloud Run (nur HTTP)
|
||||
|
||||
```bash
|
||||
gcloud run deploy baradb \
|
||||
--image gcr.io/PROJECT/baradb \
|
||||
--port 9470 \
|
||||
--memory 4Gi \
|
||||
--cpu 2 \
|
||||
--max-instances 10
|
||||
```
|
||||
@@ -0,0 +1,110 @@
|
||||
# Distributed Systems
|
||||
|
||||
BaraDB unterstützt verteiltes Deployment mit Raft Consensus, Sharding und Replikation.
|
||||
|
||||
## Raft Consensus
|
||||
|
||||
Leader Election und Log-Replikation:
|
||||
|
||||
```nim
|
||||
import barabadb/core/raft
|
||||
|
||||
var cluster = newRaftCluster()
|
||||
cluster.addNode("node1")
|
||||
cluster.addNode("node2")
|
||||
cluster.addNode("node3")
|
||||
|
||||
let n1 = cluster.nodes["n1"]
|
||||
n1.becomeCandidate()
|
||||
n1.becomeLeader()
|
||||
let entry = n1.appendLog("SET key1 value1")
|
||||
```
|
||||
|
||||
## Sharding
|
||||
|
||||
Daten über Knoten verteilen:
|
||||
|
||||
```nim
|
||||
import barabadb/core/sharding
|
||||
|
||||
var router = newShardRouter(ShardConfig(
|
||||
numShards: 4,
|
||||
replicas: 2,
|
||||
strategy: ssHash
|
||||
))
|
||||
router.rebalance(@["node1", "node2", "node3"])
|
||||
let shard = router.getShard("user_123")
|
||||
```
|
||||
|
||||
### Sharding-Strategien
|
||||
|
||||
| Strategie | Beschreibung |
|
||||
|----------|-------------|
|
||||
| `ssHash` | Hash-basiertes Sharding |
|
||||
| `ssRange` | Range-basiertes Sharding |
|
||||
| `ssConsistent` | Consistent Hashing |
|
||||
|
||||
## Replikation
|
||||
|
||||
```nim
|
||||
import barabadb/core/replication
|
||||
|
||||
var rm = newReplicationManager(rmSync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 9472))
|
||||
rm.connectReplica("r1")
|
||||
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||
rm.ackLsn("r1", lsn)
|
||||
```
|
||||
|
||||
### Replikationsmodi
|
||||
|
||||
| Modus | Beschreibung |
|
||||
|-------|--------------|
|
||||
| `rmSync` | Synchronous Replikation |
|
||||
| `rmAsync` | Asynchrone Replikation |
|
||||
| `rmSemiSync` | Semi-synchrone Replikation |
|
||||
|
||||
## Gossip Protocol
|
||||
|
||||
Membership und Failure-Erkennung:
|
||||
|
||||
```nim
|
||||
import barabadb/core/gossip
|
||||
|
||||
var g = newGossipManager()
|
||||
g.addNode("node1")
|
||||
g.addNode("node2")
|
||||
g.tick() # Membership-Info austauschen
|
||||
```
|
||||
|
||||
## Distributed Transactions
|
||||
|
||||
Two-Phase Commit über Knoten:
|
||||
|
||||
```nim
|
||||
import barabadb/core/disttxn
|
||||
|
||||
var dt = newDistributedTxn()
|
||||
dt.prepare(@["node1", "node2"])
|
||||
dt.commit()
|
||||
```
|
||||
|
||||
## Formale Verifikation
|
||||
|
||||
Kern-Algorithmen für Distributed Systems sind formal in TLA+ spezifiziert und model-gecheckt:
|
||||
|
||||
- **Raft Consensus** — `formal-verification/raft.tla`
|
||||
- Verifiziert: ElectionSafety, StateMachineSafety
|
||||
- **Two-Phase Commit** — `formal-verification/twopc.tla`
|
||||
- Verifiziert: Atomicity, NoOrphanBlocks
|
||||
- **Replication** — `formal-verification/replication.tla`
|
||||
- Verifiziert: MonotonicLsn, AcksRemovePending
|
||||
|
||||
TLC lokal ausführen:
|
||||
|
||||
```bash
|
||||
cd formal-verification
|
||||
java -cp tla2tools.jar tlc2.TLC -config models/raft.cfg raft.tla
|
||||
java -cp tla2tools.jar tlc2.TLC -config models/twopc.cfg twopc.tla
|
||||
java -cp tla2tools.jar tlc2.TLC -config models/replication.cfg replication.tla
|
||||
```
|
||||
@@ -0,0 +1,191 @@
|
||||
# Docker Deployment-Leitfaden
|
||||
|
||||
Dieser Leitfaden beschreibt, wie Sie BaraDB mit Docker und Docker Compose verwenden.
|
||||
|
||||
## Schnellstart
|
||||
|
||||
```bash
|
||||
# Repository klonen
|
||||
git clone https://codeberg.org/baraba/baradb
|
||||
cd barabaDB
|
||||
|
||||
# Image bauen
|
||||
docker build -t baradb:latest .
|
||||
|
||||
# Mit Docker Compose starten
|
||||
docker compose up -d
|
||||
|
||||
# Status prüfen
|
||||
docker compose ps
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
## Dateien
|
||||
|
||||
| Datei | Beschreibung |
|
||||
|-------|--------------|
|
||||
| `Dockerfile` | Multi-stage Production Build |
|
||||
| `docker-compose.yml` | Development Konfiguration |
|
||||
| `docker-compose.prod.yml` | Production Konfiguration |
|
||||
| `docker-compose.override.yml` | Development Override (wird automatisch geladen) |
|
||||
| `docker-entrypoint.sh` | Entrypoint-Skript für Initialisierung |
|
||||
| `.dockerignore` | Dateien, die nicht ins Image kopiert werden sollen |
|
||||
| `scripts/docker-build.sh` | Helper-Skript für Build |
|
||||
| `scripts/docker-run.sh` | Helper-Skript für manuelles Starten |
|
||||
|
||||
## Image erstellen
|
||||
|
||||
```bash
|
||||
# Standard Build
|
||||
docker build -t baradb:latest .
|
||||
|
||||
# Mit Skript
|
||||
./scripts/docker-build.sh
|
||||
|
||||
# Mit bestimmter Version
|
||||
IMAGE_NAME=baradb VERSION=0.1.0 ./scripts/docker-build.sh
|
||||
```
|
||||
|
||||
## Starten
|
||||
|
||||
### Development (docker compose)
|
||||
|
||||
```bash
|
||||
# Im Hintergrund starten
|
||||
docker compose up -d
|
||||
|
||||
# Stoppen
|
||||
docker compose down
|
||||
|
||||
# Stoppen und Volumes löschen (WARNUNG — löscht Daten!)
|
||||
docker compose down -v
|
||||
|
||||
# Logs ansehen
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
### Production (docker compose)
|
||||
|
||||
```bash
|
||||
# Mit Production-Konfiguration starten
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
|
||||
# Healthcheck prüfen
|
||||
docker compose -f docker-compose.prod.yml ps
|
||||
```
|
||||
|
||||
### Manuell (docker run)
|
||||
|
||||
```bash
|
||||
# Mit Skript
|
||||
./scripts/docker-run.sh
|
||||
|
||||
# Manuell
|
||||
docker run -d \
|
||||
--name baradb \
|
||||
-p 9472:9472 \
|
||||
-p 9470:9470 \
|
||||
-p 9471:9471 \
|
||||
-v baradb_data:/data \
|
||||
-e BARADB_LOG_LEVEL=info \
|
||||
baradb:latest
|
||||
```
|
||||
|
||||
## Ports
|
||||
|
||||
| Port | Beschreibung |
|
||||
|------|--------------|
|
||||
| `9472` | Binary Wire Protocol |
|
||||
| `9912` | HTTP/REST API (TCP port + 440) |
|
||||
| `9913` | WebSocket (TCP port + 441) |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Standardwert | Beschreibung |
|
||||
|----------|--------------|--------------|
|
||||
| `BARADB_ADDRESS` | `0.0.0.0` | Bind-Adresse |
|
||||
| `BARADB_PORT` | `9472` | Binary Protocol Port |
|
||||
| `BARADB_HTTP_PORT` | `9470` | HTTP Port |
|
||||
| `BARADB_WS_PORT` | `9471` | WebSocket Port |
|
||||
| `BARADB_DATA_DIR` | `/data` | Datenverzeichnis |
|
||||
| `BARADB_LOG_LEVEL` | `info` | Log-Level |
|
||||
|
||||
## Volumes
|
||||
|
||||
| Pfad im Container | Beschreibung |
|
||||
|-------------------|--------------|
|
||||
| `/data` | Hauptdatenverzeichnis |
|
||||
| `/data/server/wal` | Write-Ahead Log |
|
||||
| `/data/server/sstables` | SSTable Dateien |
|
||||
|
||||
## Production Checklist
|
||||
|
||||
- [ ] TLS-Zertifikate in `./certs/` erstellen
|
||||
- [ ] Starkes `BARADB_JWT_SECRET` setzen
|
||||
- [ ] Firewall-Regeln konfigurieren
|
||||
- [ ] Regelmäßige Backups konfigurieren
|
||||
- [ ] Resource Limits prüfen
|
||||
- [ ] Monitoring einrichten (Healthcheck, Logs)
|
||||
|
||||
## TLS in Docker
|
||||
|
||||
1. Zertifikate erstellen:
|
||||
```bash
|
||||
mkdir -p certs
|
||||
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
|
||||
-keyout certs/server.key -out certs/server.crt
|
||||
```
|
||||
|
||||
2. In `docker-compose.prod.yml` aktivieren:
|
||||
```yaml
|
||||
environment:
|
||||
- BARADB_TLS_ENABLED=true
|
||||
- BARADB_CERT_FILE=/certs/server.crt
|
||||
- BARADB_KEY_FILE=/certs/server.key
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
```
|
||||
|
||||
## Backup in Docker
|
||||
|
||||
```bash
|
||||
# Manueller Backup
|
||||
docker exec baradb /app/backup backup --data-dir=/data
|
||||
|
||||
# Backup-Liste
|
||||
docker exec baradb /app/backup list
|
||||
|
||||
# Wiederherstellung
|
||||
docker exec baradb /app/backup restore --input=backup_xxx.tar.gz
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Container startet nicht
|
||||
|
||||
```bash
|
||||
# Logs prüfen
|
||||
docker compose logs -f baradb
|
||||
|
||||
# Status prüfen
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
### Keine Verbindung zur Datenbank
|
||||
|
||||
```bash
|
||||
# Prüfen ob Ports exponiert sind
|
||||
docker port baradb
|
||||
|
||||
# Von innen prüfen
|
||||
docker exec baradb wget -qO- http://localhost:9470/health
|
||||
```
|
||||
|
||||
### Permission denied auf /data
|
||||
|
||||
Das Entrypoint-Skript erstellt automatisch Verzeichnisse und setzt korrekte Permissions. Bei Problemen:
|
||||
|
||||
```bash
|
||||
docker exec baradb ls -la /data
|
||||
docker exec baradb chown -R baradb:baradb /data
|
||||
```
|
||||
@@ -0,0 +1,87 @@
|
||||
# Volltextsuchmaschine
|
||||
|
||||
Invertierter Index mit BM25 und TF-IDF-Ranking für Textsuche.
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/fts/engine
|
||||
|
||||
var idx = newInvertedIndex()
|
||||
idx.addDocument(1, "Nim is a fast programming language")
|
||||
idx.addDocument(2, "Python is popular for data science")
|
||||
|
||||
# BM25-Suche
|
||||
let results = idx.search("programming language")
|
||||
|
||||
# TF-IDF-Suche
|
||||
let tfidf = idx.searchTfidf("programming language")
|
||||
|
||||
# Fuzzy-Suche (Tippfehlertoleranz)
|
||||
let fuzzy = idx.fuzzySearch("programing", maxDistance = 2)
|
||||
|
||||
# Platzhalter-Suche
|
||||
let wild = idx.regexSearch("prog*")
|
||||
```
|
||||
|
||||
## Ranking-Methoden
|
||||
|
||||
### BM25
|
||||
|
||||
Best-Matching-Ranking-Algorithmus:
|
||||
|
||||
```nim
|
||||
let bm25 = idx.searchBM25("query terms")
|
||||
```
|
||||
|
||||
### TF-IDF
|
||||
|
||||
Term Frequency-Inverse Document Frequency:
|
||||
|
||||
```nim
|
||||
let tfidf = idx.searchTfidf("query terms")
|
||||
```
|
||||
|
||||
## Suchfunktionen
|
||||
|
||||
| Funktion | Beschreibung |
|
||||
|---------|-------------|
|
||||
| Fuzzy-Suche | Levenshtein-Distanz-Toleranz |
|
||||
| Platzhalter | Präfix-, Suffix- und Infix-Platzhalter |
|
||||
| Regex | Reguläre Ausdrucksmuster |
|
||||
| Phrasensuche | Exakte Phrasenübereinstimmung |
|
||||
| Boolesch | AND, OR, NOT Operatoren |
|
||||
|
||||
## SQL-Schnittstelle
|
||||
|
||||
Volltextsuche ist auch direkt in BaraQL verfügbar:
|
||||
|
||||
```sql
|
||||
-- Tabelle mit Textspalte erstellen
|
||||
CREATE TABLE articles (id INT PRIMARY KEY, title TEXT, body TEXT);
|
||||
|
||||
-- FTS-Index erstellen
|
||||
CREATE INDEX idx_fts ON articles(body) USING FTS;
|
||||
|
||||
-- Suche mit dem @@ Operator (BM25-Ranking)
|
||||
SELECT * FROM articles WHERE body @@ 'machine learning';
|
||||
|
||||
-- Suche mit mehreren Begriffen
|
||||
SELECT * FROM articles WHERE body @@ 'quick brown fox';
|
||||
```
|
||||
|
||||
## Mehrsprachige Unterstützung
|
||||
|
||||
```nim
|
||||
import barabadb/fts/multilang
|
||||
|
||||
# Unterstützte Sprachen: EN, BG, DE, FR, RU
|
||||
var tokenizer = newTokenizer("de") # Deutsch
|
||||
let tokens = tokenizer.tokenize("Volltextsuche")
|
||||
```
|
||||
|
||||
Funktionen pro Sprache:
|
||||
- Tokenisierung
|
||||
- Stoppwörter
|
||||
- Stemming
|
||||
- Spracherkennung
|
||||
@@ -0,0 +1,64 @@
|
||||
# LSM-Tree Speicher-Engine
|
||||
|
||||
Die primäre Speicher-Engine in BaraDB mit Log-Structured Merge-Tree Architektur.
|
||||
|
||||
## Architektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ Writes │
|
||||
│ (append to WAL + MemTable) │
|
||||
└─────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ MemTable │
|
||||
│ (in-memory sorted buffer) │
|
||||
└─────────────────────────────────────────────┘
|
||||
│
|
||||
(when full, flush to SSTable)
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ SSTable │
|
||||
│ (sorted string table on disk) │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/storage/lsm
|
||||
|
||||
var db = newLSMTree("./data")
|
||||
|
||||
# Schreiben
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
|
||||
# Lesen
|
||||
let (found, value) = db.get("key1")
|
||||
|
||||
# Löschen
|
||||
db.delete("key1")
|
||||
|
||||
db.close()
|
||||
```
|
||||
|
||||
## Funktionen
|
||||
|
||||
- **Write-optimiert**: Append-only Log-Struktur
|
||||
- **Dauerhaftigkeit**: Write-Ahead Log (WAL) sichert Crash-Wiederherstellung
|
||||
- **Bloom-Filter**: Schnelle negative Lookups
|
||||
- **Compaction**: Size-tiered Strategie mischt SSTables
|
||||
- **Page-Cache**: LRU-Cache für häufig zugegriffene Seiten
|
||||
|
||||
## Konfiguration
|
||||
|
||||
```nim
|
||||
var db = newLSMTree(
|
||||
path = "./data",
|
||||
memTableSize = 64 * 1024 * 1024, # 64MB
|
||||
walEnabled = true,
|
||||
bloomFpRate = 0.01
|
||||
)
|
||||
```
|
||||
@@ -0,0 +1,258 @@
|
||||
# Monitoring & Observability
|
||||
|
||||
## Health Checks
|
||||
|
||||
### HTTP Health Endpoint
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/health
|
||||
```
|
||||
|
||||
Antwort:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"version": "0.1.0",
|
||||
"uptime_seconds": 86400,
|
||||
"checks": {
|
||||
"storage": "ok",
|
||||
"memory": "ok",
|
||||
"connections": "ok"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Readiness Probe
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/ready
|
||||
```
|
||||
|
||||
Gibt `200 OK` zurück wenn der Server bereit ist Traffic anzunehmen, `503` während des Starts.
|
||||
|
||||
## Metrics
|
||||
|
||||
### Prometheus-kompatible Metrics
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/metrics
|
||||
```
|
||||
|
||||
Beispielausgabe:
|
||||
|
||||
```
|
||||
# HELP baradb_queries_total Total number of queries executed
|
||||
# TYPE baradb_queries_total counter
|
||||
baradb_queries_total 152340
|
||||
|
||||
# HELP baradb_queries_duration_seconds Query duration histogram
|
||||
# TYPE baradb_queries_duration_seconds histogram
|
||||
baradb_queries_duration_seconds_bucket{le="0.001"} 45000
|
||||
baradb_queries_duration_seconds_bucket{le="0.01"} 120000
|
||||
baradb_queries_duration_seconds_bucket{le="0.1"} 148000
|
||||
|
||||
# HELP baradb_storage_lsm_size_bytes LSM-Tree total size
|
||||
# TYPE baradb_storage_lsm_size_bytes gauge
|
||||
baradb_storage_lsm_size_bytes 2147483648
|
||||
|
||||
# HELP baradb_storage_sstables Number of SSTables
|
||||
# TYPE baradb_storage_sstables gauge
|
||||
baradb_storage_sstables 12
|
||||
|
||||
# HELP baradb_cache_hit_rate Page cache hit rate
|
||||
# TYPE baradb_cache_hit_rate gauge
|
||||
baradb_cache_hit_rate 0.94
|
||||
|
||||
# HELP baradb_active_connections Active client connections
|
||||
# TYPE baradb_active_connections gauge
|
||||
baradb_active_connections 42
|
||||
|
||||
# HELP baradb_txns_active Active transactions
|
||||
# TYPE baradb_txns_active gauge
|
||||
baradb_txns_active 7
|
||||
|
||||
# HELP baradb_txns_committed_total Total committed transactions
|
||||
# TYPE baradb_txns_committed_total counter
|
||||
baradb_txns_committed_total 89123
|
||||
```
|
||||
|
||||
### JSON Metrics
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/metrics?format=json
|
||||
```
|
||||
|
||||
## Logging
|
||||
|
||||
### Log-Level
|
||||
|
||||
| Level | Beschreibung |
|
||||
|-------|--------------|
|
||||
| `debug` | Detaillierte interne Operationen |
|
||||
| `info` | Normale Operationen |
|
||||
| `warn` | Behebbare Probleme |
|
||||
| `error` | Fehler die Aufmerksamkeit erfordern |
|
||||
|
||||
### Strukturiertes JSON Logging
|
||||
|
||||
```bash
|
||||
BARADB_LOG_LEVEL=info \
|
||||
BARADB_LOG_FORMAT=json \
|
||||
BARADB_LOG_FILE=/var/log/baradb/baradb.log \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
Beispiel-Log-Eintrag:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2025-01-15T10:30:00.123Z",
|
||||
"level": "info",
|
||||
"component": "server",
|
||||
"message": "Query executed",
|
||||
"query": "SELECT * FROM users",
|
||||
"duration_ms": 12,
|
||||
"client_ip": "10.0.0.15"
|
||||
}
|
||||
```
|
||||
|
||||
### Textformat
|
||||
|
||||
```bash
|
||||
BARADB_LOG_FORMAT=text ./build/baradadb
|
||||
```
|
||||
|
||||
Ausgabe:
|
||||
|
||||
```
|
||||
2025-01-15T10:30:00.123Z [INFO] server: Query executed | query="SELECT * FROM users" duration_ms=12
|
||||
```
|
||||
|
||||
## Alerting-Regeln
|
||||
|
||||
### Prometheus AlertManager
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: baradb
|
||||
rules:
|
||||
- alert: BaraDBHighErrorRate
|
||||
expr: rate(baradb_errors_total[5m]) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "BaraDB error rate is high"
|
||||
|
||||
- alert: BaraDBLowCacheHitRate
|
||||
expr: baradb_cache_hit_rate < 0.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "BaraDB cache hit rate below 80%"
|
||||
|
||||
- alert: BaraDBHighConnections
|
||||
expr: baradb_active_connections > 800
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "BaraDB connection count is high"
|
||||
|
||||
- alert: BaraDBDown
|
||||
expr: up{job="baradb"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "BaraDB instance is down"
|
||||
```
|
||||
|
||||
## Grafana Dashboard
|
||||
|
||||
Dashboard ID `baradb-001` importieren oder das bereitgestellte JSON in `monitoring/grafana-dashboard.json` verwenden.
|
||||
|
||||
Wichtige Panels:
|
||||
- Queries pro Sekunde
|
||||
- Query-Latenz Perzentile (p50, p95, p99)
|
||||
- Speichergröße und SSTable-Anzahl
|
||||
- Cache Hit Rate
|
||||
- Aktive Verbindungen
|
||||
- Transaktionsrate
|
||||
- Fehlerrate
|
||||
|
||||
## Distributed Monitoring
|
||||
|
||||
### Cluster Metrics
|
||||
|
||||
Für Raft-Cluster überwachen:
|
||||
|
||||
```bash
|
||||
curl http://node1:9470/metrics/cluster
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"cluster_id": "baradb-cluster-1",
|
||||
"nodes": [
|
||||
{"id": "node1", "role": "leader", "health": "healthy"},
|
||||
{"id": "node2", "role": "follower", "health": "healthy"},
|
||||
{"id": "node3", "role": "follower", "health": "healthy"}
|
||||
],
|
||||
"raft_log_index": 15420,
|
||||
"raft_commit_index": 15420,
|
||||
"shards": 4,
|
||||
"replication_lag_ms": 5
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Profiling
|
||||
|
||||
### Eingebauter CPU Profiler
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:9470/debug/pprof/cpu?seconds=30 > cpu.prof
|
||||
```
|
||||
|
||||
### Memory Profiler
|
||||
|
||||
```bash
|
||||
curl http://localhost:9470/debug/pprof/heap > heap.prof
|
||||
```
|
||||
|
||||
### Trace
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:9470/debug/pprof/trace?seconds=5 > trace.out
|
||||
```
|
||||
|
||||
## Log-Aggregation
|
||||
|
||||
### Fluent Bit Konfiguration
|
||||
|
||||
```ini
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/baradb/baradb.log
|
||||
Parser json
|
||||
Tag baradb
|
||||
|
||||
[OUTPUT]
|
||||
Name elasticsearch
|
||||
Match baradb
|
||||
Host elasticsearch
|
||||
Port 9200
|
||||
Index baradb-logs
|
||||
```
|
||||
|
||||
## Troubleshooting mit Metrics
|
||||
|
||||
| Symptom | Metrik | Aktion |
|
||||
|---------|--------|--------|
|
||||
| Langsame Abfragen | `baradb_queries_duration_seconds` | Cache Hit Rate prüfen, Indizes in Betracht ziehen |
|
||||
| Hoher Speicherverbrauch | `process_resident_memory_bytes` | Memtable/Cache-Größen reduzieren |
|
||||
| Speicher wächst | `baradb_storage_lsm_size_bytes` | Manuelle Compaction ausführen |
|
||||
| Verbindungsfehler | `baradb_active_connections` | Connection Pool erhöhen oder Knoten hinzufügen |
|
||||
| Replikations-Lag | `baradb_replication_lag_ms` | Netzwerk prüfen, Ressourcen erhöhen |
|
||||
@@ -0,0 +1,174 @@
|
||||
# BaraDB Performance-Leitfaden
|
||||
|
||||
## Benchmark-Methodik
|
||||
|
||||
Alle Benchmarks wurden ausgeführt mit:
|
||||
- **Compiler**: Nim 2.2.0 mit `-d:release --opt:speed`
|
||||
- **CPU**: AMD Ryzen 9 5900X (12 Kerne / 24 Threads)
|
||||
- **Memory**: 64 GB DDR4-3600
|
||||
- **Storage**: Samsung 980 Pro NVMe SSD
|
||||
- **OS**: Ubuntu 24.04 LTS
|
||||
|
||||
Die vollständige Benchmark-Suite ausführen:
|
||||
|
||||
```bash
|
||||
nim c -d:ssl -d:release -r benchmarks/bench_all.nim
|
||||
```
|
||||
|
||||
## Storage Engine Benchmarks
|
||||
|
||||
### LSM-Tree Key-Value
|
||||
|
||||
| Metrik | Wert |
|
||||
|--------|------|
|
||||
| Write Throughput | ~580,000 ops/s |
|
||||
| Read Throughput | ~720,000 ops/s |
|
||||
| Durchschnittliche Write-Latenz | 1.7 µs |
|
||||
| Durchschnittliche Read-Latenz | 1.4 µs |
|
||||
| Testdatensatz | 100,000 Keys (16-Byte Keys, 64-Byte Values) |
|
||||
|
||||
Der LSM-Tree verwendet eine 64MB MemTable, WAL fsync bei jedem Write und size-tiered
|
||||
Compaction mit 6 Levels.
|
||||
|
||||
### B-Tree Index
|
||||
|
||||
| Metrik | Wert |
|
||||
|--------|------|
|
||||
| Insert Throughput | ~1,200,000 ops/s |
|
||||
| Point Lookup Throughput | ~1,500,000 ops/s |
|
||||
| Range Scan (1000 Keys) | ~0.3 ms |
|
||||
| Baumhöhe (100K Keys) | 4 |
|
||||
|
||||
B-Tree Knoten sind 4KB mit Copy-on-Write für MVCC-Kompatibilität.
|
||||
|
||||
## Vector Engine Benchmarks
|
||||
|
||||
### HNSW Index
|
||||
|
||||
| Metrik | Wert |
|
||||
|--------|------|
|
||||
| Insert (dim=128) | ~45,000 vectors/s |
|
||||
| Search top-10 (dim=128, n=10K) | ~2 ms |
|
||||
| Search top-10 (dim=128, n=100K) | ~8 ms |
|
||||
| Speicher pro Vektor (dim=128) | ~580 bytes |
|
||||
|
||||
Parameter: `M=16`, `efConstruction=200`, `efSearch=64`.
|
||||
|
||||
### SIMD Distanzfunktionen
|
||||
|
||||
| Operation | dim=128 | dim=768 | dim=1536 |
|
||||
|-----------|---------|---------|----------|
|
||||
| Cosine Distance | 4.2M/s | 850K/s | 420K/s |
|
||||
| L2 (Euclidean) | 4.5M/s | 920K/s | 450K/s |
|
||||
| Dot Product | 4.8M/s | 980K/s | 480K/s |
|
||||
|
||||
SIMD verwendet AVX2 256-Bit Vektoren mit Loop Unrolling.
|
||||
|
||||
### Quantization
|
||||
|
||||
| Methode | Genauigkeitsverlust | Speicherreduzierung |
|
||||
|---------|--------------------|--------------------|
|
||||
| Scalar 8-bit | <1% | 4× |
|
||||
| Scalar 4-bit | ~3% | 8× |
|
||||
| Product Quantization (PQ16) | ~5% | 16× |
|
||||
| Binary | ~15% | 32× |
|
||||
|
||||
## Full-Text Search Benchmarks
|
||||
|
||||
| Metrik | Wert |
|
||||
|--------|------|
|
||||
| Index Throughput | ~320,000 docs/s |
|
||||
| BM25 Search | ~28,000 queries/s |
|
||||
| Fuzzy Search (distance=2) | ~850 queries/s |
|
||||
| Wildcard Regex Search | ~4,200 queries/s |
|
||||
|
||||
Testkorpus: 5 einzigartige Dokumente × 2,000 Wiederholungen (~50 Wörter/Dok).
|
||||
|
||||
## Graph Engine Benchmarks
|
||||
|
||||
| Operation | Throughput | Latenz |
|
||||
|-----------|------------|--------|
|
||||
| Knoten hinzufügen | ~2.5M ops/s | 0.4 µs |
|
||||
| Kante hinzufügen | ~1.8M ops/s | 0.55 µs |
|
||||
| BFS (1K Knoten, 5K Kanten) | ~12K Traversierungen/s | 83 µs |
|
||||
| DFS (1K Knoten, 5K Kanten) | ~15K Traversierungen/s | 67 µs |
|
||||
| Dijkstra kürzester Pfad | — | ~120 µs |
|
||||
| PageRank (10 Iterationen) | ~450 Graphen/s | 2.2 ms |
|
||||
| Louvain Community Detection | — | ~45 ms |
|
||||
|
||||
## Protokoll-Benchmarks
|
||||
|
||||
| Protokoll | Verbindungen | Queries/sec | Latenz p99 |
|
||||
|-----------|--------------|-------------|------------|
|
||||
| Binary (localhost) | 1 | 45,000 | 0.4 ms |
|
||||
| Binary (localhost) | 100 | 380,000 | 1.2 ms |
|
||||
| HTTP/REST | 1 | 12,000 | 2.1 ms |
|
||||
| HTTP/REST | 100 | 95,000 | 5.8 ms |
|
||||
| WebSocket | 1 | 18,000 | 1.8 ms |
|
||||
|
||||
## Query Engine Benchmarks
|
||||
|
||||
| Abfragetyp | Zeilen | Zeit |
|
||||
|------------|--------|------|
|
||||
| Simple SELECT | 100K | 12 ms |
|
||||
| SELECT + WHERE | 100K | 18 ms |
|
||||
| SELECT + ORDER BY | 100K | 35 ms |
|
||||
| GROUP BY + Aggregates | 100K | 42 ms |
|
||||
| INNER JOIN (1K × 1K) | 1M Ergebnis | 85 ms |
|
||||
| CTE (2 Ebenen) | 100K | 28 ms |
|
||||
| Subquery (EXISTS) | 100K | 22 ms |
|
||||
|
||||
## Skalierungsverhalten
|
||||
|
||||
### Vertikale Skalierung
|
||||
|
||||
| Kerne | LSM Write | LSM Read | Vector Search |
|
||||
|-------|-----------|----------|---------------|
|
||||
| 1 | 580K | 720K | 2.0 ms |
|
||||
| 4 | 1.9M | 2.6M | 1.1 ms |
|
||||
| 8 | 3.4M | 4.8M | 0.7 ms |
|
||||
| 16 | 5.8M | 7.2M | 0.5 ms |
|
||||
|
||||
### Speicherverbrauch
|
||||
|
||||
| Komponente | Basis-Speicher | Pro-Entity Overhead |
|
||||
|------------|----------------|---------------------|
|
||||
| LSM MemTable | 64 MB (fest) | ~1.2× Rohdaten |
|
||||
| B-Tree | 8 MB (fest) | ~8 bytes/Key |
|
||||
| HNSW Index | — | ~580 bytes/Vektor (dim=128) |
|
||||
| Graph | — | ~32 bytes/Knoten, ~24 bytes/Kante |
|
||||
| FTS Index | — | ~40% von Rohtext |
|
||||
| Page Cache | 256 MB (konfigurierbar) | — |
|
||||
|
||||
## Tuning-Leitfaden
|
||||
|
||||
### Für Write-intensive Workloads
|
||||
|
||||
```bash
|
||||
export BARADB_MEMTABLE_SIZE_MB=256
|
||||
export BARADB_WAL_SYNC_INTERVAL_MS=10
|
||||
export BARADB_COMPACTION_INTERVAL_MS=30000
|
||||
```
|
||||
|
||||
### Für Read-intensive Workloads
|
||||
|
||||
```bash
|
||||
export BARADB_CACHE_SIZE_MB=1024
|
||||
export BARADB_BLOOM_BITS_PER_KEY=10
|
||||
export BARADB_COMPACTION_INTERVAL_MS=120000
|
||||
```
|
||||
|
||||
### Für Vector Search
|
||||
|
||||
```bash
|
||||
export BARADB_VECTOR_EF_CONSTRUCTION=200
|
||||
export BARADB_VECTOR_EF_SEARCH=128
|
||||
export BARADB_VECTOR_M=32
|
||||
```
|
||||
|
||||
### Für Graph Analytics
|
||||
|
||||
```bash
|
||||
export BARADB_GRAPH_PAGE_RANK_ITERATIONS=20
|
||||
export BARADB_GRAPH_LOUVAIN_RESOLUTION=1.0
|
||||
```
|
||||
@@ -0,0 +1,240 @@
|
||||
# Protokoll-Referenz
|
||||
|
||||
BaraDB unterstützt mehrere Protokolle für Client-Kommunikation:
|
||||
- **Binär Wire Protokoll** — hochperformant, niedrige Latenz
|
||||
- **HTTP/REST API** — sprachunabhängig, einfach zu debuggen
|
||||
- **WebSocket** — Streaming und Pub/Sub
|
||||
|
||||
---
|
||||
|
||||
## Binär Wire Protokoll
|
||||
|
||||
Das Binärprotokoll verwendet Big-Endian-Kodierung für alle Multi-Byte-Werte.
|
||||
|
||||
### Verbindungslebenszyklus
|
||||
|
||||
```
|
||||
Client Server
|
||||
| |
|
||||
|─── TCP connect ──────────────>|
|
||||
|<── TLS handshake (optional) ──|
|
||||
|─── Auth message ─────────────>|
|
||||
|<── Auth_OK / Error ───────────|
|
||||
|─── Query message ────────────>|
|
||||
|<── Data / Complete / Error ───|
|
||||
|─── Close message ────────────>|
|
||||
|<── TCP close ─────────────────|
|
||||
```
|
||||
|
||||
### Nachrichtenformat
|
||||
|
||||
Jede Nachricht beginnt mit einem 8-Byte-Header:
|
||||
|
||||
```
|
||||
┌─────────────┬─────────────┬─────────────┬─────────────────────┐
|
||||
│ Length │ Type │ Sequence │ Payload │
|
||||
│ (4 bytes) │ (1 byte) │ (1 byte) │ (Length - 6 bytes) │
|
||||
│ uint32 BE │ uint8 │ uint8 │ │
|
||||
└─────────────┴─────────────┴─────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
### Nachrichtentypen
|
||||
|
||||
| Typ | ID | Richtung | Beschreibung |
|
||||
|------|----|-----------|-------------|
|
||||
| Query | 0x01 | C→S | Abfrage ausführen |
|
||||
| Insert | 0x02 | C→S | Daten einfügen |
|
||||
| Update | 0x03 | C→S | Daten aktualisieren |
|
||||
| Delete | 0x04 | C→S | Daten löschen |
|
||||
| Ready | 0x05 | S→C | Bereit für nächsten Befehl |
|
||||
| Error | 0x06 | S→C | Fehlerantwort |
|
||||
| Auth | 0x07 | C→S | Authentifizierungsanfrage |
|
||||
| Batch | 0x08 | C→S | Batch-Operationen |
|
||||
| Ping | 0x09 | C→S | Keepalive Ping |
|
||||
| Data | 0x81 | S→C | Abfrageergebnis-Daten |
|
||||
| Complete | 0x82 | S→C | Abfrage abgeschlossen |
|
||||
| Auth_OK | 0x83 | S→C | Authentifizierung erfolgreich |
|
||||
| Pong | 0x84 | S→C | Keepalive-Antwort |
|
||||
|
||||
### Query-Nachrichten-Payload
|
||||
|
||||
```
|
||||
┌──────────────┬──────────────┬────────────────────────────┐
|
||||
│ Result Format│ Query Length │ Query String │
|
||||
│ (1 byte) │ (4 bytes) │ (Query Length bytes) │
|
||||
│ 0x00=Binary │ uint32 BE │ UTF-8 │
|
||||
│ 0x01=JSON │ │ │
|
||||
│ 0x02=Text │ │ │
|
||||
└──────────────┴──────────────┴────────────────────────────┘
|
||||
```
|
||||
|
||||
### Data-Nachrichten-Payload
|
||||
|
||||
```
|
||||
┌──────────────┬─────────────────────────────────────────────┐
|
||||
│ Column Count │ Column Definitions + Row Data │
|
||||
│ (2 bytes) │ │
|
||||
│ uint16 BE │ │
|
||||
└──────────────┴─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Spaltendefinition
|
||||
|
||||
```
|
||||
┌──────────────┬──────────────┬────────────────────────────┐
|
||||
│ Name Length │ Name │ Type │
|
||||
│ (2 bytes) │ (N bytes) │ (1 byte) │
|
||||
│ uint16 BE │ UTF-8 │ Siehe FieldKind-Tabelle │
|
||||
└──────────────┴──────────────┴────────────────────────────┘
|
||||
```
|
||||
|
||||
### Feldtypen
|
||||
|
||||
| Typ | ID | Größe | Beschreibung |
|
||||
|------|----|------|-------------|
|
||||
| NULL | 0x00 | 0 | NULL-Wert |
|
||||
| BOOL | 0x01 | 1 | true/false |
|
||||
| INT8 | 0x02 | 1 | Signed 8-bit Integer |
|
||||
| INT16 | 0x03 | 2 | Signed 16-bit Integer |
|
||||
| INT32 | 0x04 | 4 | Signed 32-bit Integer |
|
||||
| INT64 | 0x05 | 8 | Signed 64-bit Integer |
|
||||
| FLOAT32 | 0x06 | 4 | IEEE 754 Single Precision (Big-Endian) |
|
||||
| FLOAT64 | 0x07 | 8 | IEEE 754 Double Precision (Big-Endian) |
|
||||
| STRING | 0x08 | variable | UTF-8 String (4-Byte Längenpräfix) |
|
||||
| BYTES | 0x09 | variable | Raw Bytes (4-Byte Längenpräfix) |
|
||||
| ARRAY | 0x0A | variable | Array von Werten |
|
||||
| OBJECT | 0x0B | variable | Key-Value Objekt |
|
||||
| VECTOR | 0x0C | variable | Float32 Array (4-Byte Längenpräfix, Big-Endian Floats) |
|
||||
|
||||
### Fehler-Nachrichten-Payload
|
||||
|
||||
```
|
||||
┌──────────────┬──────────────┬────────────────────────────┐
|
||||
│ Error Code │ Message Len │ Error Message │
|
||||
│ (4 bytes) │ (4 bytes) │ (Message Len bytes) │
|
||||
│ uint32 BE │ uint32 BE │ UTF-8 │
|
||||
└──────────────┴──────────────┴────────────────────────────┘
|
||||
```
|
||||
|
||||
### Beispiel: Raw TCP-Session
|
||||
|
||||
```bash
|
||||
# Verbinden
|
||||
nc localhost 9472
|
||||
|
||||
# Senden: Auth-Anfrage (Token "mytoken")
|
||||
# Header: length=15, type=0x07, seq=1
|
||||
# Payload: token length=7, token="mytoken"
|
||||
printf '\x00\x00\x00\x0f\x07\x01\x00\x00\x00\x07mytoken' > /dev/tcp/localhost/9472
|
||||
|
||||
# Empfangen: Auth_OK
|
||||
# \x00\x00\x00\x06\x83\x01
|
||||
|
||||
# Senden: Query "SELECT 1"
|
||||
printf '\x00\x00\x00\x12\x01\x02\x00\x00\x00\x00\x08SELECT 1' > /dev/tcp/localhost/9472
|
||||
|
||||
# Empfangen: Data + Complete
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## HTTP/REST API
|
||||
|
||||
Basis-URL: `http://localhost:9470/api/v1`
|
||||
|
||||
### Endpoints
|
||||
|
||||
#### Health
|
||||
|
||||
```http
|
||||
GET /health
|
||||
```
|
||||
|
||||
Antwort:
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"version": "0.1.0",
|
||||
"uptime_seconds": 86400
|
||||
}
|
||||
```
|
||||
|
||||
#### Ready
|
||||
|
||||
```http
|
||||
GET /ready
|
||||
```
|
||||
|
||||
Gibt `200` zurück wenn bereit, `503` während des Starts.
|
||||
|
||||
#### Query
|
||||
|
||||
```http
|
||||
POST /query
|
||||
Content-Type: application/json
|
||||
Authorization: Bearer <token>
|
||||
|
||||
{
|
||||
"query": "SELECT name, age FROM users WHERE age > 18",
|
||||
"params": [],
|
||||
"format": "json"
|
||||
}
|
||||
```
|
||||
|
||||
#### Batch
|
||||
|
||||
```http
|
||||
POST /batch
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"queries": [
|
||||
"INSERT users { name := 'Alice', age := 30 }",
|
||||
"INSERT users { name := 'Bob', age := 25 }"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### HTTP-Statuscodes
|
||||
|
||||
| Code | Bedeutung |
|
||||
|------|----------|
|
||||
| 200 | Erfolg |
|
||||
| 400 | Bad request (Syntaxfehler) |
|
||||
| 401 | Unauthorized (Auth erforderlich) |
|
||||
| 403 | Forbidden (ungenügende Berechtigungen) |
|
||||
| 404 | Not found (Tabelle/Typ existiert nicht) |
|
||||
| 429 | Too many requests (Rate limitiert) |
|
||||
| 500 | Internal server error |
|
||||
| 503 | Service unavailable (wird gestartet) |
|
||||
|
||||
---
|
||||
|
||||
## WebSocket-Protokoll
|
||||
|
||||
URL: `ws://localhost:9471`
|
||||
|
||||
### Frame-Format
|
||||
|
||||
WebSocket Text-Frames enthalten JSON-Nachrichten:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"type": "query",
|
||||
"query": "SELECT * FROM users"
|
||||
}
|
||||
```
|
||||
|
||||
### Nachrichtentypen
|
||||
|
||||
| Typ | Richtung | Beschreibung |
|
||||
|------|-----------|--------------|
|
||||
| `query` | C→S | Abfrage ausführen |
|
||||
| `subscribe` | C→S | Änderungen abonnieren |
|
||||
| `unsubscribe` | C→S | Abonnement beenden |
|
||||
| `ping` | C→S | Keepalive |
|
||||
| `result` | S→C | Abfrageergebnis |
|
||||
| `notification` | S→C | Änderungsbenachrichtigung |
|
||||
| `error` | S→C | Fehler |
|
||||
| `pong` | S→C | Keepalive-Antwort |
|
||||
@@ -0,0 +1,55 @@
|
||||
# Schema-System
|
||||
|
||||
BaraDB verwendet ein Schema-first Design mit Typvererbung und automatischen Migrationen.
|
||||
|
||||
## Typen definieren
|
||||
|
||||
```nim
|
||||
import barabadb/schema/schema
|
||||
|
||||
var s = newSchema()
|
||||
|
||||
let person = newType("Person")
|
||||
person.addProperty("name", "str", required = true)
|
||||
person.addProperty("age", "int32")
|
||||
s.addType("default", person)
|
||||
```
|
||||
|
||||
## Typvererbung
|
||||
|
||||
```nim
|
||||
let employee = newType("Employee")
|
||||
employee.setBases(@["Person"])
|
||||
employee.addProperty("department", "str")
|
||||
s.addType("default", employee)
|
||||
|
||||
# Vererbung auflösen — Employee erhält name, age, department
|
||||
let resolved = s.resolveInheritance(employee)
|
||||
```
|
||||
|
||||
## Schema-Operationen
|
||||
|
||||
### Diff
|
||||
|
||||
Zwei Schemata vergleichen:
|
||||
|
||||
```nim
|
||||
let diff = s.diff(oldSchema, newSchema)
|
||||
```
|
||||
|
||||
### Migrationen
|
||||
|
||||
Schema-Änderungen werden verfolgt und können Migrationsskripte generieren.
|
||||
|
||||
## Eigenschaftstypen
|
||||
|
||||
| Typ | Beschreibung |
|
||||
|------|-------------|
|
||||
| `str` | String |
|
||||
| `int32` | 32-Bit Integer |
|
||||
| `int64` | 64-Bit Integer |
|
||||
| `float32` | 32-Bit Float |
|
||||
| `float64` | 64-Bit Float |
|
||||
| `bool` | Boolean |
|
||||
| `datetime` | Datums-/Zeitwert |
|
||||
| `bytes` | Binärdaten |
|
||||
@@ -0,0 +1,184 @@
|
||||
# Sicherheitsleitfaden
|
||||
|
||||
## TLS/SSL-Verschlüsselung
|
||||
|
||||
BaraDB unterstützt TLS 1.3 für alle Protokolle (Binary, HTTP, WebSocket). Wenn kein
|
||||
Zertifikat bereitgestellt wird, generiert der Server automatisch ein selbstsigniertes Zertifikat
|
||||
beim Start für Zero-Configuration-Verschlüsselung.
|
||||
|
||||
### Eigene Zertifikate verwenden
|
||||
|
||||
```bash
|
||||
# Vorhandene Zertifikate bereitstellen
|
||||
BARADB_TLS_ENABLED=true \
|
||||
BARADB_CERT_FILE=/etc/baradb/server.crt \
|
||||
BARADB_KEY_FILE=/etc/baradb/server.key \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
### Selbstsignierte Zertifikate generieren
|
||||
|
||||
```bash
|
||||
openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt \
|
||||
-days 365 -nodes -subj "/CN=localhost"
|
||||
```
|
||||
|
||||
### Let's Encrypt (Production)
|
||||
|
||||
Certbot verwenden und BaraDB auf die generierten Dateien zeigen:
|
||||
|
||||
```bash
|
||||
sudo certbot certonly --standalone -d db.example.com
|
||||
|
||||
BARADB_CERT_FILE=/etc/letsencrypt/live/db.example.com/fullchain.pem \
|
||||
BARADB_KEY_FILE=/etc/letsencrypt/live/db.example.com/privkey.pem \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Authentifizierung
|
||||
|
||||
### JWT-basierte Authentifizierung
|
||||
|
||||
BaraDB verwendet JWT (JSON Web Tokens) mit HMAC-SHA256-Signatur.
|
||||
|
||||
#### Authentifizierung aktivieren
|
||||
|
||||
```bash
|
||||
BARADB_AUTH_ENABLED=true \
|
||||
BARADB_JWT_SECRET="$(openssl rand -hex 32)" \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
#### Tokens erstellen
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/auth
|
||||
|
||||
var am = newAuthManager("your-secret-key")
|
||||
let token = am.createToken(JWTClaims(
|
||||
sub: "user1",
|
||||
role: "admin",
|
||||
exp: getTime() + 24.hours
|
||||
))
|
||||
```
|
||||
|
||||
#### Rollenbasierte Zugriffskontrolle
|
||||
|
||||
| Rolle | Berechtigungen |
|
||||
|------|---------------|
|
||||
| `admin` | Voller Zugriff |
|
||||
| `write` | Lesen + Schreiben |
|
||||
| `read` | Nur Lesen |
|
||||
| `monitor` | Nur Metrics und Health |
|
||||
|
||||
#### Tokens verwenden
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
http://localhost:9470/api/query \
|
||||
-d '{"query": "SELECT * FROM users"}'
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
Token-Bucket Rate Limiting verhindert Missbrauch:
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/ratelimit
|
||||
|
||||
var rl = newRateLimiter(
|
||||
rlaTokenBucket,
|
||||
globalRate = 10000, # 10K req/s global
|
||||
perClientRate = 1000, # 1K req/s pro IP/Token
|
||||
burstSize = 100 # 100 req Burst erlauben
|
||||
)
|
||||
|
||||
if not rl.allowRequest("client-ip"):
|
||||
return error("Rate limit exceeded")
|
||||
```
|
||||
|
||||
## Netzwerksicherheit
|
||||
|
||||
### Bind-Adresse
|
||||
|
||||
Standardmäßig bindet BaraDB an `127.0.0.1` (nur Localhost). Für Production:
|
||||
|
||||
```bash
|
||||
# An alle Interfaces binden (hinter Firewall oder Reverse Proxy)
|
||||
BARADB_ADDRESS=0.0.0.0 ./build/baradadb
|
||||
|
||||
# An spezifisches internes Interface binden
|
||||
BARADB_ADDRESS=10.0.0.5 ./build/baradadb
|
||||
```
|
||||
|
||||
### Firewall-Regeln
|
||||
|
||||
```bash
|
||||
# Nur Application-Server erlauben
|
||||
sudo ufw allow from 10.0.0.0/8 to any port 9472
|
||||
sudo ufw allow from 10.0.0.0/8 to any port 9470
|
||||
|
||||
# Externen Zugriff auf Management-Ports blockieren
|
||||
sudo ufw deny 9471 # WebSocket (nur für internen Gebrauch)
|
||||
```
|
||||
|
||||
## Datenverschlüsselung at Rest
|
||||
|
||||
### OS-Level-Verschlüsselung
|
||||
|
||||
LUKS für Vollständige-Festplatten-Verschlüsselung verwenden:
|
||||
|
||||
```bash
|
||||
cryptsetup luksFormat /dev/nvme0n1p2
|
||||
cryptsetup open /dev/nvme0n1p2 baradb-crypt
|
||||
mkfs.ext4 /dev/mapper/baradb-crypt
|
||||
mount /dev/mapper/baradb-crypt /var/lib/baradb
|
||||
```
|
||||
|
||||
### Applikations-Level-Verschlüsselung
|
||||
|
||||
BaraDB unterstützt transparente Verschlüsselung von SSTable-Dateien:
|
||||
|
||||
```bash
|
||||
BARADB_STORAGE_ENCRYPTION_KEY="$(openssl rand -hex 32)" \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Audit Logging
|
||||
|
||||
Alle Abfragen und administrativen Aktionen werden geloggt:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2025-01-15T10:30:00Z",
|
||||
"level": "info",
|
||||
"event": "query_executed",
|
||||
"client_ip": "10.0.0.15",
|
||||
"user": "app_user",
|
||||
"query": "SELECT * FROM users WHERE id = ?",
|
||||
"duration_ms": 12,
|
||||
"rows_returned": 1
|
||||
}
|
||||
```
|
||||
|
||||
Audit Logging aktivieren:
|
||||
|
||||
```bash
|
||||
BARADB_LOG_LEVEL=info \
|
||||
BARADB_LOG_FORMAT=json \
|
||||
BARADB_LOG_FILE=/var/log/baradb/audit.log \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Sicherheits-Checkliste
|
||||
|
||||
- [ ] Standard-JWT-Geheimnis ändern
|
||||
- [ ] TLS mit gültigen Zertifikaten aktivieren
|
||||
- [ ] An spezifische Interfaces binden
|
||||
- [ ] Authentifizierung in Production aktivieren
|
||||
- [ ] Rate Limiting konfigurieren
|
||||
- [ ] Audit Logging aktivieren
|
||||
- [ ] Daten at Rest verschlüsseln (LUKS oder App-Level)
|
||||
- [ ] BaraDB als Non-Root-Benutzer ausführen
|
||||
- [ ] Firewall-Regeln restriktiv halten
|
||||
- [ ] JWT-Geheimnisse regelmäßig rotieren
|
||||
@@ -0,0 +1,78 @@
|
||||
# Speicher-Engines
|
||||
|
||||
BaraDB bietet mehrere Speicher-Engines, optimiert für verschiedene Zugriffsmuster.
|
||||
|
||||
## LSM-Tree (Key-Value)
|
||||
|
||||
Die primäre Speicher-Engine mit write-optimierter Append-only Log-Struktur.
|
||||
|
||||
### Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/storage/lsm
|
||||
|
||||
var db = newLSMTree("./data")
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
let (found, value) = db.get("key1")
|
||||
db.close()
|
||||
```
|
||||
|
||||
### Komponenten
|
||||
|
||||
- **MemTable**: In-Memory sortierter Puffer
|
||||
- **WAL**: Write-Ahead Log für Dauerhaftigkeit
|
||||
- **SSTable**: Sortierte String-Tabellen auf Disk
|
||||
- **Bloom-Filter**: Probabilistische Mengenmitgliedschaft
|
||||
- **Compaction**: Size-tiered Strategie mit Level-Management
|
||||
- **Page-Cache**: LRU-Cache mit Trefferraten-Verfolgung
|
||||
|
||||
## B-Tree Index
|
||||
|
||||
Geordneter Index für Bereichsabfragen und Point-Lookups.
|
||||
|
||||
### Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/storage/btree
|
||||
|
||||
var btree = newBTreeIndex[string, string]()
|
||||
btree.insert("key1", "value1")
|
||||
let values = btree.get("key1")
|
||||
let range = btree.scan("key_a", "key_z")
|
||||
```
|
||||
|
||||
## Write-Ahead Log (WAL)
|
||||
|
||||
Sichert Dauerhaftigkeit von Schreiboperationen.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/wal
|
||||
|
||||
var wal = newWAL("./wal")
|
||||
wal.append("txn1", "SET key1 value1")
|
||||
wal.flush()
|
||||
```
|
||||
|
||||
## Bloom-Filter
|
||||
|
||||
Probabilistische Datenstruktur für schnelle negative Lookups.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/bloom
|
||||
|
||||
var filter = newBloomFilter(10000, 0.01)
|
||||
filter.add("key1")
|
||||
if filter.mightContain("key1"):
|
||||
echo "possibly exists"
|
||||
```
|
||||
|
||||
## Memory-mapped I/O
|
||||
|
||||
Effizienter Dateizugriff mittels mmap.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/mmap
|
||||
|
||||
var mapped = mmapFile("./data/file.dat")
|
||||
let data = mapped.read(0, 100)
|
||||
```
|
||||
@@ -0,0 +1,78 @@
|
||||
# Transaktionen & MVCC
|
||||
|
||||
MVCC (Multi-Version Concurrency Control) mit Snapshot-Isolation und Deadlock-Erkennung.
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/core/mvcc
|
||||
|
||||
var tm = newTxnManager()
|
||||
let txn = tm.beginTxn()
|
||||
|
||||
# Schreiboperationen
|
||||
discard tm.write(txn, "key1", cast[seq[byte]]("value1"))
|
||||
discard tm.write(txn, "key2", cast[seq[byte]]("value2"))
|
||||
|
||||
# Savepoint
|
||||
tm.savepoint(txn)
|
||||
discard tm.write(txn, "key3", cast[seq[byte]]("value3"))
|
||||
discard tm.rollbackToSavepoint(txn) # rückgängig machen key3
|
||||
|
||||
# Commit
|
||||
discard tm.commit(txn)
|
||||
```
|
||||
|
||||
## Transaktionsisolation
|
||||
|
||||
BaraDB verwendet **Snapshot-Isolation**:
|
||||
- Leser blockieren keine Schreiber
|
||||
- Schreiber blockieren keine Leser
|
||||
- Jede Transaktion sieht einen konsistenten Snapshot
|
||||
|
||||
## Deadlock-Erkennung
|
||||
|
||||
```nim
|
||||
import barabadb/core/deadlock
|
||||
|
||||
var detector = newDeadlockDetector()
|
||||
if detector.detectCycle(txn1, txn2):
|
||||
echo "Deadlock erkannt!"
|
||||
```
|
||||
|
||||
## Write-Ahead Log
|
||||
|
||||
```nim
|
||||
import barabadb/storage/wal
|
||||
|
||||
var wal = newWAL("./wal")
|
||||
wal.append(txnId, "SET key value")
|
||||
wal.flush()
|
||||
```
|
||||
|
||||
## Savepoints
|
||||
|
||||
Verschachtelte Transaktions-Savepoints:
|
||||
|
||||
```nim
|
||||
tm.savepoint(txn, "sp1")
|
||||
# ... Operationen ...
|
||||
tm.rollbackToSavepoint(txn, "sp1")
|
||||
```
|
||||
|
||||
## Formale Verifikation
|
||||
|
||||
Das MVCC / Snapshot-Isolation Protokoll ist formal in TLA+ spezifiziert:
|
||||
|
||||
- **Spec:** `formal-verification/mvcc.tla`
|
||||
- **Verifizierte Eigenschaften:**
|
||||
- `NoDirtyReads` — Transaktionen lesen niemals nicht-committete Daten
|
||||
- `ReadOwnWrites` — Transaktionen sehen immer ihre eigenen Schreiboperationen
|
||||
- `WriteWriteConflict` — First-committer-wins (keine zwei committete Transaktionen schreiben denselben Schlüssel)
|
||||
|
||||
Lokale TLC-Ausführung:
|
||||
|
||||
```bash
|
||||
cd formal-verification
|
||||
java -cp tla2tools.jar tlc2.TLC -config models/mvcc.cfg mvcc.tla
|
||||
```
|
||||
@@ -0,0 +1,335 @@
|
||||
# Fehlerbehebungsleitfaden
|
||||
|
||||
## Installationsprobleme
|
||||
|
||||
### Nim nicht gefunden
|
||||
|
||||
```
|
||||
im: command not found
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Linux/macOS
|
||||
curl https://nim-lang.org/choosenim/init.sh -sSf | sh
|
||||
|
||||
# Zu PATH hinzufügen
|
||||
echo 'export PATH=$HOME/.nimble/bin:$PATH' >> ~/.bashrc
|
||||
source ~/.bashrc
|
||||
```
|
||||
|
||||
### SSL-Kompilierungsfehler
|
||||
|
||||
```
|
||||
Error: BaraDB requires SSL support. Compile with -d:ssl
|
||||
```
|
||||
|
||||
**Lösung:** Immer mit `-d:ssl` kompilieren:
|
||||
|
||||
```bash
|
||||
nim c -d:ssl -d:release -o:build/baradadb src/baradadb.nim
|
||||
```
|
||||
|
||||
### Fehlende Abhängigkeiten
|
||||
|
||||
```
|
||||
Error: cannot open file: hunos
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
nimble install -d -y
|
||||
```
|
||||
|
||||
## Laufzeitprobleme
|
||||
|
||||
### Port bereits in Verwendung
|
||||
|
||||
```
|
||||
Error: unhandled exception: Address already in use [OSError]
|
||||
```
|
||||
|
||||
**Lösung 1:** Port ändern:
|
||||
|
||||
```bash
|
||||
BARADB_PORT=5433 ./build/baradadb
|
||||
```
|
||||
|
||||
**Lösung 2:** Bestehenden Prozess beenden:
|
||||
|
||||
```bash
|
||||
lsof -ti:9472 | xargs kill -9
|
||||
# oder
|
||||
fuser -k 9472/tcp
|
||||
```
|
||||
|
||||
### Permission Denied auf Datenverzeichnis
|
||||
|
||||
```
|
||||
Error: cannot create directory: Permission denied
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
chmod 755 ./data
|
||||
# Oder anderes Verzeichnis verwenden
|
||||
BARADB_DATA_DIR=/tmp/baradb_data ./build/baradadb
|
||||
```
|
||||
|
||||
### Out of Memory
|
||||
|
||||
```
|
||||
Error: out of memory
|
||||
```
|
||||
|
||||
**Lösung:** Speicherverbrauch reduzieren:
|
||||
|
||||
```bash
|
||||
BARADB_MEMTABLE_SIZE_MB=32 \
|
||||
BARADB_CACHE_SIZE_MB=128 \
|
||||
BARADB_VECTOR_EF_CONSTRUCTION=100 \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
### Disk Full
|
||||
|
||||
```
|
||||
Error: No space left on device
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Disk-Nutzung prüfen
|
||||
df -h
|
||||
|
||||
# Compaction auslösen um Platz freizugeben
|
||||
curl -X POST http://localhost:9470/api/admin/compact
|
||||
|
||||
# Oder manuell
|
||||
./build/baradadb --compact
|
||||
```
|
||||
|
||||
## Abfrageprobleme
|
||||
|
||||
### Syntaxfehler
|
||||
|
||||
```
|
||||
Error: Syntax error at position 15: unexpected token
|
||||
```
|
||||
|
||||
**Lösung:** Abfragesyntax prüfen:
|
||||
|
||||
```sql
|
||||
-- Korrekt
|
||||
SELECT name, age FROM users WHERE age > 18;
|
||||
|
||||
-- Inkorrekt (fehlendes Komma)
|
||||
SELECT name age FROM users WHERE age > 18;
|
||||
```
|
||||
|
||||
### Tabelle nicht gefunden
|
||||
|
||||
```
|
||||
Error: Table 'users' does not exist
|
||||
```
|
||||
|
||||
**Lösung:** Zuerst Schema erstellen:
|
||||
|
||||
```sql
|
||||
CREATE TYPE User {
|
||||
name: str,
|
||||
age: int32
|
||||
};
|
||||
```
|
||||
|
||||
### Typ-Mismatch
|
||||
|
||||
```
|
||||
Error: Cannot compare int32 with str
|
||||
```
|
||||
|
||||
**Lösung:** Korrekte Typen verwenden:
|
||||
|
||||
```sql
|
||||
-- Korrekt
|
||||
SELECT * FROM users WHERE age > 18;
|
||||
|
||||
-- Inkorrekt
|
||||
SELECT * FROM users WHERE age > '18';
|
||||
```
|
||||
|
||||
### Timeout
|
||||
|
||||
```
|
||||
Error: Query execution timeout
|
||||
```
|
||||
|
||||
**Lösung:** LIMIT hinzufügen oder optimieren:
|
||||
|
||||
```sql
|
||||
-- Limit hinzufügen
|
||||
SELECT * FROM large_table LIMIT 1000;
|
||||
|
||||
-- Index verwenden
|
||||
SELECT * FROM users WHERE id = 123;
|
||||
```
|
||||
|
||||
## Verbindungsprobleme
|
||||
|
||||
### Connection Refused
|
||||
|
||||
```
|
||||
Connection refused: localhost:9472
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Prüfen ob Server läuft
|
||||
ps aux | grep baradadb
|
||||
|
||||
# Server starten
|
||||
./build/baradadb
|
||||
|
||||
# Firewall prüfen
|
||||
sudo ufw status
|
||||
sudo ufw allow 9472
|
||||
```
|
||||
|
||||
### Authentifizierung fehlgeschlagen
|
||||
|
||||
```
|
||||
Error: Authentication failed
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Prüfen ob JWT-Geheimnis übereinstimmt
|
||||
BARADB_AUTH_ENABLED=true \
|
||||
BARADB_JWT_SECRET="correct-secret" \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Performance-Probleme
|
||||
|
||||
### Langsame Abfragen
|
||||
|
||||
**Diagnose:**
|
||||
|
||||
```bash
|
||||
# Abfrageplan prüfen
|
||||
curl -X POST http://localhost:9470/api/explain \
|
||||
-d '{"query": "SELECT * FROM large_table"}'
|
||||
```
|
||||
|
||||
**Lösungen:**
|
||||
|
||||
1. Indizes hinzufügen:
|
||||
```sql
|
||||
CREATE INDEX idx_users_name ON users(name);
|
||||
```
|
||||
|
||||
2. LIMIT verwenden:
|
||||
```sql
|
||||
SELECT * FROM users LIMIT 100;
|
||||
```
|
||||
|
||||
3. Cache erhöhen:
|
||||
```bash
|
||||
BARADB_CACHE_SIZE_MB=1024 ./build/baradadb
|
||||
```
|
||||
|
||||
### Hohe CPU-Nutzung
|
||||
|
||||
**Ursachen:**
|
||||
- Compaction läuft
|
||||
- Große Vektor-Suche ohne HNSW
|
||||
- Komplexe Graph-Traversierung
|
||||
|
||||
**Lösungen:**
|
||||
|
||||
```bash
|
||||
# Compaction-Intervall anpassen
|
||||
BARADB_COMPACTION_INTERVAL_MS=300000 ./build/baradadb
|
||||
|
||||
# Approximative Vektor-Suche verwenden
|
||||
SELECT /*+ APPROXIMATE */ * FROM vectors
|
||||
ORDER BY cosine_distance(embedding, [...])
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
## Cluster-Probleme
|
||||
|
||||
### Raft Split-Brain
|
||||
|
||||
```
|
||||
Warning: Multiple leaders detected
|
||||
```
|
||||
|
||||
**Lösung:** Ungerade Anzahl von Knoten sicherstellen (3, 5, 7). Minority-Partition neu starten.
|
||||
|
||||
### Replikations-Lag
|
||||
|
||||
```
|
||||
Warning: Replication lag > 10s
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Netzwerk-Latenz prüfen
|
||||
ping replica-node
|
||||
|
||||
# Replikations-Threads erhöhen
|
||||
BARADB_REPLICATION_THREADS=4 ./build/baradadb
|
||||
|
||||
# Auf Async-Replikation umschalten
|
||||
BARADB_REPLICATION_MODE=async
|
||||
```
|
||||
|
||||
## Datenkorruption
|
||||
|
||||
### Prüfsummen-Mismatch
|
||||
|
||||
```
|
||||
Error: SSTable checksum mismatch
|
||||
```
|
||||
|
||||
**Lösung:**
|
||||
|
||||
```bash
|
||||
# Korrupte SSTable entfernen (Daten werden aus WAL wiederhergestellt)
|
||||
rm ./data/sstables/corrupted.sst
|
||||
|
||||
# Neustarten und wiederherstellen
|
||||
./build/baradadb --recover
|
||||
```
|
||||
|
||||
## Debug-Modus
|
||||
|
||||
Debug-Logging für detaillierte Diagnostik aktivieren:
|
||||
|
||||
```bash
|
||||
BARADB_LOG_LEVEL=debug \
|
||||
BARADB_LOG_FILE=/tmp/baradb_debug.log \
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
## Hilfe erhalten
|
||||
|
||||
Wenn das Problem weiterhin besteht:
|
||||
|
||||
1. Logs prüfen: `tail -f /var/log/baradb/baradb.log`
|
||||
2. Metrics prüfen: `curl http://localhost:9470/metrics`
|
||||
3. Diagnostik ausführen: `./build/baradadb --diagnose`
|
||||
4. Issue öffnen mit:
|
||||
- BaraDB Version (`./build/baradadb --version`)
|
||||
- OS und Architektur
|
||||
- Relevante Log-Auszüge
|
||||
- Schritte zur Reproduktion
|
||||
@@ -0,0 +1,56 @@
|
||||
# Benutzerdefinierte Funktionen
|
||||
|
||||
BaraQL mit benutzerdefinierten Funktionen erweitern.
|
||||
|
||||
## Verwendung
|
||||
|
||||
```nim
|
||||
import barabadb/query/udf
|
||||
|
||||
var reg = newUDFRegistry()
|
||||
|
||||
# Standard-Bibliothek registrieren
|
||||
reg.registerStdlib() # abs, sqrt, pow, lower, upper, len, trim, substr, toString, toInt
|
||||
|
||||
# Benutzerdefinierte Funktion
|
||||
reg.register("greet", @[UDFParam(name: "name", typeName: "str")],
|
||||
"str", proc(args: seq[Value]): Value =
|
||||
return Value(kind: vkString, strVal: "Hello, " & args[0].strVal & "!"))
|
||||
```
|
||||
|
||||
## Standard-Bibliotheksfunktionen
|
||||
|
||||
| Funktion | Beschreibung | Beispiel |
|
||||
|----------|-------------|---------|
|
||||
| `abs(n)` | Absoluter Wert | `abs(-5)` → 5 |
|
||||
| `sqrt(n)` | Quadratwurzel | `sqrt(16)` → 4 |
|
||||
| `pow(n, e)` | Potenz | `pow(2, 3)` → 8 |
|
||||
| `lower(s)` | Kleinbuchstaben | `lower('ABC')` → 'abc' |
|
||||
| `upper(s)` | Großbuchstaben | `upper('abc')` → 'ABC' |
|
||||
| `len(s)` | Länge | `len('hello')` → 5 |
|
||||
| `trim(s)` | Leerzeichen trimmen | `trim(' hello ')` → 'hello' |
|
||||
| `substr(s, start, len)` | Substring | `substr('hello', 0, 3)` → 'hel' |
|
||||
| `toString(n)` | In String konvertieren | `toString(123)` → '123' |
|
||||
| `toInt(s)` | In Integer konvertieren | `toInt('123')` → 123 |
|
||||
|
||||
## Funktionsregistrierung
|
||||
|
||||
```nim
|
||||
reg.register(
|
||||
name: "my_function",
|
||||
params: @[
|
||||
UDFParam(name: "arg1", typeName: "str"),
|
||||
UDFParam(name: "arg2", typeName: "int32")
|
||||
],
|
||||
returnType: "str",
|
||||
body: proc(args: seq[Value]): Value =
|
||||
# Implementierung
|
||||
result = Value(kind: vkString, strVal: "")
|
||||
)
|
||||
```
|
||||
|
||||
## UDFs in Abfragen verwenden
|
||||
|
||||
```sql
|
||||
SELECT greet(name) FROM users;
|
||||
```
|
||||
Reference in New Issue
Block a user