Add comprehensive documentation with i18n support (EN/BG)
- Add docs/ folder with English (en/) and Bulgarian (bg/) documentation - Create index.md with language switching and links - English docs: installation, quickstart, architecture, baraql, storage, schema, lsm, btree, vector, graph, fts, columnar, transactions, distributed, protocol, udf, api-binary, api-http, api-websocket - Bulgarian docs: installation, quickstart, architecture, baraql, schema, lsm, btree, vector, graph, fts, transactions, distributed - Update README license to BSD 3-Clause - Add LICENSE file with BSD 3-Clause text
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# Binary Protocol API
|
||||
|
||||
Low-level wire protocol for high-performance client connections.
|
||||
|
||||
## Message Format
|
||||
|
||||
All messages use big-endian byte order:
|
||||
|
||||
```
|
||||
┌────────┬────────┬────────┬────────┬─────────────┐
|
||||
│ Length │ Type │ Seq │ Status │ Payload │
|
||||
│ 4 bytes│ 1 byte │ 2 bytes│ 1 byte │ N bytes │
|
||||
└────────┴────────┴────────┴────────┴─────────────┘
|
||||
```
|
||||
|
||||
## Message Types
|
||||
|
||||
### Query (0x01)
|
||||
|
||||
```nim
|
||||
let msg = makeQueryMessage(seq, "SELECT * FROM users")
|
||||
```
|
||||
|
||||
### Insert (0x02)
|
||||
|
||||
```nim
|
||||
let msg = makeInsertMessage(seq, "users", data)
|
||||
```
|
||||
|
||||
### Update (0x03)
|
||||
|
||||
```nim
|
||||
let msg = makeUpdateMessage(seq, "users", updates, where)
|
||||
```
|
||||
|
||||
### Delete (0x04)
|
||||
|
||||
```nim
|
||||
let msg = makeDeleteMessage(seq, "users", where)
|
||||
```
|
||||
|
||||
### Ready (0x05)
|
||||
|
||||
```nim
|
||||
let msg = makeReadyMessage(seq)
|
||||
```
|
||||
|
||||
### Error (0x06)
|
||||
|
||||
```nim
|
||||
let msg = makeErrorMessage(seq, code, message)
|
||||
```
|
||||
|
||||
## Response Codes
|
||||
|
||||
| Code | Name | Description |
|
||||
|------|------|-------------|
|
||||
| 0x00 | OK | Success |
|
||||
| 0x01 | ERROR | General error |
|
||||
| 0x02 | AUTH_REQUIRED | Authentication needed |
|
||||
| 0x03 | INVALID_QUERY | Query syntax error |
|
||||
| 0x04 | NOT_FOUND | Resource not found |
|
||||
|
||||
## Serialization
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/wire
|
||||
|
||||
# Serialize value
|
||||
let bytes = serializeValue(Value(kind: vkString, strVal: "test"))
|
||||
|
||||
# Deserialize value
|
||||
let value = deserializeValue(bytes)
|
||||
```
|
||||
@@ -0,0 +1,91 @@
|
||||
# HTTP/REST API
|
||||
|
||||
JSON-based REST API for web applications.
|
||||
|
||||
## Base URL
|
||||
|
||||
```
|
||||
http://localhost:8080/api
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
### GET /api/users
|
||||
|
||||
List all users:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/api/users
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
[
|
||||
{"id": 1, "name": "Alice", "age": 30},
|
||||
{"id": 2, "name": "Bob", "age": 25}
|
||||
]
|
||||
```
|
||||
|
||||
### GET /api/users/:id
|
||||
|
||||
Get user by ID:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/api/users/1
|
||||
```
|
||||
|
||||
### POST /api/users
|
||||
|
||||
Create user:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8080/api/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Charlie", "age": 35}'
|
||||
```
|
||||
|
||||
### PUT /api/users/:id
|
||||
|
||||
Update user:
|
||||
|
||||
```bash
|
||||
curl -X PUT http://localhost:8080/api/users/1 \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Alice", "age": 31}'
|
||||
```
|
||||
|
||||
### DELETE /api/users/:id
|
||||
|
||||
Delete user:
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:8080/api/users/1
|
||||
```
|
||||
|
||||
## Query Endpoint
|
||||
|
||||
Execute BaraQL queries via HTTP:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8080/api/query \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"sql": "SELECT * FROM users WHERE age > 18"}'
|
||||
```
|
||||
|
||||
## Error Response
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "INVALID_QUERY",
|
||||
"message": "Syntax error at line 1"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer <token>" \
|
||||
http://localhost:8080/api/users
|
||||
```
|
||||
@@ -0,0 +1,120 @@
|
||||
# WebSocket API
|
||||
|
||||
Full-duplex streaming for real-time data feeds and push notifications.
|
||||
|
||||
## Connection
|
||||
|
||||
```
|
||||
ws://localhost:8081/ws
|
||||
```
|
||||
|
||||
## Client Example
|
||||
|
||||
```javascript
|
||||
const ws = new WebSocket('ws://localhost:8081/ws');
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('Connected');
|
||||
ws.send(JSON.stringify({
|
||||
type: 'query',
|
||||
sql: 'SELECT * FROM users'
|
||||
}));
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
console.log('Received:', data);
|
||||
};
|
||||
```
|
||||
|
||||
## Message Format
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "query",
|
||||
"id": "uuid",
|
||||
"sql": "SELECT * FROM users"
|
||||
}
|
||||
```
|
||||
|
||||
## Message Types
|
||||
|
||||
### Query Request
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "query",
|
||||
"id": "123",
|
||||
"sql": "SELECT * FROM users"
|
||||
}
|
||||
```
|
||||
|
||||
### Query Response
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "result",
|
||||
"id": "123",
|
||||
"data": [
|
||||
{"id": 1, "name": "Alice"},
|
||||
{"id": 2, "name": "Bob"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Error Response
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "error",
|
||||
"id": "123",
|
||||
"error": {
|
||||
"code": "INVALID_QUERY",
|
||||
"message": "Syntax error"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Subscription
|
||||
|
||||
Subscribe to changes:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "subscribe",
|
||||
"id": "sub1",
|
||||
"table": "users"
|
||||
}
|
||||
```
|
||||
|
||||
### Push Notification
|
||||
|
||||
Server push:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "push",
|
||||
"table": "users",
|
||||
"action": "insert",
|
||||
"data": {"id": 3, "name": "Charlie"}
|
||||
}
|
||||
```
|
||||
|
||||
## JavaScript Client
|
||||
|
||||
```javascript
|
||||
class BaraDBClient {
|
||||
constructor(url) {
|
||||
this.ws = new WebSocket(url);
|
||||
this.pending = new Map();
|
||||
}
|
||||
|
||||
query(sql) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = crypto.randomUUID();
|
||||
this.pending.set(id, { resolve, reject });
|
||||
this.ws.send(JSON.stringify({ type: 'query', id, sql }));
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,119 @@
|
||||
# BaraDB Architecture
|
||||
|
||||
## Overview
|
||||
|
||||
BaraDB is a **multimodal database engine** written in Nim that combines document (KV), graph, vector, columnar, and full-text search storage in a single engine with a unified query language called **BaraQL**.
|
||||
|
||||
## Layer Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ 1. CLIENT LAYER │
|
||||
│ Binary Protocol │ HTTP/REST │ WebSocket │ Embedded │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 2. QUERY LAYER (BaraQL) │
|
||||
│ Lexer → Parser → AST → IR → Optimizer → Codegen │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 3. EXECUTION ENGINE │
|
||||
│ Document │ Graph │ Vector │ Columnar │ FTS │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 4. STORAGE │
|
||||
│ LSM-Tree │ B-Tree │ WAL │ Bloom │ Compaction │ Cache │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ 5. DISTRIBUTED │
|
||||
│ Raft Consensus │ Sharding │ Replication │ Gossip │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Layer 1: Client Layer
|
||||
|
||||
Multiple communication protocols:
|
||||
|
||||
- **Binary Protocol** (`protocol/wire.nim`): Efficient big-endian binary protocol with 16 message types
|
||||
- **HTTP/REST** (`protocol/http.nim`): JSON-based REST API
|
||||
- **WebSocket** (`protocol/websocket.nim`): Full-duplex streaming
|
||||
- **Embedded** (`storage/lsm.nim`): Direct in-process access
|
||||
|
||||
## Layer 2: Query Layer (BaraQL)
|
||||
|
||||
The BaraQL pipeline:
|
||||
|
||||
1. **Lexer** (`query/lexer.nim`): Tokenizes input into 80+ token types
|
||||
2. **Parser** (`query/parser.nim`): Recursive descent parser producing AST
|
||||
3. **AST** (`query/ast.nim`): 300+ lines covering 25+ node kinds
|
||||
4. **IR** (`query/ir.nim`): Intermediate representation for execution plans
|
||||
5. **Optimizer/Codegen** (`query/codegen.nim`): Translates IR to storage operations
|
||||
|
||||
## Layer 3: Execution Engine
|
||||
|
||||
### Document/KV Engine
|
||||
- **LSM-Tree** (`storage/lsm.nim`): Write-optimized storage
|
||||
- **B-Tree Index** (`storage/btree.nim`): Ordered index for range scans
|
||||
|
||||
### Vector Engine (`vector/`)
|
||||
- **HNSW Index**: Hierarchical Navigable Small World graph
|
||||
- **IVF-PQ Index**: Inverted File Index with Product Quantization
|
||||
- **SIMD Operations**: Unrolled distance computations
|
||||
|
||||
### Graph Engine (`graph/`)
|
||||
- **Adjacency List**: Edge-weighted directed graph
|
||||
- **Algorithms**: BFS, DFS, Dijkstra, PageRank, Louvain
|
||||
|
||||
### Full-Text Search (`fts/`)
|
||||
- **Inverted Index**: Term-document index
|
||||
- **Ranking**: BM25 and TF-IDF scoring
|
||||
- **Multi-Language**: Tokenizers for EN, BG, DE, FR, RU
|
||||
|
||||
### Columnar Engine (`core/columnar.nim`)
|
||||
- Per-column storage for analytical queries
|
||||
- RLE and dictionary encoding
|
||||
|
||||
## Layer 4: Storage
|
||||
|
||||
- **LSM-Tree**: MemTable, WAL, SSTable, Bloom Filter, Compaction
|
||||
- **Page Cache**: LRU cache with hit rate tracking
|
||||
- **Memory-mapped I/O**: mmap-based file access
|
||||
|
||||
## Layer 5: Distributed
|
||||
|
||||
- **Raft Consensus**: Leader election, log replication
|
||||
- **Sharding**: Hash, range, and consistent hashing
|
||||
- **Replication**: Sync, async, semi-sync modes
|
||||
- **Gossip Protocol**: Membership management
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Write Path
|
||||
```
|
||||
Client → Protocol → Auth → Parser → AST → IR → Codegen
|
||||
→ StorageOp → MVCC Txn → WAL Write → MemTable → Commit
|
||||
```
|
||||
|
||||
### Read Path
|
||||
```
|
||||
Client → Protocol → Auth → Parser → AST → IR → Codegen
|
||||
→ StorageOp → MVCC Snapshot → MemTable → SSTable → Result
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Pure Nim**: No Cython, Python, or Rust dependencies
|
||||
2. **Unified Storage**: One engine handles KV, graph, vector, FTS, and columnar
|
||||
3. **Embedded Mode**: Can run as library or server
|
||||
4. **Binary Protocol**: Custom efficient wire protocol
|
||||
5. **MVCC**: Multi-version concurrency control
|
||||
6. **Schema-First**: Strongly typed schema system with inheritance
|
||||
|
||||
## Module Statistics
|
||||
|
||||
| Category | Modules |
|
||||
|----------|---------|
|
||||
| Core | 10 |
|
||||
| Storage | 7 |
|
||||
| Query | 7 |
|
||||
| Vector | 3 |
|
||||
| Graph | 3 |
|
||||
| FTS | 2 |
|
||||
| Protocol | 7 |
|
||||
| Distributed | 5 |
|
||||
| **Total** | **48** |
|
||||
@@ -0,0 +1,156 @@
|
||||
# BaraQL - Query Language Reference
|
||||
|
||||
BaraQL is a SQL-compatible query language with extensions for graph, vector, and document operations.
|
||||
|
||||
## Basic Queries
|
||||
|
||||
### SELECT
|
||||
|
||||
```sql
|
||||
SELECT name, age FROM users WHERE age > 18 ORDER BY name LIMIT 10;
|
||||
```
|
||||
|
||||
### INSERT
|
||||
|
||||
```sql
|
||||
INSERT users { name := 'Alice', age := 30 };
|
||||
```
|
||||
|
||||
### UPDATE
|
||||
|
||||
```sql
|
||||
UPDATE users SET age = 31 WHERE name = 'Alice';
|
||||
```
|
||||
|
||||
### DELETE
|
||||
|
||||
```sql
|
||||
DELETE FROM users WHERE name = 'Alice';
|
||||
```
|
||||
|
||||
## Aggregates and Grouping
|
||||
|
||||
```sql
|
||||
SELECT department, count(*), avg(salary)
|
||||
FROM employees
|
||||
GROUP BY department
|
||||
HAVING count(*) > 5;
|
||||
|
||||
SELECT count(*), sum(amount), avg(price) FROM orders;
|
||||
```
|
||||
|
||||
## JOINs
|
||||
|
||||
```sql
|
||||
-- INNER JOIN
|
||||
SELECT u.name, o.total
|
||||
FROM users u
|
||||
INNER JOIN orders o ON u.id = o.user_id;
|
||||
|
||||
-- LEFT JOIN
|
||||
SELECT u.name, o.total
|
||||
FROM users u
|
||||
LEFT JOIN orders o ON u.id = o.user_id;
|
||||
|
||||
-- Multiple JOINs
|
||||
SELECT *
|
||||
FROM orders o
|
||||
JOIN users u ON o.user_id = u.id
|
||||
JOIN products p ON o.product_id = p.id;
|
||||
```
|
||||
|
||||
## CTEs (Common Table Expressions)
|
||||
|
||||
```sql
|
||||
-- Single CTE
|
||||
WITH active_users AS (
|
||||
SELECT * FROM users WHERE active = true
|
||||
)
|
||||
SELECT * FROM active_users;
|
||||
|
||||
-- Multiple CTEs
|
||||
WITH
|
||||
recent AS (SELECT * FROM orders WHERE date > '2025-01-01'),
|
||||
totals AS (SELECT user_id, sum(amount) as total FROM recent GROUP BY user_id)
|
||||
SELECT u.name, t.total FROM users u JOIN totals t ON u.id = t.user_id;
|
||||
```
|
||||
|
||||
## Subqueries
|
||||
|
||||
```sql
|
||||
-- Subquery in FROM
|
||||
SELECT * FROM (SELECT id, name FROM users WHERE active = true) AS active;
|
||||
|
||||
-- EXISTS subquery
|
||||
SELECT name FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id);
|
||||
```
|
||||
|
||||
## CASE Expressions
|
||||
|
||||
```sql
|
||||
SELECT name,
|
||||
CASE
|
||||
WHEN age < 18 THEN 'minor'
|
||||
WHEN age < 65 THEN 'adult'
|
||||
ELSE 'senior'
|
||||
END AS category
|
||||
FROM users;
|
||||
```
|
||||
|
||||
## Schema Definition
|
||||
|
||||
```sql
|
||||
CREATE TYPE Person {
|
||||
name: str,
|
||||
age: int32
|
||||
};
|
||||
|
||||
CREATE TYPE Movie {
|
||||
title: str,
|
||||
director: Person
|
||||
};
|
||||
```
|
||||
|
||||
## Vector Search
|
||||
|
||||
```sql
|
||||
-- Insert with vector
|
||||
INSERT articles {
|
||||
title := 'Nim Programming',
|
||||
embedding := [0.1, 0.2, 0.3, ...]
|
||||
};
|
||||
|
||||
-- Similarity search
|
||||
SELECT title FROM articles
|
||||
ORDER BY cosine_distance(embedding, [0.1, 0.2, 0.3, ...])
|
||||
LIMIT 5;
|
||||
```
|
||||
|
||||
## Graph Patterns
|
||||
|
||||
```sql
|
||||
MATCH (p:Person)-[:KNOWS]->(friend:Person)
|
||||
WHERE p.name = 'Alice'
|
||||
RETURN friend.name;
|
||||
```
|
||||
|
||||
## Full-Text Search
|
||||
|
||||
```sql
|
||||
SELECT * FROM articles
|
||||
WHERE MATCH(title, body) AGAINST('database programming');
|
||||
```
|
||||
|
||||
## Supported Keywords
|
||||
|
||||
| Category | Keywords |
|
||||
|----------|----------|
|
||||
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET |
|
||||
| DML | INSERT, UPDATE, DELETE, SET |
|
||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX |
|
||||
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN |
|
||||
| Set | UNION, INTERSECT, EXCEPT |
|
||||
| CTEs | WITH, AS |
|
||||
| Case | CASE, WHEN, THEN, ELSE, END |
|
||||
| Graph | MATCH, RETURN, WHERE |
|
||||
| FTS | MATCH, AGAINST |
|
||||
@@ -0,0 +1,38 @@
|
||||
# B-Tree Index
|
||||
|
||||
Ordered index structure for efficient range scans and point lookups.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/storage/btree
|
||||
|
||||
var btree = newBTreeIndex[string, string]()
|
||||
|
||||
# Insert
|
||||
btree.insert("key1", "value1")
|
||||
btree.insert("key2", "value2")
|
||||
|
||||
# Point lookup
|
||||
let values = btree.get("key1")
|
||||
|
||||
# Range scan
|
||||
let range = btree.scan("key_a", "key_z")
|
||||
|
||||
# Delete
|
||||
btree.delete("key1")
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- Ordered key-value storage
|
||||
- Range queries (BETWEEN, >, <, >=, <=)
|
||||
- Prefix scans
|
||||
- Configurable page size
|
||||
- Iterator support
|
||||
|
||||
## Use Cases
|
||||
|
||||
- Primary key indexes
|
||||
- Secondary indexes for frequently queried columns
|
||||
- Range-partitioned data
|
||||
@@ -0,0 +1,58 @@
|
||||
# Columnar Storage
|
||||
|
||||
Column-oriented storage for analytical queries and aggregations.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/core/columnar
|
||||
|
||||
var batch = newColumnBatch()
|
||||
var ageCol = batch.addInt64Col("age")
|
||||
var nameCol = batch.addStringCol("name")
|
||||
|
||||
ageCol.appendInt64(25)
|
||||
nameCol.appendString("Alice")
|
||||
```
|
||||
|
||||
## Aggregations
|
||||
|
||||
```nim
|
||||
echo ageCol.sumInt64()
|
||||
echo ageCol.avgInt64()
|
||||
echo ageCol.minInt64()
|
||||
echo ageCol.maxInt64()
|
||||
echo ageCol.count()
|
||||
```
|
||||
|
||||
## Encoding
|
||||
|
||||
### RLE (Run-Length Encoding)
|
||||
|
||||
```nim
|
||||
let rle = rleEncode(@[1'i64, 1, 1, 2, 2, 3])
|
||||
```
|
||||
|
||||
### Dictionary Encoding
|
||||
|
||||
```nim
|
||||
let dict = dictEncode(@["apple", "banana", "apple"])
|
||||
```
|
||||
|
||||
## Column Types
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `int32` | 32-bit integer |
|
||||
| `int64` | 64-bit integer |
|
||||
| `float32` | 32-bit float |
|
||||
| `float64` | 64-bit float |
|
||||
| `string` | Variable-length string |
|
||||
| `bool` | Boolean |
|
||||
|
||||
## Use Cases
|
||||
|
||||
- OLAP workloads
|
||||
- Large-scale aggregations
|
||||
- Data warehousing
|
||||
- Time-series analysis
|
||||
@@ -0,0 +1,90 @@
|
||||
# Distributed Systems
|
||||
|
||||
BaraDB supports distributed deployment with Raft consensus, sharding, and replication.
|
||||
|
||||
## Raft Consensus
|
||||
|
||||
Leader election and log replication:
|
||||
|
||||
```nim
|
||||
import barabadb/core/raft
|
||||
|
||||
var cluster = newRaftCluster()
|
||||
cluster.addNode("node1")
|
||||
cluster.addNode("node2")
|
||||
cluster.addNode("node3")
|
||||
|
||||
let n1 = cluster.nodes["n1"]
|
||||
n1.becomeCandidate()
|
||||
n1.becomeLeader()
|
||||
let entry = n1.appendLog("SET key1 value1")
|
||||
```
|
||||
|
||||
## Sharding
|
||||
|
||||
Distribute data across nodes:
|
||||
|
||||
```nim
|
||||
import barabadb/core/sharding
|
||||
|
||||
var router = newShardRouter(ShardConfig(
|
||||
numShards: 4,
|
||||
replicas: 2,
|
||||
strategy: ssHash
|
||||
))
|
||||
router.rebalance(@["node1", "node2", "node3"])
|
||||
let shard = router.getShard("user_123")
|
||||
```
|
||||
|
||||
### Sharding Strategies
|
||||
|
||||
| Strategy | Description |
|
||||
|----------|-------------|
|
||||
| `ssHash` | Hash-based sharding |
|
||||
| `ssRange` | Range-based sharding |
|
||||
| `ssConsistent` | Consistent hashing |
|
||||
|
||||
## Replication
|
||||
|
||||
```nim
|
||||
import barabadb/core/replication
|
||||
|
||||
var rm = newReplicationManager(rmSync)
|
||||
rm.addReplica(newReplica("r1", "10.0.0.1", 5432))
|
||||
rm.connectReplica("r1")
|
||||
let lsn = rm.writeLsn(@[1'u8, 2, 3])
|
||||
rm.ackLsn("r1", lsn)
|
||||
```
|
||||
|
||||
### Replication Modes
|
||||
|
||||
| Mode | Description |
|
||||
|------|-------------|
|
||||
| `rmSync` | Synchronous replication |
|
||||
| `rmAsync` | Asynchronous replication |
|
||||
| `rmSemiSync` | Semi-synchronous replication |
|
||||
|
||||
## Gossip Protocol
|
||||
|
||||
Membership and failure detection:
|
||||
|
||||
```nim
|
||||
import barabadb/core/gossip
|
||||
|
||||
var g = newGossipManager()
|
||||
g.addNode("node1")
|
||||
g.addNode("node2")
|
||||
g.tick() # Exchange membership info
|
||||
```
|
||||
|
||||
## Distributed Transactions
|
||||
|
||||
Two-phase commit across nodes:
|
||||
|
||||
```nim
|
||||
import barabadb/core/disttxn
|
||||
|
||||
var dt = newDistributedTxn()
|
||||
dt.prepare(@["node1", "node2"])
|
||||
dt.commit()
|
||||
```
|
||||
@@ -0,0 +1,69 @@
|
||||
# Full-Text Search Engine
|
||||
|
||||
Inverted index with BM25 and TF-IDF ranking for text search.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/fts/engine
|
||||
|
||||
var idx = newInvertedIndex()
|
||||
idx.addDocument(1, "Nim is a fast programming language")
|
||||
idx.addDocument(2, "Python is popular for data science")
|
||||
|
||||
# BM25 search
|
||||
let results = idx.search("programming language")
|
||||
|
||||
# TF-IDF search
|
||||
let tfidf = idx.searchTfidf("programming language")
|
||||
|
||||
# Fuzzy search (typo tolerance)
|
||||
let fuzzy = idx.fuzzySearch("programing", maxDistance = 2)
|
||||
|
||||
# Wildcard search
|
||||
let wild = idx.regexSearch("prog*")
|
||||
```
|
||||
|
||||
## Ranking Methods
|
||||
|
||||
### BM25
|
||||
|
||||
Best matching ranking algorithm:
|
||||
|
||||
```nim
|
||||
let bm25 = idx.searchBM25("query terms")
|
||||
```
|
||||
|
||||
### TF-IDF
|
||||
|
||||
Term Frequency-Inverse Document Frequency:
|
||||
|
||||
```nim
|
||||
let tfidf = idx.searchTfidf("query terms")
|
||||
```
|
||||
|
||||
## Search Features
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Fuzzy search | Levenshtein distance tolerance |
|
||||
| Wildcard | Prefix, suffix, and infix wildcards |
|
||||
| Regex | Regular expression patterns |
|
||||
| Phrase search | Exact phrase matching |
|
||||
| Boolean | AND, OR, NOT operators |
|
||||
|
||||
## Multi-Language Support
|
||||
|
||||
```nim
|
||||
import barabadb/fts/multilang
|
||||
|
||||
# Supported languages: EN, BG, DE, FR, RU
|
||||
var tokenizer = newTokenizer("bg") # Bulgarian
|
||||
let tokens = tokenizer.tokenize("Търсене в пълен текст")
|
||||
```
|
||||
|
||||
Features per language:
|
||||
- Tokenization
|
||||
- Stop words
|
||||
- Stemming
|
||||
- Language detection
|
||||
@@ -0,0 +1,52 @@
|
||||
# Graph Engine
|
||||
|
||||
Adjacency list storage with built-in algorithms for graph traversal and analysis.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/graph/engine
|
||||
|
||||
var g = newGraph()
|
||||
let alice = g.addNode("Person", {"name": "Alice"}.toTable)
|
||||
let bob = g.addNode("Person", {"name": "Bob"}.toTable)
|
||||
discard g.addEdge(alice, bob, "knows")
|
||||
|
||||
# Traversal
|
||||
let bfs = g.bfs(alice)
|
||||
let dfs = g.dfs(alice)
|
||||
let path = g.shortestPath(alice, bob)
|
||||
let ranks = g.pageRank()
|
||||
```
|
||||
|
||||
## Algorithms
|
||||
|
||||
| Algorithm | Description |
|
||||
|-----------|-------------|
|
||||
| `bfs` | Breadth-first traversal |
|
||||
| `dfs` | Depth-first traversal |
|
||||
| `dijkstra` | Shortest weighted path |
|
||||
| `pageRank` | Node importance ranking |
|
||||
| `louvain` | Community detection |
|
||||
| `patternMatch` | Subgraph isomorphism |
|
||||
|
||||
## Cypher Query
|
||||
|
||||
```nim
|
||||
import barabadb/graph/cypher
|
||||
|
||||
var engine = newCypherEngine(g)
|
||||
let results = engine.execute("""
|
||||
MATCH (p:Person)-[:KNOWS]->(friend:Person)
|
||||
WHERE p.name = 'Alice'
|
||||
RETURN friend.name
|
||||
""")
|
||||
```
|
||||
|
||||
## Pattern Matching
|
||||
|
||||
```sql
|
||||
MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person)
|
||||
WHERE a.name = 'Alice'
|
||||
RETURN b.name, c.name
|
||||
```
|
||||
@@ -0,0 +1,113 @@
|
||||
# BaraDB - Installation Guide
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Nim Compiler** >= 2.0.0
|
||||
- **Operating System**: Linux, macOS, Windows
|
||||
|
||||
## Installing Nim
|
||||
|
||||
### Linux/macOS
|
||||
|
||||
```bash
|
||||
curl https://nim-lang.org/choosenim/init.sh -sSf | sh
|
||||
```
|
||||
|
||||
Or via package manager:
|
||||
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
apt-get install nim
|
||||
|
||||
# macOS
|
||||
brew install nim
|
||||
```
|
||||
|
||||
### Windows
|
||||
|
||||
Download the installer from [nim-lang.org](https://nim-lang.org/install.html) or use winget:
|
||||
|
||||
```powershell
|
||||
winget install nim
|
||||
```
|
||||
|
||||
## Building BaraDB
|
||||
|
||||
### Clone the Repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/katehonz/barabaDB.git
|
||||
cd barabaDB
|
||||
```
|
||||
|
||||
### Build the Project
|
||||
|
||||
```bash
|
||||
# Debug build
|
||||
nim c -o:build/baradadb src/baradadb.nim
|
||||
|
||||
# Release build (optimized)
|
||||
nim c -d:release -o:build/baradadb src/baradadb.nim
|
||||
```
|
||||
|
||||
### Run Tests
|
||||
|
||||
```bash
|
||||
nim c --path:src -r tests/test_all.nim
|
||||
```
|
||||
|
||||
### Run Benchmarks
|
||||
|
||||
```bash
|
||||
nim c -d:release -r benchmarks/bench_all.nim
|
||||
```
|
||||
|
||||
## Installation Options
|
||||
|
||||
### Pre-built Binary
|
||||
|
||||
Download the latest release from the [GitHub Releases](https://github.com/katehonz/barabaDB/releases) page.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker pull barabadb/barabadb
|
||||
docker run -it barabadb/barabadb
|
||||
```
|
||||
|
||||
### Embedded Usage
|
||||
|
||||
Add to your `.nimble` file:
|
||||
|
||||
```nim
|
||||
requires "barabadb >= 1.0.0"
|
||||
```
|
||||
|
||||
Then import in your code:
|
||||
|
||||
```nim
|
||||
import barabadb
|
||||
|
||||
var db = newLSMTree("./data")
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
db.close()
|
||||
```
|
||||
|
||||
## Verifying Installation
|
||||
|
||||
```bash
|
||||
./build/baradadb --version
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
BaraDB v1.0.0
|
||||
multimodal database engine
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Quick Start Guide](en/quickstart.md)
|
||||
- [Architecture Overview](en/architecture.md)
|
||||
- [BaraQL Query Language](en/baraql.md)
|
||||
@@ -0,0 +1,64 @@
|
||||
# LSM-Tree Storage Engine
|
||||
|
||||
The primary storage engine in BaraDB using the Log-Structured Merge-Tree architecture.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ Writes │
|
||||
│ (append to WAL + MemTable) │
|
||||
└─────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ MemTable │
|
||||
│ (in-memory sorted buffer) │
|
||||
└─────────────────────────────────────────────┘
|
||||
│
|
||||
(when full, flush to SSTable)
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ SSTable │
|
||||
│ (sorted string table on disk) │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/storage/lsm
|
||||
|
||||
var db = newLSMTree("./data")
|
||||
|
||||
# Write
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
|
||||
# Read
|
||||
let (found, value) = db.get("key1")
|
||||
|
||||
# Delete
|
||||
db.delete("key1")
|
||||
|
||||
db.close()
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- **Write-optimized**: Append-only log structure
|
||||
- **Durability**: Write-ahead log (WAL) ensures crash recovery
|
||||
- **Bloom Filter**: Fast negative lookups
|
||||
- **Compaction**: Size-tiered strategy merges SSTables
|
||||
- **Page Cache**: LRU cache for frequently accessed pages
|
||||
|
||||
## Configuration
|
||||
|
||||
```nim
|
||||
var db = newLSMTree(
|
||||
path = "./data",
|
||||
memTableSize = 64 * 1024 * 1024, # 64MB
|
||||
walEnabled = true,
|
||||
bloomFpRate = 0.01
|
||||
)
|
||||
```
|
||||
@@ -0,0 +1,99 @@
|
||||
# Protocol Reference
|
||||
|
||||
BaraDB supports multiple protocols for client communication.
|
||||
|
||||
## Binary Wire Protocol
|
||||
|
||||
Efficient big-endian binary protocol:
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/wire
|
||||
|
||||
# Query message
|
||||
let msg = makeQueryMessage(1, "SELECT * FROM users")
|
||||
|
||||
# Ready message
|
||||
let ready = makeReadyMessage(1)
|
||||
|
||||
# Error message
|
||||
let error = makeErrorMessage(1, 42, "Syntax error")
|
||||
```
|
||||
|
||||
### Message Types
|
||||
|
||||
| Type | ID | Description |
|
||||
|------|-----|-------------|
|
||||
| Query | 0x01 | Execute query |
|
||||
| Insert | 0x02 | Insert data |
|
||||
| Update | 0x03 | Update data |
|
||||
| Delete | 0x04 | Delete data |
|
||||
| Ready | 0x05 | Ready for next command |
|
||||
| Error | 0x06 | Error response |
|
||||
| Auth | 0x07 | Authentication |
|
||||
| Batch | 0x08 | Batch operations |
|
||||
|
||||
## HTTP/REST API
|
||||
|
||||
JSON-based REST API:
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/http
|
||||
|
||||
var router = newHttpRouter(port = 8080)
|
||||
|
||||
router.get("/api/users", proc(req: Request): Future[JsonNode] {.async.} =
|
||||
return %*[{"id": 1, "name": "Alice"}])
|
||||
|
||||
router.post("/api/users", proc(req: Request): Future[JsonNode] {.async.} =
|
||||
return %*{"status": "created"})
|
||||
```
|
||||
|
||||
## WebSocket API
|
||||
|
||||
Full-duplex streaming:
|
||||
|
||||
```nim
|
||||
import barabadb/core/websocket
|
||||
|
||||
var server = newWsServer(port = 8081)
|
||||
server.onMessage = proc(ws: WebSocket, data: seq[byte]) {.gcsafe.} =
|
||||
echo "Received: ", cast[string](data)
|
||||
asyncCheck server.run()
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
JWT-based authentication:
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/auth
|
||||
|
||||
var am = newAuthManager("secret-key")
|
||||
let token = am.createToken(JWTClaims(sub: "user1", role: "admin"))
|
||||
let result = am.validateCredentials(AuthCredentials(authMethod: amToken, payload: token))
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
Token bucket rate limiting:
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/ratelimit
|
||||
|
||||
var rl = newRateLimiter(rlaTokenBucket, globalRate = 1000, perClientRate = 100)
|
||||
if rl.allowRequest("client-123"):
|
||||
echo "Request allowed"
|
||||
```
|
||||
|
||||
## Connection Pooling
|
||||
|
||||
```nim
|
||||
import barabadb/protocol/pool
|
||||
|
||||
var pool = newConnectionPool(
|
||||
minConnections = 5,
|
||||
maxConnections = 100
|
||||
)
|
||||
let conn = pool.acquire()
|
||||
pool.release(conn)
|
||||
```
|
||||
@@ -0,0 +1,134 @@
|
||||
# BaraDB - Quick Start Guide
|
||||
|
||||
## Starting the Server
|
||||
|
||||
After building BaraDB, start the server:
|
||||
|
||||
```bash
|
||||
./build/baradadb
|
||||
```
|
||||
|
||||
The server will start on `localhost:8080` by default.
|
||||
|
||||
## Connecting via CLI
|
||||
|
||||
BaraDB includes an interactive shell:
|
||||
|
||||
```bash
|
||||
./build/baradadb --shell
|
||||
```
|
||||
|
||||
## Basic Operations
|
||||
|
||||
### Create Schema
|
||||
|
||||
```sql
|
||||
CREATE TYPE Person {
|
||||
name: str,
|
||||
age: int32
|
||||
};
|
||||
|
||||
CREATE TYPE Movie {
|
||||
title: str,
|
||||
year: int32,
|
||||
director: Person
|
||||
};
|
||||
```
|
||||
|
||||
### Insert Data
|
||||
|
||||
```sql
|
||||
INSERT Person { name := 'Alice', age := 30 };
|
||||
INSERT Person { name := 'Bob', age := 25 };
|
||||
```
|
||||
|
||||
### Query Data
|
||||
|
||||
```sql
|
||||
SELECT name, age FROM Person WHERE age > 18;
|
||||
```
|
||||
|
||||
### Update Data
|
||||
|
||||
```sql
|
||||
UPDATE Person SET age = 31 WHERE name = 'Alice';
|
||||
```
|
||||
|
||||
### Delete Data
|
||||
|
||||
```sql
|
||||
DELETE FROM Person WHERE name = 'Bob';
|
||||
```
|
||||
|
||||
## Advanced Queries
|
||||
|
||||
### JOIN
|
||||
|
||||
```sql
|
||||
SELECT u.name, o.total
|
||||
FROM users u
|
||||
INNER JOIN orders o ON u.id = o.user_id;
|
||||
```
|
||||
|
||||
### Aggregates
|
||||
|
||||
```sql
|
||||
SELECT department, count(*), avg(salary)
|
||||
FROM employees
|
||||
GROUP BY department
|
||||
HAVING count(*) > 5;
|
||||
```
|
||||
|
||||
### CTEs
|
||||
|
||||
```sql
|
||||
WITH active_users AS (
|
||||
SELECT * FROM users WHERE active = true
|
||||
)
|
||||
SELECT * FROM active_users;
|
||||
```
|
||||
|
||||
## Vector Search
|
||||
|
||||
```sql
|
||||
-- Insert vector
|
||||
INSERT vectors { id := 1, embedding := [0.1, 0.2, 0.3] };
|
||||
|
||||
-- Search similar
|
||||
SELECT * FROM vectors ORDER BY cosine_distance(embedding, [0.1, 0.2, 0.3]) LIMIT 10;
|
||||
```
|
||||
|
||||
## Graph Operations
|
||||
|
||||
```sql
|
||||
-- Match graph pattern
|
||||
MATCH (p:Person)-[:KNOWS]->(other:Person)
|
||||
WHERE p.name = 'Alice'
|
||||
RETURN other.name;
|
||||
```
|
||||
|
||||
## Full-Text Search
|
||||
|
||||
```sql
|
||||
-- Search documents
|
||||
SELECT * FROM articles WHERE MATCH(title, body) AGAINST('database');
|
||||
```
|
||||
|
||||
## HTTP/REST API
|
||||
|
||||
```bash
|
||||
# GET request
|
||||
curl http://localhost:8080/api/users
|
||||
|
||||
# POST request
|
||||
curl -X POST http://localhost:8080/api/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Alice", "age": 30}'
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [BaraQL Reference](en/baraql.md)
|
||||
- [Storage Engines](en/storage.md)
|
||||
- [Architecture Overview](en/architecture.md)
|
||||
- [Protocol Reference](en/protocol.md)
|
||||
@@ -0,0 +1,55 @@
|
||||
# Schema System
|
||||
|
||||
BaraDB uses a schema-first design with type inheritance and automatic migrations.
|
||||
|
||||
## Defining Types
|
||||
|
||||
```nim
|
||||
import barabadb/schema/schema
|
||||
|
||||
var s = newSchema()
|
||||
|
||||
let person = newType("Person")
|
||||
person.addProperty("name", "str", required = true)
|
||||
person.addProperty("age", "int32")
|
||||
s.addType("default", person)
|
||||
```
|
||||
|
||||
## Type Inheritance
|
||||
|
||||
```nim
|
||||
let employee = newType("Employee")
|
||||
employee.setBases(@["Person"])
|
||||
employee.addProperty("department", "str")
|
||||
s.addType("default", employee)
|
||||
|
||||
# Resolve inheritance — Employee gets name, age, department
|
||||
let resolved = s.resolveInheritance(employee)
|
||||
```
|
||||
|
||||
## Schema Operations
|
||||
|
||||
### Diff
|
||||
|
||||
Compare two schemas:
|
||||
|
||||
```nim
|
||||
let diff = s.diff(oldSchema, newSchema)
|
||||
```
|
||||
|
||||
### Migrations
|
||||
|
||||
Schema changes are tracked and can generate migration scripts.
|
||||
|
||||
## Property Types
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `str` | String |
|
||||
| `int32` | 32-bit integer |
|
||||
| `int64` | 64-bit integer |
|
||||
| `float32` | 32-bit float |
|
||||
| `float64` | 64-bit float |
|
||||
| `bool` | Boolean |
|
||||
| `datetime` | Date/time value |
|
||||
| `bytes` | Binary data |
|
||||
@@ -0,0 +1,78 @@
|
||||
# Storage Engines
|
||||
|
||||
BaraDB provides multiple storage engines optimized for different access patterns.
|
||||
|
||||
## LSM-Tree (Key-Value)
|
||||
|
||||
The primary storage engine with write-optimized append-only log structure.
|
||||
|
||||
### Usage
|
||||
|
||||
```nim
|
||||
import barabadb/storage/lsm
|
||||
|
||||
var db = newLSMTree("./data")
|
||||
db.put("key1", cast[seq[byte]]("value1"))
|
||||
let (found, value) = db.get("key1")
|
||||
db.close()
|
||||
```
|
||||
|
||||
### Components
|
||||
|
||||
- **MemTable**: In-memory sorted buffer
|
||||
- **WAL**: Write-ahead log for durability
|
||||
- **SSTable**: Sorted string tables on disk
|
||||
- **Bloom Filter**: Probabilistic set membership
|
||||
- **Compaction**: Size-tiered strategy with level management
|
||||
- **Page Cache**: LRU cache with hit rate tracking
|
||||
|
||||
## B-Tree Index
|
||||
|
||||
Ordered index for range scans and point lookups.
|
||||
|
||||
### Usage
|
||||
|
||||
```nim
|
||||
import barabadb/storage/btree
|
||||
|
||||
var btree = newBTreeIndex[string, string]()
|
||||
btree.insert("key1", "value1")
|
||||
let values = btree.get("key1")
|
||||
let range = btree.scan("key_a", "key_z")
|
||||
```
|
||||
|
||||
## Write-Ahead Log (WAL)
|
||||
|
||||
Ensures durability of write operations.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/wal
|
||||
|
||||
var wal = newWAL("./wal")
|
||||
wal.append("txn1", "SET key1 value1")
|
||||
wal.flush()
|
||||
```
|
||||
|
||||
## Bloom Filter
|
||||
|
||||
Probabilistic data structure for fast negative lookups.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/bloom
|
||||
|
||||
var filter = newBloomFilter(10000, 0.01)
|
||||
filter.add("key1")
|
||||
if filter.mightContain("key1"):
|
||||
echo "possibly exists"
|
||||
```
|
||||
|
||||
## Memory-mapped I/O
|
||||
|
||||
Efficient file access using mmap.
|
||||
|
||||
```nim
|
||||
import barabadb/storage/mmap
|
||||
|
||||
var mapped = mmapFile("./data/file.dat")
|
||||
let data = mapped.read(0, 100)
|
||||
```
|
||||
@@ -0,0 +1,61 @@
|
||||
# Transactions & MVCC
|
||||
|
||||
MVCC (Multi-Version Concurrency Control) with snapshot isolation and deadlock detection.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/core/mvcc
|
||||
|
||||
var tm = newTxnManager()
|
||||
let txn = tm.beginTxn()
|
||||
|
||||
# Write operations
|
||||
discard tm.write(txn, "key1", cast[seq[byte]]("value1"))
|
||||
discard tm.write(txn, "key2", cast[seq[byte]]("value2"))
|
||||
|
||||
# Savepoint
|
||||
tm.savepoint(txn)
|
||||
discard tm.write(txn, "key3", cast[seq[byte]]("value3"))
|
||||
discard tm.rollbackToSavepoint(txn) # undo key3
|
||||
|
||||
# Commit
|
||||
discard tm.commit(txn)
|
||||
```
|
||||
|
||||
## Transaction Isolation
|
||||
|
||||
BaraDB uses **snapshot isolation**:
|
||||
- Readers don't block writers
|
||||
- Writers don't block readers
|
||||
- Each transaction sees a consistent snapshot
|
||||
|
||||
## Deadlock Detection
|
||||
|
||||
```nim
|
||||
import barabadb/core/deadlock
|
||||
|
||||
var detector = newDeadlockDetector()
|
||||
if detector.detectCycle(txn1, txn2):
|
||||
echo "Deadlock detected!"
|
||||
```
|
||||
|
||||
## Write-Ahead Log
|
||||
|
||||
```nim
|
||||
import barabadb/storage/wal
|
||||
|
||||
var wal = newWAL("./wal")
|
||||
wal.append(txnId, "SET key value")
|
||||
wal.flush()
|
||||
```
|
||||
|
||||
## Savepoints
|
||||
|
||||
Nested transaction savepoints:
|
||||
|
||||
```nim
|
||||
tm.savepoint(txn, "sp1")
|
||||
# ... operations ...
|
||||
tm.rollbackToSavepoint(txn, "sp1")
|
||||
```
|
||||
@@ -0,0 +1,56 @@
|
||||
# User Defined Functions
|
||||
|
||||
Extend BaraQL with custom functions.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/query/udf
|
||||
|
||||
var reg = newUDFRegistry()
|
||||
|
||||
# Register standard library
|
||||
reg.registerStdlib() # abs, sqrt, pow, lower, upper, len, trim, substr, toString, toInt
|
||||
|
||||
# Custom function
|
||||
reg.register("greet", @[UDFParam(name: "name", typeName: "str")],
|
||||
"str", proc(args: seq[Value]): Value =
|
||||
return Value(kind: vkString, strVal: "Hello, " & args[0].strVal & "!"))
|
||||
```
|
||||
|
||||
## Standard Library Functions
|
||||
|
||||
| Function | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `abs(n)` | Absolute value | `abs(-5)` → 5 |
|
||||
| `sqrt(n)` | Square root | `sqrt(16)` → 4 |
|
||||
| `pow(n, e)` | Power | `pow(2, 3)` → 8 |
|
||||
| `lower(s)` | Lowercase | `lower('ABC')` → 'abc' |
|
||||
| `upper(s)` | Uppercase | `upper('abc')` → 'ABC' |
|
||||
| `len(s)` | Length | `len('hello')` → 5 |
|
||||
| `trim(s)` | Trim whitespace | `trim(' hello ')` → 'hello' |
|
||||
| `substr(s, start, len)` | Substring | `substr('hello', 0, 3)` → 'hel' |
|
||||
| `toString(n)` | Convert to string | `toString(123)` → '123' |
|
||||
| `toInt(s)` | Convert to integer | `toInt('123')` → 123 |
|
||||
|
||||
## Function Registration
|
||||
|
||||
```nim
|
||||
reg.register(
|
||||
name: "my_function",
|
||||
params: @[
|
||||
UDFParam(name: "arg1", typeName: "str"),
|
||||
UDFParam(name: "arg2", typeName: "int32")
|
||||
],
|
||||
returnType: "str",
|
||||
body: proc(args: seq[Value]): Value =
|
||||
# Implementation
|
||||
result = Value(kind: vkString, strVal: "")
|
||||
)
|
||||
```
|
||||
|
||||
## Using UDFs in Queries
|
||||
|
||||
```sql
|
||||
SELECT greet(name) FROM users;
|
||||
```
|
||||
@@ -0,0 +1,76 @@
|
||||
# Vector Search Engine
|
||||
|
||||
Native HNSW and IVF-PQ indexes for similarity search.
|
||||
|
||||
## Usage
|
||||
|
||||
```nim
|
||||
import barabadb/vector/engine
|
||||
|
||||
var idx = newHNSWIndex(dimensions = 128)
|
||||
idx.insert(1, @[1.0'f32, 0.0'f32, ...], {"category": "A"}.toTable)
|
||||
|
||||
# Search
|
||||
let results = idx.search(queryVector, k = 10)
|
||||
|
||||
# With metadata filtering
|
||||
let filtered = idx.searchWithFilter(queryVector, k = 10,
|
||||
filter = proc(meta: Table[string, string]): bool =
|
||||
return meta.getOrDefault("category") == "A")
|
||||
```
|
||||
|
||||
## Index Types
|
||||
|
||||
### HNSW
|
||||
|
||||
Hierarchical Navigable Small World graph for approximate nearest neighbor search.
|
||||
|
||||
```nim
|
||||
var hnsw = newHNSWIndex(
|
||||
dimensions = 128,
|
||||
m = 16, # connections per layer
|
||||
efConstruction = 200, # search width during construction
|
||||
efSearch = 100 # search width during query
|
||||
)
|
||||
```
|
||||
|
||||
### IVF-PQ
|
||||
|
||||
Inverted File Index with Product Quantization for compression.
|
||||
|
||||
```nim
|
||||
var ivfpq = newIVFPQIndex(
|
||||
dimensions = 128,
|
||||
numCentroids = 256,
|
||||
subQuantizers = 8
|
||||
)
|
||||
```
|
||||
|
||||
## Distance Metrics
|
||||
|
||||
| Metric | Description |
|
||||
|--------|-------------|
|
||||
| `cosine` | Cosine similarity |
|
||||
| `euclidean` | L2 distance |
|
||||
| `dotproduct` | Dot product similarity |
|
||||
| `manhattan` | L1 distance |
|
||||
|
||||
## Quantization
|
||||
|
||||
```nim
|
||||
import barabadb/vector/quant
|
||||
|
||||
# Scalar quantization
|
||||
let scalar = scalarQuantize(data, bits = 8)
|
||||
|
||||
# Product quantization
|
||||
let pq = productQuantize(data, subVectors = 8, bits = 8)
|
||||
```
|
||||
|
||||
## SIMD Acceleration
|
||||
|
||||
```nim
|
||||
import barabadb/vector/simd
|
||||
|
||||
let dist = simdCosineDistance(vec1, vec2)
|
||||
```
|
||||
Reference in New Issue
Block a user