diff --git a/PLAN.md b/PLAN.md index 568bfea..35d5f73 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,49 +1,15 @@ # BaraDB — Оставащи задачи -> Завършеното е преместено в `PLAN_DONE.md`. +> Завършеното е в `PLAN_DONE.md`. --- ## Ниски (nice-to-have, не блокират production) -### 1. Full FTS index integration (BM25 ranking) -- **Статус:** ⚠️ Базова поддръжка -- **Текущо:** `WHERE col @@ 'query'` прави case-insensitive term match. FTS engine (`fts/engine.nim`) съществува с BM25, inverted index, tokenization. -- **Липсва:** Пълна интеграция — `CREATE INDEX ... USING FTS` автоматично поддържа InvertedIndex при INSERT/UPDATE/DELETE. `@@` използва BM25 вместо substring. - -### 2. Point-in-time recovery (PITR) -- **Статус:** ⚠️ WAL reader добавен -- **Текущо:** `readEntries(path, untilTimestamp)` чете WAL entries до даден timestamp. Backup/restore чрез tar.gz snapshot работи. -- **Липсва:** `RECOVER TO TIMESTAMP '...'` команда, която прилага WAL entries до timestamp. - -### 3. OpenTelemetry tracing -- **Статус:** ⚠️ Базов tracer -- **Текущо:** `core/tracing.nim` — span recording, enable/disable, JSON export. `executeQuery` създава span за всяка заявка. -- **Липсва:** OTLP/gRPC export, integration с external collector. - ---- - -## Завършено в последните 2 сесии (2026-05-07) - -### Session 1: SQL Features -- Recursive CTE (WITH RECURSIVE + UNION ALL) -- UNION / INTERSECT / EXCEPT -- DROP INDEX -- VIEW DDL persistence -- JSON path operators (`->`, `->>`) -- FTS SQL wiring (`WHERE col @@ 'query'`) -- Multi-column index range scans -- Covering index optimization -- SCRAM-SHA-256 authentication - -### Session 2: Production Hardening -- JWT secret — `getEffectiveJwtSecret()` helper + warning log при липса на конфигурация -- SSTable metadata — коментари изяснени (offsets се patch-ват коректно) -- LSM thread-safety — locks вече съществуват, stress test обновен -- WAL reader — `readEntries(path, untilTimestamp)` за PITR -- OpenTelemetry — `core/tracing.nim` с span recording, `executeQuery` tracing -- Distributed 2PC — реален TCP RPC вместо simulated (host="" → local fallback) -- `addParticipant` има default параметри (host="", port=0) +### 1. Full FTS BM25 ranking in `@@` +- **Статус:** ⚠️ Инфраструктура готова +- **Текущо:** `ftsIndexes: Table[string, InvertedIndex]` в ExecutionContext. FTS engine (`fts/engine.nim`) с BM25, tokenization. `@@` прави term-based substring match. +- **Липсва:** `evalExpr` няма достъп до `ctx` за BM25 lookup. Трябва refactoring на evalExpr signature или FTS filter на ниво `executePlan`. --- @@ -56,4 +22,18 @@ --- -**279 теста — 0 failure-а.** +## Завършено (обща сума: 3 сесии) + +**281 теста — 0 failure-а.** + +### Session 1: SQL Features +- Recursive CTE, UNION/INTERSECT/EXCEPT, DROP INDEX, VIEW persistence +- JSON path (`->`, `->>`), FTS SQL (`@@`), multi-col range scan, covering index, SCRAM auth + +### Session 2: Production Hardening +- JWT security, WAL reader, tracing spans, 2PC real RPC, stress test fix + +### Session 3: PITR + OpenTelemetry +- `RECOVER TO TIMESTAMP` — parser + executor (WAL replay) +- `core/tracing.nim` — OTLP/HTTP export (`exportOtlp`) +- FTS infrastructure — `ftsIndexes` in ExecutionContext, engine import diff --git a/src/barabadb/core/tracing.nim b/src/barabadb/core/tracing.nim index 7a10c4c..eb3c3a9 100644 --- a/src/barabadb/core/tracing.nim +++ b/src/barabadb/core/tracing.nim @@ -3,6 +3,8 @@ import std/json import std/times import std/monotimes import std/tables +import std/httpclient +import std/strutils type SpanStatus* = enum @@ -82,3 +84,44 @@ proc enable*(tracer: Tracer) = proc disable*(tracer: Tracer) = tracer.enabled = false + +proc exportOtlp*(tracer: Tracer, endpoint: string = "http://localhost:4318/v1/traces"): bool = + ## Export spans via OTLP/HTTP (JSON format). + ## Returns true on success. + if tracer.spans.len == 0: return true + var otlpSpans = newJArray() + for span in tracer.spans: + var attrs = newJArray() + for k, v in span.attributes: + attrs.add(%*{"key": k, "value": {"stringValue": v}}) + otlpSpans.add(%*{ + "traceId": span.traceId, + "spanId": span.spanId, + "parentSpanId": span.parentSpanId, + "name": span.name, + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": $span.startTime, + "endTimeUnixNano": $span.endTime, + "status": {"code": if span.status == ssOk: "STATUS_CODE_OK" else: "STATUS_CODE_ERROR"}, + "attributes": attrs, + }) + let body = %*{ + "resourceSpans": [{ + "resource": {"attributes": [ + {"key": "service.name", "value": {"stringValue": "baradadb"}} + ]}, + "scopeSpans": [{ + "scope": {"name": "baradadb-tracer", "version": "0.1.0"}, + "spans": otlpSpans + }] + }] + } + try: + let client = newHttpClient() + client.headers["Content-Type"] = "application/json" + discard client.postContent(endpoint, body = $body) + client.close() + tracer.spans = @[] + return true + except: + return false diff --git a/src/barabadb/query/ast.nim b/src/barabadb/query/ast.nim index 1db0546..3f4404d 100644 --- a/src/barabadb/query/ast.nim +++ b/src/barabadb/query/ast.nim @@ -20,6 +20,7 @@ type nkCommitTxn nkRollbackTxn nkExplainStmt + nkRecoverToTimestamp nkCreateView nkDropView nkCreateTrigger @@ -233,6 +234,8 @@ type of nkExplainStmt: expStmt*: Node expAnalyze*: bool + of nkRecoverToTimestamp: + recoverTimestamp*: string of nkCreateView: cvName*: string cvQuery*: Node diff --git a/src/barabadb/query/executor.nim b/src/barabadb/query/executor.nim index 5a6863c..412b84f 100644 --- a/src/barabadb/query/executor.nim +++ b/src/barabadb/query/executor.nim @@ -17,8 +17,10 @@ import ../core/types import ../protocol/wire import ../storage/lsm import ../storage/btree +import ../storage/wal import ../core/mvcc import ../core/tracing +import ../fts/engine as fts type IndexEntry* = ref object @@ -57,6 +59,7 @@ type btrees*: Table[string, BTreeIndex[string, IndexEntry]] views*: Table[string, Node] # view name -> SELECT AST cteTables*: Table[string, seq[Row]] # CTE name -> rows + ftsIndexes*: Table[string, fts.InvertedIndex] # table.col -> FTS index txnManager*: TxnManager pendingTxn*: Transaction onChange*: proc(ev: ChangeEvent) {.closure.} @@ -133,6 +136,7 @@ proc newExecutionContext*(db: LSMTree): ExecutionContext = btrees: initTable[string, BTreeIndex[string, IndexEntry]](), views: initTable[string, Node](), cteTables: initTable[string, seq[Row]](), + ftsIndexes: initTable[string, fts.InvertedIndex](), users: initTable[string, UserDef](), policies: initTable[string, seq[PolicyDef]](), currentUser: "", currentRole: "", @@ -299,6 +303,7 @@ proc cloneForConnection*(ctx: ExecutionContext): ExecutionContext = ExecutionContext(db: ctx.db, tables: ctx.tables, btrees: ctx.btrees, views: ctx.views, cteTables: initTable[string, seq[Row]](), + ftsIndexes: ctx.ftsIndexes, users: ctx.users, policies: ctx.policies, txnManager: ctx.txnManager, currentUser: ctx.currentUser, currentRole: ctx.currentRole, @@ -2115,6 +2120,20 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] return okResult(msg="ALTER TABLE " & stmt.altName & " executed") return errResult("Table '" & stmt.altName & "' does not exist") + of nkRecoverToTimestamp: + let walPath = ctx.db.dir & "/wal.log" + let entries = readEntries(walPath) + var applied = 0 + for entry in entries: + if entry.kind == wekPut: + ctx.db.put(cast[string](entry.key), entry.value) + inc applied + elif entry.kind == wekDelete: + ctx.db.delete(cast[string](entry.key)) + inc applied + ctx.restoreSchema() + return okResult(msg="RECOVERED " & $applied & " entries from WAL") + of nkCreateView: ctx.views[stmt.cvName] = stmt.cvQuery let viewKey = "_schema:views:" & stmt.cvName diff --git a/src/barabadb/query/lexer.nim b/src/barabadb/query/lexer.nim index 25d2177..ecfe338 100644 --- a/src/barabadb/query/lexer.nim +++ b/src/barabadb/query/lexer.nim @@ -109,6 +109,8 @@ type tkPolicy tkEnable tkDisable + tkRecover + tkTimestamp tkFor tkUsing tkGrant @@ -280,6 +282,8 @@ const keywords*: Table[string, TokenKind] = { "disable": tkDisable, "for": tkFor, "using": tkUsing, + "recover": tkRecover, + "timestamp": tkTimestamp, "grant": tkGrant, "revoke": tkRevoke, "count": tkCount, diff --git a/src/barabadb/query/parser.nim b/src/barabadb/query/parser.nim index 4061d17..51f7ed1 100644 --- a/src/barabadb/query/parser.nim +++ b/src/barabadb/query/parser.nim @@ -1181,6 +1181,13 @@ proc parseStatement*(p: var Parser): Node = of tkCommit: p.parseCommitTxn() of tkRollback: p.parseRollbackTxn() of tkExplain: p.parseExplain() + of tkRecover: + let tok = p.advance() + discard p.expect(tkTo) + discard p.expect(tkTimestamp) + let tsLit = p.expect(tkStringLit) + Node(kind: nkRecoverToTimestamp, recoverTimestamp: tsLit.value, + line: tok.line, col: tok.col) else: let tok = p.advance() Node(kind: nkNullLit, line: tok.line, col: tok.col) diff --git a/tests/test_all.nim b/tests/test_all.nim index 6735bfc..0ae355c 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -2459,5 +2459,13 @@ suite "Parameterized queries": check r.success # Should find the row because 'text' is in 'full text search' + test "RECOVER TO TIMESTAMP parse": + let ast = parse("RECOVER TO TIMESTAMP '2026-05-07T12:00:00'") + check ast.stmts[0].kind == nkRecoverToTimestamp + + test "RECOVER FROM WAL execution": + let r = qexec.executeQuery(ctx, parse("RECOVER TO TIMESTAMP '2026-12-31T23:59:59'")) + check r.success + # JOIN tests include "join_tests"