Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0c7847abba | |||
| eb62b381a1 |
+2
-2
@@ -9,8 +9,8 @@ binDir = "build"
|
|||||||
|
|
||||||
# Dependencies
|
# Dependencies
|
||||||
requires "nim >= 2.2.0"
|
requires "nim >= 2.2.0"
|
||||||
requires "https://github.com/katehonz/hunos >= 1.2.0"
|
requires "https://github.com/katehonz/hunos >= 1.3.0"
|
||||||
requires "jwt >= 0.3.0"
|
requires "https://github.com/katehonz/jwt-nim-baraba >= 2.1.0"
|
||||||
requires "checksums >= 0.2.0"
|
requires "checksums >= 0.2.0"
|
||||||
|
|
||||||
# Tasks
|
# Tasks
|
||||||
|
|||||||
@@ -102,8 +102,8 @@ services:
|
|||||||
sh -c '
|
sh -c '
|
||||||
while true; do
|
while true; do
|
||||||
sleep 86400;
|
sleep 86400;
|
||||||
/app/backup backup --data-dir=/data --output=/backups/baradb_$$(date +%Y%m%d_%H%M%S).tar.gz --level=6;
|
/app/backup backup --all-databases --data-root=/data/databases --output=/backups/baradb_$$(date +%Y%m%d_%H%M%S).tar.gz --level=6;
|
||||||
/app/backup cleanup --keep=7;
|
/app/backup cleanup --data-root=/data/databases --keep=7;
|
||||||
done
|
done
|
||||||
'
|
'
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
+87
-60
@@ -5,7 +5,7 @@ JSON-базиран REST API за уеб приложения.
|
|||||||
## Базов URL
|
## Базов URL
|
||||||
|
|
||||||
```
|
```
|
||||||
http://localhost:9470
|
http://localhost:9912
|
||||||
```
|
```
|
||||||
|
|
||||||
## Endpoints
|
## Endpoints
|
||||||
@@ -15,53 +15,7 @@ http://localhost:9470
|
|||||||
Health проверка:
|
Health проверка:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:9470/health
|
curl http://localhost:9912/health
|
||||||
```
|
|
||||||
|
|
||||||
### GET /ready
|
|
||||||
|
|
||||||
Readiness проверка:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:9470/ready
|
|
||||||
```
|
|
||||||
|
|
||||||
### POST /query
|
|
||||||
|
|
||||||
Изпълнение на BaraQL заявка:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:9470/api/query \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"query": "SELECT * FROM users WHERE age > 18"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Отговор:
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"columns": ["name", "age"],
|
|
||||||
"rows": [["Alice", 30], ["Bob", 25]],
|
|
||||||
"row_count": 2,
|
|
||||||
"duration_ms": 12
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### POST /batch
|
|
||||||
|
|
||||||
Групови заявки:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:9470/api/batch \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"queries": ["INSERT users { name := \"Alice\" }", "INSERT users { name := \"Bob\" }"]}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### GET /schema
|
|
||||||
|
|
||||||
Преглед на схемата:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:9470/api/schema
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### GET /metrics
|
### GET /metrics
|
||||||
@@ -69,37 +23,110 @@ curl http://localhost:9470/api/schema
|
|||||||
Prometheus метрики:
|
Prometheus метрики:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:9470/metrics
|
curl http://localhost:9912/metrics
|
||||||
```
|
```
|
||||||
|
|
||||||
### POST /explain
|
### POST /auth
|
||||||
|
|
||||||
Обяснение на план за изпълнение:
|
JWT автентикация:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:9470/api/explain \
|
curl -X POST http://localhost:9912/auth \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"username": "admin", "password": "secret"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /query
|
||||||
|
|
||||||
|
Изпълнение на SQL заявка:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:9912/query \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Database: default" \
|
||||||
-d '{"query": "SELECT * FROM users WHERE age > 18"}'
|
-d '{"query": "SELECT * FROM users WHERE age > 18"}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Отговор:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"columns": ["name", "age"],
|
||||||
|
"rows": [{"name": "Alice", "age": 30}],
|
||||||
|
"affectedRows": 0
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Забележка:** Header `X-Database` избира към коя база данни да се изпълни заявката. Ако липсва, се използва `default`.
|
||||||
|
|
||||||
|
### GET /tables
|
||||||
|
|
||||||
|
Списък с таблици в избраната база данни:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:9912/tables -H "X-Database: default"
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /databases
|
||||||
|
|
||||||
|
Списък с налични бази данни:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:9912/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /databases
|
||||||
|
|
||||||
|
Създаване на нова база данни (изисква admin права):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:9912/databases \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-d '{"name": "mydb"}'
|
||||||
|
```
|
||||||
|
|
||||||
### POST /backup
|
### POST /backup
|
||||||
|
|
||||||
Създаване на backup:
|
Създаване на backup (изисква admin права):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:9470/api/backup \
|
# Backup на всички бази данни
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d '{"destination": "/backup/snapshot.db"}'
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-d '{"all": true}'
|
||||||
|
|
||||||
|
# Backup на единична база
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-d '{"database": "default"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /backups
|
||||||
|
|
||||||
|
Списък с налични архиви:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:9912/backups -H "Authorization: Bearer <token>"
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /restore
|
||||||
|
|
||||||
|
Възстановяване от архив (изисква admin права):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:9912/restore \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-d '{"input": "backup_1234567890.tar.gz", "all": true}'
|
||||||
```
|
```
|
||||||
|
|
||||||
## Грешки
|
## Грешки
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"error": {
|
"error": "Unauthorized"
|
||||||
"code": "INVALID_QUERY",
|
|
||||||
"message": "Грешка в синтаксиса"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -107,5 +134,5 @@ curl -X POST http://localhost:9470/api/backup \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -H "Authorization: Bearer <token>" \
|
curl -H "Authorization: Bearer <token>" \
|
||||||
http://localhost:9470/api/users
|
http://localhost:9912/metrics
|
||||||
```
|
```
|
||||||
|
|||||||
+127
-138
@@ -1,36 +1,125 @@
|
|||||||
# Backup и Възстановяване
|
# Backup и Възстановяване
|
||||||
|
|
||||||
BaraDB предоставя няколко стратегии за backup — от пълни snapshot-ове до инкрементални и online consistent backups.
|
BaraDB предоставя няколко стратегии за backup — от пълни snapshot-ове до инкрементални, online consistent backups и multi-database архивиране.
|
||||||
|
|
||||||
> ⚠️ **Настройка с множество бази данни**
|
|
||||||
> BaraDB поддържа множество бази данни (`CREATE DATABASE`). Всяка база има собствена изолирана data директория (напр. `data/databases/<име>/`). Командите за backup, repair, checkpoint и migration работят върху **една директория наведнъж**. Ако използвате множество бази, пускайте командите за всяка база отделно или архивирайте цялата `data/databases/` директория.
|
|
||||||
|
|
||||||
## Архитектура
|
## Архитектура
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────────────┐
|
┌─────────────────────────────────────────┐
|
||||||
│ Data Directory │
|
│ Data Root │
|
||||||
│ ├── MANIFEST (atomic catalog) │
|
│ └── databases/ │
|
||||||
│ ├── sstables/ (SSTable v3 CRC) │
|
│ ├── default/ │
|
||||||
│ │ ├── 1.sst │
|
│ │ ├── MANIFEST │
|
||||||
│ │ └── 2.sst │
|
│ │ ├── sstables/ │
|
||||||
│ └── wal/ │
|
│ │ └── wal/ │
|
||||||
│ ├── wal.log (активен сегмент)│
|
│ ├── mydb/ │
|
||||||
│ └── wal_archive/ (ротирани сегменти
|
│ │ ├── MANIFEST │
|
||||||
|
│ │ ├── sstables/ │
|
||||||
|
│ │ └── wal/ │
|
||||||
|
│ └── ... │
|
||||||
└─────────────────────────────────────────┘
|
└─────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Backup Инструмент
|
## Backup Инструмент
|
||||||
|
|
||||||
Backup инструментът е в `src/barabadb/core/backup.nim`. Компилирайте го преди употреба:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
nim c -o:build/backup src/barabadb/core/backup.nim
|
nim c -o:build/backup src/barabadb/core/backup.nim
|
||||||
```
|
```
|
||||||
|
|
||||||
## SSTable Integrity (v3 CRC Footer)
|
## Multi-Database Backup (Препоръчително)
|
||||||
|
|
||||||
Всеки SSTable файл, записан от BaraDB, включва CRC32 footer:
|
### Архивиране на всички бази
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --all-databases --data-root=./data/databases --output=all_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Архивът съдържа:
|
||||||
|
- `backup.json` — метаданни (версия, timestamp, списък бази)
|
||||||
|
- `databases/<име>/` — всяка база със своя MANIFEST, SSTables и WAL
|
||||||
|
|
||||||
|
### Backup на единична база
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --database=default --data-root=./data/databases --output=default_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
### Възстановяване на всички бази
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup restore --input=all_1234567890.tar.gz --all-databases --data-root=./data/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
### Възстановяване на единична база
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup restore --input=default_1234567890.tar.gz --database=default --data-root=./data/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
## Legacy Single-Directory Backup
|
||||||
|
|
||||||
|
За обратна съвместимост със стари инсталации (една база в `data/server`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --data-dir=./data/server --output=legacy_$(date +%s).tar.gz
|
||||||
|
./build/backup restore --input=legacy_1234567890.tar.gz --data-dir=./data/server
|
||||||
|
```
|
||||||
|
|
||||||
|
## Инкрементален Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup incremental --database=default --data-root=./data/databases --output=inc_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Включва само:
|
||||||
|
- `MANIFEST`
|
||||||
|
- Активни SSTables (от MANIFEST)
|
||||||
|
- Текущ WAL (`wal/wal.log`)
|
||||||
|
- WAL архив (`wal/wal_archive/*.log`)
|
||||||
|
|
||||||
|
Всички SSTables се **проверяват с CRC** преди архивиране.
|
||||||
|
|
||||||
|
## Online Consistent Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --online --database=default --data-root=./data/databases --output=online_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Еквивалентно на:
|
||||||
|
1. `checkpoint` (freeze memtable, flush, rotate WAL)
|
||||||
|
2. `incremental backup`
|
||||||
|
|
||||||
|
Безопасно за пускане докато сървърът работи.
|
||||||
|
|
||||||
|
## HTTP API Backup
|
||||||
|
|
||||||
|
Backup/restore е достъпен и през REST API (изисква admin JWT токен):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backup на всички бази
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"all": true}'
|
||||||
|
|
||||||
|
# Backup на единична база
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"database": "default"}'
|
||||||
|
|
||||||
|
# Списък с архиви
|
||||||
|
curl http://localhost:9912/backups \
|
||||||
|
-H "Authorization: Bearer <token>"
|
||||||
|
|
||||||
|
# Restore
|
||||||
|
curl -X POST http://localhost:9912/restore \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"input": "backup_1234567890.tar.gz", "all": true}'
|
||||||
|
```
|
||||||
|
|
||||||
|
## SSTable Integrity (v3 CRC Footer)
|
||||||
|
|
||||||
```
|
```
|
||||||
[Header] 36 байта
|
[Header] 36 байта
|
||||||
@@ -43,36 +132,18 @@ nim c -o:build/backup src/barabadb/core/backup.nim
|
|||||||
dataCrc32, indexCrc32, bloomCrc32, reserved
|
dataCrc32, indexCrc32, bloomCrc32, reserved
|
||||||
```
|
```
|
||||||
|
|
||||||
Това позволява независима проверка на всеки SSTable:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Чрез Nim API
|
|
||||||
import barabadb/storage/lsm
|
|
||||||
let (ok, msg) = verifySSTable("data/databases/default/sstables/1.sst")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Storage Repair (`baradadb repair`)
|
## Storage Repair (`baradadb repair`)
|
||||||
|
|
||||||
При съмнение за повреда, пуснете repair инструмента за конкретна база данни:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Dry run — само преглед
|
# Dry run — само преглед
|
||||||
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
||||||
|
|
||||||
# Пълен ремонт — проверка, преместване на битите файлове, WAL replay
|
# Пълен ремонт
|
||||||
./build/baradadb repair --data-dir=./data/databases/default
|
./build/baradadb repair --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
**Какво прави repair:**
|
|
||||||
1. Сканира всички `sstables/*.sst` и проверява CRC
|
|
||||||
2. Премества корумпираните SSTables в `<data-dir>/corrupt/`
|
|
||||||
3. Пуска WAL replay за възстановяване на незаписани данни
|
|
||||||
4. Докладва резултати
|
|
||||||
|
|
||||||
## MANIFEST Каталог
|
## MANIFEST Каталог
|
||||||
|
|
||||||
Файлът `MANIFEST` е единственият източник на истина за активните SSTables. Обновява се атомично при всеки flush и compaction.
|
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
@@ -84,137 +155,55 @@ let (ok, msg) = verifySSTable("data/databases/default/sstables/1.sst")
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Предимства:
|
|
||||||
- **Консистентен изглед** — няма orphan SSTables след crash
|
|
||||||
- **Бързо стартиране** — зарежда от MANIFEST вместо scan на директория
|
|
||||||
- **Откриване на orphans** — `checkStorageConsistency()` докладва излишни/липсващи файлове
|
|
||||||
|
|
||||||
При настройка с множество бази данни, всяка база поддържа собствен независим MANIFEST в своята data директория.
|
|
||||||
|
|
||||||
## WAL Ротация
|
|
||||||
|
|
||||||
Write-Ahead Log се ротира при достигане на 64MB:
|
|
||||||
|
|
||||||
```
|
|
||||||
wal/wal.log → активен сегмент
|
|
||||||
wal/wal_archive/
|
|
||||||
├── wal.000001.log
|
|
||||||
├── wal.000002.log
|
|
||||||
└── wal.000003.log
|
|
||||||
```
|
|
||||||
|
|
||||||
Ротацията се случва:
|
|
||||||
- На всеки 1000 WAL записа (лека проверка на размер)
|
|
||||||
- При всеки `flush` / `checkpoint`
|
|
||||||
|
|
||||||
## Checkpoint
|
## Checkpoint
|
||||||
|
|
||||||
Checkpoint създава консистентна граница на storage без спиране на сървъра:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build/baradadb checkpoint --data-dir=./data/databases/default
|
./build/baradadb checkpoint --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
**Как работи:**
|
**Как работи:**
|
||||||
1. Freeze на memtable (swap към immutable, нов memtable за writes)
|
1. Freeze на memtable (< 1ms)
|
||||||
2. Flush на frozen memtable към SSTable
|
2. Flush към SSTable
|
||||||
3. Ротация на WAL
|
3. Ротация на WAL
|
||||||
4. Запис на MANIFEST
|
4. Запис на MANIFEST
|
||||||
|
|
||||||
Freeze-ът отнема **< 1ms**; flush-ът продължава паралелно с writes.
|
|
||||||
|
|
||||||
## Backup Команди
|
|
||||||
|
|
||||||
> Компилирайте backup инструмента първо: `nim c -o:build/backup src/barabadb/core/backup.nim`
|
|
||||||
|
|
||||||
### Пълен Backup (tar.gz)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup backup --data-dir=./data/databases/default --output=backup_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Архивира цялата data директория на посочената база данни.
|
|
||||||
|
|
||||||
### Инкрементален Backup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup incremental --data-dir=./data/databases/default --output=backup_inc_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Включва само:
|
|
||||||
- `MANIFEST`
|
|
||||||
- Активни SSTables (от MANIFEST)
|
|
||||||
- Текущ WAL (`wal/wal.log`)
|
|
||||||
- WAL архив (`wal/wal_archive/*.log`)
|
|
||||||
|
|
||||||
Всички SSTables се **проверяват с CRC** преди архивиране.
|
|
||||||
|
|
||||||
### Online Consistent Backup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup backup --online --data-dir=./data/databases/default --output=backup_online_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Еквивалентно на:
|
|
||||||
1. `checkpoint`
|
|
||||||
2. `incremental backup`
|
|
||||||
|
|
||||||
Безопасно за пускане докато сървърът работи. Checkpoint-ът създава консистентен snapshot, след което инкременталният backup го архивира.
|
|
||||||
|
|
||||||
## Миграция на SSTable Версии
|
## Миграция на SSTable Версии
|
||||||
|
|
||||||
Ако имате legacy v1/v2 SSTables, мигрирайте ги към v3 за всяка база данни:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Преглед
|
|
||||||
./build/baradadb migrate --data-dir=./data/databases/default --dry-run
|
./build/baradadb migrate --data-dir=./data/databases/default --dry-run
|
||||||
|
|
||||||
# Миграция
|
|
||||||
./build/baradadb migrate --data-dir=./data/databases/default
|
./build/baradadb migrate --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
Миграцията пренаписва всеки legacy SSTable в текущия v3 формат (CRC footer) и обновява MANIFEST.
|
|
||||||
|
|
||||||
## Процедури за Възстановяване
|
## Процедури за Възстановяване
|
||||||
|
|
||||||
### Сценарий 1: Открит е Корумпиран SSTable
|
### Сценарий 1: Корумпиран SSTable
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Repair премества битите файлове и пуска WAL replay
|
|
||||||
./build/baradadb repair --data-dir=./data/databases/default
|
./build/baradadb repair --data-dir=./data/databases/default
|
||||||
|
|
||||||
# Проверка на консистентност
|
|
||||||
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Сценарий 2: Възстановяване от Backup (Единична База)
|
### Сценарий 2: Възстановяване от Multi-Database Backup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Спиране на сървъра
|
# 1. Разархивиране
|
||||||
# Разархивиране на backup в директорията на базата
|
./build/backup restore --input=backup_latest.tar.gz --all-databases --data-root=./data/databases
|
||||||
tar -xzf backup_1234567890.tar.gz -C ./data/databases/default
|
|
||||||
|
|
||||||
# Рестарт — LSMTree зарежда от MANIFEST
|
# 2. Repair за всяка база
|
||||||
./build/baradadb
|
|
||||||
```
|
|
||||||
|
|
||||||
### Сценарий 3: Възстановяване на Всички Бази
|
|
||||||
|
|
||||||
Ако сте архивирали цялото `data/databases/` дърво:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Разархивиране на последния backup
|
|
||||||
tar -xzf backup_latest.tar.gz -C ./data
|
|
||||||
|
|
||||||
# 2. Repair за всяка база за replay на наличния WAL
|
|
||||||
for db in ./data/databases/*/; do
|
for db in ./data/databases/*/; do
|
||||||
./build/baradadb repair --data-dir="$db" --dry-run
|
./build/baradadb repair --data-dir="$db" --dry-run
|
||||||
done
|
done
|
||||||
|
|
||||||
# 3. Стартиране на сървъра
|
# 3. Стартиране
|
||||||
./build/baradadb
|
./build/baradadb
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Сценарий 3: Ръчно разархивиране
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tar -xzf backup_latest.tar.gz -C ./data
|
||||||
|
# Архивът съдържа: databases/<име>/ + backup.json
|
||||||
|
```
|
||||||
|
|
||||||
## Изисквания за Съхранение
|
## Изисквания за Съхранение
|
||||||
|
|
||||||
| Тип Backup | Размер | Честота | Задържане |
|
| Тип Backup | Размер | Честота | Задържане |
|
||||||
@@ -225,9 +214,9 @@ done
|
|||||||
|
|
||||||
## Най-добри Практики
|
## Най-добри Практики
|
||||||
|
|
||||||
1. **Пускайте repair след некоректно спиране** — `./build/baradadb repair`
|
1. **Използвайте `--all-databases`** за пълен backup в multi-DB сетъп
|
||||||
2. **Мигрирайте legacy SSTables** — `./build/baradadb migrate`
|
2. **Тествайте възстановяването редовно** — Backup, който не може да бъде възстановен, е безполезен
|
||||||
3. **Тествайте възстановяването редовно** — Backup, който не може да бъде възстановен, е безполезен
|
3. **Пускайте repair след некоректно спиране**
|
||||||
4. **Използвайте incremental + checkpoint** — За чести консистентни snapshot-ове
|
4. **Съхранявайте backups извън локацията** — S3, GCS или друг сървър
|
||||||
5. **Съхранявайте backups извън локацията** — S3, GCS или друг сървър
|
5. **Използвайте incremental + checkpoint** — За чести консистентни snapshot-ове
|
||||||
6. **Следете MANIFEST sequence** — Трябва да расте монотонно с flush-овете
|
6. **Мониторирайте `/backups` endpoint** — През админ панела или API
|
||||||
|
|||||||
+11
-1
@@ -240,9 +240,19 @@ Content-Type: application/json
|
|||||||
```http
|
```http
|
||||||
POST /backup
|
POST /backup
|
||||||
Content-Type: application/json
|
Content-Type: application/json
|
||||||
|
Authorization: Bearer <token>
|
||||||
|
|
||||||
{
|
{
|
||||||
"destination": "/backup/snapshot.db"
|
"all": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Отговор:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"output": "backup_1234567890.tar.gz",
|
||||||
|
"message": "Backup created"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+126
-137
@@ -1,36 +1,125 @@
|
|||||||
# Backup & Recovery
|
# Backup & Recovery
|
||||||
|
|
||||||
BaraDB provides multiple backup strategies ranging from full snapshots to incremental and online consistent backups.
|
BaraDB provides multiple backup strategies ranging from full snapshots to incremental, online consistent backups, and multi-database archiving.
|
||||||
|
|
||||||
> ⚠️ **Multi-Database Setup**
|
|
||||||
> BaraDB supports multiple databases (`CREATE DATABASE`). Each database has its own isolated data directory (e.g. `data/databases/<name>/`). Backup, repair, checkpoint, and migration commands operate on **one directory at a time**. If you use multiple databases, run the commands for each database separately or back up the entire `data/databases/` parent directory.
|
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────────────┐
|
┌─────────────────────────────────────────┐
|
||||||
│ Data Directory │
|
│ Data Root │
|
||||||
│ ├── MANIFEST (atomic catalog) │
|
│ └── databases/ │
|
||||||
│ ├── sstables/ (SSTable v3 CRC) │
|
│ ├── default/ │
|
||||||
│ │ ├── 1.sst │
|
│ │ ├── MANIFEST │
|
||||||
│ │ └── 2.sst │
|
│ │ ├── sstables/ │
|
||||||
│ └── wal/ │
|
│ │ └── wal/ │
|
||||||
│ ├── wal.log (active segment) │
|
│ ├── mydb/ │
|
||||||
│ └── wal_archive/ (rotated segments│
|
│ │ ├── MANIFEST │
|
||||||
|
│ │ ├── sstables/ │
|
||||||
|
│ │ └── wal/ │
|
||||||
|
│ └── ... │
|
||||||
└─────────────────────────────────────────┘
|
└─────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Backup Tool
|
## Backup Tool
|
||||||
|
|
||||||
The backup tool is shipped as `src/barabadb/core/backup.nim`. Build it before use:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
nim c -o:build/backup src/barabadb/core/backup.nim
|
nim c -o:build/backup src/barabadb/core/backup.nim
|
||||||
```
|
```
|
||||||
|
|
||||||
## SSTable Integrity (v3 CRC Footer)
|
## Multi-Database Backup (Recommended)
|
||||||
|
|
||||||
Every SSTable file written by BaraDB includes a CRC32 footer:
|
### Backup all databases
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --all-databases --data-root=./data/databases --output=all_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
The archive contains:
|
||||||
|
- `backup.json` — metadata (version, timestamp, database list)
|
||||||
|
- `databases/<name>/` — each database with its MANIFEST, SSTables, and WAL
|
||||||
|
|
||||||
|
### Backup a single database
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --database=default --data-root=./data/databases --output=default_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore all databases
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup restore --input=all_1234567890.tar.gz --all-databases --data-root=./data/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore a single database
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup restore --input=default_1234567890.tar.gz --database=default --data-root=./data/databases
|
||||||
|
```
|
||||||
|
|
||||||
|
## Legacy Single-Directory Backup
|
||||||
|
|
||||||
|
For backward compatibility with older installations (single database in `data/server`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --data-dir=./data/server --output=legacy_$(date +%s).tar.gz
|
||||||
|
./build/backup restore --input=legacy_1234567890.tar.gz --data-dir=./data/server
|
||||||
|
```
|
||||||
|
|
||||||
|
## Incremental Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup incremental --database=default --data-root=./data/databases --output=inc_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Includes only:
|
||||||
|
- `MANIFEST`
|
||||||
|
- Active SSTables (from MANIFEST)
|
||||||
|
- Current WAL (`wal/wal.log`)
|
||||||
|
- WAL archive (`wal/wal_archive/*.log`)
|
||||||
|
|
||||||
|
All SSTables are **CRC-verified** before archiving.
|
||||||
|
|
||||||
|
## Online Consistent Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/backup backup --online --database=default --data-root=./data/databases --output=online_$(date +%s).tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalent to:
|
||||||
|
1. `checkpoint` (freeze memtable, flush, rotate WAL)
|
||||||
|
2. `incremental backup`
|
||||||
|
|
||||||
|
Safe to run while the server is running.
|
||||||
|
|
||||||
|
## HTTP API Backup
|
||||||
|
|
||||||
|
Backup/restore is also available via REST API (requires admin JWT token):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backup all databases
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"all": true}'
|
||||||
|
|
||||||
|
# Backup single database
|
||||||
|
curl -X POST http://localhost:9912/backup \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"database": "default"}'
|
||||||
|
|
||||||
|
# List backups
|
||||||
|
curl http://localhost:9912/backups \
|
||||||
|
-H "Authorization: Bearer <token>"
|
||||||
|
|
||||||
|
# Restore
|
||||||
|
curl -X POST http://localhost:9912/restore \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"input": "backup_1234567890.tar.gz", "all": true}'
|
||||||
|
```
|
||||||
|
|
||||||
|
## SSTable Integrity (v3 CRC Footer)
|
||||||
|
|
||||||
```
|
```
|
||||||
[Header] 36 bytes
|
[Header] 36 bytes
|
||||||
@@ -43,36 +132,18 @@ Every SSTable file written by BaraDB includes a CRC32 footer:
|
|||||||
dataCrc32, indexCrc32, bloomCrc32, reserved
|
dataCrc32, indexCrc32, bloomCrc32, reserved
|
||||||
```
|
```
|
||||||
|
|
||||||
This enables independent verification of each SSTable:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Via Nim API
|
|
||||||
import barabadb/storage/lsm
|
|
||||||
let (ok, msg) = verifySSTable("data/databases/default/sstables/1.sst")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Storage Repair (`baradadb repair`)
|
## Storage Repair (`baradadb repair`)
|
||||||
|
|
||||||
If corruption is suspected, run the repair tool against a specific database directory:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Dry run — preview only
|
# Dry run — preview only
|
||||||
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
||||||
|
|
||||||
# Full repair — verify, move corrupt files, replay WAL
|
# Full repair
|
||||||
./build/baradadb repair --data-dir=./data/databases/default
|
./build/baradadb repair --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
**What repair does:**
|
|
||||||
1. Scans all `sstables/*.sst` and verifies CRC
|
|
||||||
2. Moves corrupt SSTables to `<data-dir>/corrupt/`
|
|
||||||
3. Replays WAL to recover unflushed committed data
|
|
||||||
4. Reports results
|
|
||||||
|
|
||||||
## MANIFEST Catalog
|
## MANIFEST Catalog
|
||||||
|
|
||||||
The `MANIFEST` file is the single source of truth for active SSTables. It is updated atomically on every flush and compaction.
|
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
@@ -84,129 +155,40 @@ The `MANIFEST` file is the single source of truth for active SSTables. It is upd
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Benefits:
|
|
||||||
- **Consistent view** — no orphan SSTables after crash
|
|
||||||
- **Fast startup** — load from MANIFEST instead of directory scan
|
|
||||||
- **Orphan detection** — `checkStorageConsistency()` reports extra/missing files
|
|
||||||
|
|
||||||
In a multi-database setup, each database maintains its own independent MANIFEST inside its data directory.
|
|
||||||
|
|
||||||
## WAL Rotation
|
|
||||||
|
|
||||||
The Write-Ahead Log rotates when it reaches 64MB:
|
|
||||||
|
|
||||||
```
|
|
||||||
wal/wal.log → active segment
|
|
||||||
wal/wal_archive/
|
|
||||||
├── wal.000001.log
|
|
||||||
├── wal.000002.log
|
|
||||||
└── wal.000003.log
|
|
||||||
```
|
|
||||||
|
|
||||||
Rotation happens:
|
|
||||||
- Every 1000 WAL entries (lightweight size check)
|
|
||||||
- On every `flush` / `checkpoint`
|
|
||||||
|
|
||||||
## Checkpoint
|
## Checkpoint
|
||||||
|
|
||||||
A checkpoint creates a consistent storage boundary without stopping the server:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build/baradadb checkpoint --data-dir=./data/databases/default
|
./build/baradadb checkpoint --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
**How it works:**
|
**How it works:**
|
||||||
1. Freeze memtable (swap to immutable, new memtable for writes)
|
1. Freeze memtable (< 1ms)
|
||||||
2. Flush frozen memtable to SSTable
|
2. Flush to SSTable
|
||||||
3. Rotate WAL
|
3. Rotate WAL
|
||||||
4. Write MANIFEST
|
4. Write MANIFEST
|
||||||
|
|
||||||
The freeze takes **< 1ms**; the flush proceeds concurrently with writes.
|
|
||||||
|
|
||||||
## Backup Commands
|
|
||||||
|
|
||||||
> Build the backup tool first: `nim c -o:build/backup src/barabadb/core/backup.nim`
|
|
||||||
|
|
||||||
### Full Backup (tar.gz)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup backup --data-dir=./data/databases/default --output=backup_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Archives the entire data directory of the specified database.
|
|
||||||
|
|
||||||
### Incremental Backup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup incremental --data-dir=./data/databases/default --output=backup_inc_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Includes only:
|
|
||||||
- `MANIFEST`
|
|
||||||
- Active SSTables (from MANIFEST)
|
|
||||||
- Current WAL (`wal/wal.log`)
|
|
||||||
- WAL archive (`wal/wal_archive/*.log`)
|
|
||||||
|
|
||||||
All SSTables are **CRC-verified** before archiving.
|
|
||||||
|
|
||||||
### Online Consistent Backup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./build/backup backup --online --data-dir=./data/databases/default --output=backup_online_$(date +%s).tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Equivalent to:
|
|
||||||
1. `checkpoint`
|
|
||||||
2. `incremental backup`
|
|
||||||
|
|
||||||
Safe to run while the server is running. The checkpoint creates a consistent snapshot, then the incremental backup archives it.
|
|
||||||
|
|
||||||
## SSTable Version Migration
|
## SSTable Version Migration
|
||||||
|
|
||||||
If you have legacy v1/v2 SSTables, migrate them to v3 per database:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Preview
|
|
||||||
./build/baradadb migrate --data-dir=./data/databases/default --dry-run
|
./build/baradadb migrate --data-dir=./data/databases/default --dry-run
|
||||||
|
|
||||||
# Migrate
|
|
||||||
./build/baradadb migrate --data-dir=./data/databases/default
|
./build/baradadb migrate --data-dir=./data/databases/default
|
||||||
```
|
```
|
||||||
|
|
||||||
Migration rewrites each legacy SSTable with the current v3 format (CRC footer) and updates the MANIFEST.
|
|
||||||
|
|
||||||
## Recovery Procedures
|
## Recovery Procedures
|
||||||
|
|
||||||
### Scenario 1: Corrupt SSTable Detected
|
### Scenario 1: Corrupt SSTable
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Repair moves corrupt files and replays WAL
|
|
||||||
./build/baradadb repair --data-dir=./data/databases/default
|
./build/baradadb repair --data-dir=./data/databases/default
|
||||||
|
|
||||||
# Verify consistency
|
|
||||||
./build/baradadb repair --data-dir=./data/databases/default --dry-run
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scenario 2: Restore from Backup (Single Database)
|
### Scenario 2: Restore from Multi-Database Backup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Stop the server
|
# 1. Extract
|
||||||
# Extract backup into the database directory
|
./build/backup restore --input=backup_latest.tar.gz --all-databases --data-root=./data/databases
|
||||||
tar -xzf backup_1234567890.tar.gz -C ./data/databases/default
|
|
||||||
|
|
||||||
# Restart — LSMTree loads from MANIFEST
|
# 2. Repair each database
|
||||||
./build/baradadb
|
|
||||||
```
|
|
||||||
|
|
||||||
### Scenario 3: Restore All Databases
|
|
||||||
|
|
||||||
If you back up the entire `data/databases/` tree:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Extract latest backup
|
|
||||||
tar -xzf backup_latest.tar.gz -C ./data
|
|
||||||
|
|
||||||
# 2. Run repair for each database to replay any available WAL
|
|
||||||
for db in ./data/databases/*/; do
|
for db in ./data/databases/*/; do
|
||||||
./build/baradadb repair --data-dir="$db" --dry-run
|
./build/baradadb repair --data-dir="$db" --dry-run
|
||||||
done
|
done
|
||||||
@@ -215,6 +197,13 @@ done
|
|||||||
./build/baradadb
|
./build/baradadb
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Scenario 3: Manual extraction
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tar -xzf backup_latest.tar.gz -C ./data
|
||||||
|
# Archive contains: databases/<name>/ + backup.json
|
||||||
|
```
|
||||||
|
|
||||||
## Storage Requirements
|
## Storage Requirements
|
||||||
|
|
||||||
| Backup Type | Size | Frequency | Retention |
|
| Backup Type | Size | Frequency | Retention |
|
||||||
@@ -225,9 +214,9 @@ done
|
|||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
1. **Run repair after unclean shutdown** — `./build/baradadb repair`
|
1. **Use `--all-databases`** for full backups in multi-DB setups
|
||||||
2. **Migrate legacy SSTables** — `./build/baradadb migrate`
|
2. **Test restores regularly** — A backup you can't restore is useless
|
||||||
3. **Test restores regularly** — A backup you can't restore is useless
|
3. **Run repair after unclean shutdown**
|
||||||
4. **Use incremental + checkpoint** — For frequent consistent snapshots
|
4. **Store backups offsite** — S3, GCS, or another server
|
||||||
5. **Store backups offsite** — S3, GCS, or another server
|
5. **Use incremental + checkpoint** — For frequent consistent snapshots
|
||||||
6. **Monitor MANIFEST sequence** — Should grow monotonically with flushes
|
6. **Monitor `/backups` endpoint** — Via admin panel or API
|
||||||
|
|||||||
+11
-1
@@ -290,9 +290,19 @@ Response:
|
|||||||
```http
|
```http
|
||||||
POST /backup
|
POST /backup
|
||||||
Content-Type: application/json
|
Content-Type: application/json
|
||||||
|
Authorization: Bearer <token>
|
||||||
|
|
||||||
{
|
{
|
||||||
"destination": "/backup/snapshot.db"
|
"all": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Response:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"output": "backup_1234567890.tar.gz",
|
||||||
|
"message": "Backup created"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -38,9 +38,11 @@ type
|
|||||||
|
|
||||||
const
|
const
|
||||||
DEFAULT_DATA_DIR = "data/server"
|
DEFAULT_DATA_DIR = "data/server"
|
||||||
|
DEFAULT_DATA_ROOT = "data/databases"
|
||||||
DEFAULT_KEEP_COUNT = 5
|
DEFAULT_KEEP_COUNT = 5
|
||||||
DEFAULT_COMPRESSION = 6
|
DEFAULT_COMPRESSION = 6
|
||||||
HISTORY_FILE = "backup_history.log"
|
HISTORY_FILE = "backup_history.log"
|
||||||
|
BACKUP_META_FILE = "backup.json"
|
||||||
HELP_TEXT = """
|
HELP_TEXT = """
|
||||||
BaraDB Backup Manager — Archive and restore your database safely
|
BaraDB Backup Manager — Archive and restore your database safely
|
||||||
================================================================
|
================================================================
|
||||||
@@ -73,20 +75,33 @@ COMMANDS:
|
|||||||
help Show this help message.
|
help Show this help message.
|
||||||
|
|
||||||
OPTIONS:
|
OPTIONS:
|
||||||
-d, --data-dir <DIR> Path to data directory (default: data/server)
|
-d, --data-dir <DIR> Path to single data directory (default: data/server)
|
||||||
|
-r, --data-root <DIR> Path to multi-database root (default: data/databases)
|
||||||
-o, --output <FILE> Destination path for new backup archive
|
-o, --output <FILE> Destination path for new backup archive
|
||||||
-i, --input <FILE> Source archive for restore or verify
|
-i, --input <FILE> Source archive for restore or verify
|
||||||
-k, --keep <N> Retention count for cleanup (default: 5)
|
-k, --keep <N> Retention count for cleanup (default: 5)
|
||||||
-e, --exclude <PAT> Exclude files matching pattern (repeatable)
|
-e, --exclude <PAT> Exclude files matching pattern (repeatable)
|
||||||
-l, --level <0-9> Gzip compression level (default: 6, max: 9)
|
-l, --level <0-9> Gzip compression level (default: 6, max: 9)
|
||||||
|
--all-databases Backup/restore all databases under --data-root
|
||||||
|
--database <NAME> Backup/restore a specific database under --data-root
|
||||||
|
--online Create consistent backup via checkpoint (freeze + flush)
|
||||||
--dry-run Show what restore would do without changing anything
|
--dry-run Show what restore would do without changing anything
|
||||||
-f, --force Skip confirmation prompts (use with caution!)
|
-f, --force Skip confirmation prompts (use with caution!)
|
||||||
-v, --verbose Print detailed progress information
|
-v, --verbose Print detailed progress information
|
||||||
|
|
||||||
EXAMPLES:
|
EXAMPLES:
|
||||||
# Quick backup with default settings
|
# Quick backup of single directory (legacy)
|
||||||
backup backup
|
backup backup
|
||||||
|
|
||||||
|
# Backup all databases (recommended for multi-DB setups)
|
||||||
|
backup backup --all-databases --output=all_backup.tar.gz
|
||||||
|
|
||||||
|
# Backup a specific database
|
||||||
|
backup backup --database=default --output=default_backup.tar.gz
|
||||||
|
|
||||||
|
# Restore all databases
|
||||||
|
backup restore --input=all_backup.tar.gz --all-databases
|
||||||
|
|
||||||
# Backup with maximum compression and custom name
|
# Backup with maximum compression and custom name
|
||||||
backup backup --output=prod_backup_$(date +%F).tar.gz --level=9
|
backup backup --output=prod_backup_$(date +%F).tar.gz --level=9
|
||||||
|
|
||||||
@@ -481,6 +496,168 @@ proc incrementalBackupDataDir*(dataDir: string, output: string, verbose: bool =
|
|||||||
echo " Files: ", filesToInclude.len
|
echo " Files: ", filesToInclude.len
|
||||||
return true
|
return true
|
||||||
|
|
||||||
|
proc discoverDatabases*(dataRoot: string): seq[string] =
|
||||||
|
## Scan dataRoot for database directories
|
||||||
|
result = @[]
|
||||||
|
if not dirExists(dataRoot):
|
||||||
|
return
|
||||||
|
for kind, path in walkDir(dataRoot):
|
||||||
|
if kind == pcDir:
|
||||||
|
let dbName = lastPathPart(path)
|
||||||
|
if dbName.len > 0 and dbName notin [".", ".."]:
|
||||||
|
result.add(dbName)
|
||||||
|
result.sort()
|
||||||
|
|
||||||
|
proc backupAllDatabases*(dataRoot: string, output: string, excludes: seq[string] = @[], compression: int = DEFAULT_COMPRESSION, verbose: bool = false): bool =
|
||||||
|
## Create a tar.gz backup of all databases under dataRoot.
|
||||||
|
## Archive layout: databases/<name>/... + backup.json
|
||||||
|
if not dirExists(dataRoot):
|
||||||
|
echo "ERROR: Data root directory not found: ", dataRoot
|
||||||
|
return false
|
||||||
|
|
||||||
|
let databases = discoverDatabases(dataRoot)
|
||||||
|
if databases.len == 0:
|
||||||
|
echo "ERROR: No databases found in ", dataRoot
|
||||||
|
return false
|
||||||
|
|
||||||
|
if fileExists(output):
|
||||||
|
echo "WARNING: Overwriting existing file: ", output
|
||||||
|
|
||||||
|
let workDir = getCurrentDir()
|
||||||
|
let tempDir = getTempDir() / "baradb_backup_" & $getTime().toUnix()
|
||||||
|
createDir(tempDir / "databases")
|
||||||
|
|
||||||
|
var meta = %*{
|
||||||
|
"version": 1,
|
||||||
|
"timestamp": getTime().toUnix(),
|
||||||
|
"databases": databases,
|
||||||
|
"createdBy": "baradb-backup"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Copy each database into temp staging area
|
||||||
|
for dbName in databases:
|
||||||
|
let src = dataRoot / dbName
|
||||||
|
let dst = tempDir / "databases" / dbName
|
||||||
|
if verbose:
|
||||||
|
echo "Staging database: ", dbName
|
||||||
|
copyDir(src, dst)
|
||||||
|
|
||||||
|
# Write metadata
|
||||||
|
writeFile(tempDir / BACKUP_META_FILE, $meta)
|
||||||
|
|
||||||
|
var excludeArgs = ""
|
||||||
|
for pattern in excludes:
|
||||||
|
excludeArgs.add(" --exclude=" & quoteShell(pattern))
|
||||||
|
|
||||||
|
let tarCmd = "tar -cf -" & excludeArgs & " -C " & quoteShell(tempDir) & " ."
|
||||||
|
let gzipCmd = "gzip -" & $compression
|
||||||
|
let cmd = tarCmd & " | " & gzipCmd & " > " & quoteShell(output)
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
echo "Running: ", cmd
|
||||||
|
echo "Databases: ", databases.join(", ")
|
||||||
|
echo "Output: ", output
|
||||||
|
|
||||||
|
let (outputStr, exitCode) = execCmdEx("bash -c " & quoteShell(cmd))
|
||||||
|
removeDir(tempDir)
|
||||||
|
|
||||||
|
if exitCode != 0:
|
||||||
|
echo "ERROR: tar command failed with exit code ", exitCode
|
||||||
|
if outputStr.len > 0:
|
||||||
|
echo outputStr
|
||||||
|
return false
|
||||||
|
|
||||||
|
let size = getFileSize(output)
|
||||||
|
echo "Multi-database backup created successfully:"
|
||||||
|
echo " File: ", output
|
||||||
|
echo " Size: ", formatBytes(size)
|
||||||
|
echo " Databases: ", databases.len, " (", databases.join(", "), ")"
|
||||||
|
return true
|
||||||
|
|
||||||
|
proc restoreAllDatabases*(input: string, dataRoot: string, verbose: bool = false, dryRun: bool = false): bool =
|
||||||
|
## Restore all databases from a multi-database archive.
|
||||||
|
if not fileExists(input):
|
||||||
|
echo "ERROR: Backup file not found: ", input
|
||||||
|
return false
|
||||||
|
|
||||||
|
let archiveSize = getFileSize(input)
|
||||||
|
let freeSpace = getFreeSpace(parentDir(dataRoot))
|
||||||
|
let oldBackupPath = dataRoot & ".old_" & $getTime().toUnix()
|
||||||
|
|
||||||
|
if dryRun:
|
||||||
|
echo "DRY-RUN: The following actions would be performed:"
|
||||||
|
echo " 1. Verify archive integrity: ", input
|
||||||
|
echo " 2. Move existing data root to: ", oldBackupPath
|
||||||
|
echo " 3. Extract archive to: ", dataRoot
|
||||||
|
echo " Archive size: ", formatBytes(archiveSize)
|
||||||
|
if freeSpace >= 0:
|
||||||
|
echo " Free space: ", formatBytes(freeSpace)
|
||||||
|
else:
|
||||||
|
echo " Free space: unable to determine"
|
||||||
|
return true
|
||||||
|
|
||||||
|
# Check free space
|
||||||
|
if freeSpace >= 0 and freeSpace < archiveSize * 2:
|
||||||
|
echo "WARNING: Free space (", formatBytes(freeSpace), ") may be insufficient."
|
||||||
|
echo " Archive is ", formatBytes(archiveSize), " — at least 2x is recommended."
|
||||||
|
|
||||||
|
if dirExists(dataRoot):
|
||||||
|
if verbose:
|
||||||
|
echo "Moving existing data root to: ", oldBackupPath
|
||||||
|
moveDir(dataRoot, oldBackupPath)
|
||||||
|
|
||||||
|
createDir(dataRoot)
|
||||||
|
|
||||||
|
let cmd = "tar -xzf " & quoteShell(input) & " -C " & quoteShell(dataRoot)
|
||||||
|
if verbose:
|
||||||
|
echo "Running: ", cmd
|
||||||
|
|
||||||
|
let (outputStr, exitCode) = execCmdEx(cmd)
|
||||||
|
if exitCode != 0:
|
||||||
|
echo "ERROR: tar extraction failed with exit code ", exitCode
|
||||||
|
if outputStr.len > 0:
|
||||||
|
echo outputStr
|
||||||
|
# Attempt rollback
|
||||||
|
if dirExists(oldBackupPath):
|
||||||
|
echo "Attempting rollback..."
|
||||||
|
removeDir(dataRoot)
|
||||||
|
moveDir(oldBackupPath, dataRoot)
|
||||||
|
echo "Rollback complete. Data restored to previous state."
|
||||||
|
return false
|
||||||
|
|
||||||
|
# Verify metadata
|
||||||
|
let metaPath = dataRoot / BACKUP_META_FILE
|
||||||
|
if fileExists(metaPath):
|
||||||
|
try:
|
||||||
|
let meta = parseJson(readFile(metaPath))
|
||||||
|
let dbList = meta{"databases"}
|
||||||
|
if dbList != nil and dbList.kind == JArray:
|
||||||
|
echo "Restored databases: ", dbList.len
|
||||||
|
for db in dbList:
|
||||||
|
echo " - ", db.getStr()
|
||||||
|
except CatchableError as e:
|
||||||
|
echo "WARNING: Could not parse backup metadata: ", e.msg
|
||||||
|
|
||||||
|
echo "Restored successfully from: ", input
|
||||||
|
echo " Target: ", dataRoot
|
||||||
|
if dirExists(oldBackupPath):
|
||||||
|
echo " Old data preserved at: ", oldBackupPath
|
||||||
|
return true
|
||||||
|
|
||||||
|
proc readBackupMeta*(input: string): JsonNode =
|
||||||
|
## Read backup metadata without full extraction.
|
||||||
|
## Returns nil if not a multi-db archive or no metadata.
|
||||||
|
result = nil
|
||||||
|
if not fileExists(input):
|
||||||
|
return
|
||||||
|
let cmd = "tar -xzf " & quoteShell(input) & " -O ./" & BACKUP_META_FILE & " 2>/dev/null"
|
||||||
|
let (outStr, exitCode) = execCmdEx(cmd)
|
||||||
|
if exitCode == 0 and outStr.len > 0:
|
||||||
|
try:
|
||||||
|
result = parseJson(outStr)
|
||||||
|
except CatchableError:
|
||||||
|
discard
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# CLI Entry Point
|
# CLI Entry Point
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -488,6 +665,7 @@ when isMainModule:
|
|||||||
var
|
var
|
||||||
command = ""
|
command = ""
|
||||||
dataDir = DEFAULT_DATA_DIR
|
dataDir = DEFAULT_DATA_DIR
|
||||||
|
dataRoot = DEFAULT_DATA_ROOT
|
||||||
target = ""
|
target = ""
|
||||||
keepCount = DEFAULT_KEEP_COUNT
|
keepCount = DEFAULT_KEEP_COUNT
|
||||||
excludes: seq[string] = @[]
|
excludes: seq[string] = @[]
|
||||||
@@ -496,6 +674,8 @@ when isMainModule:
|
|||||||
dryRun = false
|
dryRun = false
|
||||||
force = false
|
force = false
|
||||||
online = false
|
online = false
|
||||||
|
allDatabases = false
|
||||||
|
databaseName = ""
|
||||||
|
|
||||||
for kind, key, val in getopt():
|
for kind, key, val in getopt():
|
||||||
case kind
|
case kind
|
||||||
@@ -507,6 +687,7 @@ when isMainModule:
|
|||||||
of cmdLongOption, cmdShortOption:
|
of cmdLongOption, cmdShortOption:
|
||||||
case key
|
case key
|
||||||
of "data-dir", "d": dataDir = val
|
of "data-dir", "d": dataDir = val
|
||||||
|
of "data-root", "r": dataRoot = val
|
||||||
of "output", "o": target = val
|
of "output", "o": target = val
|
||||||
of "input", "i": target = val
|
of "input", "i": target = val
|
||||||
of "keep", "k":
|
of "keep", "k":
|
||||||
@@ -522,6 +703,8 @@ when isMainModule:
|
|||||||
of "dry-run": dryRun = true
|
of "dry-run": dryRun = true
|
||||||
of "force", "f": force = true
|
of "force", "f": force = true
|
||||||
of "online": online = true
|
of "online": online = true
|
||||||
|
of "all-databases": allDatabases = true
|
||||||
|
of "database": databaseName = val
|
||||||
of "verbose", "v": verbose = true
|
of "verbose", "v": verbose = true
|
||||||
of "help", "h":
|
of "help", "h":
|
||||||
echo HELP_TEXT
|
echo HELP_TEXT
|
||||||
@@ -537,6 +720,33 @@ when isMainModule:
|
|||||||
case command
|
case command
|
||||||
of "backup":
|
of "backup":
|
||||||
let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
|
let outputFile = if target.len > 0: target else: "backup_" & $getTime().toUnix() & ".tar.gz"
|
||||||
|
if allDatabases:
|
||||||
|
let ok = backupAllDatabases(dataRoot, outputFile, excludes, compression, verbose)
|
||||||
|
if not ok:
|
||||||
|
quit("Multi-database backup failed", 1)
|
||||||
|
elif databaseName.len > 0:
|
||||||
|
let dbDir = dataRoot / databaseName
|
||||||
|
if online:
|
||||||
|
echo "Creating online backup with checkpoint..."
|
||||||
|
echo " Database: ", databaseName
|
||||||
|
echo " Data dir: ", dbDir
|
||||||
|
echo " Output: ", outputFile
|
||||||
|
try:
|
||||||
|
var db = newLSMTree(dbDir)
|
||||||
|
db.checkpoint()
|
||||||
|
db.close()
|
||||||
|
echo "Checkpoint complete."
|
||||||
|
except CatchableError as e:
|
||||||
|
echo "ERROR: Checkpoint failed: ", e.msg
|
||||||
|
quit(1)
|
||||||
|
let ok = incrementalBackupDataDir(dbDir, outputFile, verbose)
|
||||||
|
if not ok:
|
||||||
|
quit("Online backup failed", 1)
|
||||||
|
else:
|
||||||
|
let ok = backupDataDir(dbDir, outputFile, excludes, compression, verbose)
|
||||||
|
if not ok:
|
||||||
|
quit("Backup failed", 1)
|
||||||
|
else:
|
||||||
if online:
|
if online:
|
||||||
echo "Creating online backup with checkpoint..."
|
echo "Creating online backup with checkpoint..."
|
||||||
echo " Data dir: ", dataDir
|
echo " Data dir: ", dataDir
|
||||||
@@ -559,6 +769,12 @@ when isMainModule:
|
|||||||
|
|
||||||
of "incremental":
|
of "incremental":
|
||||||
let outputFile = if target.len > 0: target else: "backup_inc_" & $getTime().toUnix() & ".tar.gz"
|
let outputFile = if target.len > 0: target else: "backup_inc_" & $getTime().toUnix() & ".tar.gz"
|
||||||
|
if databaseName.len > 0:
|
||||||
|
let dbDir = dataRoot / databaseName
|
||||||
|
let ok = incrementalBackupDataDir(dbDir, outputFile, verbose)
|
||||||
|
if not ok:
|
||||||
|
quit("Incremental backup failed", 1)
|
||||||
|
else:
|
||||||
let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
|
let ok = incrementalBackupDataDir(dataDir, outputFile, verbose)
|
||||||
if not ok:
|
if not ok:
|
||||||
quit("Incremental backup failed", 1)
|
quit("Incremental backup failed", 1)
|
||||||
@@ -567,6 +783,16 @@ when isMainModule:
|
|||||||
if target.len == 0:
|
if target.len == 0:
|
||||||
quit("ERROR: restore requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
|
quit("ERROR: restore requires --input=<file.tar.gz>\nUse 'backup help' for usage.", 1)
|
||||||
|
|
||||||
|
# Detect archive type from metadata
|
||||||
|
let meta = readBackupMeta(target)
|
||||||
|
let isMultiDb = meta != nil and meta{"databases"} != nil
|
||||||
|
|
||||||
|
if isMultiDb and not allDatabases and databaseName.len == 0:
|
||||||
|
echo "Detected multi-database archive containing:"
|
||||||
|
for db in meta{"databases"}:
|
||||||
|
echo " - ", db.getStr()
|
||||||
|
echo "Use --all-databases to restore all, or --database=<name> for a single database."
|
||||||
|
|
||||||
# Always verify first unless dry-run
|
# Always verify first unless dry-run
|
||||||
if not dryRun:
|
if not dryRun:
|
||||||
echo "Verifying archive before restore..."
|
echo "Verifying archive before restore..."
|
||||||
@@ -575,6 +801,20 @@ when isMainModule:
|
|||||||
logRestore(target, dataDir, false)
|
logRestore(target, dataDir, false)
|
||||||
quit("Restore aborted: archive verification failed.", 1)
|
quit("Restore aborted: archive verification failed.", 1)
|
||||||
|
|
||||||
|
if allDatabases or isMultiDb:
|
||||||
|
if not dryRun and not force:
|
||||||
|
echo "WARNING: This will REPLACE all databases in: ", dataRoot
|
||||||
|
echo "Continue? [y/N] "
|
||||||
|
let answer = readLine(stdin)
|
||||||
|
if answer.toLowerAscii() notin ["y", "yes"]:
|
||||||
|
echo "Restore cancelled."
|
||||||
|
logRestore(target, dataRoot, false, dryRun = false)
|
||||||
|
quit(0)
|
||||||
|
let ok = restoreAllDatabases(target, dataRoot, verbose, dryRun)
|
||||||
|
logRestore(target, dataRoot, ok, dryRun)
|
||||||
|
if not ok and not dryRun:
|
||||||
|
quit("Restore failed", 1)
|
||||||
|
else:
|
||||||
if not dryRun and not force:
|
if not dryRun and not force:
|
||||||
echo "WARNING: This will REPLACE the data in: ", dataDir
|
echo "WARNING: This will REPLACE the data in: ", dataDir
|
||||||
echo "Continue? [y/N] "
|
echo "Continue? [y/N] "
|
||||||
@@ -583,7 +823,6 @@ when isMainModule:
|
|||||||
echo "Restore cancelled."
|
echo "Restore cancelled."
|
||||||
logRestore(target, dataDir, false, dryRun = false)
|
logRestore(target, dataDir, false, dryRun = false)
|
||||||
quit(0)
|
quit(0)
|
||||||
|
|
||||||
let ok = restoreDataDir(target, dataDir, verbose, dryRun)
|
let ok = restoreDataDir(target, dataDir, verbose, dryRun)
|
||||||
logRestore(target, dataDir, ok, dryRun)
|
logRestore(target, dataDir, ok, dryRun)
|
||||||
if not ok and not dryRun:
|
if not ok and not dryRun:
|
||||||
@@ -601,6 +840,9 @@ when isMainModule:
|
|||||||
quit("Verification failed", 1)
|
quit("Verification failed", 1)
|
||||||
|
|
||||||
of "cleanup":
|
of "cleanup":
|
||||||
|
if allDatabases:
|
||||||
|
cleanupOldBackups(dataRoot, keepCount, verbose)
|
||||||
|
else:
|
||||||
cleanupOldBackups(dataDir, keepCount, verbose)
|
cleanupOldBackups(dataDir, keepCount, verbose)
|
||||||
|
|
||||||
of "history":
|
of "history":
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import json
|
|||||||
import tables
|
import tables
|
||||||
import strutils
|
import strutils
|
||||||
import times
|
import times
|
||||||
|
import os
|
||||||
import std/asyncdispatch
|
import std/asyncdispatch
|
||||||
import config
|
import config
|
||||||
import ../query/lexer
|
import ../query/lexer
|
||||||
@@ -21,6 +22,7 @@ import jwt as jwtlib
|
|||||||
import ../protocol/auth
|
import ../protocol/auth
|
||||||
import ../protocol/ratelimit
|
import ../protocol/ratelimit
|
||||||
import ../core/registry
|
import ../core/registry
|
||||||
|
import ../core/backup
|
||||||
|
|
||||||
type
|
type
|
||||||
HttpServer* = ref object
|
HttpServer* = ref object
|
||||||
@@ -121,6 +123,36 @@ proc checkAuth(server: HttpServer, request: Request, ctx: Context): bool =
|
|||||||
return false
|
return false
|
||||||
return true
|
return true
|
||||||
|
|
||||||
|
proc checkAdmin(server: HttpServer, request: Request, ctx: Context): bool =
|
||||||
|
if not server.config.authEnabled:
|
||||||
|
return true
|
||||||
|
let authHeader = request.headers["Authorization"]
|
||||||
|
if authHeader.len == 0 or not authHeader.startsWith("Bearer "):
|
||||||
|
ctx.json(%*{"error": "Unauthorized"}, 401)
|
||||||
|
return false
|
||||||
|
let tokenStr = authHeader[7..^1]
|
||||||
|
let (valid, _, role) = server.verifyToken(tokenStr)
|
||||||
|
if not valid:
|
||||||
|
ctx.json(%*{"error": "Unauthorized"}, 401)
|
||||||
|
return false
|
||||||
|
if role != "admin":
|
||||||
|
ctx.json(%*{"error": "Forbidden: admin role required"}, 403)
|
||||||
|
return false
|
||||||
|
return true
|
||||||
|
|
||||||
|
proc getRequestDatabaseContext(server: HttpServer, request: Request): ExecutionContext =
|
||||||
|
## Return execution context for the requested database (via X-Database header)
|
||||||
|
## Falls back to server.ctx (default database) if not specified.
|
||||||
|
let dbName = request.headers["X-Database"]
|
||||||
|
if dbName.len > 0 and server.registry != nil:
|
||||||
|
try:
|
||||||
|
let dbInfo = getOrCreateDatabase(server.registry, dbName)
|
||||||
|
if dbInfo != nil and dbInfo.db != nil:
|
||||||
|
return newExecutionContext(dbInfo.db, server.registry)
|
||||||
|
except CatchableError:
|
||||||
|
discard
|
||||||
|
return newExecutionContext(server.db, server.registry)
|
||||||
|
|
||||||
# ----------------------------------------------------------------------
|
# ----------------------------------------------------------------------
|
||||||
# Handlers
|
# Handlers
|
||||||
# ----------------------------------------------------------------------
|
# ----------------------------------------------------------------------
|
||||||
@@ -164,7 +196,7 @@ proc queryHandler(server: HttpServer): RequestHandler =
|
|||||||
ctx.json(%*{"error": "Empty query"}, 400)
|
ctx.json(%*{"error": "Empty query"}, 400)
|
||||||
return
|
return
|
||||||
|
|
||||||
var reqCtx = cloneForConnection(server.ctx)
|
var reqCtx = getRequestDatabaseContext(server, request)
|
||||||
reqCtx.currentUser = userId
|
reqCtx.currentUser = userId
|
||||||
reqCtx.currentRole = role
|
reqCtx.currentRole = role
|
||||||
let tokens = tokenize(queryStr)
|
let tokens = tokenize(queryStr)
|
||||||
@@ -297,11 +329,14 @@ proc openApiHandler(): RequestHandler =
|
|||||||
let ctx = newContext(request)
|
let ctx = newContext(request)
|
||||||
ctx.json(%*{
|
ctx.json(%*{
|
||||||
"openapi": "3.0.0",
|
"openapi": "3.0.0",
|
||||||
"info": {"title": "BaraDB API", "version": "0.1.0"},
|
"info": {"title": "BaraDB API", "version": "1.1.6"},
|
||||||
"paths": {
|
"paths": {
|
||||||
"/query": {
|
"/query": {
|
||||||
"post": {
|
"post": {
|
||||||
"summary": "Execute SQL query",
|
"summary": "Execute SQL query",
|
||||||
|
"parameters": [
|
||||||
|
{"name": "X-Database", "in": "header", "schema": {"type": "string"}, "description": "Target database (default: default)"}
|
||||||
|
],
|
||||||
"requestBody": {
|
"requestBody": {
|
||||||
"content": {
|
"content": {
|
||||||
"application/json": {
|
"application/json": {
|
||||||
@@ -315,6 +350,20 @@ proc openApiHandler(): RequestHandler =
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/tables": {"get": {"summary": "List tables", "parameters": [{"name": "X-Database", "in": "header", "schema": {"type": "string"}}]}},
|
||||||
|
"/databases": {
|
||||||
|
"get": {"summary": "List databases"},
|
||||||
|
"post": {"summary": "Create database (admin)"}
|
||||||
|
},
|
||||||
|
"/backup": {
|
||||||
|
"post": {"summary": "Create backup (admin)"}
|
||||||
|
},
|
||||||
|
"/backups": {
|
||||||
|
"get": {"summary": "List backups"}
|
||||||
|
},
|
||||||
|
"/restore": {
|
||||||
|
"post": {"summary": "Restore from backup (admin)"}
|
||||||
|
},
|
||||||
"/health": {"get": {"summary": "Health check"}},
|
"/health": {"get": {"summary": "Health check"}},
|
||||||
"/metrics": {"get": {"summary": "Prometheus metrics"}},
|
"/metrics": {"get": {"summary": "Prometheus metrics"}},
|
||||||
"/auth": {"post": {"summary": "Authenticate and get JWT token"}}
|
"/auth": {"post": {"summary": "Authenticate and get JWT token"}}
|
||||||
@@ -327,8 +376,9 @@ proc tablesHandler(server: HttpServer): RequestHandler =
|
|||||||
let ctx = newContext(request)
|
let ctx = newContext(request)
|
||||||
if not server.checkAuth(request, ctx):
|
if not server.checkAuth(request, ctx):
|
||||||
return
|
return
|
||||||
|
let reqCtx = getRequestDatabaseContext(server, request)
|
||||||
var tables = newJArray()
|
var tables = newJArray()
|
||||||
for name, tbl in server.ctx.tables:
|
for name, tbl in reqCtx.tables:
|
||||||
var cols = newJArray()
|
var cols = newJArray()
|
||||||
for col in tbl.columns:
|
for col in tbl.columns:
|
||||||
cols.add(%*{"name": col.name, "type": col.colType,
|
cols.add(%*{"name": col.name, "type": col.colType,
|
||||||
@@ -337,6 +387,162 @@ proc tablesHandler(server: HttpServer): RequestHandler =
|
|||||||
"pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
|
"pkColumns": tbl.pkColumns, "fkCount": tbl.foreignKeys.len})
|
||||||
ctx.json(%*{"tables": tables})
|
ctx.json(%*{"tables": tables})
|
||||||
|
|
||||||
|
proc databasesHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAuth(request, ctx):
|
||||||
|
return
|
||||||
|
let dbs = server.registry.listDatabases()
|
||||||
|
var arr = newJArray()
|
||||||
|
for dbName in dbs:
|
||||||
|
var obj = newJObject()
|
||||||
|
obj["name"] = %dbName
|
||||||
|
try:
|
||||||
|
let dbInfo = getDatabaseInfo(server.registry, dbName)
|
||||||
|
if dbInfo != nil and dbInfo.ctx != nil:
|
||||||
|
let dbCtx = cast[ExecutionContext](cast[pointer](dbInfo.ctx))
|
||||||
|
obj["tables"] = %dbCtx.tables.len
|
||||||
|
obj["connections"] = %getConnectionCount(server.registry, dbName)
|
||||||
|
else:
|
||||||
|
obj["tables"] = %0
|
||||||
|
obj["connections"] = %0
|
||||||
|
except CatchableError:
|
||||||
|
obj["tables"] = %0
|
||||||
|
obj["connections"] = %0
|
||||||
|
arr.add(obj)
|
||||||
|
ctx.json(%*{"databases": arr})
|
||||||
|
|
||||||
|
proc createDatabaseHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAdmin(request, ctx):
|
||||||
|
return
|
||||||
|
let body = parseJson(request.body)
|
||||||
|
if body == nil or "name" notin body:
|
||||||
|
ctx.json(%*{"error": "Missing 'name' in request body"}, 400)
|
||||||
|
return
|
||||||
|
let dbName = body["name"].getStr()
|
||||||
|
if dbName.len == 0:
|
||||||
|
ctx.json(%*{"error": "Empty database name"}, 400)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
discard getOrCreateDatabase(server.registry, dbName)
|
||||||
|
ctx.json(%*{"success": true, "name": dbName, "message": "Database created"})
|
||||||
|
except CatchableError as e:
|
||||||
|
ctx.json(%*{"error": e.msg}, 400)
|
||||||
|
|
||||||
|
proc dropDatabaseHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAdmin(request, ctx):
|
||||||
|
return
|
||||||
|
let dbName = request.pathParams.getOrDefault("name", "")
|
||||||
|
if dbName.len == 0:
|
||||||
|
ctx.json(%*{"error": "Missing database name"}, 400)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
let ok = dropDatabase(server.registry, dbName)
|
||||||
|
if ok:
|
||||||
|
ctx.json(%*{"success": true, "name": dbName, "message": "Database dropped"})
|
||||||
|
else:
|
||||||
|
ctx.json(%*{"error": "Database not found"}, 404)
|
||||||
|
except CatchableError as e:
|
||||||
|
ctx.json(%*{"error": e.msg}, 400)
|
||||||
|
|
||||||
|
proc backupHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAdmin(request, ctx):
|
||||||
|
return
|
||||||
|
let body = parseJson(request.body)
|
||||||
|
let dataRoot = server.registry.dataRoot
|
||||||
|
let allDatabases = if body != nil and "all" in body: body["all"].getBool() else: false
|
||||||
|
let dbName = if body != nil and "database" in body: body["database"].getStr() else: ""
|
||||||
|
let outputFile = if body != nil and "output" in body: body["output"].getStr() else: "backup_" & $getTime().toUnix() & ".tar.gz"
|
||||||
|
let compression = if body != nil and "level" in body: body["level"].getInt() else: 6
|
||||||
|
try:
|
||||||
|
var ok = false
|
||||||
|
if allDatabases:
|
||||||
|
ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
|
||||||
|
elif dbName.len > 0:
|
||||||
|
let dbDir = dataRoot / dbName
|
||||||
|
ok = backupDataDir(dbDir, outputFile, @[], compression, false)
|
||||||
|
else:
|
||||||
|
ok = backupAllDatabases(dataRoot, outputFile, @[], compression, false)
|
||||||
|
if ok:
|
||||||
|
ctx.json(%*{"success": true, "output": outputFile, "message": "Backup created"})
|
||||||
|
else:
|
||||||
|
ctx.json(%*{"error": "Backup failed"}, 500)
|
||||||
|
except CatchableError as e:
|
||||||
|
ctx.json(%*{"error": e.msg}, 500)
|
||||||
|
|
||||||
|
proc listBackupsHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAuth(request, ctx):
|
||||||
|
return
|
||||||
|
let dataRoot = server.registry.dataRoot
|
||||||
|
let backups = listBackups(dataRoot)
|
||||||
|
var arr = newJArray()
|
||||||
|
for b in backups:
|
||||||
|
var meta: JsonNode = nil
|
||||||
|
try:
|
||||||
|
meta = readBackupMeta(b.path)
|
||||||
|
except CatchableError:
|
||||||
|
discard
|
||||||
|
var obj = newJObject()
|
||||||
|
obj["path"] = %b.path
|
||||||
|
obj["size"] = %b.size
|
||||||
|
obj["sizeHuman"] = %formatBytes(b.size)
|
||||||
|
obj["timestamp"] = %b.timestamp
|
||||||
|
obj["compressed"] = %b.compressed
|
||||||
|
if meta != nil and meta{"databases"} != nil:
|
||||||
|
obj["databases"] = meta{"databases"}
|
||||||
|
obj["type"] = %"multi"
|
||||||
|
else:
|
||||||
|
obj["type"] = %"single"
|
||||||
|
arr.add(obj)
|
||||||
|
ctx.json(%*{"backups": arr})
|
||||||
|
|
||||||
|
proc restoreHandler(server: HttpServer): RequestHandler =
|
||||||
|
return proc(request: Request) {.gcsafe.} =
|
||||||
|
{.cast(gcsafe).}:
|
||||||
|
let ctx = newContext(request)
|
||||||
|
if not server.checkAdmin(request, ctx):
|
||||||
|
return
|
||||||
|
let body = parseJson(request.body)
|
||||||
|
if body == nil or "input" notin body:
|
||||||
|
ctx.json(%*{"error": "Missing 'input' in request body"}, 400)
|
||||||
|
return
|
||||||
|
let inputFile = body["input"].getStr()
|
||||||
|
let allDatabases = if body != nil and "all" in body: body["all"].getBool() else: false
|
||||||
|
let dbName = if body != nil and "database" in body: body["database"].getStr() else: ""
|
||||||
|
let dataRoot = server.registry.dataRoot
|
||||||
|
try:
|
||||||
|
let meta = readBackupMeta(inputFile)
|
||||||
|
let isMultiDb = meta != nil and meta{"databases"} != nil
|
||||||
|
var ok = false
|
||||||
|
if isMultiDb or allDatabases:
|
||||||
|
ok = restoreAllDatabases(inputFile, dataRoot, false, false)
|
||||||
|
elif dbName.len > 0:
|
||||||
|
let dbDir = dataRoot / dbName
|
||||||
|
ok = restoreDataDir(inputFile, dbDir, false, false)
|
||||||
|
else:
|
||||||
|
ok = restoreAllDatabases(inputFile, dataRoot, false, false)
|
||||||
|
if ok:
|
||||||
|
# Reload databases after restore
|
||||||
|
server.registry.loadExistingDatabases()
|
||||||
|
ctx.json(%*{"success": true, "message": "Restore completed"})
|
||||||
|
else:
|
||||||
|
ctx.json(%*{"error": "Restore failed"}, 500)
|
||||||
|
except CatchableError as e:
|
||||||
|
ctx.json(%*{"error": e.msg}, 500)
|
||||||
|
|
||||||
proc adminHandler(server: HttpServer): RequestHandler =
|
proc adminHandler(server: HttpServer): RequestHandler =
|
||||||
return proc(request: Request) {.gcsafe.} =
|
return proc(request: Request) {.gcsafe.} =
|
||||||
let html = """
|
let html = """
|
||||||
@@ -389,7 +595,7 @@ tr:hover td{background:#1a2744}
|
|||||||
<button onclick='doLogin()' style='width:100%'>Login</button>
|
<button onclick='doLogin()' style='width:100%'>Login</button>
|
||||||
<p class='status' id='login-status'></p>
|
<p class='status' id='login-status'></p>
|
||||||
</div></div>
|
</div></div>
|
||||||
<header><h1>BaraDB Admin</h1><span id='user-info'></span></header>
|
<header><h1>BaraDB Admin</h1><div style='display:flex;align-items:center;gap:12px'><select id='db-select' onchange='switchDatabase()' style='width:auto;padding:4px 8px;font-size:13px'><option value=''>Loading...</option></select><span id='user-info'></span></div></header>
|
||||||
<div id='tabs'>
|
<div id='tabs'>
|
||||||
<span class='tab active' onclick='showTab(0)'>SQL Playground</span>
|
<span class='tab active' onclick='showTab(0)'>SQL Playground</span>
|
||||||
<span class='tab' onclick='showTab(1)'>Tables</span>
|
<span class='tab' onclick='showTab(1)'>Tables</span>
|
||||||
@@ -397,6 +603,8 @@ tr:hover td{background:#1a2744}
|
|||||||
<span class='tab' onclick='showTab(3)'>Live</span>
|
<span class='tab' onclick='showTab(3)'>Live</span>
|
||||||
<span class='tab' onclick='showTab(4)'>Metrics</span>
|
<span class='tab' onclick='showTab(4)'>Metrics</span>
|
||||||
<span class='tab' onclick='showTab(5)'>Cluster</span>
|
<span class='tab' onclick='showTab(5)'>Cluster</span>
|
||||||
|
<span class='tab' onclick='showTab(6)'>Databases</span>
|
||||||
|
<span class='tab' onclick='showTab(7)'>Backups</span>
|
||||||
</div>
|
</div>
|
||||||
<div class='panel active'>
|
<div class='panel active'>
|
||||||
<div class='card'><textarea id='sql' placeholder='SELECT * FROM users'></textarea>
|
<div class='card'><textarea id='sql' placeholder='SELECT * FROM users'></textarea>
|
||||||
@@ -434,10 +642,33 @@ tr:hover td{background:#1a2744}
|
|||||||
<div class='panel'>
|
<div class='panel'>
|
||||||
<div class='card'><h3>Cluster Status</h3><div id='cluster-data'>Cluster status not available</div></div>
|
<div class='card'><h3>Cluster Status</h3><div id='cluster-data'>Cluster status not available</div></div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class='panel'>
|
||||||
|
<div class='card'><h3>Databases</h3>
|
||||||
|
<div style='display:flex;gap:8px;margin-bottom:12px'>
|
||||||
|
<input id='new-db-name' placeholder='Database name' style='flex:1'>
|
||||||
|
<button onclick='createDatabase()'>Create</button>
|
||||||
|
</div>
|
||||||
|
<div id='db-list'>Loading...</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class='panel'>
|
||||||
|
<div class='card'><h3>Backups</h3>
|
||||||
|
<div style='display:flex;gap:8px;margin-bottom:12px'>
|
||||||
|
<button onclick='createBackup()'>Backup All</button>
|
||||||
|
<button class='secondary' onclick='createBackupSingle()'>Backup Current DB</button>
|
||||||
|
</div>
|
||||||
|
<div id='backup-list'>Loading...</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<script>
|
<script>
|
||||||
let token = localStorage.getItem('baradb_token') || ''
|
let token = localStorage.getItem('baradb_token') || ''
|
||||||
|
let currentDatabase = localStorage.getItem('baradb_db') || 'default'
|
||||||
let ws = null
|
let ws = null
|
||||||
function authHeader(){ return token ? {'Authorization':'Bearer '+token,'Content-Type':'application/json'} : {'Content-Type':'application/json'} }
|
function authHeader(){
|
||||||
|
let h = token ? {'Authorization':'Bearer '+token,'Content-Type':'application/json'} : {'Content-Type':'application/json'}
|
||||||
|
if(currentDatabase) h['X-Database'] = currentDatabase
|
||||||
|
return h
|
||||||
|
}
|
||||||
async function api(path, body, method='POST') {
|
async function api(path, body, method='POST') {
|
||||||
const r = await fetch(path, {method, headers:authHeader(), body: body?JSON.stringify(body):undefined})
|
const r = await fetch(path, {method, headers:authHeader(), body: body?JSON.stringify(body):undefined})
|
||||||
return r.json().catch(() => r.text())
|
return r.json().catch(() => r.text())
|
||||||
@@ -446,13 +677,67 @@ async function doLogin(){
|
|||||||
const u = document.getElementById('username').value
|
const u = document.getElementById('username').value
|
||||||
const p = document.getElementById('password').value
|
const p = document.getElementById('password').value
|
||||||
const r = await api('/auth', {username:u, password:p})
|
const r = await api('/auth', {username:u, password:p})
|
||||||
if(r.token){ token = r.token; localStorage.setItem('baradb_token', token); hideLogin(); showUser(r.user); connectWS() }
|
if(r.token){ token = r.token; localStorage.setItem('baradb_token', token); hideLogin(); showUser(r.user); connectWS(); loadDatabases() }
|
||||||
else { document.getElementById('login-status').textContent = r.error || 'Login failed' }
|
else { document.getElementById('login-status').textContent = r.error || 'Login failed' }
|
||||||
}
|
}
|
||||||
function hideLogin(){ document.getElementById('login-overlay').style.display='none' }
|
function hideLogin(){ document.getElementById('login-overlay').style.display='none' }
|
||||||
function showUser(u){ document.getElementById('user-info').innerHTML = 'User: <b>'+u+'</b> <a href="#" onclick="logout()">logout</a>' }
|
function showUser(u){ document.getElementById('user-info').innerHTML = 'User: <b>'+u+'</b> <a href="#" onclick="logout()">logout</a>' }
|
||||||
function logout(){ token=''; localStorage.removeItem('baradb_token'); location.reload() }
|
function logout(){ token=''; localStorage.removeItem('baradb_token'); localStorage.removeItem('baradb_db'); location.reload() }
|
||||||
if(token){ hideLogin(); showUser('...'); api('/health',null,'GET').then(()=>showUser('authed')).catch(()=>logout()); connectWS() }
|
if(token){ hideLogin(); showUser('...'); api('/health',null,'GET').then(()=>{showUser('authed'); loadDatabases()}).catch(()=>logout()); connectWS() }
|
||||||
|
function switchDatabase(){
|
||||||
|
currentDatabase = document.getElementById('db-select').value
|
||||||
|
localStorage.setItem('baradb_db', currentDatabase)
|
||||||
|
loadTables()
|
||||||
|
loadSchema()
|
||||||
|
}
|
||||||
|
async function loadDatabases(){
|
||||||
|
try{
|
||||||
|
const r = await api('/databases', null, 'GET')
|
||||||
|
const sel = document.getElementById('db-select')
|
||||||
|
if(!r.databases){ sel.innerHTML = '<option value="default">default</option>'; return }
|
||||||
|
sel.innerHTML = r.databases.map(d => '<option value="'+d.name+'"'+(d.name===currentDatabase?' selected':'')+'>'+d.name+' ('+d.tables+' tables)</option>').join('')
|
||||||
|
// Databases tab
|
||||||
|
const el = document.getElementById('db-list')
|
||||||
|
el.innerHTML = '<table><tr><th>Name</th><th>Tables</th><th>Connections</th><th>Action</th></tr>'+r.databases.map(d => '<tr><td>'+d.name+'</td><td>'+d.tables+'</td><td>'+d.connections+'</td><td>'+(d.name!=="default"?'<button class="secondary" onclick="dropDatabase(\''+d.name+'\')">Drop</button>':'')+'</td></tr>').join('')+'</table>'
|
||||||
|
}catch(e){}
|
||||||
|
}
|
||||||
|
async function createDatabase(){
|
||||||
|
const name = document.getElementById('new-db-name').value.trim()
|
||||||
|
if(!name) return alert('Enter database name')
|
||||||
|
const r = await api('/databases', {name:name})
|
||||||
|
if(r.success){ document.getElementById('new-db-name').value=''; loadDatabases() }
|
||||||
|
else alert(r.error || 'Failed')
|
||||||
|
}
|
||||||
|
async function dropDatabase(name){
|
||||||
|
if(!confirm('Drop database '+name+'?')) return
|
||||||
|
const r = await api('/databases/'+name, null, 'DELETE')
|
||||||
|
if(r.success){ loadDatabases() }
|
||||||
|
else alert(r.error || 'Failed')
|
||||||
|
}
|
||||||
|
async function createBackup(){
|
||||||
|
const r = await api('/backup', {all:true})
|
||||||
|
if(r.success){ alert('Backup created: '+r.output); loadBackups() }
|
||||||
|
else alert(r.error || 'Failed')
|
||||||
|
}
|
||||||
|
async function createBackupSingle(){
|
||||||
|
const r = await api('/backup', {database:currentDatabase})
|
||||||
|
if(r.success){ alert('Backup created: '+r.output); loadBackups() }
|
||||||
|
else alert(r.error || 'Failed')
|
||||||
|
}
|
||||||
|
async function loadBackups(){
|
||||||
|
try{
|
||||||
|
const r = await api('/backups', null, 'GET')
|
||||||
|
const el = document.getElementById('backup-list')
|
||||||
|
if(!r.backups || !r.backups.length){ el.innerHTML = 'No backups'; return }
|
||||||
|
el.innerHTML = '<table><tr><th>File</th><th>Type</th><th>Size</th><th>Date</th><th>Databases</th><th>Action</th></tr>'+r.backups.map(b => '<tr><td>'+b.path.split('/').pop()+'</td><td>'+b.type+'</td><td>'+b.sizeHuman+'</td><td>'+new Date(b.timestamp*1000).toLocaleString()+'</td><td>'+(b.databases?b.databases.join(', '):'-')+'</td><td><button class="secondary" onclick="restoreBackup(\''+b.path+'\')">Restore</button></td></tr>').join('')+'</table>'
|
||||||
|
}catch(e){}
|
||||||
|
}
|
||||||
|
async function restoreBackup(path){
|
||||||
|
if(!confirm('Restore from '+path+'? This replaces current data.')) return
|
||||||
|
const r = await api('/restore', {input:path, all:true})
|
||||||
|
if(r.success){ alert('Restore completed'); loadDatabases(); loadBackups() }
|
||||||
|
else alert(r.error || 'Failed')
|
||||||
|
}
|
||||||
async function runQuery(){
|
async function runQuery(){
|
||||||
const sql = document.getElementById('sql').value
|
const sql = document.getElementById('sql').value
|
||||||
const res = await api('/query', {query: sql})
|
const res = await api('/query', {query: sql})
|
||||||
@@ -548,6 +833,8 @@ function showTab(idx){
|
|||||||
if(idx===1) loadTables()
|
if(idx===1) loadTables()
|
||||||
if(idx===2) loadSchema()
|
if(idx===2) loadSchema()
|
||||||
if(idx===4) loadMetrics()
|
if(idx===4) loadMetrics()
|
||||||
|
if(idx===6) loadDatabases()
|
||||||
|
if(idx===7) loadBackups()
|
||||||
}
|
}
|
||||||
setInterval(() => { if(document.querySelectorAll('.panel')[4].classList.contains('active')) loadMetrics() }, 5000)
|
setInterval(() => { if(document.querySelectorAll('.panel')[4].classList.contains('active')) loadMetrics() }, 5000)
|
||||||
</script>
|
</script>
|
||||||
@@ -567,6 +854,12 @@ proc run*(server: HttpServer, port: int = 9470) =
|
|||||||
router.post("/auth/scram/finish", server.scramFinishHandler())
|
router.post("/auth/scram/finish", server.scramFinishHandler())
|
||||||
router.get("/tables", server.tablesHandler())
|
router.get("/tables", server.tablesHandler())
|
||||||
router.get("/api", openApiHandler())
|
router.get("/api", openApiHandler())
|
||||||
|
router.get("/databases", server.databasesHandler())
|
||||||
|
router.post("/databases", server.createDatabaseHandler())
|
||||||
|
# router.delete("/databases/@name", server.dropDatabaseHandler()) -- disabled due to ORC memory issue with LSMTree close
|
||||||
|
router.post("/backup", server.backupHandler())
|
||||||
|
router.get("/backups", server.listBackupsHandler())
|
||||||
|
router.post("/restore", server.restoreHandler())
|
||||||
|
|
||||||
var stack = newMiddlewareStack(router)
|
var stack = newMiddlewareStack(router)
|
||||||
stack.use(corsMiddleware())
|
stack.use(corsMiddleware())
|
||||||
|
|||||||
@@ -158,13 +158,16 @@ proc dropDatabase*(reg: DatabaseRegistry, name: string): bool =
|
|||||||
"Cannot drop database '" & name & "': " &
|
"Cannot drop database '" & name & "': " &
|
||||||
$info.activeConnections & " active connections")
|
$info.activeConnections & " active connections")
|
||||||
|
|
||||||
|
# Copy needed references before deletion
|
||||||
|
let db = info.db
|
||||||
|
let dbDir = reg.dataRoot / name
|
||||||
|
|
||||||
# Remove from registry first so no new references can be obtained
|
# Remove from registry first so no new references can be obtained
|
||||||
reg.databases.del(name)
|
reg.databases.del(name)
|
||||||
release(reg.lock)
|
release(reg.lock)
|
||||||
|
|
||||||
# Close LSMTree and remove directory outside the lock
|
# Close LSMTree and remove directory outside the lock
|
||||||
info.db.close()
|
db.close()
|
||||||
let dbDir = reg.dataRoot / name
|
|
||||||
if dirExists(dbDir):
|
if dirExists(dbDir):
|
||||||
removeDir(dbDir)
|
removeDir(dbDir)
|
||||||
true
|
true
|
||||||
|
|||||||
Reference in New Issue
Block a user