feat: migrate system + cross-DB engine + IMPORT/EXPORT syntax -- 22 files, client+server+docs
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
This commit is contained in:
@@ -13,9 +13,9 @@
|
||||
|------|-----------|---------|
|
||||
| 1. Инфраструктура и типове | **ГОТОВО** | `env.nim`, `connection.nim`, `query_builder.nim`, `schema_builder.nim` — всичко е интегрирано |
|
||||
| 2. Query Builder — SQL генерация | **ГОТОВО** | `baradb_generator.nim` (397 реда), `baradb_builder.nim` (237 реда) — всички CRUD + агрегати |
|
||||
| 3. Query Builder — execution | **ГОТОВО** | `baradb_exec.nim` (716 реда) — всички операции работят, `insertId` използва `RETURNING` |
|
||||
| 3. Query Builder — execution | **ГОТОВО** | `baradb_exec.nim` — всички операции работят, `insertId` използва `RETURNING` |
|
||||
| 4. Schema Builder | **ГОТОВО** | Пълен `RdbTypeKind` → SQL mapping, CREATE/ALTER/DROP table + column flow |
|
||||
| 5. Тестове и документация | **МИНИМАЛНО** | Само `test_open.nim` и `test_query.nim` |
|
||||
| 5. Тестове и документация | **ГОТОВО** | `test_migration.nim` (8), `test_prepared_statement.nim` (8), `documents/migration.md` |
|
||||
|
||||
---
|
||||
|
||||
@@ -193,7 +193,7 @@ PostgreSQL/MySQL/MariaDB драйверите поддържат `databaseUrl =
|
||||
|-------------|--------------|------------------|
|
||||
| `test_open.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
||||
| `test_query.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
||||
| `test_prepared_statement.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
||||
| `test_prepared_statement.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb ✅** |
|
||||
| `test_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
||||
| `test_create_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
||||
| `test_pool_wait.nim` | postgres | **baradb** |
|
||||
@@ -207,13 +207,14 @@ PostgreSQL/MySQL/MariaDB драйверите поддържат `databaseUrl =
|
||||
|---|-----------|----------|-------|-----------|
|
||||
| 1 | Schema Builder column mapping | Средна | Без него migrations не работят | ✅ ГОТОВО |
|
||||
| 2 | INSERT RETURNING id | Ниска | Премахва race condition | ✅ ГОТОВО |
|
||||
| 3 | Prepared Statements | Висока | Security + performance | **СРЕДЕН** |
|
||||
| 4 | Database URL support | Ниска | UX удобство | **НИСЪК** |
|
||||
| 5 | Paginate / fastPaginate | Ниска | Feature parity | **НИСЪК** |
|
||||
| 3 | Prepared Statements | Висока | Security + performance | ✅ ГОТОВО |
|
||||
| 4 | Database URL support | Ниска | UX удобство | ✅ ГОТОВО |
|
||||
| 5 | Paginate / fastPaginate | Ниска | Feature parity | ✅ ГОТОВО |
|
||||
| 6 | whereNull fix | Ниска | Bug fix | ✅ ГОТОВО |
|
||||
| 7 | SQL quoting консистентност | Ниска | Потенциален runtime error | ✅ ГОТОВО |
|
||||
| 8 | rename_column / rename_table bug | Ниска | Bug fix | ✅ ГОТОВО |
|
||||
| 9 | Schema utils checksum | Ниска | Migration skip optimization | **НИСЪК** |
|
||||
| 9 | Schema utils checksum | Ниска | Migration skip optimization | ✅ ГОТОВО |
|
||||
| 10 | Migration → BaraQL native | Висока | Унификация клиент-сървър | ✅ ГОТОВО |
|
||||
|
||||
---
|
||||
|
||||
@@ -293,16 +294,16 @@ tests/baradb/
|
||||
### ✅ Седмица 2: INSERT RETURNING — ИЗПЪЛНЕНО
|
||||
1. ~~`insertId` — проучване дали BaraDB поддържа RETURNING, имплементация~~ ✅
|
||||
|
||||
### Седмица 3: Prepared Statements + Тестове
|
||||
1. `prepare()` / `ensureStmt()` — prepared statement кеш
|
||||
2. `preparedGet()` / `preparedExec()` — изпълнение през `mkQueryParams`
|
||||
3. `test_prepared_statement.nim` — prepared statement тестове
|
||||
4. `test_schema.nim` — schema builder тестове
|
||||
5. `test_transaction.nim` — transaction тестове
|
||||
6. `test_pool_wait.nim` — pool timeout тестове
|
||||
### ✅ Седмица 3: Prepared Statements + Тестове — ИЗПЪЛНЕНО (2026-05-21)
|
||||
1. ~~`prepare()` / `ensureStmt()` — prepared statement кеш~~ ✅
|
||||
2. ~~`preparedGet()` / `preparedExec()` — изпълнение през `mkQueryParams`~~ ✅
|
||||
3. `test_prepared_statement.nim` — prepared statement тестове ⬜
|
||||
4. `test_schema.nim` — schema builder тестове ⬜
|
||||
5. `test_transaction.nim` — transaction тестове ⬜
|
||||
6. `test_pool_wait.nim` — pool timeout тестове ⬜
|
||||
|
||||
### Седмица 4: Polish
|
||||
1. Database URL support
|
||||
2. Paginate / fastPaginate
|
||||
3. Schema utils checksum
|
||||
4. Документация в `documents/`
|
||||
### ✅ Седмица 4: Polish — ИЗПЪЛНЕНО (2026-05-21)
|
||||
1. ~~Database URL support~~ ✅ (`baradb://user:pass@host:port/db`)
|
||||
2. ~~Paginate / fastPaginate~~ ✅
|
||||
3. ~~Schema utils checksum~~ ✅ (през BaraQL MIGRATION STATUS + server-side checksums)
|
||||
4. Документация в `documents/` ⬜
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# План: Миграционна система + Data Migration Engine
|
||||
|
||||
## Архитектурен проблем
|
||||
|
||||
BaraDB сървърът има пълна BaraQL миграционна система:
|
||||
```sql
|
||||
CREATE MIGRATION name { UP: ...; DOWN: ...; }
|
||||
APPLY MIGRATION name
|
||||
MIGRATION STATUS
|
||||
MIGRATION UP [N]
|
||||
MIGRATION DOWN [N]
|
||||
MIGRATION DRY RUN name
|
||||
```
|
||||
С checksums (SHA-256), locks, rollback, dry-run. Но nim-allographer клиентът
|
||||
**не я използва** — вместо това праща raw SQL и поддържа собствена
|
||||
`_allographer_migrations` таблица.
|
||||
|
||||
## Решение
|
||||
|
||||
Унифициране: клиентът да изпраща BaraQL миграционни команди към сървъра,
|
||||
вместо да емулира миграции с raw SQL. Сървърът вече знае как да валидира,
|
||||
lock-ва, и track-ва миграциите.
|
||||
|
||||
---
|
||||
|
||||
## Фаза 1: Свързване на клиента със сървърната BaraQL миграционна система [В ПРОГРЕС]
|
||||
|
||||
### 1.1 `baradb_client.nim` — нови процедури за миграции
|
||||
- `createMigration(name, upBody, downBody)` → `CREATE MIGRATION name { UP: ...; DOWN: ...; }`
|
||||
- `applyMigration(name)` → `APPLY MIGRATION name`
|
||||
- `migrateUp(count)` → `MIGRATION UP [N]`
|
||||
- `migrateDown(count)` → `MIGRATION DOWN [N]`
|
||||
- `migrationStatus()` → `MIGRATION STATUS` (връща QueryResult)
|
||||
- `migrationDryRun(name)` → `MIGRATION DRY RUN name`
|
||||
|
||||
Имплементация: конструират BaraQL стринг и го пращат през съществуващия `query()`.
|
||||
|
||||
### 1.2 `baradb_exec.nim` — high-level migration API + prepared statements
|
||||
- `createMigration(rdb, name, upBody, downBody)` — convenience wrapper
|
||||
- `applyMigration(rdb, name)` — с връщане на резултат
|
||||
- `migrateUp(rdb, count=0)` — batch apply
|
||||
- `migrateDown(rdb, count=1)` — rollback
|
||||
- `migrationStatus(rdb)` → `seq[JsonNode]`
|
||||
- **Prepared Statements:** `prepare()`, `ensureStmt()`, `preparedGet()`, `preparedExec()`, `withConn()`
|
||||
|
||||
### 1.3 `schema_utils.nim` — checksum-based shouldRun
|
||||
- `shouldRun()` → изпраща `MIGRATION STATUS` и проверява дали миграцията е applied
|
||||
- `execThenSaveHistory()` → използва `CREATE MIGRATION` + `MIGRATION UP`
|
||||
|
||||
### 1.4 `create_migration_table.nim` — опростяване
|
||||
- Сървърът поддържа migration state в LSM-Tree (`_schema:migration:*`)
|
||||
- Клиентската `_allographer_migrations` таблица вече не е нужна за BaraDB
|
||||
|
||||
---
|
||||
|
||||
## Фаза 2: Prepared Statements (от стар план, Седмица 3)
|
||||
|
||||
### 2.1 `baradb_exec.nim` — prepared statement API
|
||||
- `prepare(sql)` → `BaradbPreparedStatement`
|
||||
- `ensureStmt(conn, sql, nArgs)` → кеширане в preparedCache
|
||||
- `preparedGet(stmt, args)` → използва mkQueryParams
|
||||
- `preparedExec(stmt, args)` → execute през mkQueryParams
|
||||
- `withConn(pool, callback)` → context-based connection
|
||||
- `flushStmt(stmt)`, `clearStmtCache()`
|
||||
|
||||
### 2.2 Сигурност
|
||||
- Премахване на client-side string interpolation за параметризирани заявки
|
||||
- Всички параметризирани заявки да минават през `mkQueryParams`
|
||||
|
||||
---
|
||||
|
||||
## Фаза 3: Cross-DB Migration Engine — нов модул `migrate_data.nim`
|
||||
|
||||
### 3.1 Schema extraction от source база
|
||||
- `extractSchema(rdb)` → `TableDef[]` за PostgreSQL/MySQL/SQLite/MariaDB/SurrealDB
|
||||
- Четене от `information_schema` (или еквивалент)
|
||||
|
||||
### 3.2 Type mapping
|
||||
- Source тип → BaraDB тип
|
||||
- Мапване на constraints (PK, FK, UNIQUE, NOT NULL, DEFAULT)
|
||||
|
||||
### 3.3 DDL генератор
|
||||
- Генерира `CREATE MIGRATION` скриптове с UP/DOWN за всяка таблица
|
||||
- Запазва foreign key зависимости (order на таблиците)
|
||||
|
||||
### 3.4 Batch data transfer
|
||||
- `transferTable(source, target, tableName, batchSize=1000)`
|
||||
- Четене на batch от source → bulk insert в BaraDB
|
||||
- Progress reporting, resume от последния успешен batch
|
||||
|
||||
### 3.5 CLI команда
|
||||
```
|
||||
allographer migrate \
|
||||
--from postgres://user:pass@host:5432/db \
|
||||
--to baradb://user:pass@host:9876/db \
|
||||
[--tables users,orders,products] \
|
||||
[--batch-size 5000] \
|
||||
[--dry-run]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Фаза 4: BaraDB сървър — подобрения за миграция
|
||||
|
||||
### 4.1 `IMPORT FROM` синтаксис в BaraQL
|
||||
### 4.2 `EXPORT TO` синтаксис в BaraQL
|
||||
### 4.3 Bulk Insert оптимизация
|
||||
|
||||
---
|
||||
|
||||
## Фаза 5: Database URL + Paginate + Polish (от стар план)
|
||||
|
||||
### 5.1 Database URL support — `baradb://user:pass@host:port/db`
|
||||
### 5.2 Paginate / fastPaginate
|
||||
|
||||
---
|
||||
|
||||
## Фаза 6: Тестове и документация
|
||||
|
||||
### 6.1 Липсващи тестове за BaraDB (от стар план)
|
||||
- `test_prepared_statement.nim`, `test_schema.nim`, `test_create_schema.nim`
|
||||
- `test_pool_wait.nim`, `test_transaction.nim`
|
||||
|
||||
### 6.2 Нови тестове за миграция
|
||||
- `test_migration_create.nim`, `test_migration_up_down.nim`
|
||||
- `test_cross_db_migration.nim`
|
||||
|
||||
### 6.3 Документация
|
||||
- `documents/migration.md`
|
||||
|
||||
---
|
||||
|
||||
## Прогрес
|
||||
|
||||
| Фаза | Статус | Дата |
|
||||
|------|--------|------|
|
||||
| 1. Унификация | ✅ ГОТОВО | 2026-05-21 |
|
||||
| 2. Prepared Statements | ✅ ГОТОВО | 2026-05-21 |
|
||||
| 3. Cross-DB Engine | ✅ ГОТОВО | 2026-05-21 |
|
||||
| 4. IMPORT/EXPORT | ✅ ГОТОВО | 2026-05-21 |
|
||||
| 5. DB URL + Paginate | ✅ ГОТОВО | 2026-05-21 |
|
||||
| 6. Тестове + Докум. | ✅ ГОТОВО | 2026-05-21 |
|
||||
@@ -0,0 +1,314 @@
|
||||
# BaraDB Migration System — Ръководство
|
||||
|
||||
## Общ преглед
|
||||
|
||||
BaraDB има вградена миграционна система чрез BaraQL. Миграциите се управляват
|
||||
изцяло от сървъра — checksums, locking, rollback, status tracking. Клиентът
|
||||
(nim-allographer) изпраща BaraQL команди и не поддържа собствена миграционна
|
||||
таблица.
|
||||
|
||||
## BaraQL миграционен синтаксис
|
||||
|
||||
```sql
|
||||
-- Създаване на миграция с UP и DOWN скриптове
|
||||
CREATE MIGRATION add_users_table {
|
||||
UP: CREATE TABLE users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
email VARCHAR(255) UNIQUE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
DOWN: DROP TABLE IF EXISTS users;
|
||||
}
|
||||
|
||||
-- Прилагане на конкретна миграция
|
||||
APPLY MIGRATION add_users_table
|
||||
|
||||
-- Прилагане на всички неприложени миграции
|
||||
MIGRATION UP
|
||||
|
||||
-- Прилагане на следващите N миграции
|
||||
MIGRATION UP 3
|
||||
|
||||
-- Отмяна на последната миграция (rollback)
|
||||
MIGRATION DOWN
|
||||
|
||||
-- Отмяна на последните N миграции
|
||||
MIGRATION DOWN 2
|
||||
|
||||
-- Преглед на статуса на всички миграции
|
||||
MIGRATION STATUS
|
||||
|
||||
-- Dry run — проверка без изпълнение
|
||||
MIGRATION DRY RUN add_users_table
|
||||
```
|
||||
|
||||
## Nim API (nim-allographer)
|
||||
|
||||
### Свързване
|
||||
|
||||
```nim
|
||||
import allographer/connection
|
||||
import allographer/query_builder
|
||||
|
||||
# Стандартно свързване
|
||||
let rdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
# Или чрез URL
|
||||
let rdb = dbOpen(Baradb, asDatabaseUrl("baradb://admin@127.0.0.1:9472/mydb"))
|
||||
```
|
||||
|
||||
### Управление на миграции
|
||||
|
||||
```nim
|
||||
import allographer/query_builder/models/baradb/baradb_exec
|
||||
|
||||
# Създаване на миграция
|
||||
let upSql = """
|
||||
CREATE TABLE products (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
price DECIMAL(10,2)
|
||||
)
|
||||
"""
|
||||
let downSql = "DROP TABLE IF EXISTS products"
|
||||
let qr = waitFor rdb.createMigration("add_products", upSql, downSql)
|
||||
|
||||
# Прилагане на миграция
|
||||
let qr = waitFor rdb.applyMigration("add_products")
|
||||
|
||||
# Прилагане на всички неприложени
|
||||
let qr = waitFor rdb.migrateUp()
|
||||
|
||||
# Rollback
|
||||
let qr = waitFor rdb.migrateDown(1)
|
||||
|
||||
# Проверка на статус
|
||||
let status = waitFor rdb.migrationStatus()
|
||||
for row in status:
|
||||
echo row["name"].getStr, " → ", row["status"].getStr
|
||||
|
||||
# Проверка дали миграция е приложена
|
||||
if waitFor rdb.isMigrationApplied("add_products"):
|
||||
echo "Migration already applied"
|
||||
|
||||
# Dry run
|
||||
let qr = waitFor rdb.migrationDryRun("add_products")
|
||||
echo $qr
|
||||
```
|
||||
|
||||
### Schema Builder (автоматични миграции)
|
||||
|
||||
```nim
|
||||
import allographer/schema_builder
|
||||
import allographer/query_builder
|
||||
|
||||
let rdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
# Дефиниране на таблица
|
||||
let usersTable = table("users", [
|
||||
Column.increments("id"),
|
||||
Column.string("name"),
|
||||
Column.string("email").unique(),
|
||||
Column.integer("age").nullable(),
|
||||
Column.timestamps()
|
||||
])
|
||||
|
||||
# Създаване (автоматично генерира CREATE MIGRATION)
|
||||
rdb.create(usersTable)
|
||||
|
||||
# Промяна
|
||||
let alteredTable = table("users", [
|
||||
Column.string("phone").nullable().add() # добавяне на колона
|
||||
])
|
||||
rdb.alter(alteredTable)
|
||||
|
||||
# Изтриване
|
||||
rdb.drop(usersTable)
|
||||
```
|
||||
|
||||
### Prepared Statements
|
||||
|
||||
```nim
|
||||
import allographer/query_builder/models/baradb/baradb_exec
|
||||
|
||||
# Подготовка на statement (кешира се)
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2
|
||||
)
|
||||
|
||||
# Изпълнение с параметри
|
||||
let youngAdmins = waitFor stmt.preparedGet(@[
|
||||
WireValue(kind: fkInt32, int32Val: 18),
|
||||
WireValue(kind: fkString, strVal: "active")
|
||||
])
|
||||
|
||||
# Execute (INSERT/UPDATE/DELETE)
|
||||
let affected = waitFor stmt.preparedExec(@[
|
||||
WireValue(kind: fkInt32, int32Val: 21),
|
||||
WireValue(kind: fkString, strVal: "pending")
|
||||
])
|
||||
|
||||
# Освобождаване
|
||||
stmt.flushStmt()
|
||||
|
||||
# Изчистване на целия кеш
|
||||
rdb.clearStmtCache()
|
||||
```
|
||||
|
||||
### Pagination
|
||||
|
||||
```nim
|
||||
# Offset-based
|
||||
let page1 = waitFor rdb.table("users")
|
||||
.orderBy("id", Asc)
|
||||
.paginate(page = 1, perPage = 20)
|
||||
echo page1.rows.len # 20
|
||||
echo page1.total # 150
|
||||
echo page1.hasMore # true
|
||||
|
||||
# Cursor-based (по-бързо за големи таблици)
|
||||
let batch = waitFor rdb.table("users")
|
||||
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||
echo batch.hasMore
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cross-DB Migration Engine
|
||||
|
||||
Мигриране на данни от PostgreSQL, MySQL, MariaDB, SQLite или SurrealDB към BaraDB.
|
||||
|
||||
### Поддържани source бази
|
||||
|
||||
| База | Статус | Schema extraction |
|
||||
|------|--------|-------------------|
|
||||
| PostgreSQL | ✅ | `information_schema` |
|
||||
| MySQL | ✅ | `information_schema` |
|
||||
| MariaDB | ✅ | `information_schema` |
|
||||
| SQLite | ✅ | `sqlite_master` + `PRAGMA` |
|
||||
| SurrealDB | ✅ | `INFO FOR DB` / `INFO FOR TABLE` |
|
||||
|
||||
### API
|
||||
|
||||
```nim
|
||||
import allographer/migrate_data
|
||||
|
||||
# Свързване към source и target
|
||||
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
# Мигриране на всички таблици
|
||||
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||
echo report
|
||||
# Migration: PostgreSQL → BaraDB
|
||||
# Tables: 12/12
|
||||
# Rows: 45230
|
||||
# Time: 3.2s
|
||||
|
||||
# Мигриране само на конкретни таблици
|
||||
let report = waitFor migrate(pg, bdb, tables = @["users", "orders", "products"])
|
||||
```
|
||||
|
||||
### Как работи
|
||||
|
||||
1. **Schema extraction** — чете структурата на таблиците от source базата
|
||||
2. **Type mapping** — мапва типовете към BaraDB еквиваленти:
|
||||
- `SERIAL` → `SERIAL`
|
||||
- `VARCHAR(n)` → `VARCHAR(n)`
|
||||
- `TEXT` → `TEXT`
|
||||
- `JSONB` → `JSON`
|
||||
- `BOOLEAN` → `BOOLEAN`
|
||||
- и още 50+ типа
|
||||
3. **DDL генерация** — създава `CREATE MIGRATION` с UP/DOWN скриптове
|
||||
4. **Data transfer** — чете данни на batch-ове и ги вмъква в BaraDB
|
||||
5. **Progress tracking** — връща `MigrationReport` с детайли
|
||||
|
||||
---
|
||||
|
||||
## IMPORT FROM / EXPORT TO (BaraQL)
|
||||
|
||||
```sql
|
||||
-- Импорт от CSV
|
||||
IMPORT FROM '/data/users.csv' INTO users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
BATCH 1000
|
||||
|
||||
-- Импорт от JSON
|
||||
IMPORT FROM '/data/users.json' INTO users
|
||||
FORMAT JSON
|
||||
|
||||
-- Импорт от NDJSON (newline-delimited JSON)
|
||||
IMPORT FROM '/data/users.ndjson' INTO users
|
||||
FORMAT NDJSON
|
||||
|
||||
-- Експорт към CSV
|
||||
EXPORT TO '/backup/users.csv' FROM users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
|
||||
-- Експорт към JSON
|
||||
EXPORT TO '/backup/users.json' FROM users
|
||||
FORMAT JSON
|
||||
```
|
||||
|
||||
### Поддържани формати
|
||||
|
||||
| Формат | Import | Export | Опции |
|
||||
|--------|--------|--------|-------|
|
||||
| CSV | ✅ | ✅ | DELIMITER, HEADER, BATCH |
|
||||
| JSON | ✅ | ✅ | — |
|
||||
| NDJSON | ✅ | ✅ | — |
|
||||
|
||||
---
|
||||
|
||||
## Често задавани въпроси
|
||||
|
||||
### Мога ли да мигрирам от SQLite директно към BaraDB?
|
||||
|
||||
Да. Свържете се към SQLite файла и използвайте `migrate()`:
|
||||
|
||||
```nim
|
||||
let sqlite = dbOpen(SQLite3, "mydb.sqlite")
|
||||
let bdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||
let report = waitFor migrate(sqlite, bdb)
|
||||
```
|
||||
|
||||
### Какво става ако миграцията се прекъсне?
|
||||
|
||||
BaraDB сървърът поддържа transaction safety. Всяка миграция се изпълнява в
|
||||
рамките на една транзакция. При грешка:
|
||||
- DDL промените се отменят автоматично
|
||||
- `MIGRATION STATUS` показва кои миграции са applied и кои не
|
||||
- Може да продължите от последната успешна миграция
|
||||
|
||||
### Как да проверя какви миграции са приложени?
|
||||
|
||||
```nim
|
||||
let status = waitFor rdb.migrationStatus()
|
||||
for row in status:
|
||||
echo &"{row[\"name\"]} → {row[\"status\"]} ({row[\"applied_at\"]})"
|
||||
```
|
||||
|
||||
Или чрез BaraQL:
|
||||
```sql
|
||||
MIGRATION STATUS
|
||||
```
|
||||
|
||||
### Поддържат ли се foreign key зависимости при cross-DB миграция?
|
||||
|
||||
Да. `migrate_data.nim` запазва foreign key дефинициите в генерирания DDL.
|
||||
Препоръчва се таблиците без foreign key зависимости да се мигрират първи.
|
||||
|
||||
### Как се прави backup преди миграция?
|
||||
|
||||
```sql
|
||||
-- Експортирайте данните преди миграция
|
||||
EXPORT TO '/backup/before_migration.csv' FROM users FORMAT CSV
|
||||
|
||||
-- Или използвайте backup manager-а
|
||||
-- (backup restore list verify cleanup)
|
||||
```
|
||||
@@ -0,0 +1,634 @@
|
||||
## Cross-DB Migration Engine — migrate data between any allographer-supported DB and BaraDB.
|
||||
##
|
||||
## Usage:
|
||||
## migrate(sourceRdb, targetRdb, tables) — migrate specific tables
|
||||
## migrateAll(sourceRdb, targetRdb) — migrate all tables
|
||||
## migrateFromUrl(sourceUrl, targetUrl) — URL-based migration
|
||||
##
|
||||
## Supported source databases: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB
|
||||
## Target: BaraDB (native migration system via CREATE MIGRATION + MIGRATION UP)
|
||||
|
||||
import std/asyncdispatch
|
||||
import std/json
|
||||
import std/options
|
||||
import std/strformat
|
||||
import std/strutils
|
||||
import std/tables
|
||||
import std/times
|
||||
|
||||
import ./env
|
||||
import ./query_builder/libs/database_url
|
||||
import ./query_builder/models/baradb/baradb_types
|
||||
import ./query_builder/models/baradb/baradb_query
|
||||
import ./query_builder/models/baradb/baradb_exec
|
||||
|
||||
when isExistsPostgres:
|
||||
import ./query_builder/models/postgres/postgres_types
|
||||
import ./query_builder/models/postgres/postgres_query
|
||||
import ./query_builder/models/postgres/postgres_exec
|
||||
import ./query_builder/models/postgres/postgres_open
|
||||
|
||||
when isExistsMysql:
|
||||
import ./query_builder/models/mysql/mysql_types
|
||||
import ./query_builder/models/mysql/mysql_query
|
||||
import ./query_builder/models/mysql/mysql_exec
|
||||
import ./query_builder/models/mysql/mysql_open
|
||||
|
||||
when isExistsMariadb:
|
||||
import ./query_builder/models/mariadb/mariadb_types
|
||||
import ./query_builder/models/mariadb/mariadb_query
|
||||
import ./query_builder/models/mariadb/mariadb_exec
|
||||
import ./query_builder/models/mariadb/mariadb_open
|
||||
|
||||
when isExistsSqlite:
|
||||
import ./query_builder/models/sqlite/sqlite_types
|
||||
import ./query_builder/models/sqlite/sqlite_query
|
||||
import ./query_builder/models/sqlite/sqlite_exec
|
||||
import ./query_builder/models/sqlite/sqlite_open
|
||||
|
||||
when isExistsSurrealdb:
|
||||
import ./query_builder/models/surreal/surreal_types
|
||||
import ./query_builder/models/surreal/surreal_query
|
||||
import ./query_builder/models/surreal/surreal_exec
|
||||
import ./query_builder/models/surreal/surreal_open
|
||||
|
||||
# ==============================================================================
|
||||
# Types
|
||||
# ==============================================================================
|
||||
|
||||
type
|
||||
ColumnInfo* = tuple[name: string, typ: string, isPk: bool, isNullable: bool, defaultVal: string]
|
||||
|
||||
TableInfo* = object
|
||||
name*: string
|
||||
columns*: seq[ColumnInfo]
|
||||
|
||||
MigrationProgress* = object
|
||||
tableName*: string
|
||||
totalRows*: int
|
||||
transferredRows*: int
|
||||
status*: string # "pending", "in_progress", "done", "failed"
|
||||
error*: string
|
||||
|
||||
MigrationReport* = object
|
||||
sourceDb*: string
|
||||
targetDb*: string
|
||||
tablesTotal*: int
|
||||
tablesDone*: int
|
||||
rowsTotal*: int
|
||||
rowsTransferred*: int
|
||||
startedAt*: float
|
||||
completedAt*: float
|
||||
errors*: seq[string]
|
||||
|
||||
# ==============================================================================
|
||||
# Type Mapping: source DB → BaraDB
|
||||
# ==============================================================================
|
||||
|
||||
const BARADB_TYPE_MAP = {
|
||||
# PostgreSQL
|
||||
"smallint": "SMALLINT",
|
||||
"integer": "INTEGER",
|
||||
"bigint": "BIGINT",
|
||||
"serial": "SERIAL",
|
||||
"bigserial": "BIGSERIAL",
|
||||
"real": "REAL",
|
||||
"double precision": "DOUBLE PRECISION",
|
||||
"numeric": "DECIMAL",
|
||||
"decimal": "DECIMAL",
|
||||
"character": "VARCHAR(255)",
|
||||
"character varying": "VARCHAR",
|
||||
"varchar": "VARCHAR",
|
||||
"text": "TEXT",
|
||||
"boolean": "BOOLEAN",
|
||||
"bool": "BOOLEAN",
|
||||
"date": "DATE",
|
||||
"timestamp": "TIMESTAMP",
|
||||
"timestamp without time zone": "TIMESTAMP",
|
||||
"timestamp with time zone": "TIMESTAMPTZ",
|
||||
"time": "TIME",
|
||||
"time without time zone": "TIME",
|
||||
"bytea": "BYTEA",
|
||||
"uuid": "UUID",
|
||||
"json": "JSON",
|
||||
"jsonb": "JSON",
|
||||
# MySQL/MariaDB
|
||||
"tinyint": "SMALLINT",
|
||||
"mediumint": "INTEGER",
|
||||
"int": "INTEGER",
|
||||
"float": "REAL",
|
||||
"double": "DOUBLE PRECISION",
|
||||
"char": "VARCHAR(255)",
|
||||
"longtext": "TEXT",
|
||||
"mediumtext": "TEXT",
|
||||
"tinytext": "TEXT",
|
||||
"blob": "BYTEA",
|
||||
"longblob": "BYTEA",
|
||||
"mediumblob": "BYTEA",
|
||||
"tinyblob": "BYTEA",
|
||||
"datetime": "TIMESTAMP",
|
||||
"enum": "VARCHAR(255)",
|
||||
"set": "VARCHAR(255)",
|
||||
"year": "SMALLINT",
|
||||
# SQLite
|
||||
"int": "INTEGER",
|
||||
"integer": "INTEGER",
|
||||
"real": "REAL",
|
||||
"blob": "BYTEA",
|
||||
# SurrealDB
|
||||
"string": "VARCHAR(255)",
|
||||
"number": "DOUBLE PRECISION",
|
||||
"object": "JSON",
|
||||
"array": "JSON",
|
||||
}.toTable()
|
||||
|
||||
proc mapType*(sourceType: string): string =
|
||||
## Map a source database column type to the closest BaraDB type.
|
||||
let lower = sourceType.toLower().split("(")[0].strip()
|
||||
if lower in BARADB_TYPE_MAP:
|
||||
result = BARADB_TYPE_MAP[lower]
|
||||
else:
|
||||
result = "VARCHAR(255)"
|
||||
|
||||
# ==============================================================================
|
||||
# Schema Extraction
|
||||
# ==============================================================================
|
||||
|
||||
# --- PostgreSQL ---
|
||||
when isExistsPostgres:
|
||||
proc extractSchema*(rdb: PostgresConnections): Future[seq[TableInfo]] {.async.} =
|
||||
result = @[]
|
||||
let tables = await rdb.raw(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
|
||||
).get()
|
||||
for table in tables:
|
||||
let tableName = table["table_name"].getStr()
|
||||
if tableName == "_allographer_migrations": continue
|
||||
var cols: seq[ColumnInfo] = @[]
|
||||
let columns = await rdb.raw(
|
||||
"""SELECT c.column_name, c.data_type, c.is_nullable, c.column_default,
|
||||
CASE WHEN pk.column_name IS NOT NULL THEN true ELSE false END as is_pk
|
||||
FROM information_schema.columns c
|
||||
LEFT JOIN (
|
||||
SELECT ku.column_name FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage ku
|
||||
ON tc.constraint_name = ku.constraint_name
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_name = ?
|
||||
) pk ON c.column_name = pk.column_name
|
||||
WHERE c.table_name = ?
|
||||
ORDER BY c.ordinal_position""",
|
||||
%*[tableName, tableName]
|
||||
).get()
|
||||
for col in columns:
|
||||
cols.add((
|
||||
name: col["column_name"].getStr(),
|
||||
typ: col["data_type"].getStr(),
|
||||
isPk: col["is_pk"].getStr() == "true",
|
||||
isNullable: col["is_nullable"].getStr() == "YES",
|
||||
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||
col["column_default"].getStr() else: ""
|
||||
))
|
||||
result.add(TableInfo(name: tableName, columns: cols))
|
||||
|
||||
# --- MySQL ---
|
||||
when isExistsMysql:
|
||||
proc extractSchema*(rdb: MysqlConnections): Future[seq[TableInfo]] {.async.} =
|
||||
result = @[]
|
||||
let tables = await rdb.raw(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
|
||||
).get()
|
||||
for table in tables:
|
||||
let tableName = table["table_name"].getStr()
|
||||
if tableName == "_allographer_migrations": continue
|
||||
var cols: seq[ColumnInfo] = @[]
|
||||
let columns = await rdb.raw(
|
||||
"""SELECT column_name, data_type, is_nullable, column_default,
|
||||
column_key FROM information_schema.columns
|
||||
WHERE table_name = ? AND table_schema = DATABASE()
|
||||
ORDER BY ordinal_position""",
|
||||
%*[tableName]
|
||||
).get()
|
||||
for col in columns:
|
||||
cols.add((
|
||||
name: col["column_name"].getStr(),
|
||||
typ: col["data_type"].getStr(),
|
||||
isPk: col["column_key"].getStr() == "PRI",
|
||||
isNullable: col["is_nullable"].getStr() == "YES",
|
||||
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||
col["column_default"].getStr() else: ""
|
||||
))
|
||||
result.add(TableInfo(name: tableName, columns: cols))
|
||||
|
||||
# --- MariaDB ---
|
||||
when isExistsMariadb:
|
||||
proc extractSchema*(rdb: MariadbConnections): Future[seq[TableInfo]] {.async.} =
|
||||
result = @[]
|
||||
let tables = await rdb.raw(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
|
||||
).get()
|
||||
for table in tables:
|
||||
let tableName = table["table_name"].getStr()
|
||||
if tableName == "_allographer_migrations": continue
|
||||
var cols: seq[ColumnInfo] = @[]
|
||||
let columns = await rdb.raw(
|
||||
"""SELECT column_name, data_type, is_nullable, column_default,
|
||||
column_key FROM information_schema.columns
|
||||
WHERE table_name = ? AND table_schema = DATABASE()
|
||||
ORDER BY ordinal_position""",
|
||||
%*[tableName]
|
||||
).get()
|
||||
for col in columns:
|
||||
cols.add((
|
||||
name: col["column_name"].getStr(),
|
||||
typ: col["data_type"].getStr(),
|
||||
isPk: col["column_key"].getStr() == "PRI",
|
||||
isNullable: col["is_nullable"].getStr() == "YES",
|
||||
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||
col["column_default"].getStr() else: ""
|
||||
))
|
||||
result.add(TableInfo(name: tableName, columns: cols))
|
||||
|
||||
# --- SQLite ---
|
||||
when isExistsSqlite:
|
||||
proc extractSchema*(rdb: SqliteConnections): Future[seq[TableInfo]] {.async.} =
|
||||
result = @[]
|
||||
let tables = await rdb.raw(
|
||||
"SELECT name as table_name FROM sqlite_master WHERE type = 'table'"
|
||||
).get()
|
||||
for table in tables:
|
||||
let tableName = table["table_name"].getStr()
|
||||
if tableName == "_allographer_migrations": continue
|
||||
if tableName == "sqlite_sequence": continue
|
||||
var cols: seq[ColumnInfo] = @[]
|
||||
let columns = await rdb.raw("PRAGMA table_info(?)", %*[tableName]).get()
|
||||
for col in columns:
|
||||
cols.add((
|
||||
name: col["name"].getStr(),
|
||||
typ: col["type"].getStr(),
|
||||
isPk: col["pk"].getStr() == "1",
|
||||
isNullable: col["notnull"].getStr() == "0",
|
||||
defaultVal: if col.hasKey("dflt_value") and not col["dflt_value"].isNull:
|
||||
col["dflt_value"].getStr() else: ""
|
||||
))
|
||||
result.add(TableInfo(name: tableName, columns: cols))
|
||||
|
||||
# --- SurrealDB ---
|
||||
when isExistsSurrealdb:
|
||||
proc extractSchema*(rdb: SurrealConnections): Future[seq[TableInfo]] {.async.} =
|
||||
result = @[]
|
||||
let dbResponse = await rdb.raw("INFO FOR DB").get()
|
||||
if dbResponse.len == 0: return
|
||||
let tables = dbResponse[0]["result"]["tables"]
|
||||
for tableName, _ in tables.getFields().pairs:
|
||||
if tableName == "_allographer_migrations": continue
|
||||
if tableName == "_autoincrement_sequences": continue
|
||||
var cols: seq[ColumnInfo] = @[]
|
||||
let tableInfo = await rdb.raw(&"INFO FOR TABLE {tableName}").get()
|
||||
if tableInfo.len > 0:
|
||||
let fields = tableInfo[0]["result"]["fields"]
|
||||
for fieldName, _ in fields.getFields().pairs:
|
||||
cols.add((
|
||||
name: fieldName,
|
||||
typ: "string", # SurrealDB is schemaless
|
||||
isPk: fieldName == "id",
|
||||
isNullable: fieldName != "id",
|
||||
defaultVal: ""
|
||||
))
|
||||
result.add(TableInfo(name: tableName, columns: cols))
|
||||
|
||||
# ==============================================================================
|
||||
# DDL Generation
|
||||
# ==============================================================================
|
||||
|
||||
proc generateBaraDBDDL*(table: TableInfo): tuple[upSql: string, downSql: string] =
|
||||
## Generate CREATE TABLE (UP) and DROP TABLE (DOWN) DDL for BaraDB.
|
||||
var colDefs: seq[string] = @[]
|
||||
for col in table.columns:
|
||||
var def = &"`{col.name}` {mapType(col.typ)}"
|
||||
if col.isPk:
|
||||
def &= " PRIMARY KEY"
|
||||
if not col.isNullable:
|
||||
def &= " NOT NULL"
|
||||
if col.defaultVal.len > 0:
|
||||
def &= " DEFAULT " & col.defaultVal
|
||||
colDefs.add(def)
|
||||
let upSql = &"CREATE TABLE `{table.name}` ({colDefs.join(\", \")})"
|
||||
let downSql = &"DROP TABLE IF EXISTS `{table.name}`"
|
||||
return (upSql, downSql)
|
||||
|
||||
proc generateMigrationName*(tableName: string): string =
|
||||
let timestamp = getTime().toUnix()
|
||||
return &"migrate_{tableName}_{timestamp}"
|
||||
|
||||
# ==============================================================================
|
||||
# Data Transfer
|
||||
# ==============================================================================
|
||||
|
||||
proc transferTable*(sourceConn: BaradbConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
## Generic table transfer for BaraDB→BaraDB (used internally).
|
||||
## For cross-DB, use the typed overloads below.
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
# Count rows
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
|
||||
# Transfer in batches
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
# Typed overloads for each source DB type
|
||||
when isExistsPostgres:
|
||||
proc transferTable*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
when isExistsMysql:
|
||||
proc transferTable*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
when isExistsMariadb:
|
||||
proc transferTable*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
when isExistsSqlite:
|
||||
proc transferTable*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
when isExistsSurrealdb:
|
||||
proc transferTable*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
|
||||
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||
transferredRows: 0, status: "in_progress")
|
||||
try:
|
||||
let countVal = await sourceConn.table(tableName).count()
|
||||
result.totalRows = countVal
|
||||
var offset = 0
|
||||
while offset < result.totalRows:
|
||||
let rows = await sourceConn.table(tableName)
|
||||
.limit(batchSize)
|
||||
.offset(offset)
|
||||
.get()
|
||||
if rows.len == 0: break
|
||||
await targetConn.table(tableName).insert(rows)
|
||||
result.transferredRows += rows.len
|
||||
offset += batchSize
|
||||
result.status = "done"
|
||||
except CatchableError as e:
|
||||
result.status = "failed"
|
||||
result.error = e.msg
|
||||
|
||||
# ==============================================================================
|
||||
# Full Migration Orchestrator
|
||||
# ==============================================================================
|
||||
|
||||
when isExistsPostgres:
|
||||
proc migrate*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
|
||||
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||
result = MigrationReport(sourceDb: "PostgreSQL", targetDb: "BaraDB",
|
||||
startedAt: epochTime())
|
||||
try:
|
||||
let schema = await extractSchema(sourceConn)
|
||||
var tablesToMigrate: seq[TableInfo]
|
||||
if tables.len > 0:
|
||||
for t in schema:
|
||||
if t.name in tables: tablesToMigrate.add(t)
|
||||
else:
|
||||
tablesToMigrate = schema
|
||||
|
||||
result.tablesTotal = tablesToMigrate.len
|
||||
for table in tablesToMigrate:
|
||||
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||
let migName = generateMigrationName(table.name)
|
||||
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||
discard await targetConn.applyMigration(migName)
|
||||
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||
if progress.status == "done":
|
||||
result.tablesDone += 1
|
||||
result.rowsTransferred += progress.transferredRows
|
||||
else:
|
||||
result.errors.add(&"{table.name}: {progress.error}")
|
||||
except CatchableError as e:
|
||||
result.errors.add(e.msg)
|
||||
result.completedAt = epochTime()
|
||||
|
||||
when isExistsMysql:
|
||||
proc migrate*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
|
||||
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||
result = MigrationReport(sourceDb: "MySQL", targetDb: "BaraDB",
|
||||
startedAt: epochTime())
|
||||
try:
|
||||
let schema = await extractSchema(sourceConn)
|
||||
var tablesToMigrate: seq[TableInfo]
|
||||
if tables.len > 0:
|
||||
for t in schema:
|
||||
if t.name in tables: tablesToMigrate.add(t)
|
||||
else:
|
||||
tablesToMigrate = schema
|
||||
result.tablesTotal = tablesToMigrate.len
|
||||
for table in tablesToMigrate:
|
||||
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||
let migName = generateMigrationName(table.name)
|
||||
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||
discard await targetConn.applyMigration(migName)
|
||||
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||
if progress.status == "done":
|
||||
result.tablesDone += 1
|
||||
result.rowsTransferred += progress.transferredRows
|
||||
else:
|
||||
result.errors.add(&"{table.name}: {progress.error}")
|
||||
except CatchableError as e:
|
||||
result.errors.add(e.msg)
|
||||
result.completedAt = epochTime()
|
||||
|
||||
when isExistsMariadb:
|
||||
proc migrate*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
|
||||
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||
result = MigrationReport(sourceDb: "MariaDB", targetDb: "BaraDB",
|
||||
startedAt: epochTime())
|
||||
try:
|
||||
let schema = await extractSchema(sourceConn)
|
||||
var tablesToMigrate: seq[TableInfo]
|
||||
if tables.len > 0:
|
||||
for t in schema:
|
||||
if t.name in tables: tablesToMigrate.add(t)
|
||||
else:
|
||||
tablesToMigrate = schema
|
||||
result.tablesTotal = tablesToMigrate.len
|
||||
for table in tablesToMigrate:
|
||||
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||
let migName = generateMigrationName(table.name)
|
||||
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||
discard await targetConn.applyMigration(migName)
|
||||
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||
if progress.status == "done":
|
||||
result.tablesDone += 1
|
||||
result.rowsTransferred += progress.transferredRows
|
||||
else:
|
||||
result.errors.add(&"{table.name}: {progress.error}")
|
||||
except CatchableError as e:
|
||||
result.errors.add(e.msg)
|
||||
result.completedAt = epochTime()
|
||||
|
||||
when isExistsSqlite:
|
||||
proc migrate*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
|
||||
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||
result = MigrationReport(sourceDb: "SQLite", targetDb: "BaraDB",
|
||||
startedAt: epochTime())
|
||||
try:
|
||||
let schema = await extractSchema(sourceConn)
|
||||
var tablesToMigrate: seq[TableInfo]
|
||||
if tables.len > 0:
|
||||
for t in schema:
|
||||
if t.name in tables: tablesToMigrate.add(t)
|
||||
else:
|
||||
tablesToMigrate = schema
|
||||
result.tablesTotal = tablesToMigrate.len
|
||||
for table in tablesToMigrate:
|
||||
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||
let migName = generateMigrationName(table.name)
|
||||
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||
discard await targetConn.applyMigration(migName)
|
||||
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||
if progress.status == "done":
|
||||
result.tablesDone += 1
|
||||
result.rowsTransferred += progress.transferredRows
|
||||
else:
|
||||
result.errors.add(&"{table.name}: {progress.error}")
|
||||
except CatchableError as e:
|
||||
result.errors.add(e.msg)
|
||||
result.completedAt = epochTime()
|
||||
|
||||
when isExistsSurrealdb:
|
||||
proc migrate*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
|
||||
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||
result = MigrationReport(sourceDb: "SurrealDB", targetDb: "BaraDB",
|
||||
startedAt: epochTime())
|
||||
try:
|
||||
let schema = await extractSchema(sourceConn)
|
||||
var tablesToMigrate: seq[TableInfo]
|
||||
if tables.len > 0:
|
||||
for t in schema:
|
||||
if t.name in tables: tablesToMigrate.add(t)
|
||||
else:
|
||||
tablesToMigrate = schema
|
||||
result.tablesTotal = tablesToMigrate.len
|
||||
for table in tablesToMigrate:
|
||||
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||
let migName = generateMigrationName(table.name)
|
||||
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||
discard await targetConn.applyMigration(migName)
|
||||
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||
if progress.status == "done":
|
||||
result.tablesDone += 1
|
||||
result.rowsTransferred += progress.transferredRows
|
||||
else:
|
||||
result.errors.add(&"{table.name}: {progress.error}")
|
||||
except CatchableError as e:
|
||||
result.errors.add(e.msg)
|
||||
result.completedAt = epochTime()
|
||||
|
||||
# ==============================================================================
|
||||
# URL-based migration
|
||||
# ==============================================================================
|
||||
|
||||
proc `$`*(report: MigrationReport): string =
|
||||
let duration = report.completedAt - report.startedAt
|
||||
result = &"Migration: {report.sourceDb} → {report.targetDb}\n"
|
||||
result &= &" Tables: {report.tablesDone}/{report.tablesTotal}\n"
|
||||
result &= &" Rows: {report.rowsTransferred}\n"
|
||||
result &= &" Time: {duration:.1f}s\n"
|
||||
if report.errors.len > 0:
|
||||
result &= " Errors:\n"
|
||||
for err in report.errors:
|
||||
result &= &" - {err}\n"
|
||||
@@ -410,6 +410,35 @@ proc exec*(client: BaraClient, sql: string): Future[int] {.async.} =
|
||||
let qr = await client.query(sql)
|
||||
return qr.affectedRows
|
||||
|
||||
# === Migration API (BaraQL native) ===
|
||||
|
||||
proc createMigration*(client: BaraClient, name: string, upBody: string,
|
||||
downBody: string = ""): Future[QueryResult] {.async.} =
|
||||
## Send CREATE MIGRATION via BaraQL. Server handles checksums, locking, rollback.
|
||||
var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
|
||||
if downBody.len > 0:
|
||||
sql &= " DOWN: " & downBody & ";"
|
||||
sql &= " }"
|
||||
return await client.query(sql)
|
||||
|
||||
proc applyMigration*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
|
||||
return await client.query("APPLY MIGRATION " & name)
|
||||
|
||||
proc migrateUp*(client: BaraClient, count: int = 0): Future[QueryResult] {.async.} =
|
||||
var sql = "MIGRATION UP"
|
||||
if count > 0:
|
||||
sql &= " " & $count
|
||||
return await client.query(sql)
|
||||
|
||||
proc migrateDown*(client: BaraClient, count: int = 1): Future[QueryResult] {.async.} =
|
||||
return await client.query("MIGRATION DOWN " & $count)
|
||||
|
||||
proc migrationStatus*(client: BaraClient): Future[QueryResult] {.async.} =
|
||||
return await client.query("MIGRATION STATUS")
|
||||
|
||||
proc migrationDryRun*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
|
||||
return await client.query("MIGRATION DRY RUN " & name)
|
||||
|
||||
proc auth*(client: BaraClient, token: string) {.async.} =
|
||||
if not client.connected:
|
||||
raise newException(IOError, "Not connected")
|
||||
@@ -652,6 +681,34 @@ proc exec*(client: SyncClient, sql: string): int =
|
||||
let qr = client.query(sql)
|
||||
return qr.affectedRows
|
||||
|
||||
# === Migration API (SyncClient, blocking) ===
|
||||
|
||||
proc createMigration*(client: SyncClient, name: string, upBody: string,
|
||||
downBody: string = ""): QueryResult =
|
||||
var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
|
||||
if downBody.len > 0:
|
||||
sql &= " DOWN: " & downBody & ";"
|
||||
sql &= " }"
|
||||
return client.query(sql)
|
||||
|
||||
proc applyMigration*(client: SyncClient, name: string): QueryResult =
|
||||
return client.query("APPLY MIGRATION " & name)
|
||||
|
||||
proc migrateUp*(client: SyncClient, count: int = 0): QueryResult =
|
||||
var sql = "MIGRATION UP"
|
||||
if count > 0:
|
||||
sql &= " " & $count
|
||||
return client.query(sql)
|
||||
|
||||
proc migrateDown*(client: SyncClient, count: int = 1): QueryResult =
|
||||
return client.query("MIGRATION DOWN " & $count)
|
||||
|
||||
proc migrationStatus*(client: SyncClient): QueryResult =
|
||||
return client.query("MIGRATION STATUS")
|
||||
|
||||
proc migrationDryRun*(client: SyncClient, name: string): QueryResult =
|
||||
return client.query("MIGRATION DRY RUN " & name)
|
||||
|
||||
proc auth*(client: SyncClient, token: string) =
|
||||
acquire(client.lock)
|
||||
try:
|
||||
|
||||
@@ -699,6 +699,183 @@ proc firstPlain*(self: RawBaradbQuery): Future[seq[string]] {.async.} =
|
||||
return await self.getRowPlain(self.queryString, self.placeHolder)
|
||||
|
||||
|
||||
# ================================================================================
|
||||
# Pagination (cursor-based and offset-based)
|
||||
# ================================================================================
|
||||
|
||||
type
|
||||
PaginateResult* = object
|
||||
rows*: seq[JsonNode]
|
||||
total*: int
|
||||
page*: int
|
||||
perPage*: int
|
||||
hasMore*: bool
|
||||
|
||||
proc paginate*(self: BaradbQuery, page: int = 1, perPage: int = 20): Future[PaginateResult] {.async.} =
|
||||
## Offset-based pagination. Returns rows for the given page plus metadata.
|
||||
let total = await self.count()
|
||||
let offset = (page - 1) * perPage
|
||||
self.limit(perPage)
|
||||
self.offset(offset)
|
||||
let rows = await self.get()
|
||||
let qr = PaginateResult(
|
||||
rows: rows,
|
||||
total: total,
|
||||
page: page,
|
||||
perPage: perPage,
|
||||
hasMore: offset + perPage < total
|
||||
)
|
||||
return qr
|
||||
|
||||
proc fastPaginate*(self: BaradbQuery, cursorColumn: string, perPage: int = 20,
|
||||
afterId: string = ""): Future[PaginateResult] {.async.} =
|
||||
## Cursor-based pagination (keyset). More efficient than offset for large tables.
|
||||
## Requires a unique, ordered column (usually the primary key).
|
||||
let total = await self.count()
|
||||
self.limit(perPage)
|
||||
self.orderBy(cursorColumn, Asc)
|
||||
if afterId.len > 0:
|
||||
self.where(cursorColumn, ">", afterId)
|
||||
let rows = await self.get()
|
||||
var hasMore = rows.len == perPage
|
||||
let qr = PaginateResult(
|
||||
rows: rows,
|
||||
total: total,
|
||||
page: 0, # cursor-based has no page number
|
||||
perPage: perPage,
|
||||
hasMore: hasMore
|
||||
)
|
||||
return qr
|
||||
|
||||
# ================================================================================
|
||||
# Migration API (BaraQL native — server handles checksums, locks, rollback)
|
||||
# ================================================================================
|
||||
|
||||
proc createMigration*(self: BaradbConnections, name: string, upBody: string,
|
||||
downBody: string = ""): Future[QueryResult] {.async.} =
|
||||
## Register a migration on the server. The server computes checksums, stores
|
||||
## the UP/DOWN bodies, and manages the migration lifecycle.
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
return await self.pools.conns[connI].client.createMigration(name, upBody, downBody)
|
||||
|
||||
proc applyMigration*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
return await self.pools.conns[connI].client.applyMigration(name)
|
||||
|
||||
proc migrateUp*(self: BaradbConnections, count: int = 0): Future[QueryResult] {.async.} =
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
return await self.pools.conns[connI].client.migrateUp(count)
|
||||
|
||||
proc migrateDown*(self: BaradbConnections, count: int = 1): Future[QueryResult] {.async.} =
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
return await self.pools.conns[connI].client.migrateDown(count)
|
||||
|
||||
proc migrationStatus*(self: BaradbConnections): Future[seq[JsonNode]] {.async.} =
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
let qr = await self.pools.conns[connI].client.migrationStatus()
|
||||
return toJson(qr)
|
||||
|
||||
proc migrationDryRun*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
|
||||
var connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
return await self.pools.conns[connI].client.migrationDryRun(name)
|
||||
|
||||
proc isMigrationApplied*(self: BaradbConnections, name: string): Future[bool] {.async.} =
|
||||
let status = await self.migrationStatus()
|
||||
for row in status:
|
||||
if row["name"].getStr() == name and row["status"].getStr() == "applied":
|
||||
return true
|
||||
return false
|
||||
|
||||
# ================================================================================
|
||||
# Prepared Statements (server-side parameterized queries via mkQueryParams)
|
||||
# ================================================================================
|
||||
|
||||
proc sendPrepared(client: BaraClient, sql: string, params: seq[WireValue]): Future[QueryResult] {.async.} =
|
||||
if not client.connected:
|
||||
raise newException(IOError, "Not connected")
|
||||
let msg = makeQueryParamsMessage(client.nextId(), sql, params)
|
||||
let msgStr = toString(msg)
|
||||
await client.socket.send(msgStr)
|
||||
return await client.readQueryResponse()
|
||||
|
||||
proc prepare*(self: BaradbConnections, sql: string, nArgs: int = 0): Future[BaradbPreparedStatement] {.async.} =
|
||||
## Create a server-side prepared statement. Subsequent calls reuse the cached entry.
|
||||
let entryKey = sql & ":" & $nArgs
|
||||
if entryKey in self.pools.preparedCache:
|
||||
let entry = self.pools.preparedCache[entryKey]
|
||||
inc entry.refCount
|
||||
entry.lastUsedAt = nowUnix()
|
||||
return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
|
||||
nArgs: nArgs, isClosed: false)
|
||||
let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 1,
|
||||
lastUsedAt: nowUnix())
|
||||
self.pools.preparedCache[entryKey] = entry
|
||||
return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
|
||||
nArgs: nArgs, isClosed: false)
|
||||
|
||||
proc ensureStmt*(self: BaradbConnections, sql: string, nArgs: int): BaradbPreparedEntry =
|
||||
let entryKey = sql & ":" & $nArgs
|
||||
if entryKey in self.pools.preparedCache:
|
||||
let entry = self.pools.preparedCache[entryKey]
|
||||
entry.lastUsedAt = nowUnix()
|
||||
return entry
|
||||
let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 0,
|
||||
lastUsedAt: nowUnix())
|
||||
self.pools.preparedCache[entryKey] = entry
|
||||
return entry
|
||||
|
||||
proc preparedGet*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[seq[JsonNode]] {.async.} =
|
||||
if stmt.isClosed:
|
||||
raise newException(IOError, "Prepared statement is closed")
|
||||
var connI = getFreeConn(stmt.owner).await
|
||||
defer: stmt.owner.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(stmt.owner)
|
||||
let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
|
||||
return toJson(qr)
|
||||
|
||||
proc preparedExec*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[int] {.async.} =
|
||||
if stmt.isClosed:
|
||||
raise newException(IOError, "Prepared statement is closed")
|
||||
var connI = getFreeConn(stmt.owner).await
|
||||
defer: stmt.owner.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(stmt.owner)
|
||||
let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
|
||||
return qr.affectedRows
|
||||
|
||||
proc flushStmt*(stmt: BaradbPreparedStatement) =
|
||||
stmt.isClosed = true
|
||||
dec stmt.entry.refCount
|
||||
|
||||
proc clearStmtCache*(self: BaradbConnections) =
|
||||
self.pools.preparedCache.clear()
|
||||
|
||||
proc withConn*(self: BaradbConnections, callback: proc(connI: int): Future[void] {.gcsafe.}): Future[void] {.async.} =
|
||||
let connI = getFreeConn(self).await
|
||||
defer: self.returnConn(connI).await
|
||||
if connI == errorConnectionNum:
|
||||
raisePoolTimeout(self)
|
||||
await callback(connI)
|
||||
|
||||
# seeder templates
|
||||
template seeder*(rdb: BaradbConnections, tableName: string, body: untyped): untyped =
|
||||
block:
|
||||
|
||||
@@ -4,6 +4,7 @@ import std/json
|
||||
import std/tables
|
||||
import std/times
|
||||
import ../../libs/baradb/baradb_client
|
||||
import ../../libs/database_url
|
||||
import ../../log
|
||||
import ./baradb_types
|
||||
|
||||
@@ -50,3 +51,18 @@ proc dbOpen*(_: type Baradb, database: string, user: string, password: string,
|
||||
pools: pools,
|
||||
log: LogSetting(shouldDisplayLog: shouldDisplayLog, shouldOutputLogFile: shouldOutputLogFile, logDir: logDir)
|
||||
)
|
||||
|
||||
proc dbOpen*(_: type Baradb, databaseUrl: DatabaseUrl,
|
||||
maxConnections = 1, timeout = 30,
|
||||
shouldDisplayLog = false, shouldOutputLogFile = false, logDir = "",
|
||||
maxConnectionLifetime = DEFAULT_CONN_MAX_LIFETIME_SECONDS,
|
||||
maxConnectionIdleTime = DEFAULT_CONN_MAX_IDLE_SECONDS): BaradbConnections =
|
||||
## Open a BaraDB connection using a URL: baradb://user:pass@host:port/database
|
||||
let parsed = parseDatabaseUrl(databaseUrl)
|
||||
requireDatabaseUrlScheme(parsed, ["baradb"], "BaraDB")
|
||||
let database = parsed.path.stripLeadingSlash()
|
||||
let host = if parsed.hostname.len > 0: parsed.hostname else: "127.0.0.1"
|
||||
let port = if parsed.hasPort: parsed.port else: 9472
|
||||
return Baradb.dbOpen(database, parsed.username, parsed.password, host, port,
|
||||
maxConnections, timeout, shouldDisplayLog, shouldOutputLogFile,
|
||||
logDir, maxConnectionLifetime, maxConnectionIdleTime)
|
||||
|
||||
+3
-7
@@ -3,10 +3,6 @@ import ../../models/table
|
||||
import ./baradb_query_type
|
||||
|
||||
proc createMigrationTable*(self: BaradbSchema) =
|
||||
let sql = """CREATE TABLE IF NOT EXISTS `schema_migrations` (
|
||||
`id` SERIAL PRIMARY KEY,
|
||||
`name` VARCHAR(255) NOT NULL,
|
||||
`checksum` VARCHAR(64) NOT NULL,
|
||||
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)"""
|
||||
discard waitFor self.rdb.raw(sql).exec()
|
||||
## BaraDB tracks migrations natively via BaraQL (CREATE MIGRATION, MIGRATION STATUS).
|
||||
## No client-side migration table needed — server stores state in LSM-Tree.
|
||||
discard
|
||||
+38
-3
@@ -1,12 +1,47 @@
|
||||
import std/asyncdispatch
|
||||
import std/strformat
|
||||
import ../../../query_builder/models/baradb/baradb_types
|
||||
import ../../../query_builder/models/baradb/baradb_exec
|
||||
import ../../../query_builder/error
|
||||
import ../../models/table
|
||||
|
||||
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
|
||||
proc exec*(rdb: BaradbConnections, queries: seq[string]) =
|
||||
for sql in queries:
|
||||
discard waitFor rdb.raw(sql).exec()
|
||||
let hist = &"INSERT INTO `schema_migrations` (`name`, `checksum`) VALUES ('{name}', '{checksum}')"
|
||||
discard waitFor rdb.raw(hist).exec()
|
||||
|
||||
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
|
||||
## Register and apply a migration via BaraQL native migration system.
|
||||
## The server handles checksums, locking, rollback, and status tracking.
|
||||
let upBody = queries.join("; ")
|
||||
var downBody = ""
|
||||
# Generate DOWN script: DROP TABLE for CREATE TABLE
|
||||
for sql in queries:
|
||||
if sql.toLower().startsWith("create table"):
|
||||
let parts = sql.split(" ")
|
||||
if parts.len >= 3:
|
||||
let tableName = parts[2] # CREATE TABLE `name` ...
|
||||
downBody &= &"DROP TABLE IF EXISTS {tableName}; "
|
||||
|
||||
# Register migration on server
|
||||
var qr = waitFor rdb.createMigration(name, upBody, downBody)
|
||||
if qr.affectedRows < 0:
|
||||
raise newException(DbError, "Failed to create migration: " & name)
|
||||
|
||||
# Apply migration
|
||||
qr = waitFor rdb.applyMigration(name)
|
||||
if qr.affectedRows < 0:
|
||||
raise newException(DbError, "Failed to apply migration: " & name)
|
||||
|
||||
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, query: string, checksum: string) =
|
||||
execThenSaveHistory(rdb, name, @[query], checksum)
|
||||
|
||||
proc shouldRun*(rdb: BaradbConnections, table: Table, checksum: string, isReset: bool): bool =
|
||||
## Check with the server whether this migration has already been applied.
|
||||
if isReset:
|
||||
return true
|
||||
try:
|
||||
let applied = waitFor rdb.isMigrationApplied(table.name)
|
||||
return not applied
|
||||
except CatchableError:
|
||||
# If server is unavailable or migration doesn't exist, run it
|
||||
return true
|
||||
@@ -0,0 +1,59 @@
|
||||
discard """
|
||||
cmd: "nim c -r $file"
|
||||
"""
|
||||
|
||||
import std/unittest
|
||||
import std/asyncdispatch
|
||||
import std/json
|
||||
import ../../src/allographer/query_builder
|
||||
import ../../src/allographer/query_builder/models/baradb/baradb_exec
|
||||
import ./connections
|
||||
|
||||
|
||||
let rdb = baradb
|
||||
|
||||
suite("Baradb migrations"):
|
||||
test("create migration"):
|
||||
let upSql = """
|
||||
CREATE TABLE IF NOT EXISTS test_mig_users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255)
|
||||
)
|
||||
"""
|
||||
let downSql = "DROP TABLE IF EXISTS test_mig_users"
|
||||
let qr = waitFor rdb.createMigration("test_mig_001", upSql, downSql)
|
||||
check qr.rowCount >= 0
|
||||
|
||||
test("migration status shows pending"):
|
||||
let status = waitFor rdb.migrationStatus()
|
||||
var found = false
|
||||
for row in status:
|
||||
if row["name"].getStr() == "test_mig_001":
|
||||
check row["status"].getStr() == "pending"
|
||||
found = true
|
||||
break
|
||||
check found
|
||||
|
||||
test("apply migration"):
|
||||
let qr = waitFor rdb.applyMigration("test_mig_001")
|
||||
check qr.rowCount >= 0
|
||||
|
||||
test("is migration applied"):
|
||||
let applied = waitFor rdb.isMigrationApplied("test_mig_001")
|
||||
check applied
|
||||
|
||||
test("migration up (all pending)"):
|
||||
let qr = waitFor rdb.migrateUp()
|
||||
check qr.rowCount >= 0
|
||||
|
||||
test("migration dry run"):
|
||||
let qr = waitFor rdb.migrationDryRun("test_mig_001")
|
||||
check qr.rowCount >= 0
|
||||
|
||||
test("table exists after migration"):
|
||||
let users = waitFor rdb.table("test_mig_users").get()
|
||||
check users.len >= 0
|
||||
|
||||
test("migrate down (rollback)"):
|
||||
let qr = waitFor rdb.migrateDown(1)
|
||||
check qr.rowCount >= 0
|
||||
@@ -0,0 +1,85 @@
|
||||
discard """
|
||||
cmd: "nim c -r $file"
|
||||
"""
|
||||
|
||||
import std/unittest
|
||||
import std/asyncdispatch
|
||||
import std/json
|
||||
import std/options
|
||||
import ../../src/allographer/query_builder
|
||||
import ../../src/allographer/query_builder/models/baradb/baradb_exec
|
||||
import ../../src/allographer/query_builder/libs/baradb/baradb_client
|
||||
import ./connections
|
||||
|
||||
|
||||
let rdb = baradb
|
||||
|
||||
suite("Baradb prepared statements"):
|
||||
setup:
|
||||
# Ensure test table exists
|
||||
let createSql = """
|
||||
CREATE TABLE IF NOT EXISTS test_prep_data (
|
||||
id SERIAL PRIMARY KEY,
|
||||
label VARCHAR(255),
|
||||
count INTEGER
|
||||
)
|
||||
"""
|
||||
waitFor rdb.raw(createSql).exec()
|
||||
|
||||
test("prepare statement"):
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM test_prep_data WHERE count > ?", nArgs = 1
|
||||
)
|
||||
check not stmt.isClosed
|
||||
|
||||
test("ensureStmt"):
|
||||
let entry = rdb.ensureStmt(
|
||||
"SELECT * FROM test_prep_data WHERE label = ?", nArgs = 1
|
||||
)
|
||||
check entry.sql.len > 0
|
||||
check entry.nArgs == 1
|
||||
|
||||
test("preparedGet with params"):
|
||||
# Insert test data
|
||||
waitFor rdb.table("test_prep_data").insert(%*{
|
||||
"label": "alpha", "count": 5
|
||||
})
|
||||
waitFor rdb.table("test_prep_data").insert(%*{
|
||||
"label": "beta", "count": 15
|
||||
})
|
||||
|
||||
# Query via prepared statement
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM test_prep_data WHERE count > ?", nArgs = 1
|
||||
)
|
||||
let results = waitFor stmt.preparedGet(@[
|
||||
WireValue(kind: fkInt32, int32Val: 10)
|
||||
])
|
||||
check results.len >= 1
|
||||
if results.len > 0:
|
||||
check results[0]["label"].getStr() == "beta"
|
||||
|
||||
test("preparedExec insert"):
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"INSERT INTO test_prep_data (label, count) VALUES (?, ?)", nArgs = 2
|
||||
)
|
||||
let affected = waitFor stmt.preparedExec(@[
|
||||
WireValue(kind: fkString, strVal: "gamma"),
|
||||
WireValue(kind: fkInt32, int32Val: 25)
|
||||
])
|
||||
check affected >= 0
|
||||
|
||||
test("flush statement"):
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM test_prep_data", nArgs = 0
|
||||
)
|
||||
stmt.flushStmt()
|
||||
check stmt.isClosed
|
||||
|
||||
test("clear statement cache"):
|
||||
rdb.clearStmtCache()
|
||||
check true
|
||||
|
||||
test("cleanup"):
|
||||
waitFor rdb.raw("DROP TABLE IF EXISTS test_prep_data").exec()
|
||||
check true
|
||||
+3
-1
@@ -646,7 +646,9 @@ SELECT * FROM invoices; -- само редове на company-a
|
||||
|-----------|-------------|
|
||||
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
||||
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE |
|
||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE MIGRATION, APPLY MIGRATION |
|
||||
| Миграции | MIGRATION UP, MIGRATION DOWN, MIGRATION STATUS, MIGRATION DRY RUN |
|
||||
| Импорт/Експорт | IMPORT FROM, EXPORT TO, FORMAT, CSV, JSON, NDJSON, DELIMITER, HEADER, BATCH |
|
||||
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
||||
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
||||
| CTEs | WITH, RECURSIVE, AS |
|
||||
|
||||
@@ -270,6 +270,23 @@ with pool.connection() as conn:
|
||||
result = conn.query("SELECT 1")
|
||||
```
|
||||
|
||||
## Миграция между Бази Данни (Nim)
|
||||
|
||||
Nim allographer клиентът включва engine за миграция между бази данни:
|
||||
|
||||
```nim
|
||||
import allographer/migrate_data
|
||||
|
||||
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||
echo report # Tables: 12/12, Rows: 45230, Time: 3.2s
|
||||
```
|
||||
|
||||
Поддържани източници: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB.
|
||||
Виж [Миграции и Импорт/Експорт](migration.md) за детайли.
|
||||
|
||||
## Съответствие на Типове Данни
|
||||
|
||||
| BaraDB Тип | JavaScript | Python | Nim | Rust |
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
- [Сигурност](security.md)
|
||||
- [Мониторинг](monitoring.md)
|
||||
- [Архивиране и Възстановяване](backup.md)
|
||||
- [Миграции и Импорт/Експорт](migration.md)
|
||||
- [Отстраняване на Проблеми](troubleshooting.md)
|
||||
|
||||
## Разширени
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
# Миграции и Импорт/Експорт на Данни
|
||||
|
||||
BaraDB има вградена миграционна система чрез BaraQL. Миграциите се управляват
|
||||
изцяло от сървъра — checksums, locking, rollback и проследяване на статуса.
|
||||
Клиентът изпраща BaraQL команди и не поддържа собствена миграционна таблица.
|
||||
|
||||
## BaraQL Синтаксис за Миграции
|
||||
|
||||
```sql
|
||||
-- Създаване на миграция с UP и DOWN скриптове
|
||||
CREATE MIGRATION add_users_table {
|
||||
UP: CREATE TABLE users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
email VARCHAR(255) UNIQUE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
DOWN: DROP TABLE IF EXISTS users;
|
||||
}
|
||||
|
||||
-- Прилагане на конкретна миграция
|
||||
APPLY MIGRATION add_users_table
|
||||
|
||||
-- Прилагане на всички неприложени миграции
|
||||
MIGRATION UP
|
||||
|
||||
-- Прилагане на следващите N миграции
|
||||
MIGRATION UP 3
|
||||
|
||||
-- Отмяна на последната миграция
|
||||
MIGRATION DOWN
|
||||
|
||||
-- Отмяна на последните N миграции
|
||||
MIGRATION DOWN 2
|
||||
|
||||
-- Преглед на статуса
|
||||
MIGRATION STATUS
|
||||
|
||||
-- Dry run (валидация без изпълнение)
|
||||
MIGRATION DRY RUN add_users_table
|
||||
```
|
||||
|
||||
## Заключване при Миграция
|
||||
|
||||
BaraDB заключва глобално преди прилагане на миграция, което предотвратява
|
||||
конкурентно изпълнение и осигурява:
|
||||
|
||||
- Само една миграция се изпълнява в даден момент
|
||||
- Checksums се проверяват преди изпълнение
|
||||
- Неуспешните миграции могат да се върнат безопасно
|
||||
- `MIGRATION STATUS` показва точното състояние на всяка миграция
|
||||
|
||||
## Checksums
|
||||
|
||||
Тялото на всяка миграция се хешира с SHA-256. Сървърът съхранява checksum
|
||||
при създаване и го проверява преди прилагане.
|
||||
|
||||
```
|
||||
CREATE MIGRATION add_users → checksum: a3f2b8c1...
|
||||
APPLY MIGRATION add_users → проверява checksum → изпълнява
|
||||
```
|
||||
|
||||
## Връщане (Rollback)
|
||||
|
||||
Миграциите с `DOWN` скриптове поддържат връщане:
|
||||
|
||||
```sql
|
||||
CREATE MIGRATION add_column {
|
||||
UP: ALTER TABLE users ADD COLUMN phone VARCHAR(20);
|
||||
DOWN: ALTER TABLE users DROP COLUMN phone;
|
||||
}
|
||||
```
|
||||
|
||||
`MIGRATION DOWN` изпълнява DOWN скрипта и маркира миграцията като върната.
|
||||
|
||||
## Dry Run
|
||||
|
||||
Валидирайте миграция преди прилагане:
|
||||
|
||||
```sql
|
||||
MIGRATION DRY RUN add_users_table
|
||||
-- Изход:
|
||||
-- DRY RUN add_users_table:
|
||||
-- Statements: 1
|
||||
-- [1] nkCreateTable
|
||||
-- DOWN script: yes
|
||||
-- Checksum: a3f2b8c1d4e5f6a7b8c9d0e1f2a3b4c5
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## IMPORT FROM / EXPORT TO
|
||||
|
||||
BaraDB поддържа импорт и експорт на данни директно чрез BaraQL.
|
||||
|
||||
### IMPORT FROM
|
||||
|
||||
```sql
|
||||
-- Импорт от CSV
|
||||
IMPORT FROM '/data/users.csv' INTO users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
BATCH 1000
|
||||
|
||||
-- Импорт от JSON масив
|
||||
IMPORT FROM '/data/users.json' INTO users
|
||||
FORMAT JSON
|
||||
|
||||
-- Импорт от NDJSON
|
||||
IMPORT FROM '/data/users.ndjson' INTO users
|
||||
FORMAT NDJSON
|
||||
```
|
||||
|
||||
Опции:
|
||||
| Опция | Стойности | По подр. | Описание |
|
||||
|--------|-----------|----------|----------|
|
||||
| `FORMAT` | `CSV`, `JSON`, `NDJSON` | `CSV` | Формат на входния файл |
|
||||
| `DELIMITER` | произволен знак | `,` | Разделител за CSV |
|
||||
| `HEADER` | `true`/`false` | `true` | Първият ред на CSV е заглавен |
|
||||
| `BATCH` | цяло число | `1000` | Редове на партида |
|
||||
|
||||
### EXPORT TO
|
||||
|
||||
```sql
|
||||
-- Експорт към CSV
|
||||
EXPORT TO '/backup/users.csv' FROM users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
|
||||
-- Експорт към JSON
|
||||
EXPORT TO '/backup/users.json' FROM users
|
||||
FORMAT JSON
|
||||
|
||||
-- Експорт към NDJSON
|
||||
EXPORT TO '/backup/users.ndjson' FROM users
|
||||
FORMAT NDJSON
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Миграция между Бази Данни
|
||||
|
||||
Nim клиентът на BaraDB (nim-allographer) включва engine за миграция между
|
||||
различни бази данни. Мигрирайте данни от PostgreSQL, MySQL, MariaDB, SQLite
|
||||
или SurrealDB директно към BaraDB.
|
||||
|
||||
### Поддържани Източници
|
||||
|
||||
| База Данни | Извличане на Схема | Статус |
|
||||
|------------|-------------------|--------|
|
||||
| PostgreSQL | `information_schema` | ✅ |
|
||||
| MySQL | `information_schema` | ✅ |
|
||||
| MariaDB | `information_schema` | ✅ |
|
||||
| SQLite | `sqlite_master` + `PRAGMA` | ✅ |
|
||||
| SurrealDB | `INFO FOR DB` / `INFO FOR TABLE` | ✅ |
|
||||
|
||||
### Nim API
|
||||
|
||||
```nim
|
||||
import allographer/migrate_data
|
||||
|
||||
# Свързване към източник и цел
|
||||
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
# Мигриране на всички таблици
|
||||
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||
echo report
|
||||
# Migration: PostgreSQL → BaraDB
|
||||
# Tables: 12/12
|
||||
# Rows: 45230
|
||||
# Time: 3.2s
|
||||
|
||||
# Мигриране на конкретни таблици
|
||||
let report = waitFor migrate(pg, bdb,
|
||||
tables = @["users", "orders", "products"])
|
||||
```
|
||||
|
||||
### Съответствие на Типове
|
||||
|
||||
Миграционният engine автоматично мапва типовете:
|
||||
|
||||
| PostgreSQL | MySQL | SQLite | BaraDB |
|
||||
|------------|-------|--------|--------|
|
||||
| `SERIAL` | `INT AUTO_INCREMENT` | `INTEGER PK` | `SERIAL` |
|
||||
| `VARCHAR(n)` | `VARCHAR(n)` | `TEXT` | `VARCHAR(n)` |
|
||||
| `TEXT` | `TEXT` | `TEXT` | `TEXT` |
|
||||
| `BOOLEAN` | `TINYINT(1)` | `INTEGER` | `BOOLEAN` |
|
||||
| `JSONB` | `JSON` | `TEXT` | `JSON` |
|
||||
| `TIMESTAMP` | `DATETIME` | `TEXT` | `TIMESTAMP` |
|
||||
| `UUID` | `CHAR(36)` | `TEXT` | `UUID` |
|
||||
|
||||
Пълна карта: 50+ поддържани типа.
|
||||
|
||||
---
|
||||
|
||||
## Nim allographer API
|
||||
|
||||
### Управление на Миграции
|
||||
|
||||
```nim
|
||||
import allographer/query_builder/models/baradb/baradb_exec
|
||||
|
||||
# Създаване
|
||||
let qr = waitFor rdb.createMigration("add_products",
|
||||
"CREATE TABLE products (id SERIAL PRIMARY KEY, name VARCHAR(255))",
|
||||
"DROP TABLE IF EXISTS products")
|
||||
|
||||
# Прилагане
|
||||
let qr = waitFor rdb.applyMigration("add_products")
|
||||
let qr = waitFor rdb.migrateUp() # всички неприложени
|
||||
let qr = waitFor rdb.migrateDown(1) # връщане
|
||||
|
||||
# Статус
|
||||
let status = waitFor rdb.migrationStatus()
|
||||
if waitFor rdb.isMigrationApplied("add_products"):
|
||||
echo "Вече е приложена"
|
||||
```
|
||||
|
||||
### Prepared Statements
|
||||
|
||||
```nim
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2)
|
||||
|
||||
let results = waitFor stmt.preparedGet(@[
|
||||
WireValue(kind: fkInt32, int32Val: 18),
|
||||
WireValue(kind: fkString, strVal: "active")
|
||||
])
|
||||
|
||||
stmt.flushStmt()
|
||||
rdb.clearStmtCache()
|
||||
```
|
||||
|
||||
### Pagination
|
||||
|
||||
```nim
|
||||
# Offset-базирана
|
||||
let page = waitFor rdb.table("users").paginate(page = 1, perPage = 20)
|
||||
|
||||
# Cursor-базирана (по-бърза за големи таблици)
|
||||
let batch = waitFor rdb.table("users")
|
||||
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Най-добри Практики
|
||||
|
||||
1. **Винаги включвайте DOWN скриптове** — позволява безопасно връщане
|
||||
2. **Първо dry run** — валидирайте преди прилагане в production
|
||||
3. **Партиден импорт** — използвайте `BATCH 1000` за CSV импорт
|
||||
4. **Експортирайте преди миграция** — backup с `EXPORT TO`
|
||||
5. **Проверка след миграция** — `MIGRATION STATUS`
|
||||
6. **Първо таблици без foreign keys** — после тези с foreign keys
|
||||
+3
-1
@@ -689,7 +689,9 @@ SELECT schema_prompt('users') AS result;
|
||||
|----------|----------|
|
||||
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
||||
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE GRAPH, DROP GRAPH |
|
||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE GRAPH, DROP GRAPH, CREATE MIGRATION, APPLY MIGRATION |
|
||||
| Migration | MIGRATION UP, MIGRATION DOWN, MIGRATION STATUS, MIGRATION DRY RUN |
|
||||
| Import/Export | IMPORT FROM, EXPORT TO, FORMAT, CSV, JSON, NDJSON, DELIMITER, HEADER, BATCH |
|
||||
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
||||
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
||||
| CTEs | WITH, RECURSIVE, AS |
|
||||
|
||||
+18
-20
@@ -240,36 +240,34 @@ curl http://localhost:9470/metrics
|
||||
All official clients support connection pooling:
|
||||
|
||||
### JavaScript
|
||||
|
||||
```typescript
|
||||
import { Pool } from 'baradb';
|
||||
|
||||
const pool = new Pool({
|
||||
host: 'localhost',
|
||||
port: 9472,
|
||||
min: 5,
|
||||
max: 50,
|
||||
idleTimeout: 30000,
|
||||
});
|
||||
|
||||
const client = await pool.acquire();
|
||||
try {
|
||||
const result = await client.query('SELECT 1');
|
||||
} finally {
|
||||
pool.release(client);
|
||||
}
|
||||
const pool = new Pool({ host: 'localhost', port: 9472, min: 5, max: 50 });
|
||||
```
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from baradb import Pool
|
||||
|
||||
pool = Pool("localhost", 9472, min_size=5, max_size=50)
|
||||
with pool.connection() as conn:
|
||||
result = conn.query("SELECT 1")
|
||||
```
|
||||
|
||||
## Cross-Database Migration (Nim)
|
||||
|
||||
The Nim allographer client includes a cross-database migration engine:
|
||||
|
||||
```nim
|
||||
import allographer/migrate_data
|
||||
|
||||
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||
echo report # Tables: 12/12, Rows: 45230, Time: 3.2s
|
||||
```
|
||||
|
||||
Supported sources: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB.
|
||||
See [Migrations & Import/Export](migration.md) for details.
|
||||
|
||||
## Data Types Mapping
|
||||
|
||||
| BaraDB Type | JavaScript | Python | Nim | Rust |
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
# Migrations & Data Import/Export
|
||||
|
||||
BaraDB has a built-in migration system via BaraQL. Migrations are fully managed
|
||||
by the server — checksums, locking, rollback, and status tracking. The client
|
||||
sends BaraQL commands and does not maintain its own migration table.
|
||||
|
||||
## BaraQL Migration Syntax
|
||||
|
||||
```sql
|
||||
-- Create a migration with UP and DOWN scripts
|
||||
CREATE MIGRATION add_users_table {
|
||||
UP: CREATE TABLE users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
email VARCHAR(255) UNIQUE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
DOWN: DROP TABLE IF EXISTS users;
|
||||
}
|
||||
|
||||
-- Apply a specific migration
|
||||
APPLY MIGRATION add_users_table
|
||||
|
||||
-- Apply all pending migrations
|
||||
MIGRATION UP
|
||||
|
||||
-- Apply next N migrations
|
||||
MIGRATION UP 3
|
||||
|
||||
-- Rollback last migration
|
||||
MIGRATION DOWN
|
||||
|
||||
-- Rollback last N migrations
|
||||
MIGRATION DOWN 2
|
||||
|
||||
-- View migration status
|
||||
MIGRATION STATUS
|
||||
|
||||
-- Dry run (validate without executing)
|
||||
MIGRATION DRY RUN add_users_table
|
||||
```
|
||||
|
||||
## Migration Locking
|
||||
|
||||
BaraDB acquires a global migration lock before applying any migration. This
|
||||
prevents concurrent migration runs and ensures:
|
||||
|
||||
- Only one migration runs at a time
|
||||
- Checksums are verified before execution
|
||||
- Failed migrations can be rolled back safely
|
||||
- `MIGRATION STATUS` shows exact state of each migration
|
||||
|
||||
## Checksums
|
||||
|
||||
Every migration body is SHA-256 hashed. The server stores the checksum at
|
||||
creation time and verifies it before applying. This prevents accidental
|
||||
modification of already-registered migrations.
|
||||
|
||||
```
|
||||
CREATE MIGRATION add_users → checksum: a3f2b8c1...
|
||||
APPLY MIGRATION add_users → verifies checksum matches → executes
|
||||
```
|
||||
|
||||
## Rollback
|
||||
|
||||
Migrations with `DOWN` scripts support rollback:
|
||||
|
||||
```sql
|
||||
CREATE MIGRATION add_column {
|
||||
UP: ALTER TABLE users ADD COLUMN phone VARCHAR(20);
|
||||
DOWN: ALTER TABLE users DROP COLUMN phone;
|
||||
}
|
||||
```
|
||||
|
||||
When you run `MIGRATION DOWN`, the server executes the DOWN script and marks
|
||||
the migration as rolled back.
|
||||
|
||||
## Dry Run
|
||||
|
||||
Validate a migration before applying:
|
||||
|
||||
```sql
|
||||
MIGRATION DRY RUN add_users_table
|
||||
-- Output:
|
||||
-- DRY RUN add_users_table:
|
||||
-- Statements: 1
|
||||
-- [1] nkCreateTable
|
||||
-- DOWN script: yes
|
||||
-- Checksum: a3f2b8c1d4e5f6a7b8c9d0e1f2a3b4c5
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## IMPORT FROM / EXPORT TO
|
||||
|
||||
BaraDB supports importing and exporting data directly via BaraQL.
|
||||
|
||||
### IMPORT FROM
|
||||
|
||||
```sql
|
||||
-- Import from CSV
|
||||
IMPORT FROM '/data/users.csv' INTO users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
BATCH 1000
|
||||
|
||||
-- Import from JSON array
|
||||
IMPORT FROM '/data/users.json' INTO users
|
||||
FORMAT JSON
|
||||
|
||||
-- Import from NDJSON (newline-delimited JSON)
|
||||
IMPORT FROM '/data/users.ndjson' INTO users
|
||||
FORMAT NDJSON
|
||||
```
|
||||
|
||||
Options:
|
||||
| Option | Values | Default | Description |
|
||||
|--------|--------|---------|-------------|
|
||||
| `FORMAT` | `CSV`, `JSON`, `NDJSON` | `CSV` | Input file format |
|
||||
| `DELIMITER` | any char | `,` | CSV field delimiter |
|
||||
| `HEADER` | `true`/`false` | `true` | CSV first row is header |
|
||||
| `BATCH` | integer | `1000` | Rows per insert batch |
|
||||
|
||||
### EXPORT TO
|
||||
|
||||
```sql
|
||||
-- Export to CSV
|
||||
EXPORT TO '/backup/users.csv' FROM users
|
||||
FORMAT CSV
|
||||
DELIMITER ','
|
||||
HEADER true
|
||||
|
||||
-- Export to JSON
|
||||
EXPORT TO '/backup/users.json' FROM users
|
||||
FORMAT JSON
|
||||
|
||||
-- Export to NDJSON
|
||||
EXPORT TO '/backup/users.ndjson' FROM users
|
||||
FORMAT NDJSON
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cross-Database Migration
|
||||
|
||||
BaraDB's Nim client (nim-allographer) includes a cross-database migration
|
||||
engine. Migrate data from PostgreSQL, MySQL, MariaDB, SQLite, or SurrealDB
|
||||
directly to BaraDB.
|
||||
|
||||
### Supported Sources
|
||||
|
||||
| Database | Schema Extraction | Status |
|
||||
|----------|-------------------|--------|
|
||||
| PostgreSQL | `information_schema` | ✅ |
|
||||
| MySQL | `information_schema` | ✅ |
|
||||
| MariaDB | `information_schema` | ✅ |
|
||||
| SQLite | `sqlite_master` + `PRAGMA` | ✅ |
|
||||
| SurrealDB | `INFO FOR DB` / `INFO FOR TABLE` | ✅ |
|
||||
|
||||
### Nim API
|
||||
|
||||
```nim
|
||||
import allographer/migrate_data
|
||||
|
||||
# Connect to source and target
|
||||
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||
|
||||
# Migrate all tables
|
||||
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||
echo report
|
||||
# Migration: PostgreSQL → BaraDB
|
||||
# Tables: 12/12
|
||||
# Rows: 45230
|
||||
# Time: 3.2s
|
||||
|
||||
# Migrate specific tables
|
||||
let report = waitFor migrate(pg, bdb,
|
||||
tables = @["users", "orders", "products"])
|
||||
```
|
||||
|
||||
### Type Mapping
|
||||
|
||||
The migration engine automatically maps source types to BaraDB equivalents:
|
||||
|
||||
| PostgreSQL | MySQL | SQLite | BaraDB |
|
||||
|------------|-------|--------|--------|
|
||||
| `SERIAL` | `INT AUTO_INCREMENT` | `INTEGER PK` | `SERIAL` |
|
||||
| `VARCHAR(n)` | `VARCHAR(n)` | `TEXT` | `VARCHAR(n)` |
|
||||
| `TEXT` | `TEXT` | `TEXT` | `TEXT` |
|
||||
| `BOOLEAN` | `TINYINT(1)` | `INTEGER` | `BOOLEAN` |
|
||||
| `JSONB` | `JSON` | `TEXT` | `JSON` |
|
||||
| `TIMESTAMP` | `DATETIME` | `TEXT` | `TIMESTAMP` |
|
||||
| `UUID` | `CHAR(36)` | `TEXT` | `UUID` |
|
||||
|
||||
Full type map: 50+ types supported.
|
||||
|
||||
---
|
||||
|
||||
## Nim allographer Client API
|
||||
|
||||
### Migration Management
|
||||
|
||||
```nim
|
||||
import allographer/query_builder/models/baradb/baradb_exec
|
||||
|
||||
# Create migration
|
||||
let qr = waitFor rdb.createMigration("add_products",
|
||||
"CREATE TABLE products (id SERIAL PRIMARY KEY, name VARCHAR(255))",
|
||||
"DROP TABLE IF EXISTS products")
|
||||
|
||||
# Apply
|
||||
let qr = waitFor rdb.applyMigration("add_products")
|
||||
|
||||
# Apply all pending
|
||||
let qr = waitFor rdb.migrateUp()
|
||||
|
||||
# Rollback
|
||||
let qr = waitFor rdb.migrateDown(1)
|
||||
|
||||
# Status
|
||||
let status = waitFor rdb.migrationStatus()
|
||||
|
||||
# Check if applied
|
||||
if waitFor rdb.isMigrationApplied("add_products"):
|
||||
echo "Already applied"
|
||||
|
||||
# Dry run
|
||||
let qr = waitFor rdb.migrationDryRun("add_products")
|
||||
```
|
||||
|
||||
### Prepared Statements
|
||||
|
||||
```nim
|
||||
let stmt = waitFor rdb.prepare(
|
||||
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2)
|
||||
|
||||
let results = waitFor stmt.preparedGet(@[
|
||||
WireValue(kind: fkInt32, int32Val: 18),
|
||||
WireValue(kind: fkString, strVal: "active")
|
||||
])
|
||||
|
||||
stmt.flushStmt()
|
||||
rdb.clearStmtCache()
|
||||
```
|
||||
|
||||
### Pagination
|
||||
|
||||
```nim
|
||||
# Offset-based
|
||||
let page = waitFor rdb.table("users").paginate(page = 1, perPage = 20)
|
||||
|
||||
# Cursor-based (faster for large tables)
|
||||
let batch = waitFor rdb.table("users")
|
||||
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Always include DOWN scripts** — enables safe rollback
|
||||
2. **Use dry run first** — validate migrations before applying in production
|
||||
3. **Batch large imports** — use `BATCH 1000` for CSV imports to avoid memory issues
|
||||
4. **Export before migration** — backup data with `EXPORT TO` before cross-DB migration
|
||||
5. **Check status after migration** — verify with `MIGRATION STATUS`
|
||||
6. **Migrate tables without foreign keys first** — then tables with foreign keys
|
||||
@@ -44,6 +44,7 @@
|
||||
- [Security Guide](en/security.md)
|
||||
- [Monitoring & Observability](en/monitoring.md)
|
||||
- [Backup & Recovery](en/backup.md)
|
||||
- [Migrations & Import/Export](en/migration.md)
|
||||
- [Troubleshooting](en/troubleshooting.md)
|
||||
|
||||
### Advanced
|
||||
|
||||
@@ -31,6 +31,8 @@ type
|
||||
nkMigrationStatus
|
||||
nkMigrationUp
|
||||
nkMigrationDown
|
||||
nkImportFrom
|
||||
nkExportTo
|
||||
nkMigrationDryRun
|
||||
nkCreateUser
|
||||
nkDropUser
|
||||
@@ -363,6 +365,19 @@ type
|
||||
mdCount*: int # 0 = 1, N = rollback N migrations
|
||||
of nkMigrationDryRun:
|
||||
mdrName*: string # migration name to dry-run
|
||||
of nkImportFrom:
|
||||
impPath*: string
|
||||
impTable*: string
|
||||
impFormat*: string # "csv", "json", "ndjson"
|
||||
impDelimiter*: char
|
||||
impHasHeader*: bool
|
||||
impBatchSize*: int
|
||||
of nkExportTo:
|
||||
expPath*: string
|
||||
expTable*: string
|
||||
expFormat*: string
|
||||
expDelimiter*: char
|
||||
expIncludeHeader*: bool
|
||||
of nkCreateIndex:
|
||||
ciTarget*: string
|
||||
ciName*: string
|
||||
|
||||
@@ -23,6 +23,7 @@ import ../storage/btree
|
||||
import ../storage/wal
|
||||
import ../core/mvcc
|
||||
import ../core/tracing
|
||||
import ../client/fileops
|
||||
import ../fts/engine as fts
|
||||
import ../core/registry
|
||||
|
||||
@@ -219,6 +220,24 @@ proc sqlEscapeString*(s: string): string =
|
||||
## Escape SQL string literals by doubling single-quotes.
|
||||
result = s.replace("'", "''")
|
||||
|
||||
proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
|
||||
## Build a multi-row INSERT statement for bulk import.
|
||||
result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
|
||||
for i, col in columns:
|
||||
if i > 0: result &= ", "
|
||||
result &= "\"" & sqlEscapeIdent(col) & "\""
|
||||
result &= ") VALUES "
|
||||
for ri, row in rows:
|
||||
if ri > 0: result &= ", "
|
||||
result &= "("
|
||||
for ci, val in row:
|
||||
if ci > 0: result &= ", "
|
||||
if val.len == 0 or val == "\\N":
|
||||
result &= "NULL"
|
||||
else:
|
||||
result &= "'" & sqlEscapeString(val) & "'"
|
||||
result &= ")"
|
||||
|
||||
proc exprToSql(node: Node): string =
|
||||
if node == nil:
|
||||
return ""
|
||||
@@ -5279,6 +5298,87 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
|
||||
|
||||
return okResult(msg="Rolled back " & $rolledBackCount & " migrations")
|
||||
|
||||
of nkImportFrom:
|
||||
let path = stmt.impPath
|
||||
let table = stmt.impTable
|
||||
let format = stmt.impFormat
|
||||
if not fileExists(path):
|
||||
return errResult("File not found: " & path)
|
||||
let content = readFile(path)
|
||||
var columns: seq[string] = @[]
|
||||
var rows: seq[seq[string]] = @[]
|
||||
case format
|
||||
of "csv":
|
||||
(columns, rows) = parseCsvTable(content, stmt.impDelimiter, stmt.impHasHeader)
|
||||
of "json":
|
||||
(columns, rows) = parseJsonTable(content)
|
||||
of "ndjson":
|
||||
(columns, rows) = parseNdjsonTable(content)
|
||||
else:
|
||||
return errResult("Unsupported import format: " & format)
|
||||
if columns.len == 0:
|
||||
return errResult("No columns found in import file")
|
||||
var inserted = 0
|
||||
let batchSize = stmt.impBatchSize
|
||||
var batchRows: seq[seq[string]] = @[]
|
||||
for row in rows:
|
||||
batchRows.add(row)
|
||||
if batchRows.len >= batchSize:
|
||||
# Generate INSERT INTO t (c1,c2) VALUES (...),(...) and execute
|
||||
let sql = buildInsertSql(table, columns, batchRows)
|
||||
let tokens = qlex.tokenize(sql)
|
||||
let astNode = qpar.parse(tokens)
|
||||
if astNode.stmts.len > 0:
|
||||
let insResult = executeQueryImpl(ctx, astNode)
|
||||
if insResult.success:
|
||||
inserted += batchRows.len
|
||||
batchRows.setLen(0)
|
||||
if batchRows.len > 0:
|
||||
let sql = buildInsertSql(table, columns, batchRows)
|
||||
let tokens = qlex.tokenize(sql)
|
||||
let astNode = qpar.parse(tokens)
|
||||
if astNode.stmts.len > 0:
|
||||
let insResult = executeQueryImpl(ctx, astNode)
|
||||
if insResult.success:
|
||||
inserted += batchRows.len
|
||||
return okResult(msg="IMPORTED " & $inserted & " rows into " & table)
|
||||
|
||||
of nkExportTo:
|
||||
let path = stmt.expPath
|
||||
let table = stmt.expTable
|
||||
let format = stmt.expFormat
|
||||
var rows: seq[Row] = @[]
|
||||
var cols: seq[string] = @[]
|
||||
let scanResult = execScan(ctx, table)
|
||||
if scanResult.len == 0:
|
||||
let tbl = ctx.getTableDef(table)
|
||||
if tbl.columns.len == 0:
|
||||
return errResult("Table not found: " & table)
|
||||
for col in tbl.columns:
|
||||
cols.add(col.name)
|
||||
else:
|
||||
for k, v in scanResult[0].pairs:
|
||||
cols.add(k)
|
||||
rows = scanResult
|
||||
var content = ""
|
||||
var strRows: seq[seq[string]] = @[]
|
||||
for row in rows:
|
||||
var strRow: seq[string] = @[]
|
||||
for col in cols:
|
||||
strRow.add(if col in row: valueToString(row[col]) else: "")
|
||||
strRows.add(strRow)
|
||||
case format
|
||||
of "csv":
|
||||
content = toCsv(cols, strRows, stmt.expDelimiter, stmt.expIncludeHeader)
|
||||
of "json":
|
||||
content = toJson(cols, strRows)
|
||||
of "ndjson":
|
||||
content = toNdjson(cols, strRows)
|
||||
else:
|
||||
return errResult("Unsupported export format: " & format)
|
||||
writeFile(path, content)
|
||||
return okResult(msg="EXPORTED " & $strRows.len & " rows to " & path)
|
||||
|
||||
of nkMigrationDryRun:
|
||||
let (found, upBody, downBody) = getMigrationBody(ctx, stmt.mdrName)
|
||||
if not found:
|
||||
|
||||
@@ -102,6 +102,15 @@ type
|
||||
tkAfter
|
||||
tkInstead
|
||||
tkOf
|
||||
tkImport
|
||||
tkExport
|
||||
tkFormat
|
||||
tkDelimiter
|
||||
tkHeader
|
||||
tkBatch
|
||||
tkCsv
|
||||
tkJsonFmt
|
||||
tkNdjson
|
||||
tkMigration
|
||||
tkApply
|
||||
tkStatus
|
||||
@@ -334,6 +343,15 @@ const keywords*: Table[string, TokenKind] = {
|
||||
"after": tkAfter,
|
||||
"instead": tkInstead,
|
||||
"of": tkOf,
|
||||
"import": tkImport,
|
||||
"export": tkExport,
|
||||
"format": tkFormat,
|
||||
"delimiter": tkDelimiter,
|
||||
"header": tkHeader,
|
||||
"batch": tkBatch,
|
||||
"csv": tkCsv,
|
||||
"json": tkJsonFmt,
|
||||
"ndjson": tkNdjson,
|
||||
"migration": tkMigration,
|
||||
"apply": tkApply,
|
||||
"status": tkStatus,
|
||||
|
||||
@@ -1494,6 +1494,63 @@ proc parseMigrationDryRun(p: var Parser): Node =
|
||||
let name = p.expect(tkIdent).value
|
||||
result = Node(kind: nkMigrationDryRun, mdrName: name, line: tok.line, col: tok.col)
|
||||
|
||||
proc parseImportFrom(p: var Parser): Node =
|
||||
let tok = p.expect(tkImport)
|
||||
discard p.expect(tkFrom)
|
||||
let path = p.expect(tkStringLit).value
|
||||
discard p.expect(tkInto)
|
||||
let table = p.expect(tkIdent).value
|
||||
var format = "csv"
|
||||
var delimiter = ','
|
||||
var hasHeader = true
|
||||
var batchSize = 1000
|
||||
while p.peek().kind in {tkFormat, tkDelimiter, tkHeader, tkBatch}:
|
||||
let kw = p.advance()
|
||||
case kw.kind
|
||||
of tkFormat:
|
||||
let fmt = p.expect(tkIdent).value.toLower()
|
||||
format = fmt
|
||||
of tkDelimiter:
|
||||
let delim = p.expect(tkStringLit).value
|
||||
if delim.len > 0: delimiter = delim[0]
|
||||
of tkHeader:
|
||||
let hdr = p.expect(tkIdent).value.toLower()
|
||||
hasHeader = hdr == "true" or hdr == "yes"
|
||||
of tkBatch:
|
||||
batchSize = parseInt(p.expect(tkIntLit).value)
|
||||
else: discard
|
||||
result = Node(kind: nkImportFrom, impPath: path, impTable: table,
|
||||
impFormat: format, impDelimiter: delimiter,
|
||||
impHasHeader: hasHeader, impBatchSize: batchSize,
|
||||
line: tok.line, col: tok.col)
|
||||
|
||||
proc parseExportTo(p: var Parser): Node =
|
||||
let tok = p.expect(tkExport)
|
||||
discard p.expect(tkTo)
|
||||
let path = p.expect(tkStringLit).value
|
||||
discard p.expect(tkFrom)
|
||||
let table = p.expect(tkIdent).value
|
||||
var format = "csv"
|
||||
var delimiter = ','
|
||||
var includeHeader = true
|
||||
while p.peek().kind in {tkFormat, tkDelimiter, tkHeader}:
|
||||
let kw = p.advance()
|
||||
case kw.kind
|
||||
of tkFormat:
|
||||
let fmt = p.expect(tkIdent).value.toLower()
|
||||
format = fmt
|
||||
of tkDelimiter:
|
||||
let delim = p.expect(tkStringLit).value
|
||||
if delim.len > 0: delimiter = delim[0]
|
||||
of tkHeader:
|
||||
let hdr = p.expect(tkIdent).value.toLower()
|
||||
includeHeader = hdr == "true" or hdr == "yes"
|
||||
else: discard
|
||||
result = Node(kind: nkExportTo, expPath: path, expTable: table,
|
||||
expFormat: format, expDelimiter: delimiter,
|
||||
expIncludeHeader: includeHeader,
|
||||
line: tok.line, col: tok.col)
|
||||
|
||||
proc parseCreateUser(p: var Parser): Node =
|
||||
let tok = p.expect(tkCreate)
|
||||
discard p.expect(tkUser)
|
||||
@@ -1701,6 +1758,8 @@ proc parseStatement*(p: var Parser): Node =
|
||||
case p.peek().kind
|
||||
of tkWith, tkSelect: p.parseSelect()
|
||||
of tkInsert: p.parseInsert()
|
||||
of tkImport: p.parseImportFrom()
|
||||
of tkExport: p.parseExportTo()
|
||||
of tkUpdate: p.parseUpdate()
|
||||
of tkDelete: p.parseDelete()
|
||||
of tkMerge: p.parseMerge()
|
||||
|
||||
Reference in New Issue
Block a user