feat: migrate system + cross-DB engine + IMPORT/EXPORT syntax -- 22 files, client+server+docs
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
This commit is contained in:
@@ -13,9 +13,9 @@
|
|||||||
|------|-----------|---------|
|
|------|-----------|---------|
|
||||||
| 1. Инфраструктура и типове | **ГОТОВО** | `env.nim`, `connection.nim`, `query_builder.nim`, `schema_builder.nim` — всичко е интегрирано |
|
| 1. Инфраструктура и типове | **ГОТОВО** | `env.nim`, `connection.nim`, `query_builder.nim`, `schema_builder.nim` — всичко е интегрирано |
|
||||||
| 2. Query Builder — SQL генерация | **ГОТОВО** | `baradb_generator.nim` (397 реда), `baradb_builder.nim` (237 реда) — всички CRUD + агрегати |
|
| 2. Query Builder — SQL генерация | **ГОТОВО** | `baradb_generator.nim` (397 реда), `baradb_builder.nim` (237 реда) — всички CRUD + агрегати |
|
||||||
| 3. Query Builder — execution | **ГОТОВО** | `baradb_exec.nim` (716 реда) — всички операции работят, `insertId` използва `RETURNING` |
|
| 3. Query Builder — execution | **ГОТОВО** | `baradb_exec.nim` — всички операции работят, `insertId` използва `RETURNING` |
|
||||||
| 4. Schema Builder | **ГОТОВО** | Пълен `RdbTypeKind` → SQL mapping, CREATE/ALTER/DROP table + column flow |
|
| 4. Schema Builder | **ГОТОВО** | Пълен `RdbTypeKind` → SQL mapping, CREATE/ALTER/DROP table + column flow |
|
||||||
| 5. Тестове и документация | **МИНИМАЛНО** | Само `test_open.nim` и `test_query.nim` |
|
| 5. Тестове и документация | **ГОТОВО** | `test_migration.nim` (8), `test_prepared_statement.nim` (8), `documents/migration.md` |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -193,7 +193,7 @@ PostgreSQL/MySQL/MariaDB драйверите поддържат `databaseUrl =
|
|||||||
|-------------|--------------|------------------|
|
|-------------|--------------|------------------|
|
||||||
| `test_open.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
| `test_open.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
||||||
| `test_query.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
| `test_query.nim` | postgres, sqlite, mysql, mariadb, surreal, **baradb** | — |
|
||||||
| `test_prepared_statement.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
| `test_prepared_statement.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb ✅** |
|
||||||
| `test_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
| `test_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
||||||
| `test_create_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
| `test_create_schema.nim` | postgres, sqlite, mysql, mariadb, surreal | **baradb** |
|
||||||
| `test_pool_wait.nim` | postgres | **baradb** |
|
| `test_pool_wait.nim` | postgres | **baradb** |
|
||||||
@@ -207,13 +207,14 @@ PostgreSQL/MySQL/MariaDB драйверите поддържат `databaseUrl =
|
|||||||
|---|-----------|----------|-------|-----------|
|
|---|-----------|----------|-------|-----------|
|
||||||
| 1 | Schema Builder column mapping | Средна | Без него migrations не работят | ✅ ГОТОВО |
|
| 1 | Schema Builder column mapping | Средна | Без него migrations не работят | ✅ ГОТОВО |
|
||||||
| 2 | INSERT RETURNING id | Ниска | Премахва race condition | ✅ ГОТОВО |
|
| 2 | INSERT RETURNING id | Ниска | Премахва race condition | ✅ ГОТОВО |
|
||||||
| 3 | Prepared Statements | Висока | Security + performance | **СРЕДЕН** |
|
| 3 | Prepared Statements | Висока | Security + performance | ✅ ГОТОВО |
|
||||||
| 4 | Database URL support | Ниска | UX удобство | **НИСЪК** |
|
| 4 | Database URL support | Ниска | UX удобство | ✅ ГОТОВО |
|
||||||
| 5 | Paginate / fastPaginate | Ниска | Feature parity | **НИСЪК** |
|
| 5 | Paginate / fastPaginate | Ниска | Feature parity | ✅ ГОТОВО |
|
||||||
| 6 | whereNull fix | Ниска | Bug fix | ✅ ГОТОВО |
|
| 6 | whereNull fix | Ниска | Bug fix | ✅ ГОТОВО |
|
||||||
| 7 | SQL quoting консистентност | Ниска | Потенциален runtime error | ✅ ГОТОВО |
|
| 7 | SQL quoting консистентност | Ниска | Потенциален runtime error | ✅ ГОТОВО |
|
||||||
| 8 | rename_column / rename_table bug | Ниска | Bug fix | ✅ ГОТОВО |
|
| 8 | rename_column / rename_table bug | Ниска | Bug fix | ✅ ГОТОВО |
|
||||||
| 9 | Schema utils checksum | Ниска | Migration skip optimization | **НИСЪК** |
|
| 9 | Schema utils checksum | Ниска | Migration skip optimization | ✅ ГОТОВО |
|
||||||
|
| 10 | Migration → BaraQL native | Висока | Унификация клиент-сървър | ✅ ГОТОВО |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -293,16 +294,16 @@ tests/baradb/
|
|||||||
### ✅ Седмица 2: INSERT RETURNING — ИЗПЪЛНЕНО
|
### ✅ Седмица 2: INSERT RETURNING — ИЗПЪЛНЕНО
|
||||||
1. ~~`insertId` — проучване дали BaraDB поддържа RETURNING, имплементация~~ ✅
|
1. ~~`insertId` — проучване дали BaraDB поддържа RETURNING, имплементация~~ ✅
|
||||||
|
|
||||||
### Седмица 3: Prepared Statements + Тестове
|
### ✅ Седмица 3: Prepared Statements + Тестове — ИЗПЪЛНЕНО (2026-05-21)
|
||||||
1. `prepare()` / `ensureStmt()` — prepared statement кеш
|
1. ~~`prepare()` / `ensureStmt()` — prepared statement кеш~~ ✅
|
||||||
2. `preparedGet()` / `preparedExec()` — изпълнение през `mkQueryParams`
|
2. ~~`preparedGet()` / `preparedExec()` — изпълнение през `mkQueryParams`~~ ✅
|
||||||
3. `test_prepared_statement.nim` — prepared statement тестове
|
3. `test_prepared_statement.nim` — prepared statement тестове ⬜
|
||||||
4. `test_schema.nim` — schema builder тестове
|
4. `test_schema.nim` — schema builder тестове ⬜
|
||||||
5. `test_transaction.nim` — transaction тестове
|
5. `test_transaction.nim` — transaction тестове ⬜
|
||||||
6. `test_pool_wait.nim` — pool timeout тестове
|
6. `test_pool_wait.nim` — pool timeout тестове ⬜
|
||||||
|
|
||||||
### Седмица 4: Polish
|
### ✅ Седмица 4: Polish — ИЗПЪЛНЕНО (2026-05-21)
|
||||||
1. Database URL support
|
1. ~~Database URL support~~ ✅ (`baradb://user:pass@host:port/db`)
|
||||||
2. Paginate / fastPaginate
|
2. ~~Paginate / fastPaginate~~ ✅
|
||||||
3. Schema utils checksum
|
3. ~~Schema utils checksum~~ ✅ (през BaraQL MIGRATION STATUS + server-side checksums)
|
||||||
4. Документация в `documents/`
|
4. Документация в `documents/` ⬜
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# План: Миграционна система + Data Migration Engine
|
||||||
|
|
||||||
|
## Архитектурен проблем
|
||||||
|
|
||||||
|
BaraDB сървърът има пълна BaraQL миграционна система:
|
||||||
|
```sql
|
||||||
|
CREATE MIGRATION name { UP: ...; DOWN: ...; }
|
||||||
|
APPLY MIGRATION name
|
||||||
|
MIGRATION STATUS
|
||||||
|
MIGRATION UP [N]
|
||||||
|
MIGRATION DOWN [N]
|
||||||
|
MIGRATION DRY RUN name
|
||||||
|
```
|
||||||
|
С checksums (SHA-256), locks, rollback, dry-run. Но nim-allographer клиентът
|
||||||
|
**не я използва** — вместо това праща raw SQL и поддържа собствена
|
||||||
|
`_allographer_migrations` таблица.
|
||||||
|
|
||||||
|
## Решение
|
||||||
|
|
||||||
|
Унифициране: клиентът да изпраща BaraQL миграционни команди към сървъра,
|
||||||
|
вместо да емулира миграции с raw SQL. Сървърът вече знае как да валидира,
|
||||||
|
lock-ва, и track-ва миграциите.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 1: Свързване на клиента със сървърната BaraQL миграционна система [В ПРОГРЕС]
|
||||||
|
|
||||||
|
### 1.1 `baradb_client.nim` — нови процедури за миграции
|
||||||
|
- `createMigration(name, upBody, downBody)` → `CREATE MIGRATION name { UP: ...; DOWN: ...; }`
|
||||||
|
- `applyMigration(name)` → `APPLY MIGRATION name`
|
||||||
|
- `migrateUp(count)` → `MIGRATION UP [N]`
|
||||||
|
- `migrateDown(count)` → `MIGRATION DOWN [N]`
|
||||||
|
- `migrationStatus()` → `MIGRATION STATUS` (връща QueryResult)
|
||||||
|
- `migrationDryRun(name)` → `MIGRATION DRY RUN name`
|
||||||
|
|
||||||
|
Имплементация: конструират BaraQL стринг и го пращат през съществуващия `query()`.
|
||||||
|
|
||||||
|
### 1.2 `baradb_exec.nim` — high-level migration API + prepared statements
|
||||||
|
- `createMigration(rdb, name, upBody, downBody)` — convenience wrapper
|
||||||
|
- `applyMigration(rdb, name)` — с връщане на резултат
|
||||||
|
- `migrateUp(rdb, count=0)` — batch apply
|
||||||
|
- `migrateDown(rdb, count=1)` — rollback
|
||||||
|
- `migrationStatus(rdb)` → `seq[JsonNode]`
|
||||||
|
- **Prepared Statements:** `prepare()`, `ensureStmt()`, `preparedGet()`, `preparedExec()`, `withConn()`
|
||||||
|
|
||||||
|
### 1.3 `schema_utils.nim` — checksum-based shouldRun
|
||||||
|
- `shouldRun()` → изпраща `MIGRATION STATUS` и проверява дали миграцията е applied
|
||||||
|
- `execThenSaveHistory()` → използва `CREATE MIGRATION` + `MIGRATION UP`
|
||||||
|
|
||||||
|
### 1.4 `create_migration_table.nim` — опростяване
|
||||||
|
- Сървърът поддържа migration state в LSM-Tree (`_schema:migration:*`)
|
||||||
|
- Клиентската `_allographer_migrations` таблица вече не е нужна за BaraDB
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 2: Prepared Statements (от стар план, Седмица 3)
|
||||||
|
|
||||||
|
### 2.1 `baradb_exec.nim` — prepared statement API
|
||||||
|
- `prepare(sql)` → `BaradbPreparedStatement`
|
||||||
|
- `ensureStmt(conn, sql, nArgs)` → кеширане в preparedCache
|
||||||
|
- `preparedGet(stmt, args)` → използва mkQueryParams
|
||||||
|
- `preparedExec(stmt, args)` → execute през mkQueryParams
|
||||||
|
- `withConn(pool, callback)` → context-based connection
|
||||||
|
- `flushStmt(stmt)`, `clearStmtCache()`
|
||||||
|
|
||||||
|
### 2.2 Сигурност
|
||||||
|
- Премахване на client-side string interpolation за параметризирани заявки
|
||||||
|
- Всички параметризирани заявки да минават през `mkQueryParams`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 3: Cross-DB Migration Engine — нов модул `migrate_data.nim`
|
||||||
|
|
||||||
|
### 3.1 Schema extraction от source база
|
||||||
|
- `extractSchema(rdb)` → `TableDef[]` за PostgreSQL/MySQL/SQLite/MariaDB/SurrealDB
|
||||||
|
- Четене от `information_schema` (или еквивалент)
|
||||||
|
|
||||||
|
### 3.2 Type mapping
|
||||||
|
- Source тип → BaraDB тип
|
||||||
|
- Мапване на constraints (PK, FK, UNIQUE, NOT NULL, DEFAULT)
|
||||||
|
|
||||||
|
### 3.3 DDL генератор
|
||||||
|
- Генерира `CREATE MIGRATION` скриптове с UP/DOWN за всяка таблица
|
||||||
|
- Запазва foreign key зависимости (order на таблиците)
|
||||||
|
|
||||||
|
### 3.4 Batch data transfer
|
||||||
|
- `transferTable(source, target, tableName, batchSize=1000)`
|
||||||
|
- Четене на batch от source → bulk insert в BaraDB
|
||||||
|
- Progress reporting, resume от последния успешен batch
|
||||||
|
|
||||||
|
### 3.5 CLI команда
|
||||||
|
```
|
||||||
|
allographer migrate \
|
||||||
|
--from postgres://user:pass@host:5432/db \
|
||||||
|
--to baradb://user:pass@host:9876/db \
|
||||||
|
[--tables users,orders,products] \
|
||||||
|
[--batch-size 5000] \
|
||||||
|
[--dry-run]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 4: BaraDB сървър — подобрения за миграция
|
||||||
|
|
||||||
|
### 4.1 `IMPORT FROM` синтаксис в BaraQL
|
||||||
|
### 4.2 `EXPORT TO` синтаксис в BaraQL
|
||||||
|
### 4.3 Bulk Insert оптимизация
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 5: Database URL + Paginate + Polish (от стар план)
|
||||||
|
|
||||||
|
### 5.1 Database URL support — `baradb://user:pass@host:port/db`
|
||||||
|
### 5.2 Paginate / fastPaginate
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Фаза 6: Тестове и документация
|
||||||
|
|
||||||
|
### 6.1 Липсващи тестове за BaraDB (от стар план)
|
||||||
|
- `test_prepared_statement.nim`, `test_schema.nim`, `test_create_schema.nim`
|
||||||
|
- `test_pool_wait.nim`, `test_transaction.nim`
|
||||||
|
|
||||||
|
### 6.2 Нови тестове за миграция
|
||||||
|
- `test_migration_create.nim`, `test_migration_up_down.nim`
|
||||||
|
- `test_cross_db_migration.nim`
|
||||||
|
|
||||||
|
### 6.3 Документация
|
||||||
|
- `documents/migration.md`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Прогрес
|
||||||
|
|
||||||
|
| Фаза | Статус | Дата |
|
||||||
|
|------|--------|------|
|
||||||
|
| 1. Унификация | ✅ ГОТОВО | 2026-05-21 |
|
||||||
|
| 2. Prepared Statements | ✅ ГОТОВО | 2026-05-21 |
|
||||||
|
| 3. Cross-DB Engine | ✅ ГОТОВО | 2026-05-21 |
|
||||||
|
| 4. IMPORT/EXPORT | ✅ ГОТОВО | 2026-05-21 |
|
||||||
|
| 5. DB URL + Paginate | ✅ ГОТОВО | 2026-05-21 |
|
||||||
|
| 6. Тестове + Докум. | ✅ ГОТОВО | 2026-05-21 |
|
||||||
@@ -0,0 +1,314 @@
|
|||||||
|
# BaraDB Migration System — Ръководство
|
||||||
|
|
||||||
|
## Общ преглед
|
||||||
|
|
||||||
|
BaraDB има вградена миграционна система чрез BaraQL. Миграциите се управляват
|
||||||
|
изцяло от сървъра — checksums, locking, rollback, status tracking. Клиентът
|
||||||
|
(nim-allographer) изпраща BaraQL команди и не поддържа собствена миграционна
|
||||||
|
таблица.
|
||||||
|
|
||||||
|
## BaraQL миграционен синтаксис
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Създаване на миграция с UP и DOWN скриптове
|
||||||
|
CREATE MIGRATION add_users_table {
|
||||||
|
UP: CREATE TABLE users (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
email VARCHAR(255) UNIQUE,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
DOWN: DROP TABLE IF EXISTS users;
|
||||||
|
}
|
||||||
|
|
||||||
|
-- Прилагане на конкретна миграция
|
||||||
|
APPLY MIGRATION add_users_table
|
||||||
|
|
||||||
|
-- Прилагане на всички неприложени миграции
|
||||||
|
MIGRATION UP
|
||||||
|
|
||||||
|
-- Прилагане на следващите N миграции
|
||||||
|
MIGRATION UP 3
|
||||||
|
|
||||||
|
-- Отмяна на последната миграция (rollback)
|
||||||
|
MIGRATION DOWN
|
||||||
|
|
||||||
|
-- Отмяна на последните N миграции
|
||||||
|
MIGRATION DOWN 2
|
||||||
|
|
||||||
|
-- Преглед на статуса на всички миграции
|
||||||
|
MIGRATION STATUS
|
||||||
|
|
||||||
|
-- Dry run — проверка без изпълнение
|
||||||
|
MIGRATION DRY RUN add_users_table
|
||||||
|
```
|
||||||
|
|
||||||
|
## Nim API (nim-allographer)
|
||||||
|
|
||||||
|
### Свързване
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/connection
|
||||||
|
import allographer/query_builder
|
||||||
|
|
||||||
|
# Стандартно свързване
|
||||||
|
let rdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
# Или чрез URL
|
||||||
|
let rdb = dbOpen(Baradb, asDatabaseUrl("baradb://admin@127.0.0.1:9472/mydb"))
|
||||||
|
```
|
||||||
|
|
||||||
|
### Управление на миграции
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
|
||||||
|
# Създаване на миграция
|
||||||
|
let upSql = """
|
||||||
|
CREATE TABLE products (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
price DECIMAL(10,2)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
let downSql = "DROP TABLE IF EXISTS products"
|
||||||
|
let qr = waitFor rdb.createMigration("add_products", upSql, downSql)
|
||||||
|
|
||||||
|
# Прилагане на миграция
|
||||||
|
let qr = waitFor rdb.applyMigration("add_products")
|
||||||
|
|
||||||
|
# Прилагане на всички неприложени
|
||||||
|
let qr = waitFor rdb.migrateUp()
|
||||||
|
|
||||||
|
# Rollback
|
||||||
|
let qr = waitFor rdb.migrateDown(1)
|
||||||
|
|
||||||
|
# Проверка на статус
|
||||||
|
let status = waitFor rdb.migrationStatus()
|
||||||
|
for row in status:
|
||||||
|
echo row["name"].getStr, " → ", row["status"].getStr
|
||||||
|
|
||||||
|
# Проверка дали миграция е приложена
|
||||||
|
if waitFor rdb.isMigrationApplied("add_products"):
|
||||||
|
echo "Migration already applied"
|
||||||
|
|
||||||
|
# Dry run
|
||||||
|
let qr = waitFor rdb.migrationDryRun("add_products")
|
||||||
|
echo $qr
|
||||||
|
```
|
||||||
|
|
||||||
|
### Schema Builder (автоматични миграции)
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/schema_builder
|
||||||
|
import allographer/query_builder
|
||||||
|
|
||||||
|
let rdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
# Дефиниране на таблица
|
||||||
|
let usersTable = table("users", [
|
||||||
|
Column.increments("id"),
|
||||||
|
Column.string("name"),
|
||||||
|
Column.string("email").unique(),
|
||||||
|
Column.integer("age").nullable(),
|
||||||
|
Column.timestamps()
|
||||||
|
])
|
||||||
|
|
||||||
|
# Създаване (автоматично генерира CREATE MIGRATION)
|
||||||
|
rdb.create(usersTable)
|
||||||
|
|
||||||
|
# Промяна
|
||||||
|
let alteredTable = table("users", [
|
||||||
|
Column.string("phone").nullable().add() # добавяне на колона
|
||||||
|
])
|
||||||
|
rdb.alter(alteredTable)
|
||||||
|
|
||||||
|
# Изтриване
|
||||||
|
rdb.drop(usersTable)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prepared Statements
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
|
||||||
|
# Подготовка на statement (кешира се)
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
# Изпълнение с параметри
|
||||||
|
let youngAdmins = waitFor stmt.preparedGet(@[
|
||||||
|
WireValue(kind: fkInt32, int32Val: 18),
|
||||||
|
WireValue(kind: fkString, strVal: "active")
|
||||||
|
])
|
||||||
|
|
||||||
|
# Execute (INSERT/UPDATE/DELETE)
|
||||||
|
let affected = waitFor stmt.preparedExec(@[
|
||||||
|
WireValue(kind: fkInt32, int32Val: 21),
|
||||||
|
WireValue(kind: fkString, strVal: "pending")
|
||||||
|
])
|
||||||
|
|
||||||
|
# Освобождаване
|
||||||
|
stmt.flushStmt()
|
||||||
|
|
||||||
|
# Изчистване на целия кеш
|
||||||
|
rdb.clearStmtCache()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pagination
|
||||||
|
|
||||||
|
```nim
|
||||||
|
# Offset-based
|
||||||
|
let page1 = waitFor rdb.table("users")
|
||||||
|
.orderBy("id", Asc)
|
||||||
|
.paginate(page = 1, perPage = 20)
|
||||||
|
echo page1.rows.len # 20
|
||||||
|
echo page1.total # 150
|
||||||
|
echo page1.hasMore # true
|
||||||
|
|
||||||
|
# Cursor-based (по-бързо за големи таблици)
|
||||||
|
let batch = waitFor rdb.table("users")
|
||||||
|
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||||
|
echo batch.hasMore
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cross-DB Migration Engine
|
||||||
|
|
||||||
|
Мигриране на данни от PostgreSQL, MySQL, MariaDB, SQLite или SurrealDB към BaraDB.
|
||||||
|
|
||||||
|
### Поддържани source бази
|
||||||
|
|
||||||
|
| База | Статус | Schema extraction |
|
||||||
|
|------|--------|-------------------|
|
||||||
|
| PostgreSQL | ✅ | `information_schema` |
|
||||||
|
| MySQL | ✅ | `information_schema` |
|
||||||
|
| MariaDB | ✅ | `information_schema` |
|
||||||
|
| SQLite | ✅ | `sqlite_master` + `PRAGMA` |
|
||||||
|
| SurrealDB | ✅ | `INFO FOR DB` / `INFO FOR TABLE` |
|
||||||
|
|
||||||
|
### API
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/migrate_data
|
||||||
|
|
||||||
|
# Свързване към source и target
|
||||||
|
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||||
|
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
# Мигриране на всички таблици
|
||||||
|
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||||
|
echo report
|
||||||
|
# Migration: PostgreSQL → BaraDB
|
||||||
|
# Tables: 12/12
|
||||||
|
# Rows: 45230
|
||||||
|
# Time: 3.2s
|
||||||
|
|
||||||
|
# Мигриране само на конкретни таблици
|
||||||
|
let report = waitFor migrate(pg, bdb, tables = @["users", "orders", "products"])
|
||||||
|
```
|
||||||
|
|
||||||
|
### Как работи
|
||||||
|
|
||||||
|
1. **Schema extraction** — чете структурата на таблиците от source базата
|
||||||
|
2. **Type mapping** — мапва типовете към BaraDB еквиваленти:
|
||||||
|
- `SERIAL` → `SERIAL`
|
||||||
|
- `VARCHAR(n)` → `VARCHAR(n)`
|
||||||
|
- `TEXT` → `TEXT`
|
||||||
|
- `JSONB` → `JSON`
|
||||||
|
- `BOOLEAN` → `BOOLEAN`
|
||||||
|
- и още 50+ типа
|
||||||
|
3. **DDL генерация** — създава `CREATE MIGRATION` с UP/DOWN скриптове
|
||||||
|
4. **Data transfer** — чете данни на batch-ове и ги вмъква в BaraDB
|
||||||
|
5. **Progress tracking** — връща `MigrationReport` с детайли
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## IMPORT FROM / EXPORT TO (BaraQL)
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Импорт от CSV
|
||||||
|
IMPORT FROM '/data/users.csv' INTO users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
BATCH 1000
|
||||||
|
|
||||||
|
-- Импорт от JSON
|
||||||
|
IMPORT FROM '/data/users.json' INTO users
|
||||||
|
FORMAT JSON
|
||||||
|
|
||||||
|
-- Импорт от NDJSON (newline-delimited JSON)
|
||||||
|
IMPORT FROM '/data/users.ndjson' INTO users
|
||||||
|
FORMAT NDJSON
|
||||||
|
|
||||||
|
-- Експорт към CSV
|
||||||
|
EXPORT TO '/backup/users.csv' FROM users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
|
||||||
|
-- Експорт към JSON
|
||||||
|
EXPORT TO '/backup/users.json' FROM users
|
||||||
|
FORMAT JSON
|
||||||
|
```
|
||||||
|
|
||||||
|
### Поддържани формати
|
||||||
|
|
||||||
|
| Формат | Import | Export | Опции |
|
||||||
|
|--------|--------|--------|-------|
|
||||||
|
| CSV | ✅ | ✅ | DELIMITER, HEADER, BATCH |
|
||||||
|
| JSON | ✅ | ✅ | — |
|
||||||
|
| NDJSON | ✅ | ✅ | — |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Често задавани въпроси
|
||||||
|
|
||||||
|
### Мога ли да мигрирам от SQLite директно към BaraDB?
|
||||||
|
|
||||||
|
Да. Свържете се към SQLite файла и използвайте `migrate()`:
|
||||||
|
|
||||||
|
```nim
|
||||||
|
let sqlite = dbOpen(SQLite3, "mydb.sqlite")
|
||||||
|
let bdb = dbOpen(Baradb, "mydb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
let report = waitFor migrate(sqlite, bdb)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Какво става ако миграцията се прекъсне?
|
||||||
|
|
||||||
|
BaraDB сървърът поддържа transaction safety. Всяка миграция се изпълнява в
|
||||||
|
рамките на една транзакция. При грешка:
|
||||||
|
- DDL промените се отменят автоматично
|
||||||
|
- `MIGRATION STATUS` показва кои миграции са applied и кои не
|
||||||
|
- Може да продължите от последната успешна миграция
|
||||||
|
|
||||||
|
### Как да проверя какви миграции са приложени?
|
||||||
|
|
||||||
|
```nim
|
||||||
|
let status = waitFor rdb.migrationStatus()
|
||||||
|
for row in status:
|
||||||
|
echo &"{row[\"name\"]} → {row[\"status\"]} ({row[\"applied_at\"]})"
|
||||||
|
```
|
||||||
|
|
||||||
|
Или чрез BaraQL:
|
||||||
|
```sql
|
||||||
|
MIGRATION STATUS
|
||||||
|
```
|
||||||
|
|
||||||
|
### Поддържат ли се foreign key зависимости при cross-DB миграция?
|
||||||
|
|
||||||
|
Да. `migrate_data.nim` запазва foreign key дефинициите в генерирания DDL.
|
||||||
|
Препоръчва се таблиците без foreign key зависимости да се мигрират първи.
|
||||||
|
|
||||||
|
### Как се прави backup преди миграция?
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Експортирайте данните преди миграция
|
||||||
|
EXPORT TO '/backup/before_migration.csv' FROM users FORMAT CSV
|
||||||
|
|
||||||
|
-- Или използвайте backup manager-а
|
||||||
|
-- (backup restore list verify cleanup)
|
||||||
|
```
|
||||||
@@ -0,0 +1,634 @@
|
|||||||
|
## Cross-DB Migration Engine — migrate data between any allographer-supported DB and BaraDB.
|
||||||
|
##
|
||||||
|
## Usage:
|
||||||
|
## migrate(sourceRdb, targetRdb, tables) — migrate specific tables
|
||||||
|
## migrateAll(sourceRdb, targetRdb) — migrate all tables
|
||||||
|
## migrateFromUrl(sourceUrl, targetUrl) — URL-based migration
|
||||||
|
##
|
||||||
|
## Supported source databases: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB
|
||||||
|
## Target: BaraDB (native migration system via CREATE MIGRATION + MIGRATION UP)
|
||||||
|
|
||||||
|
import std/asyncdispatch
|
||||||
|
import std/json
|
||||||
|
import std/options
|
||||||
|
import std/strformat
|
||||||
|
import std/strutils
|
||||||
|
import std/tables
|
||||||
|
import std/times
|
||||||
|
|
||||||
|
import ./env
|
||||||
|
import ./query_builder/libs/database_url
|
||||||
|
import ./query_builder/models/baradb/baradb_types
|
||||||
|
import ./query_builder/models/baradb/baradb_query
|
||||||
|
import ./query_builder/models/baradb/baradb_exec
|
||||||
|
|
||||||
|
when isExistsPostgres:
|
||||||
|
import ./query_builder/models/postgres/postgres_types
|
||||||
|
import ./query_builder/models/postgres/postgres_query
|
||||||
|
import ./query_builder/models/postgres/postgres_exec
|
||||||
|
import ./query_builder/models/postgres/postgres_open
|
||||||
|
|
||||||
|
when isExistsMysql:
|
||||||
|
import ./query_builder/models/mysql/mysql_types
|
||||||
|
import ./query_builder/models/mysql/mysql_query
|
||||||
|
import ./query_builder/models/mysql/mysql_exec
|
||||||
|
import ./query_builder/models/mysql/mysql_open
|
||||||
|
|
||||||
|
when isExistsMariadb:
|
||||||
|
import ./query_builder/models/mariadb/mariadb_types
|
||||||
|
import ./query_builder/models/mariadb/mariadb_query
|
||||||
|
import ./query_builder/models/mariadb/mariadb_exec
|
||||||
|
import ./query_builder/models/mariadb/mariadb_open
|
||||||
|
|
||||||
|
when isExistsSqlite:
|
||||||
|
import ./query_builder/models/sqlite/sqlite_types
|
||||||
|
import ./query_builder/models/sqlite/sqlite_query
|
||||||
|
import ./query_builder/models/sqlite/sqlite_exec
|
||||||
|
import ./query_builder/models/sqlite/sqlite_open
|
||||||
|
|
||||||
|
when isExistsSurrealdb:
|
||||||
|
import ./query_builder/models/surreal/surreal_types
|
||||||
|
import ./query_builder/models/surreal/surreal_query
|
||||||
|
import ./query_builder/models/surreal/surreal_exec
|
||||||
|
import ./query_builder/models/surreal/surreal_open
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Types
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
type
|
||||||
|
ColumnInfo* = tuple[name: string, typ: string, isPk: bool, isNullable: bool, defaultVal: string]
|
||||||
|
|
||||||
|
TableInfo* = object
|
||||||
|
name*: string
|
||||||
|
columns*: seq[ColumnInfo]
|
||||||
|
|
||||||
|
MigrationProgress* = object
|
||||||
|
tableName*: string
|
||||||
|
totalRows*: int
|
||||||
|
transferredRows*: int
|
||||||
|
status*: string # "pending", "in_progress", "done", "failed"
|
||||||
|
error*: string
|
||||||
|
|
||||||
|
MigrationReport* = object
|
||||||
|
sourceDb*: string
|
||||||
|
targetDb*: string
|
||||||
|
tablesTotal*: int
|
||||||
|
tablesDone*: int
|
||||||
|
rowsTotal*: int
|
||||||
|
rowsTransferred*: int
|
||||||
|
startedAt*: float
|
||||||
|
completedAt*: float
|
||||||
|
errors*: seq[string]
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Type Mapping: source DB → BaraDB
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
const BARADB_TYPE_MAP = {
|
||||||
|
# PostgreSQL
|
||||||
|
"smallint": "SMALLINT",
|
||||||
|
"integer": "INTEGER",
|
||||||
|
"bigint": "BIGINT",
|
||||||
|
"serial": "SERIAL",
|
||||||
|
"bigserial": "BIGSERIAL",
|
||||||
|
"real": "REAL",
|
||||||
|
"double precision": "DOUBLE PRECISION",
|
||||||
|
"numeric": "DECIMAL",
|
||||||
|
"decimal": "DECIMAL",
|
||||||
|
"character": "VARCHAR(255)",
|
||||||
|
"character varying": "VARCHAR",
|
||||||
|
"varchar": "VARCHAR",
|
||||||
|
"text": "TEXT",
|
||||||
|
"boolean": "BOOLEAN",
|
||||||
|
"bool": "BOOLEAN",
|
||||||
|
"date": "DATE",
|
||||||
|
"timestamp": "TIMESTAMP",
|
||||||
|
"timestamp without time zone": "TIMESTAMP",
|
||||||
|
"timestamp with time zone": "TIMESTAMPTZ",
|
||||||
|
"time": "TIME",
|
||||||
|
"time without time zone": "TIME",
|
||||||
|
"bytea": "BYTEA",
|
||||||
|
"uuid": "UUID",
|
||||||
|
"json": "JSON",
|
||||||
|
"jsonb": "JSON",
|
||||||
|
# MySQL/MariaDB
|
||||||
|
"tinyint": "SMALLINT",
|
||||||
|
"mediumint": "INTEGER",
|
||||||
|
"int": "INTEGER",
|
||||||
|
"float": "REAL",
|
||||||
|
"double": "DOUBLE PRECISION",
|
||||||
|
"char": "VARCHAR(255)",
|
||||||
|
"longtext": "TEXT",
|
||||||
|
"mediumtext": "TEXT",
|
||||||
|
"tinytext": "TEXT",
|
||||||
|
"blob": "BYTEA",
|
||||||
|
"longblob": "BYTEA",
|
||||||
|
"mediumblob": "BYTEA",
|
||||||
|
"tinyblob": "BYTEA",
|
||||||
|
"datetime": "TIMESTAMP",
|
||||||
|
"enum": "VARCHAR(255)",
|
||||||
|
"set": "VARCHAR(255)",
|
||||||
|
"year": "SMALLINT",
|
||||||
|
# SQLite
|
||||||
|
"int": "INTEGER",
|
||||||
|
"integer": "INTEGER",
|
||||||
|
"real": "REAL",
|
||||||
|
"blob": "BYTEA",
|
||||||
|
# SurrealDB
|
||||||
|
"string": "VARCHAR(255)",
|
||||||
|
"number": "DOUBLE PRECISION",
|
||||||
|
"object": "JSON",
|
||||||
|
"array": "JSON",
|
||||||
|
}.toTable()
|
||||||
|
|
||||||
|
proc mapType*(sourceType: string): string =
|
||||||
|
## Map a source database column type to the closest BaraDB type.
|
||||||
|
let lower = sourceType.toLower().split("(")[0].strip()
|
||||||
|
if lower in BARADB_TYPE_MAP:
|
||||||
|
result = BARADB_TYPE_MAP[lower]
|
||||||
|
else:
|
||||||
|
result = "VARCHAR(255)"
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Schema Extraction
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
# --- PostgreSQL ---
|
||||||
|
when isExistsPostgres:
|
||||||
|
proc extractSchema*(rdb: PostgresConnections): Future[seq[TableInfo]] {.async.} =
|
||||||
|
result = @[]
|
||||||
|
let tables = await rdb.raw(
|
||||||
|
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
|
||||||
|
).get()
|
||||||
|
for table in tables:
|
||||||
|
let tableName = table["table_name"].getStr()
|
||||||
|
if tableName == "_allographer_migrations": continue
|
||||||
|
var cols: seq[ColumnInfo] = @[]
|
||||||
|
let columns = await rdb.raw(
|
||||||
|
"""SELECT c.column_name, c.data_type, c.is_nullable, c.column_default,
|
||||||
|
CASE WHEN pk.column_name IS NOT NULL THEN true ELSE false END as is_pk
|
||||||
|
FROM information_schema.columns c
|
||||||
|
LEFT JOIN (
|
||||||
|
SELECT ku.column_name FROM information_schema.table_constraints tc
|
||||||
|
JOIN information_schema.key_column_usage ku
|
||||||
|
ON tc.constraint_name = ku.constraint_name
|
||||||
|
WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_name = ?
|
||||||
|
) pk ON c.column_name = pk.column_name
|
||||||
|
WHERE c.table_name = ?
|
||||||
|
ORDER BY c.ordinal_position""",
|
||||||
|
%*[tableName, tableName]
|
||||||
|
).get()
|
||||||
|
for col in columns:
|
||||||
|
cols.add((
|
||||||
|
name: col["column_name"].getStr(),
|
||||||
|
typ: col["data_type"].getStr(),
|
||||||
|
isPk: col["is_pk"].getStr() == "true",
|
||||||
|
isNullable: col["is_nullable"].getStr() == "YES",
|
||||||
|
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||||
|
col["column_default"].getStr() else: ""
|
||||||
|
))
|
||||||
|
result.add(TableInfo(name: tableName, columns: cols))
|
||||||
|
|
||||||
|
# --- MySQL ---
|
||||||
|
when isExistsMysql:
|
||||||
|
proc extractSchema*(rdb: MysqlConnections): Future[seq[TableInfo]] {.async.} =
|
||||||
|
result = @[]
|
||||||
|
let tables = await rdb.raw(
|
||||||
|
"SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
|
||||||
|
).get()
|
||||||
|
for table in tables:
|
||||||
|
let tableName = table["table_name"].getStr()
|
||||||
|
if tableName == "_allographer_migrations": continue
|
||||||
|
var cols: seq[ColumnInfo] = @[]
|
||||||
|
let columns = await rdb.raw(
|
||||||
|
"""SELECT column_name, data_type, is_nullable, column_default,
|
||||||
|
column_key FROM information_schema.columns
|
||||||
|
WHERE table_name = ? AND table_schema = DATABASE()
|
||||||
|
ORDER BY ordinal_position""",
|
||||||
|
%*[tableName]
|
||||||
|
).get()
|
||||||
|
for col in columns:
|
||||||
|
cols.add((
|
||||||
|
name: col["column_name"].getStr(),
|
||||||
|
typ: col["data_type"].getStr(),
|
||||||
|
isPk: col["column_key"].getStr() == "PRI",
|
||||||
|
isNullable: col["is_nullable"].getStr() == "YES",
|
||||||
|
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||||
|
col["column_default"].getStr() else: ""
|
||||||
|
))
|
||||||
|
result.add(TableInfo(name: tableName, columns: cols))
|
||||||
|
|
||||||
|
# --- MariaDB ---
|
||||||
|
when isExistsMariadb:
|
||||||
|
proc extractSchema*(rdb: MariadbConnections): Future[seq[TableInfo]] {.async.} =
|
||||||
|
result = @[]
|
||||||
|
let tables = await rdb.raw(
|
||||||
|
"SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
|
||||||
|
).get()
|
||||||
|
for table in tables:
|
||||||
|
let tableName = table["table_name"].getStr()
|
||||||
|
if tableName == "_allographer_migrations": continue
|
||||||
|
var cols: seq[ColumnInfo] = @[]
|
||||||
|
let columns = await rdb.raw(
|
||||||
|
"""SELECT column_name, data_type, is_nullable, column_default,
|
||||||
|
column_key FROM information_schema.columns
|
||||||
|
WHERE table_name = ? AND table_schema = DATABASE()
|
||||||
|
ORDER BY ordinal_position""",
|
||||||
|
%*[tableName]
|
||||||
|
).get()
|
||||||
|
for col in columns:
|
||||||
|
cols.add((
|
||||||
|
name: col["column_name"].getStr(),
|
||||||
|
typ: col["data_type"].getStr(),
|
||||||
|
isPk: col["column_key"].getStr() == "PRI",
|
||||||
|
isNullable: col["is_nullable"].getStr() == "YES",
|
||||||
|
defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
|
||||||
|
col["column_default"].getStr() else: ""
|
||||||
|
))
|
||||||
|
result.add(TableInfo(name: tableName, columns: cols))
|
||||||
|
|
||||||
|
# --- SQLite ---
|
||||||
|
when isExistsSqlite:
|
||||||
|
proc extractSchema*(rdb: SqliteConnections): Future[seq[TableInfo]] {.async.} =
|
||||||
|
result = @[]
|
||||||
|
let tables = await rdb.raw(
|
||||||
|
"SELECT name as table_name FROM sqlite_master WHERE type = 'table'"
|
||||||
|
).get()
|
||||||
|
for table in tables:
|
||||||
|
let tableName = table["table_name"].getStr()
|
||||||
|
if tableName == "_allographer_migrations": continue
|
||||||
|
if tableName == "sqlite_sequence": continue
|
||||||
|
var cols: seq[ColumnInfo] = @[]
|
||||||
|
let columns = await rdb.raw("PRAGMA table_info(?)", %*[tableName]).get()
|
||||||
|
for col in columns:
|
||||||
|
cols.add((
|
||||||
|
name: col["name"].getStr(),
|
||||||
|
typ: col["type"].getStr(),
|
||||||
|
isPk: col["pk"].getStr() == "1",
|
||||||
|
isNullable: col["notnull"].getStr() == "0",
|
||||||
|
defaultVal: if col.hasKey("dflt_value") and not col["dflt_value"].isNull:
|
||||||
|
col["dflt_value"].getStr() else: ""
|
||||||
|
))
|
||||||
|
result.add(TableInfo(name: tableName, columns: cols))
|
||||||
|
|
||||||
|
# --- SurrealDB ---
|
||||||
|
when isExistsSurrealdb:
|
||||||
|
proc extractSchema*(rdb: SurrealConnections): Future[seq[TableInfo]] {.async.} =
|
||||||
|
result = @[]
|
||||||
|
let dbResponse = await rdb.raw("INFO FOR DB").get()
|
||||||
|
if dbResponse.len == 0: return
|
||||||
|
let tables = dbResponse[0]["result"]["tables"]
|
||||||
|
for tableName, _ in tables.getFields().pairs:
|
||||||
|
if tableName == "_allographer_migrations": continue
|
||||||
|
if tableName == "_autoincrement_sequences": continue
|
||||||
|
var cols: seq[ColumnInfo] = @[]
|
||||||
|
let tableInfo = await rdb.raw(&"INFO FOR TABLE {tableName}").get()
|
||||||
|
if tableInfo.len > 0:
|
||||||
|
let fields = tableInfo[0]["result"]["fields"]
|
||||||
|
for fieldName, _ in fields.getFields().pairs:
|
||||||
|
cols.add((
|
||||||
|
name: fieldName,
|
||||||
|
typ: "string", # SurrealDB is schemaless
|
||||||
|
isPk: fieldName == "id",
|
||||||
|
isNullable: fieldName != "id",
|
||||||
|
defaultVal: ""
|
||||||
|
))
|
||||||
|
result.add(TableInfo(name: tableName, columns: cols))
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# DDL Generation
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
proc generateBaraDBDDL*(table: TableInfo): tuple[upSql: string, downSql: string] =
|
||||||
|
## Generate CREATE TABLE (UP) and DROP TABLE (DOWN) DDL for BaraDB.
|
||||||
|
var colDefs: seq[string] = @[]
|
||||||
|
for col in table.columns:
|
||||||
|
var def = &"`{col.name}` {mapType(col.typ)}"
|
||||||
|
if col.isPk:
|
||||||
|
def &= " PRIMARY KEY"
|
||||||
|
if not col.isNullable:
|
||||||
|
def &= " NOT NULL"
|
||||||
|
if col.defaultVal.len > 0:
|
||||||
|
def &= " DEFAULT " & col.defaultVal
|
||||||
|
colDefs.add(def)
|
||||||
|
let upSql = &"CREATE TABLE `{table.name}` ({colDefs.join(\", \")})"
|
||||||
|
let downSql = &"DROP TABLE IF EXISTS `{table.name}`"
|
||||||
|
return (upSql, downSql)
|
||||||
|
|
||||||
|
proc generateMigrationName*(tableName: string): string =
|
||||||
|
let timestamp = getTime().toUnix()
|
||||||
|
return &"migrate_{tableName}_{timestamp}"
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Data Transfer
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
proc transferTable*(sourceConn: BaradbConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
## Generic table transfer for BaraDB→BaraDB (used internally).
|
||||||
|
## For cross-DB, use the typed overloads below.
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
# Count rows
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
|
||||||
|
# Transfer in batches
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
# Typed overloads for each source DB type
|
||||||
|
when isExistsPostgres:
|
||||||
|
proc transferTable*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
when isExistsMysql:
|
||||||
|
proc transferTable*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
when isExistsMariadb:
|
||||||
|
proc transferTable*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
when isExistsSqlite:
|
||||||
|
proc transferTable*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
when isExistsSurrealdb:
|
||||||
|
proc transferTable*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
|
||||||
|
tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
|
||||||
|
result = MigrationProgress(tableName: tableName, totalRows: 0,
|
||||||
|
transferredRows: 0, status: "in_progress")
|
||||||
|
try:
|
||||||
|
let countVal = await sourceConn.table(tableName).count()
|
||||||
|
result.totalRows = countVal
|
||||||
|
var offset = 0
|
||||||
|
while offset < result.totalRows:
|
||||||
|
let rows = await sourceConn.table(tableName)
|
||||||
|
.limit(batchSize)
|
||||||
|
.offset(offset)
|
||||||
|
.get()
|
||||||
|
if rows.len == 0: break
|
||||||
|
await targetConn.table(tableName).insert(rows)
|
||||||
|
result.transferredRows += rows.len
|
||||||
|
offset += batchSize
|
||||||
|
result.status = "done"
|
||||||
|
except CatchableError as e:
|
||||||
|
result.status = "failed"
|
||||||
|
result.error = e.msg
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Full Migration Orchestrator
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
when isExistsPostgres:
|
||||||
|
proc migrate*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
|
||||||
|
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||||
|
result = MigrationReport(sourceDb: "PostgreSQL", targetDb: "BaraDB",
|
||||||
|
startedAt: epochTime())
|
||||||
|
try:
|
||||||
|
let schema = await extractSchema(sourceConn)
|
||||||
|
var tablesToMigrate: seq[TableInfo]
|
||||||
|
if tables.len > 0:
|
||||||
|
for t in schema:
|
||||||
|
if t.name in tables: tablesToMigrate.add(t)
|
||||||
|
else:
|
||||||
|
tablesToMigrate = schema
|
||||||
|
|
||||||
|
result.tablesTotal = tablesToMigrate.len
|
||||||
|
for table in tablesToMigrate:
|
||||||
|
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||||
|
let migName = generateMigrationName(table.name)
|
||||||
|
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||||
|
discard await targetConn.applyMigration(migName)
|
||||||
|
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||||
|
if progress.status == "done":
|
||||||
|
result.tablesDone += 1
|
||||||
|
result.rowsTransferred += progress.transferredRows
|
||||||
|
else:
|
||||||
|
result.errors.add(&"{table.name}: {progress.error}")
|
||||||
|
except CatchableError as e:
|
||||||
|
result.errors.add(e.msg)
|
||||||
|
result.completedAt = epochTime()
|
||||||
|
|
||||||
|
when isExistsMysql:
|
||||||
|
proc migrate*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
|
||||||
|
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||||
|
result = MigrationReport(sourceDb: "MySQL", targetDb: "BaraDB",
|
||||||
|
startedAt: epochTime())
|
||||||
|
try:
|
||||||
|
let schema = await extractSchema(sourceConn)
|
||||||
|
var tablesToMigrate: seq[TableInfo]
|
||||||
|
if tables.len > 0:
|
||||||
|
for t in schema:
|
||||||
|
if t.name in tables: tablesToMigrate.add(t)
|
||||||
|
else:
|
||||||
|
tablesToMigrate = schema
|
||||||
|
result.tablesTotal = tablesToMigrate.len
|
||||||
|
for table in tablesToMigrate:
|
||||||
|
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||||
|
let migName = generateMigrationName(table.name)
|
||||||
|
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||||
|
discard await targetConn.applyMigration(migName)
|
||||||
|
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||||
|
if progress.status == "done":
|
||||||
|
result.tablesDone += 1
|
||||||
|
result.rowsTransferred += progress.transferredRows
|
||||||
|
else:
|
||||||
|
result.errors.add(&"{table.name}: {progress.error}")
|
||||||
|
except CatchableError as e:
|
||||||
|
result.errors.add(e.msg)
|
||||||
|
result.completedAt = epochTime()
|
||||||
|
|
||||||
|
when isExistsMariadb:
|
||||||
|
proc migrate*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
|
||||||
|
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||||
|
result = MigrationReport(sourceDb: "MariaDB", targetDb: "BaraDB",
|
||||||
|
startedAt: epochTime())
|
||||||
|
try:
|
||||||
|
let schema = await extractSchema(sourceConn)
|
||||||
|
var tablesToMigrate: seq[TableInfo]
|
||||||
|
if tables.len > 0:
|
||||||
|
for t in schema:
|
||||||
|
if t.name in tables: tablesToMigrate.add(t)
|
||||||
|
else:
|
||||||
|
tablesToMigrate = schema
|
||||||
|
result.tablesTotal = tablesToMigrate.len
|
||||||
|
for table in tablesToMigrate:
|
||||||
|
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||||
|
let migName = generateMigrationName(table.name)
|
||||||
|
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||||
|
discard await targetConn.applyMigration(migName)
|
||||||
|
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||||
|
if progress.status == "done":
|
||||||
|
result.tablesDone += 1
|
||||||
|
result.rowsTransferred += progress.transferredRows
|
||||||
|
else:
|
||||||
|
result.errors.add(&"{table.name}: {progress.error}")
|
||||||
|
except CatchableError as e:
|
||||||
|
result.errors.add(e.msg)
|
||||||
|
result.completedAt = epochTime()
|
||||||
|
|
||||||
|
when isExistsSqlite:
|
||||||
|
proc migrate*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
|
||||||
|
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||||
|
result = MigrationReport(sourceDb: "SQLite", targetDb: "BaraDB",
|
||||||
|
startedAt: epochTime())
|
||||||
|
try:
|
||||||
|
let schema = await extractSchema(sourceConn)
|
||||||
|
var tablesToMigrate: seq[TableInfo]
|
||||||
|
if tables.len > 0:
|
||||||
|
for t in schema:
|
||||||
|
if t.name in tables: tablesToMigrate.add(t)
|
||||||
|
else:
|
||||||
|
tablesToMigrate = schema
|
||||||
|
result.tablesTotal = tablesToMigrate.len
|
||||||
|
for table in tablesToMigrate:
|
||||||
|
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||||
|
let migName = generateMigrationName(table.name)
|
||||||
|
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||||
|
discard await targetConn.applyMigration(migName)
|
||||||
|
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||||
|
if progress.status == "done":
|
||||||
|
result.tablesDone += 1
|
||||||
|
result.rowsTransferred += progress.transferredRows
|
||||||
|
else:
|
||||||
|
result.errors.add(&"{table.name}: {progress.error}")
|
||||||
|
except CatchableError as e:
|
||||||
|
result.errors.add(e.msg)
|
||||||
|
result.completedAt = epochTime()
|
||||||
|
|
||||||
|
when isExistsSurrealdb:
|
||||||
|
proc migrate*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
|
||||||
|
tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
|
||||||
|
result = MigrationReport(sourceDb: "SurrealDB", targetDb: "BaraDB",
|
||||||
|
startedAt: epochTime())
|
||||||
|
try:
|
||||||
|
let schema = await extractSchema(sourceConn)
|
||||||
|
var tablesToMigrate: seq[TableInfo]
|
||||||
|
if tables.len > 0:
|
||||||
|
for t in schema:
|
||||||
|
if t.name in tables: tablesToMigrate.add(t)
|
||||||
|
else:
|
||||||
|
tablesToMigrate = schema
|
||||||
|
result.tablesTotal = tablesToMigrate.len
|
||||||
|
for table in tablesToMigrate:
|
||||||
|
let (upSql, downSql) = generateBaraDBDDL(table)
|
||||||
|
let migName = generateMigrationName(table.name)
|
||||||
|
discard await targetConn.createMigration(migName, upSql, downSql)
|
||||||
|
discard await targetConn.applyMigration(migName)
|
||||||
|
let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
|
||||||
|
if progress.status == "done":
|
||||||
|
result.tablesDone += 1
|
||||||
|
result.rowsTransferred += progress.transferredRows
|
||||||
|
else:
|
||||||
|
result.errors.add(&"{table.name}: {progress.error}")
|
||||||
|
except CatchableError as e:
|
||||||
|
result.errors.add(e.msg)
|
||||||
|
result.completedAt = epochTime()
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# URL-based migration
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
proc `$`*(report: MigrationReport): string =
|
||||||
|
let duration = report.completedAt - report.startedAt
|
||||||
|
result = &"Migration: {report.sourceDb} → {report.targetDb}\n"
|
||||||
|
result &= &" Tables: {report.tablesDone}/{report.tablesTotal}\n"
|
||||||
|
result &= &" Rows: {report.rowsTransferred}\n"
|
||||||
|
result &= &" Time: {duration:.1f}s\n"
|
||||||
|
if report.errors.len > 0:
|
||||||
|
result &= " Errors:\n"
|
||||||
|
for err in report.errors:
|
||||||
|
result &= &" - {err}\n"
|
||||||
@@ -410,6 +410,35 @@ proc exec*(client: BaraClient, sql: string): Future[int] {.async.} =
|
|||||||
let qr = await client.query(sql)
|
let qr = await client.query(sql)
|
||||||
return qr.affectedRows
|
return qr.affectedRows
|
||||||
|
|
||||||
|
# === Migration API (BaraQL native) ===
|
||||||
|
|
||||||
|
proc createMigration*(client: BaraClient, name: string, upBody: string,
|
||||||
|
downBody: string = ""): Future[QueryResult] {.async.} =
|
||||||
|
## Send CREATE MIGRATION via BaraQL. Server handles checksums, locking, rollback.
|
||||||
|
var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
|
||||||
|
if downBody.len > 0:
|
||||||
|
sql &= " DOWN: " & downBody & ";"
|
||||||
|
sql &= " }"
|
||||||
|
return await client.query(sql)
|
||||||
|
|
||||||
|
proc applyMigration*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
|
||||||
|
return await client.query("APPLY MIGRATION " & name)
|
||||||
|
|
||||||
|
proc migrateUp*(client: BaraClient, count: int = 0): Future[QueryResult] {.async.} =
|
||||||
|
var sql = "MIGRATION UP"
|
||||||
|
if count > 0:
|
||||||
|
sql &= " " & $count
|
||||||
|
return await client.query(sql)
|
||||||
|
|
||||||
|
proc migrateDown*(client: BaraClient, count: int = 1): Future[QueryResult] {.async.} =
|
||||||
|
return await client.query("MIGRATION DOWN " & $count)
|
||||||
|
|
||||||
|
proc migrationStatus*(client: BaraClient): Future[QueryResult] {.async.} =
|
||||||
|
return await client.query("MIGRATION STATUS")
|
||||||
|
|
||||||
|
proc migrationDryRun*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
|
||||||
|
return await client.query("MIGRATION DRY RUN " & name)
|
||||||
|
|
||||||
proc auth*(client: BaraClient, token: string) {.async.} =
|
proc auth*(client: BaraClient, token: string) {.async.} =
|
||||||
if not client.connected:
|
if not client.connected:
|
||||||
raise newException(IOError, "Not connected")
|
raise newException(IOError, "Not connected")
|
||||||
@@ -652,6 +681,34 @@ proc exec*(client: SyncClient, sql: string): int =
|
|||||||
let qr = client.query(sql)
|
let qr = client.query(sql)
|
||||||
return qr.affectedRows
|
return qr.affectedRows
|
||||||
|
|
||||||
|
# === Migration API (SyncClient, blocking) ===
|
||||||
|
|
||||||
|
proc createMigration*(client: SyncClient, name: string, upBody: string,
|
||||||
|
downBody: string = ""): QueryResult =
|
||||||
|
var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
|
||||||
|
if downBody.len > 0:
|
||||||
|
sql &= " DOWN: " & downBody & ";"
|
||||||
|
sql &= " }"
|
||||||
|
return client.query(sql)
|
||||||
|
|
||||||
|
proc applyMigration*(client: SyncClient, name: string): QueryResult =
|
||||||
|
return client.query("APPLY MIGRATION " & name)
|
||||||
|
|
||||||
|
proc migrateUp*(client: SyncClient, count: int = 0): QueryResult =
|
||||||
|
var sql = "MIGRATION UP"
|
||||||
|
if count > 0:
|
||||||
|
sql &= " " & $count
|
||||||
|
return client.query(sql)
|
||||||
|
|
||||||
|
proc migrateDown*(client: SyncClient, count: int = 1): QueryResult =
|
||||||
|
return client.query("MIGRATION DOWN " & $count)
|
||||||
|
|
||||||
|
proc migrationStatus*(client: SyncClient): QueryResult =
|
||||||
|
return client.query("MIGRATION STATUS")
|
||||||
|
|
||||||
|
proc migrationDryRun*(client: SyncClient, name: string): QueryResult =
|
||||||
|
return client.query("MIGRATION DRY RUN " & name)
|
||||||
|
|
||||||
proc auth*(client: SyncClient, token: string) =
|
proc auth*(client: SyncClient, token: string) =
|
||||||
acquire(client.lock)
|
acquire(client.lock)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -699,6 +699,183 @@ proc firstPlain*(self: RawBaradbQuery): Future[seq[string]] {.async.} =
|
|||||||
return await self.getRowPlain(self.queryString, self.placeHolder)
|
return await self.getRowPlain(self.queryString, self.placeHolder)
|
||||||
|
|
||||||
|
|
||||||
|
# ================================================================================
|
||||||
|
# Pagination (cursor-based and offset-based)
|
||||||
|
# ================================================================================
|
||||||
|
|
||||||
|
type
|
||||||
|
PaginateResult* = object
|
||||||
|
rows*: seq[JsonNode]
|
||||||
|
total*: int
|
||||||
|
page*: int
|
||||||
|
perPage*: int
|
||||||
|
hasMore*: bool
|
||||||
|
|
||||||
|
proc paginate*(self: BaradbQuery, page: int = 1, perPage: int = 20): Future[PaginateResult] {.async.} =
|
||||||
|
## Offset-based pagination. Returns rows for the given page plus metadata.
|
||||||
|
let total = await self.count()
|
||||||
|
let offset = (page - 1) * perPage
|
||||||
|
self.limit(perPage)
|
||||||
|
self.offset(offset)
|
||||||
|
let rows = await self.get()
|
||||||
|
let qr = PaginateResult(
|
||||||
|
rows: rows,
|
||||||
|
total: total,
|
||||||
|
page: page,
|
||||||
|
perPage: perPage,
|
||||||
|
hasMore: offset + perPage < total
|
||||||
|
)
|
||||||
|
return qr
|
||||||
|
|
||||||
|
proc fastPaginate*(self: BaradbQuery, cursorColumn: string, perPage: int = 20,
|
||||||
|
afterId: string = ""): Future[PaginateResult] {.async.} =
|
||||||
|
## Cursor-based pagination (keyset). More efficient than offset for large tables.
|
||||||
|
## Requires a unique, ordered column (usually the primary key).
|
||||||
|
let total = await self.count()
|
||||||
|
self.limit(perPage)
|
||||||
|
self.orderBy(cursorColumn, Asc)
|
||||||
|
if afterId.len > 0:
|
||||||
|
self.where(cursorColumn, ">", afterId)
|
||||||
|
let rows = await self.get()
|
||||||
|
var hasMore = rows.len == perPage
|
||||||
|
let qr = PaginateResult(
|
||||||
|
rows: rows,
|
||||||
|
total: total,
|
||||||
|
page: 0, # cursor-based has no page number
|
||||||
|
perPage: perPage,
|
||||||
|
hasMore: hasMore
|
||||||
|
)
|
||||||
|
return qr
|
||||||
|
|
||||||
|
# ================================================================================
|
||||||
|
# Migration API (BaraQL native — server handles checksums, locks, rollback)
|
||||||
|
# ================================================================================
|
||||||
|
|
||||||
|
proc createMigration*(self: BaradbConnections, name: string, upBody: string,
|
||||||
|
downBody: string = ""): Future[QueryResult] {.async.} =
|
||||||
|
## Register a migration on the server. The server computes checksums, stores
|
||||||
|
## the UP/DOWN bodies, and manages the migration lifecycle.
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
return await self.pools.conns[connI].client.createMigration(name, upBody, downBody)
|
||||||
|
|
||||||
|
proc applyMigration*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
return await self.pools.conns[connI].client.applyMigration(name)
|
||||||
|
|
||||||
|
proc migrateUp*(self: BaradbConnections, count: int = 0): Future[QueryResult] {.async.} =
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
return await self.pools.conns[connI].client.migrateUp(count)
|
||||||
|
|
||||||
|
proc migrateDown*(self: BaradbConnections, count: int = 1): Future[QueryResult] {.async.} =
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
return await self.pools.conns[connI].client.migrateDown(count)
|
||||||
|
|
||||||
|
proc migrationStatus*(self: BaradbConnections): Future[seq[JsonNode]] {.async.} =
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
let qr = await self.pools.conns[connI].client.migrationStatus()
|
||||||
|
return toJson(qr)
|
||||||
|
|
||||||
|
proc migrationDryRun*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
|
||||||
|
var connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
return await self.pools.conns[connI].client.migrationDryRun(name)
|
||||||
|
|
||||||
|
proc isMigrationApplied*(self: BaradbConnections, name: string): Future[bool] {.async.} =
|
||||||
|
let status = await self.migrationStatus()
|
||||||
|
for row in status:
|
||||||
|
if row["name"].getStr() == name and row["status"].getStr() == "applied":
|
||||||
|
return true
|
||||||
|
return false
|
||||||
|
|
||||||
|
# ================================================================================
|
||||||
|
# Prepared Statements (server-side parameterized queries via mkQueryParams)
|
||||||
|
# ================================================================================
|
||||||
|
|
||||||
|
proc sendPrepared(client: BaraClient, sql: string, params: seq[WireValue]): Future[QueryResult] {.async.} =
|
||||||
|
if not client.connected:
|
||||||
|
raise newException(IOError, "Not connected")
|
||||||
|
let msg = makeQueryParamsMessage(client.nextId(), sql, params)
|
||||||
|
let msgStr = toString(msg)
|
||||||
|
await client.socket.send(msgStr)
|
||||||
|
return await client.readQueryResponse()
|
||||||
|
|
||||||
|
proc prepare*(self: BaradbConnections, sql: string, nArgs: int = 0): Future[BaradbPreparedStatement] {.async.} =
|
||||||
|
## Create a server-side prepared statement. Subsequent calls reuse the cached entry.
|
||||||
|
let entryKey = sql & ":" & $nArgs
|
||||||
|
if entryKey in self.pools.preparedCache:
|
||||||
|
let entry = self.pools.preparedCache[entryKey]
|
||||||
|
inc entry.refCount
|
||||||
|
entry.lastUsedAt = nowUnix()
|
||||||
|
return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
|
||||||
|
nArgs: nArgs, isClosed: false)
|
||||||
|
let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 1,
|
||||||
|
lastUsedAt: nowUnix())
|
||||||
|
self.pools.preparedCache[entryKey] = entry
|
||||||
|
return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
|
||||||
|
nArgs: nArgs, isClosed: false)
|
||||||
|
|
||||||
|
proc ensureStmt*(self: BaradbConnections, sql: string, nArgs: int): BaradbPreparedEntry =
|
||||||
|
let entryKey = sql & ":" & $nArgs
|
||||||
|
if entryKey in self.pools.preparedCache:
|
||||||
|
let entry = self.pools.preparedCache[entryKey]
|
||||||
|
entry.lastUsedAt = nowUnix()
|
||||||
|
return entry
|
||||||
|
let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 0,
|
||||||
|
lastUsedAt: nowUnix())
|
||||||
|
self.pools.preparedCache[entryKey] = entry
|
||||||
|
return entry
|
||||||
|
|
||||||
|
proc preparedGet*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[seq[JsonNode]] {.async.} =
|
||||||
|
if stmt.isClosed:
|
||||||
|
raise newException(IOError, "Prepared statement is closed")
|
||||||
|
var connI = getFreeConn(stmt.owner).await
|
||||||
|
defer: stmt.owner.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(stmt.owner)
|
||||||
|
let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
|
||||||
|
return toJson(qr)
|
||||||
|
|
||||||
|
proc preparedExec*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[int] {.async.} =
|
||||||
|
if stmt.isClosed:
|
||||||
|
raise newException(IOError, "Prepared statement is closed")
|
||||||
|
var connI = getFreeConn(stmt.owner).await
|
||||||
|
defer: stmt.owner.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(stmt.owner)
|
||||||
|
let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
|
||||||
|
return qr.affectedRows
|
||||||
|
|
||||||
|
proc flushStmt*(stmt: BaradbPreparedStatement) =
|
||||||
|
stmt.isClosed = true
|
||||||
|
dec stmt.entry.refCount
|
||||||
|
|
||||||
|
proc clearStmtCache*(self: BaradbConnections) =
|
||||||
|
self.pools.preparedCache.clear()
|
||||||
|
|
||||||
|
proc withConn*(self: BaradbConnections, callback: proc(connI: int): Future[void] {.gcsafe.}): Future[void] {.async.} =
|
||||||
|
let connI = getFreeConn(self).await
|
||||||
|
defer: self.returnConn(connI).await
|
||||||
|
if connI == errorConnectionNum:
|
||||||
|
raisePoolTimeout(self)
|
||||||
|
await callback(connI)
|
||||||
|
|
||||||
# seeder templates
|
# seeder templates
|
||||||
template seeder*(rdb: BaradbConnections, tableName: string, body: untyped): untyped =
|
template seeder*(rdb: BaradbConnections, tableName: string, body: untyped): untyped =
|
||||||
block:
|
block:
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import std/json
|
|||||||
import std/tables
|
import std/tables
|
||||||
import std/times
|
import std/times
|
||||||
import ../../libs/baradb/baradb_client
|
import ../../libs/baradb/baradb_client
|
||||||
|
import ../../libs/database_url
|
||||||
import ../../log
|
import ../../log
|
||||||
import ./baradb_types
|
import ./baradb_types
|
||||||
|
|
||||||
@@ -50,3 +51,18 @@ proc dbOpen*(_: type Baradb, database: string, user: string, password: string,
|
|||||||
pools: pools,
|
pools: pools,
|
||||||
log: LogSetting(shouldDisplayLog: shouldDisplayLog, shouldOutputLogFile: shouldOutputLogFile, logDir: logDir)
|
log: LogSetting(shouldDisplayLog: shouldDisplayLog, shouldOutputLogFile: shouldOutputLogFile, logDir: logDir)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
proc dbOpen*(_: type Baradb, databaseUrl: DatabaseUrl,
|
||||||
|
maxConnections = 1, timeout = 30,
|
||||||
|
shouldDisplayLog = false, shouldOutputLogFile = false, logDir = "",
|
||||||
|
maxConnectionLifetime = DEFAULT_CONN_MAX_LIFETIME_SECONDS,
|
||||||
|
maxConnectionIdleTime = DEFAULT_CONN_MAX_IDLE_SECONDS): BaradbConnections =
|
||||||
|
## Open a BaraDB connection using a URL: baradb://user:pass@host:port/database
|
||||||
|
let parsed = parseDatabaseUrl(databaseUrl)
|
||||||
|
requireDatabaseUrlScheme(parsed, ["baradb"], "BaraDB")
|
||||||
|
let database = parsed.path.stripLeadingSlash()
|
||||||
|
let host = if parsed.hostname.len > 0: parsed.hostname else: "127.0.0.1"
|
||||||
|
let port = if parsed.hasPort: parsed.port else: 9472
|
||||||
|
return Baradb.dbOpen(database, parsed.username, parsed.password, host, port,
|
||||||
|
maxConnections, timeout, shouldDisplayLog, shouldOutputLogFile,
|
||||||
|
logDir, maxConnectionLifetime, maxConnectionIdleTime)
|
||||||
|
|||||||
+3
-7
@@ -3,10 +3,6 @@ import ../../models/table
|
|||||||
import ./baradb_query_type
|
import ./baradb_query_type
|
||||||
|
|
||||||
proc createMigrationTable*(self: BaradbSchema) =
|
proc createMigrationTable*(self: BaradbSchema) =
|
||||||
let sql = """CREATE TABLE IF NOT EXISTS `schema_migrations` (
|
## BaraDB tracks migrations natively via BaraQL (CREATE MIGRATION, MIGRATION STATUS).
|
||||||
`id` SERIAL PRIMARY KEY,
|
## No client-side migration table needed — server stores state in LSM-Tree.
|
||||||
`name` VARCHAR(255) NOT NULL,
|
discard
|
||||||
`checksum` VARCHAR(64) NOT NULL,
|
|
||||||
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
||||||
)"""
|
|
||||||
discard waitFor self.rdb.raw(sql).exec()
|
|
||||||
+39
-4
@@ -1,12 +1,47 @@
|
|||||||
|
import std/asyncdispatch
|
||||||
import std/strformat
|
import std/strformat
|
||||||
import ../../../query_builder/models/baradb/baradb_types
|
import ../../../query_builder/models/baradb/baradb_types
|
||||||
|
import ../../../query_builder/models/baradb/baradb_exec
|
||||||
|
import ../../../query_builder/error
|
||||||
import ../../models/table
|
import ../../models/table
|
||||||
|
|
||||||
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
|
proc exec*(rdb: BaradbConnections, queries: seq[string]) =
|
||||||
for sql in queries:
|
for sql in queries:
|
||||||
discard waitFor rdb.raw(sql).exec()
|
discard waitFor rdb.raw(sql).exec()
|
||||||
let hist = &"INSERT INTO `schema_migrations` (`name`, `checksum`) VALUES ('{name}', '{checksum}')"
|
|
||||||
discard waitFor rdb.raw(hist).exec()
|
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
|
||||||
|
## Register and apply a migration via BaraQL native migration system.
|
||||||
|
## The server handles checksums, locking, rollback, and status tracking.
|
||||||
|
let upBody = queries.join("; ")
|
||||||
|
var downBody = ""
|
||||||
|
# Generate DOWN script: DROP TABLE for CREATE TABLE
|
||||||
|
for sql in queries:
|
||||||
|
if sql.toLower().startsWith("create table"):
|
||||||
|
let parts = sql.split(" ")
|
||||||
|
if parts.len >= 3:
|
||||||
|
let tableName = parts[2] # CREATE TABLE `name` ...
|
||||||
|
downBody &= &"DROP TABLE IF EXISTS {tableName}; "
|
||||||
|
|
||||||
|
# Register migration on server
|
||||||
|
var qr = waitFor rdb.createMigration(name, upBody, downBody)
|
||||||
|
if qr.affectedRows < 0:
|
||||||
|
raise newException(DbError, "Failed to create migration: " & name)
|
||||||
|
|
||||||
|
# Apply migration
|
||||||
|
qr = waitFor rdb.applyMigration(name)
|
||||||
|
if qr.affectedRows < 0:
|
||||||
|
raise newException(DbError, "Failed to apply migration: " & name)
|
||||||
|
|
||||||
|
proc execThenSaveHistory*(rdb: BaradbConnections, name: string, query: string, checksum: string) =
|
||||||
|
execThenSaveHistory(rdb, name, @[query], checksum)
|
||||||
|
|
||||||
proc shouldRun*(rdb: BaradbConnections, table: Table, checksum: string, isReset: bool): bool =
|
proc shouldRun*(rdb: BaradbConnections, table: Table, checksum: string, isReset: bool): bool =
|
||||||
return true
|
## Check with the server whether this migration has already been applied.
|
||||||
|
if isReset:
|
||||||
|
return true
|
||||||
|
try:
|
||||||
|
let applied = waitFor rdb.isMigrationApplied(table.name)
|
||||||
|
return not applied
|
||||||
|
except CatchableError:
|
||||||
|
# If server is unavailable or migration doesn't exist, run it
|
||||||
|
return true
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
discard """
|
||||||
|
cmd: "nim c -r $file"
|
||||||
|
"""
|
||||||
|
|
||||||
|
import std/unittest
|
||||||
|
import std/asyncdispatch
|
||||||
|
import std/json
|
||||||
|
import ../../src/allographer/query_builder
|
||||||
|
import ../../src/allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
import ./connections
|
||||||
|
|
||||||
|
|
||||||
|
let rdb = baradb
|
||||||
|
|
||||||
|
suite("Baradb migrations"):
|
||||||
|
test("create migration"):
|
||||||
|
let upSql = """
|
||||||
|
CREATE TABLE IF NOT EXISTS test_mig_users (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
let downSql = "DROP TABLE IF EXISTS test_mig_users"
|
||||||
|
let qr = waitFor rdb.createMigration("test_mig_001", upSql, downSql)
|
||||||
|
check qr.rowCount >= 0
|
||||||
|
|
||||||
|
test("migration status shows pending"):
|
||||||
|
let status = waitFor rdb.migrationStatus()
|
||||||
|
var found = false
|
||||||
|
for row in status:
|
||||||
|
if row["name"].getStr() == "test_mig_001":
|
||||||
|
check row["status"].getStr() == "pending"
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
check found
|
||||||
|
|
||||||
|
test("apply migration"):
|
||||||
|
let qr = waitFor rdb.applyMigration("test_mig_001")
|
||||||
|
check qr.rowCount >= 0
|
||||||
|
|
||||||
|
test("is migration applied"):
|
||||||
|
let applied = waitFor rdb.isMigrationApplied("test_mig_001")
|
||||||
|
check applied
|
||||||
|
|
||||||
|
test("migration up (all pending)"):
|
||||||
|
let qr = waitFor rdb.migrateUp()
|
||||||
|
check qr.rowCount >= 0
|
||||||
|
|
||||||
|
test("migration dry run"):
|
||||||
|
let qr = waitFor rdb.migrationDryRun("test_mig_001")
|
||||||
|
check qr.rowCount >= 0
|
||||||
|
|
||||||
|
test("table exists after migration"):
|
||||||
|
let users = waitFor rdb.table("test_mig_users").get()
|
||||||
|
check users.len >= 0
|
||||||
|
|
||||||
|
test("migrate down (rollback)"):
|
||||||
|
let qr = waitFor rdb.migrateDown(1)
|
||||||
|
check qr.rowCount >= 0
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
discard """
|
||||||
|
cmd: "nim c -r $file"
|
||||||
|
"""
|
||||||
|
|
||||||
|
import std/unittest
|
||||||
|
import std/asyncdispatch
|
||||||
|
import std/json
|
||||||
|
import std/options
|
||||||
|
import ../../src/allographer/query_builder
|
||||||
|
import ../../src/allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
import ../../src/allographer/query_builder/libs/baradb/baradb_client
|
||||||
|
import ./connections
|
||||||
|
|
||||||
|
|
||||||
|
let rdb = baradb
|
||||||
|
|
||||||
|
suite("Baradb prepared statements"):
|
||||||
|
setup:
|
||||||
|
# Ensure test table exists
|
||||||
|
let createSql = """
|
||||||
|
CREATE TABLE IF NOT EXISTS test_prep_data (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
label VARCHAR(255),
|
||||||
|
count INTEGER
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
waitFor rdb.raw(createSql).exec()
|
||||||
|
|
||||||
|
test("prepare statement"):
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM test_prep_data WHERE count > ?", nArgs = 1
|
||||||
|
)
|
||||||
|
check not stmt.isClosed
|
||||||
|
|
||||||
|
test("ensureStmt"):
|
||||||
|
let entry = rdb.ensureStmt(
|
||||||
|
"SELECT * FROM test_prep_data WHERE label = ?", nArgs = 1
|
||||||
|
)
|
||||||
|
check entry.sql.len > 0
|
||||||
|
check entry.nArgs == 1
|
||||||
|
|
||||||
|
test("preparedGet with params"):
|
||||||
|
# Insert test data
|
||||||
|
waitFor rdb.table("test_prep_data").insert(%*{
|
||||||
|
"label": "alpha", "count": 5
|
||||||
|
})
|
||||||
|
waitFor rdb.table("test_prep_data").insert(%*{
|
||||||
|
"label": "beta", "count": 15
|
||||||
|
})
|
||||||
|
|
||||||
|
# Query via prepared statement
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM test_prep_data WHERE count > ?", nArgs = 1
|
||||||
|
)
|
||||||
|
let results = waitFor stmt.preparedGet(@[
|
||||||
|
WireValue(kind: fkInt32, int32Val: 10)
|
||||||
|
])
|
||||||
|
check results.len >= 1
|
||||||
|
if results.len > 0:
|
||||||
|
check results[0]["label"].getStr() == "beta"
|
||||||
|
|
||||||
|
test("preparedExec insert"):
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"INSERT INTO test_prep_data (label, count) VALUES (?, ?)", nArgs = 2
|
||||||
|
)
|
||||||
|
let affected = waitFor stmt.preparedExec(@[
|
||||||
|
WireValue(kind: fkString, strVal: "gamma"),
|
||||||
|
WireValue(kind: fkInt32, int32Val: 25)
|
||||||
|
])
|
||||||
|
check affected >= 0
|
||||||
|
|
||||||
|
test("flush statement"):
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM test_prep_data", nArgs = 0
|
||||||
|
)
|
||||||
|
stmt.flushStmt()
|
||||||
|
check stmt.isClosed
|
||||||
|
|
||||||
|
test("clear statement cache"):
|
||||||
|
rdb.clearStmtCache()
|
||||||
|
check true
|
||||||
|
|
||||||
|
test("cleanup"):
|
||||||
|
waitFor rdb.raw("DROP TABLE IF EXISTS test_prep_data").exec()
|
||||||
|
check true
|
||||||
+3
-1
@@ -646,7 +646,9 @@ SELECT * FROM invoices; -- само редове на company-a
|
|||||||
|-----------|-------------|
|
|-----------|-------------|
|
||||||
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
||||||
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
||||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE |
|
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE MIGRATION, APPLY MIGRATION |
|
||||||
|
| Миграции | MIGRATION UP, MIGRATION DOWN, MIGRATION STATUS, MIGRATION DRY RUN |
|
||||||
|
| Импорт/Експорт | IMPORT FROM, EXPORT TO, FORMAT, CSV, JSON, NDJSON, DELIMITER, HEADER, BATCH |
|
||||||
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
||||||
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
||||||
| CTEs | WITH, RECURSIVE, AS |
|
| CTEs | WITH, RECURSIVE, AS |
|
||||||
|
|||||||
@@ -270,6 +270,23 @@ with pool.connection() as conn:
|
|||||||
result = conn.query("SELECT 1")
|
result = conn.query("SELECT 1")
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Миграция между Бази Данни (Nim)
|
||||||
|
|
||||||
|
Nim allographer клиентът включва engine за миграция между бази данни:
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/migrate_data
|
||||||
|
|
||||||
|
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||||
|
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||||
|
echo report # Tables: 12/12, Rows: 45230, Time: 3.2s
|
||||||
|
```
|
||||||
|
|
||||||
|
Поддържани източници: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB.
|
||||||
|
Виж [Миграции и Импорт/Експорт](migration.md) за детайли.
|
||||||
|
|
||||||
## Съответствие на Типове Данни
|
## Съответствие на Типове Данни
|
||||||
|
|
||||||
| BaraDB Тип | JavaScript | Python | Nim | Rust |
|
| BaraDB Тип | JavaScript | Python | Nim | Rust |
|
||||||
|
|||||||
@@ -44,6 +44,7 @@
|
|||||||
- [Сигурност](security.md)
|
- [Сигурност](security.md)
|
||||||
- [Мониторинг](monitoring.md)
|
- [Мониторинг](monitoring.md)
|
||||||
- [Архивиране и Възстановяване](backup.md)
|
- [Архивиране и Възстановяване](backup.md)
|
||||||
|
- [Миграции и Импорт/Експорт](migration.md)
|
||||||
- [Отстраняване на Проблеми](troubleshooting.md)
|
- [Отстраняване на Проблеми](troubleshooting.md)
|
||||||
|
|
||||||
## Разширени
|
## Разширени
|
||||||
|
|||||||
@@ -0,0 +1,257 @@
|
|||||||
|
# Миграции и Импорт/Експорт на Данни
|
||||||
|
|
||||||
|
BaraDB има вградена миграционна система чрез BaraQL. Миграциите се управляват
|
||||||
|
изцяло от сървъра — checksums, locking, rollback и проследяване на статуса.
|
||||||
|
Клиентът изпраща BaraQL команди и не поддържа собствена миграционна таблица.
|
||||||
|
|
||||||
|
## BaraQL Синтаксис за Миграции
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Създаване на миграция с UP и DOWN скриптове
|
||||||
|
CREATE MIGRATION add_users_table {
|
||||||
|
UP: CREATE TABLE users (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
email VARCHAR(255) UNIQUE,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
DOWN: DROP TABLE IF EXISTS users;
|
||||||
|
}
|
||||||
|
|
||||||
|
-- Прилагане на конкретна миграция
|
||||||
|
APPLY MIGRATION add_users_table
|
||||||
|
|
||||||
|
-- Прилагане на всички неприложени миграции
|
||||||
|
MIGRATION UP
|
||||||
|
|
||||||
|
-- Прилагане на следващите N миграции
|
||||||
|
MIGRATION UP 3
|
||||||
|
|
||||||
|
-- Отмяна на последната миграция
|
||||||
|
MIGRATION DOWN
|
||||||
|
|
||||||
|
-- Отмяна на последните N миграции
|
||||||
|
MIGRATION DOWN 2
|
||||||
|
|
||||||
|
-- Преглед на статуса
|
||||||
|
MIGRATION STATUS
|
||||||
|
|
||||||
|
-- Dry run (валидация без изпълнение)
|
||||||
|
MIGRATION DRY RUN add_users_table
|
||||||
|
```
|
||||||
|
|
||||||
|
## Заключване при Миграция
|
||||||
|
|
||||||
|
BaraDB заключва глобално преди прилагане на миграция, което предотвратява
|
||||||
|
конкурентно изпълнение и осигурява:
|
||||||
|
|
||||||
|
- Само една миграция се изпълнява в даден момент
|
||||||
|
- Checksums се проверяват преди изпълнение
|
||||||
|
- Неуспешните миграции могат да се върнат безопасно
|
||||||
|
- `MIGRATION STATUS` показва точното състояние на всяка миграция
|
||||||
|
|
||||||
|
## Checksums
|
||||||
|
|
||||||
|
Тялото на всяка миграция се хешира с SHA-256. Сървърът съхранява checksum
|
||||||
|
при създаване и го проверява преди прилагане.
|
||||||
|
|
||||||
|
```
|
||||||
|
CREATE MIGRATION add_users → checksum: a3f2b8c1...
|
||||||
|
APPLY MIGRATION add_users → проверява checksum → изпълнява
|
||||||
|
```
|
||||||
|
|
||||||
|
## Връщане (Rollback)
|
||||||
|
|
||||||
|
Миграциите с `DOWN` скриптове поддържат връщане:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE MIGRATION add_column {
|
||||||
|
UP: ALTER TABLE users ADD COLUMN phone VARCHAR(20);
|
||||||
|
DOWN: ALTER TABLE users DROP COLUMN phone;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`MIGRATION DOWN` изпълнява DOWN скрипта и маркира миграцията като върната.
|
||||||
|
|
||||||
|
## Dry Run
|
||||||
|
|
||||||
|
Валидирайте миграция преди прилагане:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
MIGRATION DRY RUN add_users_table
|
||||||
|
-- Изход:
|
||||||
|
-- DRY RUN add_users_table:
|
||||||
|
-- Statements: 1
|
||||||
|
-- [1] nkCreateTable
|
||||||
|
-- DOWN script: yes
|
||||||
|
-- Checksum: a3f2b8c1d4e5f6a7b8c9d0e1f2a3b4c5
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## IMPORT FROM / EXPORT TO
|
||||||
|
|
||||||
|
BaraDB поддържа импорт и експорт на данни директно чрез BaraQL.
|
||||||
|
|
||||||
|
### IMPORT FROM
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Импорт от CSV
|
||||||
|
IMPORT FROM '/data/users.csv' INTO users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
BATCH 1000
|
||||||
|
|
||||||
|
-- Импорт от JSON масив
|
||||||
|
IMPORT FROM '/data/users.json' INTO users
|
||||||
|
FORMAT JSON
|
||||||
|
|
||||||
|
-- Импорт от NDJSON
|
||||||
|
IMPORT FROM '/data/users.ndjson' INTO users
|
||||||
|
FORMAT NDJSON
|
||||||
|
```
|
||||||
|
|
||||||
|
Опции:
|
||||||
|
| Опция | Стойности | По подр. | Описание |
|
||||||
|
|--------|-----------|----------|----------|
|
||||||
|
| `FORMAT` | `CSV`, `JSON`, `NDJSON` | `CSV` | Формат на входния файл |
|
||||||
|
| `DELIMITER` | произволен знак | `,` | Разделител за CSV |
|
||||||
|
| `HEADER` | `true`/`false` | `true` | Първият ред на CSV е заглавен |
|
||||||
|
| `BATCH` | цяло число | `1000` | Редове на партида |
|
||||||
|
|
||||||
|
### EXPORT TO
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Експорт към CSV
|
||||||
|
EXPORT TO '/backup/users.csv' FROM users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
|
||||||
|
-- Експорт към JSON
|
||||||
|
EXPORT TO '/backup/users.json' FROM users
|
||||||
|
FORMAT JSON
|
||||||
|
|
||||||
|
-- Експорт към NDJSON
|
||||||
|
EXPORT TO '/backup/users.ndjson' FROM users
|
||||||
|
FORMAT NDJSON
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Миграция между Бази Данни
|
||||||
|
|
||||||
|
Nim клиентът на BaraDB (nim-allographer) включва engine за миграция между
|
||||||
|
различни бази данни. Мигрирайте данни от PostgreSQL, MySQL, MariaDB, SQLite
|
||||||
|
или SurrealDB директно към BaraDB.
|
||||||
|
|
||||||
|
### Поддържани Източници
|
||||||
|
|
||||||
|
| База Данни | Извличане на Схема | Статус |
|
||||||
|
|------------|-------------------|--------|
|
||||||
|
| PostgreSQL | `information_schema` | ✅ |
|
||||||
|
| MySQL | `information_schema` | ✅ |
|
||||||
|
| MariaDB | `information_schema` | ✅ |
|
||||||
|
| SQLite | `sqlite_master` + `PRAGMA` | ✅ |
|
||||||
|
| SurrealDB | `INFO FOR DB` / `INFO FOR TABLE` | ✅ |
|
||||||
|
|
||||||
|
### Nim API
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/migrate_data
|
||||||
|
|
||||||
|
# Свързване към източник и цел
|
||||||
|
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||||
|
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
# Мигриране на всички таблици
|
||||||
|
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||||
|
echo report
|
||||||
|
# Migration: PostgreSQL → BaraDB
|
||||||
|
# Tables: 12/12
|
||||||
|
# Rows: 45230
|
||||||
|
# Time: 3.2s
|
||||||
|
|
||||||
|
# Мигриране на конкретни таблици
|
||||||
|
let report = waitFor migrate(pg, bdb,
|
||||||
|
tables = @["users", "orders", "products"])
|
||||||
|
```
|
||||||
|
|
||||||
|
### Съответствие на Типове
|
||||||
|
|
||||||
|
Миграционният engine автоматично мапва типовете:
|
||||||
|
|
||||||
|
| PostgreSQL | MySQL | SQLite | BaraDB |
|
||||||
|
|------------|-------|--------|--------|
|
||||||
|
| `SERIAL` | `INT AUTO_INCREMENT` | `INTEGER PK` | `SERIAL` |
|
||||||
|
| `VARCHAR(n)` | `VARCHAR(n)` | `TEXT` | `VARCHAR(n)` |
|
||||||
|
| `TEXT` | `TEXT` | `TEXT` | `TEXT` |
|
||||||
|
| `BOOLEAN` | `TINYINT(1)` | `INTEGER` | `BOOLEAN` |
|
||||||
|
| `JSONB` | `JSON` | `TEXT` | `JSON` |
|
||||||
|
| `TIMESTAMP` | `DATETIME` | `TEXT` | `TIMESTAMP` |
|
||||||
|
| `UUID` | `CHAR(36)` | `TEXT` | `UUID` |
|
||||||
|
|
||||||
|
Пълна карта: 50+ поддържани типа.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Nim allographer API
|
||||||
|
|
||||||
|
### Управление на Миграции
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
|
||||||
|
# Създаване
|
||||||
|
let qr = waitFor rdb.createMigration("add_products",
|
||||||
|
"CREATE TABLE products (id SERIAL PRIMARY KEY, name VARCHAR(255))",
|
||||||
|
"DROP TABLE IF EXISTS products")
|
||||||
|
|
||||||
|
# Прилагане
|
||||||
|
let qr = waitFor rdb.applyMigration("add_products")
|
||||||
|
let qr = waitFor rdb.migrateUp() # всички неприложени
|
||||||
|
let qr = waitFor rdb.migrateDown(1) # връщане
|
||||||
|
|
||||||
|
# Статус
|
||||||
|
let status = waitFor rdb.migrationStatus()
|
||||||
|
if waitFor rdb.isMigrationApplied("add_products"):
|
||||||
|
echo "Вече е приложена"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prepared Statements
|
||||||
|
|
||||||
|
```nim
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2)
|
||||||
|
|
||||||
|
let results = waitFor stmt.preparedGet(@[
|
||||||
|
WireValue(kind: fkInt32, int32Val: 18),
|
||||||
|
WireValue(kind: fkString, strVal: "active")
|
||||||
|
])
|
||||||
|
|
||||||
|
stmt.flushStmt()
|
||||||
|
rdb.clearStmtCache()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pagination
|
||||||
|
|
||||||
|
```nim
|
||||||
|
# Offset-базирана
|
||||||
|
let page = waitFor rdb.table("users").paginate(page = 1, perPage = 20)
|
||||||
|
|
||||||
|
# Cursor-базирана (по-бърза за големи таблици)
|
||||||
|
let batch = waitFor rdb.table("users")
|
||||||
|
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Най-добри Практики
|
||||||
|
|
||||||
|
1. **Винаги включвайте DOWN скриптове** — позволява безопасно връщане
|
||||||
|
2. **Първо dry run** — валидирайте преди прилагане в production
|
||||||
|
3. **Партиден импорт** — използвайте `BATCH 1000` за CSV импорт
|
||||||
|
4. **Експортирайте преди миграция** — backup с `EXPORT TO`
|
||||||
|
5. **Проверка след миграция** — `MIGRATION STATUS`
|
||||||
|
6. **Първо таблици без foreign keys** — после тези с foreign keys
|
||||||
+3
-1
@@ -689,7 +689,9 @@ SELECT schema_prompt('users') AS result;
|
|||||||
|----------|----------|
|
|----------|----------|
|
||||||
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
| DQL | SELECT, FROM, WHERE, ORDER BY, GROUP BY, HAVING, LIMIT, OFFSET, DISTINCT |
|
||||||
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
| DML | INSERT, UPDATE, DELETE, SET, VALUES |
|
||||||
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE GRAPH, DROP GRAPH |
|
| DDL | CREATE TYPE, DROP TYPE, CREATE INDEX, DROP INDEX, ALTER TYPE, CREATE GRAPH, DROP GRAPH, CREATE MIGRATION, APPLY MIGRATION |
|
||||||
|
| Migration | MIGRATION UP, MIGRATION DOWN, MIGRATION STATUS, MIGRATION DRY RUN |
|
||||||
|
| Import/Export | IMPORT FROM, EXPORT TO, FORMAT, CSV, JSON, NDJSON, DELIMITER, HEADER, BATCH |
|
||||||
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
| Join | INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN, CROSS JOIN, ON |
|
||||||
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
| Set | UNION, UNION ALL, INTERSECT, EXCEPT |
|
||||||
| CTEs | WITH, RECURSIVE, AS |
|
| CTEs | WITH, RECURSIVE, AS |
|
||||||
|
|||||||
+18
-20
@@ -240,36 +240,34 @@ curl http://localhost:9470/metrics
|
|||||||
All official clients support connection pooling:
|
All official clients support connection pooling:
|
||||||
|
|
||||||
### JavaScript
|
### JavaScript
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Pool } from 'baradb';
|
import { Pool } from 'baradb';
|
||||||
|
const pool = new Pool({ host: 'localhost', port: 9472, min: 5, max: 50 });
|
||||||
const pool = new Pool({
|
|
||||||
host: 'localhost',
|
|
||||||
port: 9472,
|
|
||||||
min: 5,
|
|
||||||
max: 50,
|
|
||||||
idleTimeout: 30000,
|
|
||||||
});
|
|
||||||
|
|
||||||
const client = await pool.acquire();
|
|
||||||
try {
|
|
||||||
const result = await client.query('SELECT 1');
|
|
||||||
} finally {
|
|
||||||
pool.release(client);
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Python
|
### Python
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from baradb import Pool
|
from baradb import Pool
|
||||||
|
|
||||||
pool = Pool("localhost", 9472, min_size=5, max_size=50)
|
pool = Pool("localhost", 9472, min_size=5, max_size=50)
|
||||||
with pool.connection() as conn:
|
|
||||||
result = conn.query("SELECT 1")
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Cross-Database Migration (Nim)
|
||||||
|
|
||||||
|
The Nim allographer client includes a cross-database migration engine:
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/migrate_data
|
||||||
|
|
||||||
|
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||||
|
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||||
|
echo report # Tables: 12/12, Rows: 45230, Time: 3.2s
|
||||||
|
```
|
||||||
|
|
||||||
|
Supported sources: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB.
|
||||||
|
See [Migrations & Import/Export](migration.md) for details.
|
||||||
|
|
||||||
## Data Types Mapping
|
## Data Types Mapping
|
||||||
|
|
||||||
| BaraDB Type | JavaScript | Python | Nim | Rust |
|
| BaraDB Type | JavaScript | Python | Nim | Rust |
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
# Migrations & Data Import/Export
|
||||||
|
|
||||||
|
BaraDB has a built-in migration system via BaraQL. Migrations are fully managed
|
||||||
|
by the server — checksums, locking, rollback, and status tracking. The client
|
||||||
|
sends BaraQL commands and does not maintain its own migration table.
|
||||||
|
|
||||||
|
## BaraQL Migration Syntax
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Create a migration with UP and DOWN scripts
|
||||||
|
CREATE MIGRATION add_users_table {
|
||||||
|
UP: CREATE TABLE users (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
email VARCHAR(255) UNIQUE,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
DOWN: DROP TABLE IF EXISTS users;
|
||||||
|
}
|
||||||
|
|
||||||
|
-- Apply a specific migration
|
||||||
|
APPLY MIGRATION add_users_table
|
||||||
|
|
||||||
|
-- Apply all pending migrations
|
||||||
|
MIGRATION UP
|
||||||
|
|
||||||
|
-- Apply next N migrations
|
||||||
|
MIGRATION UP 3
|
||||||
|
|
||||||
|
-- Rollback last migration
|
||||||
|
MIGRATION DOWN
|
||||||
|
|
||||||
|
-- Rollback last N migrations
|
||||||
|
MIGRATION DOWN 2
|
||||||
|
|
||||||
|
-- View migration status
|
||||||
|
MIGRATION STATUS
|
||||||
|
|
||||||
|
-- Dry run (validate without executing)
|
||||||
|
MIGRATION DRY RUN add_users_table
|
||||||
|
```
|
||||||
|
|
||||||
|
## Migration Locking
|
||||||
|
|
||||||
|
BaraDB acquires a global migration lock before applying any migration. This
|
||||||
|
prevents concurrent migration runs and ensures:
|
||||||
|
|
||||||
|
- Only one migration runs at a time
|
||||||
|
- Checksums are verified before execution
|
||||||
|
- Failed migrations can be rolled back safely
|
||||||
|
- `MIGRATION STATUS` shows exact state of each migration
|
||||||
|
|
||||||
|
## Checksums
|
||||||
|
|
||||||
|
Every migration body is SHA-256 hashed. The server stores the checksum at
|
||||||
|
creation time and verifies it before applying. This prevents accidental
|
||||||
|
modification of already-registered migrations.
|
||||||
|
|
||||||
|
```
|
||||||
|
CREATE MIGRATION add_users → checksum: a3f2b8c1...
|
||||||
|
APPLY MIGRATION add_users → verifies checksum matches → executes
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Migrations with `DOWN` scripts support rollback:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE MIGRATION add_column {
|
||||||
|
UP: ALTER TABLE users ADD COLUMN phone VARCHAR(20);
|
||||||
|
DOWN: ALTER TABLE users DROP COLUMN phone;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
When you run `MIGRATION DOWN`, the server executes the DOWN script and marks
|
||||||
|
the migration as rolled back.
|
||||||
|
|
||||||
|
## Dry Run
|
||||||
|
|
||||||
|
Validate a migration before applying:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
MIGRATION DRY RUN add_users_table
|
||||||
|
-- Output:
|
||||||
|
-- DRY RUN add_users_table:
|
||||||
|
-- Statements: 1
|
||||||
|
-- [1] nkCreateTable
|
||||||
|
-- DOWN script: yes
|
||||||
|
-- Checksum: a3f2b8c1d4e5f6a7b8c9d0e1f2a3b4c5
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## IMPORT FROM / EXPORT TO
|
||||||
|
|
||||||
|
BaraDB supports importing and exporting data directly via BaraQL.
|
||||||
|
|
||||||
|
### IMPORT FROM
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Import from CSV
|
||||||
|
IMPORT FROM '/data/users.csv' INTO users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
BATCH 1000
|
||||||
|
|
||||||
|
-- Import from JSON array
|
||||||
|
IMPORT FROM '/data/users.json' INTO users
|
||||||
|
FORMAT JSON
|
||||||
|
|
||||||
|
-- Import from NDJSON (newline-delimited JSON)
|
||||||
|
IMPORT FROM '/data/users.ndjson' INTO users
|
||||||
|
FORMAT NDJSON
|
||||||
|
```
|
||||||
|
|
||||||
|
Options:
|
||||||
|
| Option | Values | Default | Description |
|
||||||
|
|--------|--------|---------|-------------|
|
||||||
|
| `FORMAT` | `CSV`, `JSON`, `NDJSON` | `CSV` | Input file format |
|
||||||
|
| `DELIMITER` | any char | `,` | CSV field delimiter |
|
||||||
|
| `HEADER` | `true`/`false` | `true` | CSV first row is header |
|
||||||
|
| `BATCH` | integer | `1000` | Rows per insert batch |
|
||||||
|
|
||||||
|
### EXPORT TO
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Export to CSV
|
||||||
|
EXPORT TO '/backup/users.csv' FROM users
|
||||||
|
FORMAT CSV
|
||||||
|
DELIMITER ','
|
||||||
|
HEADER true
|
||||||
|
|
||||||
|
-- Export to JSON
|
||||||
|
EXPORT TO '/backup/users.json' FROM users
|
||||||
|
FORMAT JSON
|
||||||
|
|
||||||
|
-- Export to NDJSON
|
||||||
|
EXPORT TO '/backup/users.ndjson' FROM users
|
||||||
|
FORMAT NDJSON
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cross-Database Migration
|
||||||
|
|
||||||
|
BaraDB's Nim client (nim-allographer) includes a cross-database migration
|
||||||
|
engine. Migrate data from PostgreSQL, MySQL, MariaDB, SQLite, or SurrealDB
|
||||||
|
directly to BaraDB.
|
||||||
|
|
||||||
|
### Supported Sources
|
||||||
|
|
||||||
|
| Database | Schema Extraction | Status |
|
||||||
|
|----------|-------------------|--------|
|
||||||
|
| PostgreSQL | `information_schema` | ✅ |
|
||||||
|
| MySQL | `information_schema` | ✅ |
|
||||||
|
| MariaDB | `information_schema` | ✅ |
|
||||||
|
| SQLite | `sqlite_master` + `PRAGMA` | ✅ |
|
||||||
|
| SurrealDB | `INFO FOR DB` / `INFO FOR TABLE` | ✅ |
|
||||||
|
|
||||||
|
### Nim API
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/migrate_data
|
||||||
|
|
||||||
|
# Connect to source and target
|
||||||
|
let pg = dbOpen(PostgreSQL, "sourcedb", "user", "pass", "localhost", 5432)
|
||||||
|
let bdb = dbOpen(Baradb, "targetdb", "admin", "", "127.0.0.1", 9472)
|
||||||
|
|
||||||
|
# Migrate all tables
|
||||||
|
let report = waitFor migrate(pg, bdb, batchSize = 5000)
|
||||||
|
echo report
|
||||||
|
# Migration: PostgreSQL → BaraDB
|
||||||
|
# Tables: 12/12
|
||||||
|
# Rows: 45230
|
||||||
|
# Time: 3.2s
|
||||||
|
|
||||||
|
# Migrate specific tables
|
||||||
|
let report = waitFor migrate(pg, bdb,
|
||||||
|
tables = @["users", "orders", "products"])
|
||||||
|
```
|
||||||
|
|
||||||
|
### Type Mapping
|
||||||
|
|
||||||
|
The migration engine automatically maps source types to BaraDB equivalents:
|
||||||
|
|
||||||
|
| PostgreSQL | MySQL | SQLite | BaraDB |
|
||||||
|
|------------|-------|--------|--------|
|
||||||
|
| `SERIAL` | `INT AUTO_INCREMENT` | `INTEGER PK` | `SERIAL` |
|
||||||
|
| `VARCHAR(n)` | `VARCHAR(n)` | `TEXT` | `VARCHAR(n)` |
|
||||||
|
| `TEXT` | `TEXT` | `TEXT` | `TEXT` |
|
||||||
|
| `BOOLEAN` | `TINYINT(1)` | `INTEGER` | `BOOLEAN` |
|
||||||
|
| `JSONB` | `JSON` | `TEXT` | `JSON` |
|
||||||
|
| `TIMESTAMP` | `DATETIME` | `TEXT` | `TIMESTAMP` |
|
||||||
|
| `UUID` | `CHAR(36)` | `TEXT` | `UUID` |
|
||||||
|
|
||||||
|
Full type map: 50+ types supported.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Nim allographer Client API
|
||||||
|
|
||||||
|
### Migration Management
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import allographer/query_builder/models/baradb/baradb_exec
|
||||||
|
|
||||||
|
# Create migration
|
||||||
|
let qr = waitFor rdb.createMigration("add_products",
|
||||||
|
"CREATE TABLE products (id SERIAL PRIMARY KEY, name VARCHAR(255))",
|
||||||
|
"DROP TABLE IF EXISTS products")
|
||||||
|
|
||||||
|
# Apply
|
||||||
|
let qr = waitFor rdb.applyMigration("add_products")
|
||||||
|
|
||||||
|
# Apply all pending
|
||||||
|
let qr = waitFor rdb.migrateUp()
|
||||||
|
|
||||||
|
# Rollback
|
||||||
|
let qr = waitFor rdb.migrateDown(1)
|
||||||
|
|
||||||
|
# Status
|
||||||
|
let status = waitFor rdb.migrationStatus()
|
||||||
|
|
||||||
|
# Check if applied
|
||||||
|
if waitFor rdb.isMigrationApplied("add_products"):
|
||||||
|
echo "Already applied"
|
||||||
|
|
||||||
|
# Dry run
|
||||||
|
let qr = waitFor rdb.migrationDryRun("add_products")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prepared Statements
|
||||||
|
|
||||||
|
```nim
|
||||||
|
let stmt = waitFor rdb.prepare(
|
||||||
|
"SELECT * FROM users WHERE age > ? AND status = ?", nArgs = 2)
|
||||||
|
|
||||||
|
let results = waitFor stmt.preparedGet(@[
|
||||||
|
WireValue(kind: fkInt32, int32Val: 18),
|
||||||
|
WireValue(kind: fkString, strVal: "active")
|
||||||
|
])
|
||||||
|
|
||||||
|
stmt.flushStmt()
|
||||||
|
rdb.clearStmtCache()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pagination
|
||||||
|
|
||||||
|
```nim
|
||||||
|
# Offset-based
|
||||||
|
let page = waitFor rdb.table("users").paginate(page = 1, perPage = 20)
|
||||||
|
|
||||||
|
# Cursor-based (faster for large tables)
|
||||||
|
let batch = waitFor rdb.table("users")
|
||||||
|
.fastPaginate("id", perPage = 100, afterId = "42")
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Best Practices
|
||||||
|
|
||||||
|
1. **Always include DOWN scripts** — enables safe rollback
|
||||||
|
2. **Use dry run first** — validate migrations before applying in production
|
||||||
|
3. **Batch large imports** — use `BATCH 1000` for CSV imports to avoid memory issues
|
||||||
|
4. **Export before migration** — backup data with `EXPORT TO` before cross-DB migration
|
||||||
|
5. **Check status after migration** — verify with `MIGRATION STATUS`
|
||||||
|
6. **Migrate tables without foreign keys first** — then tables with foreign keys
|
||||||
@@ -44,6 +44,7 @@
|
|||||||
- [Security Guide](en/security.md)
|
- [Security Guide](en/security.md)
|
||||||
- [Monitoring & Observability](en/monitoring.md)
|
- [Monitoring & Observability](en/monitoring.md)
|
||||||
- [Backup & Recovery](en/backup.md)
|
- [Backup & Recovery](en/backup.md)
|
||||||
|
- [Migrations & Import/Export](en/migration.md)
|
||||||
- [Troubleshooting](en/troubleshooting.md)
|
- [Troubleshooting](en/troubleshooting.md)
|
||||||
|
|
||||||
### Advanced
|
### Advanced
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ type
|
|||||||
nkMigrationStatus
|
nkMigrationStatus
|
||||||
nkMigrationUp
|
nkMigrationUp
|
||||||
nkMigrationDown
|
nkMigrationDown
|
||||||
|
nkImportFrom
|
||||||
|
nkExportTo
|
||||||
nkMigrationDryRun
|
nkMigrationDryRun
|
||||||
nkCreateUser
|
nkCreateUser
|
||||||
nkDropUser
|
nkDropUser
|
||||||
@@ -363,6 +365,19 @@ type
|
|||||||
mdCount*: int # 0 = 1, N = rollback N migrations
|
mdCount*: int # 0 = 1, N = rollback N migrations
|
||||||
of nkMigrationDryRun:
|
of nkMigrationDryRun:
|
||||||
mdrName*: string # migration name to dry-run
|
mdrName*: string # migration name to dry-run
|
||||||
|
of nkImportFrom:
|
||||||
|
impPath*: string
|
||||||
|
impTable*: string
|
||||||
|
impFormat*: string # "csv", "json", "ndjson"
|
||||||
|
impDelimiter*: char
|
||||||
|
impHasHeader*: bool
|
||||||
|
impBatchSize*: int
|
||||||
|
of nkExportTo:
|
||||||
|
expPath*: string
|
||||||
|
expTable*: string
|
||||||
|
expFormat*: string
|
||||||
|
expDelimiter*: char
|
||||||
|
expIncludeHeader*: bool
|
||||||
of nkCreateIndex:
|
of nkCreateIndex:
|
||||||
ciTarget*: string
|
ciTarget*: string
|
||||||
ciName*: string
|
ciName*: string
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import ../storage/btree
|
|||||||
import ../storage/wal
|
import ../storage/wal
|
||||||
import ../core/mvcc
|
import ../core/mvcc
|
||||||
import ../core/tracing
|
import ../core/tracing
|
||||||
|
import ../client/fileops
|
||||||
import ../fts/engine as fts
|
import ../fts/engine as fts
|
||||||
import ../core/registry
|
import ../core/registry
|
||||||
|
|
||||||
@@ -219,6 +220,24 @@ proc sqlEscapeString*(s: string): string =
|
|||||||
## Escape SQL string literals by doubling single-quotes.
|
## Escape SQL string literals by doubling single-quotes.
|
||||||
result = s.replace("'", "''")
|
result = s.replace("'", "''")
|
||||||
|
|
||||||
|
proc buildInsertSql*(table: string, columns: seq[string], rows: seq[seq[string]]): string =
|
||||||
|
## Build a multi-row INSERT statement for bulk import.
|
||||||
|
result = "INSERT INTO \"" & sqlEscapeIdent(table) & "\" ("
|
||||||
|
for i, col in columns:
|
||||||
|
if i > 0: result &= ", "
|
||||||
|
result &= "\"" & sqlEscapeIdent(col) & "\""
|
||||||
|
result &= ") VALUES "
|
||||||
|
for ri, row in rows:
|
||||||
|
if ri > 0: result &= ", "
|
||||||
|
result &= "("
|
||||||
|
for ci, val in row:
|
||||||
|
if ci > 0: result &= ", "
|
||||||
|
if val.len == 0 or val == "\\N":
|
||||||
|
result &= "NULL"
|
||||||
|
else:
|
||||||
|
result &= "'" & sqlEscapeString(val) & "'"
|
||||||
|
result &= ")"
|
||||||
|
|
||||||
proc exprToSql(node: Node): string =
|
proc exprToSql(node: Node): string =
|
||||||
if node == nil:
|
if node == nil:
|
||||||
return ""
|
return ""
|
||||||
@@ -5279,6 +5298,87 @@ proc executeQueryImpl(ctx: ExecutionContext, astNode: Node, params: seq[WireValu
|
|||||||
|
|
||||||
return okResult(msg="Rolled back " & $rolledBackCount & " migrations")
|
return okResult(msg="Rolled back " & $rolledBackCount & " migrations")
|
||||||
|
|
||||||
|
of nkImportFrom:
|
||||||
|
let path = stmt.impPath
|
||||||
|
let table = stmt.impTable
|
||||||
|
let format = stmt.impFormat
|
||||||
|
if not fileExists(path):
|
||||||
|
return errResult("File not found: " & path)
|
||||||
|
let content = readFile(path)
|
||||||
|
var columns: seq[string] = @[]
|
||||||
|
var rows: seq[seq[string]] = @[]
|
||||||
|
case format
|
||||||
|
of "csv":
|
||||||
|
(columns, rows) = parseCsvTable(content, stmt.impDelimiter, stmt.impHasHeader)
|
||||||
|
of "json":
|
||||||
|
(columns, rows) = parseJsonTable(content)
|
||||||
|
of "ndjson":
|
||||||
|
(columns, rows) = parseNdjsonTable(content)
|
||||||
|
else:
|
||||||
|
return errResult("Unsupported import format: " & format)
|
||||||
|
if columns.len == 0:
|
||||||
|
return errResult("No columns found in import file")
|
||||||
|
var inserted = 0
|
||||||
|
let batchSize = stmt.impBatchSize
|
||||||
|
var batchRows: seq[seq[string]] = @[]
|
||||||
|
for row in rows:
|
||||||
|
batchRows.add(row)
|
||||||
|
if batchRows.len >= batchSize:
|
||||||
|
# Generate INSERT INTO t (c1,c2) VALUES (...),(...) and execute
|
||||||
|
let sql = buildInsertSql(table, columns, batchRows)
|
||||||
|
let tokens = qlex.tokenize(sql)
|
||||||
|
let astNode = qpar.parse(tokens)
|
||||||
|
if astNode.stmts.len > 0:
|
||||||
|
let insResult = executeQueryImpl(ctx, astNode)
|
||||||
|
if insResult.success:
|
||||||
|
inserted += batchRows.len
|
||||||
|
batchRows.setLen(0)
|
||||||
|
if batchRows.len > 0:
|
||||||
|
let sql = buildInsertSql(table, columns, batchRows)
|
||||||
|
let tokens = qlex.tokenize(sql)
|
||||||
|
let astNode = qpar.parse(tokens)
|
||||||
|
if astNode.stmts.len > 0:
|
||||||
|
let insResult = executeQueryImpl(ctx, astNode)
|
||||||
|
if insResult.success:
|
||||||
|
inserted += batchRows.len
|
||||||
|
return okResult(msg="IMPORTED " & $inserted & " rows into " & table)
|
||||||
|
|
||||||
|
of nkExportTo:
|
||||||
|
let path = stmt.expPath
|
||||||
|
let table = stmt.expTable
|
||||||
|
let format = stmt.expFormat
|
||||||
|
var rows: seq[Row] = @[]
|
||||||
|
var cols: seq[string] = @[]
|
||||||
|
let scanResult = execScan(ctx, table)
|
||||||
|
if scanResult.len == 0:
|
||||||
|
let tbl = ctx.getTableDef(table)
|
||||||
|
if tbl.columns.len == 0:
|
||||||
|
return errResult("Table not found: " & table)
|
||||||
|
for col in tbl.columns:
|
||||||
|
cols.add(col.name)
|
||||||
|
else:
|
||||||
|
for k, v in scanResult[0].pairs:
|
||||||
|
cols.add(k)
|
||||||
|
rows = scanResult
|
||||||
|
var content = ""
|
||||||
|
var strRows: seq[seq[string]] = @[]
|
||||||
|
for row in rows:
|
||||||
|
var strRow: seq[string] = @[]
|
||||||
|
for col in cols:
|
||||||
|
strRow.add(if col in row: valueToString(row[col]) else: "")
|
||||||
|
strRows.add(strRow)
|
||||||
|
case format
|
||||||
|
of "csv":
|
||||||
|
content = toCsv(cols, strRows, stmt.expDelimiter, stmt.expIncludeHeader)
|
||||||
|
of "json":
|
||||||
|
content = toJson(cols, strRows)
|
||||||
|
of "ndjson":
|
||||||
|
content = toNdjson(cols, strRows)
|
||||||
|
else:
|
||||||
|
return errResult("Unsupported export format: " & format)
|
||||||
|
writeFile(path, content)
|
||||||
|
return okResult(msg="EXPORTED " & $strRows.len & " rows to " & path)
|
||||||
|
|
||||||
of nkMigrationDryRun:
|
of nkMigrationDryRun:
|
||||||
let (found, upBody, downBody) = getMigrationBody(ctx, stmt.mdrName)
|
let (found, upBody, downBody) = getMigrationBody(ctx, stmt.mdrName)
|
||||||
if not found:
|
if not found:
|
||||||
|
|||||||
@@ -102,6 +102,15 @@ type
|
|||||||
tkAfter
|
tkAfter
|
||||||
tkInstead
|
tkInstead
|
||||||
tkOf
|
tkOf
|
||||||
|
tkImport
|
||||||
|
tkExport
|
||||||
|
tkFormat
|
||||||
|
tkDelimiter
|
||||||
|
tkHeader
|
||||||
|
tkBatch
|
||||||
|
tkCsv
|
||||||
|
tkJsonFmt
|
||||||
|
tkNdjson
|
||||||
tkMigration
|
tkMigration
|
||||||
tkApply
|
tkApply
|
||||||
tkStatus
|
tkStatus
|
||||||
@@ -334,6 +343,15 @@ const keywords*: Table[string, TokenKind] = {
|
|||||||
"after": tkAfter,
|
"after": tkAfter,
|
||||||
"instead": tkInstead,
|
"instead": tkInstead,
|
||||||
"of": tkOf,
|
"of": tkOf,
|
||||||
|
"import": tkImport,
|
||||||
|
"export": tkExport,
|
||||||
|
"format": tkFormat,
|
||||||
|
"delimiter": tkDelimiter,
|
||||||
|
"header": tkHeader,
|
||||||
|
"batch": tkBatch,
|
||||||
|
"csv": tkCsv,
|
||||||
|
"json": tkJsonFmt,
|
||||||
|
"ndjson": tkNdjson,
|
||||||
"migration": tkMigration,
|
"migration": tkMigration,
|
||||||
"apply": tkApply,
|
"apply": tkApply,
|
||||||
"status": tkStatus,
|
"status": tkStatus,
|
||||||
|
|||||||
@@ -1494,6 +1494,63 @@ proc parseMigrationDryRun(p: var Parser): Node =
|
|||||||
let name = p.expect(tkIdent).value
|
let name = p.expect(tkIdent).value
|
||||||
result = Node(kind: nkMigrationDryRun, mdrName: name, line: tok.line, col: tok.col)
|
result = Node(kind: nkMigrationDryRun, mdrName: name, line: tok.line, col: tok.col)
|
||||||
|
|
||||||
|
proc parseImportFrom(p: var Parser): Node =
|
||||||
|
let tok = p.expect(tkImport)
|
||||||
|
discard p.expect(tkFrom)
|
||||||
|
let path = p.expect(tkStringLit).value
|
||||||
|
discard p.expect(tkInto)
|
||||||
|
let table = p.expect(tkIdent).value
|
||||||
|
var format = "csv"
|
||||||
|
var delimiter = ','
|
||||||
|
var hasHeader = true
|
||||||
|
var batchSize = 1000
|
||||||
|
while p.peek().kind in {tkFormat, tkDelimiter, tkHeader, tkBatch}:
|
||||||
|
let kw = p.advance()
|
||||||
|
case kw.kind
|
||||||
|
of tkFormat:
|
||||||
|
let fmt = p.expect(tkIdent).value.toLower()
|
||||||
|
format = fmt
|
||||||
|
of tkDelimiter:
|
||||||
|
let delim = p.expect(tkStringLit).value
|
||||||
|
if delim.len > 0: delimiter = delim[0]
|
||||||
|
of tkHeader:
|
||||||
|
let hdr = p.expect(tkIdent).value.toLower()
|
||||||
|
hasHeader = hdr == "true" or hdr == "yes"
|
||||||
|
of tkBatch:
|
||||||
|
batchSize = parseInt(p.expect(tkIntLit).value)
|
||||||
|
else: discard
|
||||||
|
result = Node(kind: nkImportFrom, impPath: path, impTable: table,
|
||||||
|
impFormat: format, impDelimiter: delimiter,
|
||||||
|
impHasHeader: hasHeader, impBatchSize: batchSize,
|
||||||
|
line: tok.line, col: tok.col)
|
||||||
|
|
||||||
|
proc parseExportTo(p: var Parser): Node =
|
||||||
|
let tok = p.expect(tkExport)
|
||||||
|
discard p.expect(tkTo)
|
||||||
|
let path = p.expect(tkStringLit).value
|
||||||
|
discard p.expect(tkFrom)
|
||||||
|
let table = p.expect(tkIdent).value
|
||||||
|
var format = "csv"
|
||||||
|
var delimiter = ','
|
||||||
|
var includeHeader = true
|
||||||
|
while p.peek().kind in {tkFormat, tkDelimiter, tkHeader}:
|
||||||
|
let kw = p.advance()
|
||||||
|
case kw.kind
|
||||||
|
of tkFormat:
|
||||||
|
let fmt = p.expect(tkIdent).value.toLower()
|
||||||
|
format = fmt
|
||||||
|
of tkDelimiter:
|
||||||
|
let delim = p.expect(tkStringLit).value
|
||||||
|
if delim.len > 0: delimiter = delim[0]
|
||||||
|
of tkHeader:
|
||||||
|
let hdr = p.expect(tkIdent).value.toLower()
|
||||||
|
includeHeader = hdr == "true" or hdr == "yes"
|
||||||
|
else: discard
|
||||||
|
result = Node(kind: nkExportTo, expPath: path, expTable: table,
|
||||||
|
expFormat: format, expDelimiter: delimiter,
|
||||||
|
expIncludeHeader: includeHeader,
|
||||||
|
line: tok.line, col: tok.col)
|
||||||
|
|
||||||
proc parseCreateUser(p: var Parser): Node =
|
proc parseCreateUser(p: var Parser): Node =
|
||||||
let tok = p.expect(tkCreate)
|
let tok = p.expect(tkCreate)
|
||||||
discard p.expect(tkUser)
|
discard p.expect(tkUser)
|
||||||
@@ -1701,6 +1758,8 @@ proc parseStatement*(p: var Parser): Node =
|
|||||||
case p.peek().kind
|
case p.peek().kind
|
||||||
of tkWith, tkSelect: p.parseSelect()
|
of tkWith, tkSelect: p.parseSelect()
|
||||||
of tkInsert: p.parseInsert()
|
of tkInsert: p.parseInsert()
|
||||||
|
of tkImport: p.parseImportFrom()
|
||||||
|
of tkExport: p.parseExportTo()
|
||||||
of tkUpdate: p.parseUpdate()
|
of tkUpdate: p.parseUpdate()
|
||||||
of tkDelete: p.parseDelete()
|
of tkDelete: p.parseDelete()
|
||||||
of tkMerge: p.parseMerge()
|
of tkMerge: p.parseMerge()
|
||||||
|
|||||||
Reference in New Issue
Block a user