feat: migrate system + cross-DB engine + IMPORT/EXPORT syntax -- 22 files, client+server+docs

clients/nim-allographer/src/allographer/migrate_data.nim

@@ -0,0 +1,634 @@
+## Cross-DB Migration Engine — migrate data between any allographer-supported DB and BaraDB.
+##
+## Usage:
+##   migrate(sourceRdb, targetRdb, tables)           — migrate specific tables
+##   migrateAll(sourceRdb, targetRdb)                 — migrate all tables
+##   migrateFromUrl(sourceUrl, targetUrl)             — URL-based migration
+##
+## Supported source databases: PostgreSQL, MySQL, MariaDB, SQLite, SurrealDB
+## Target: BaraDB (native migration system via CREATE MIGRATION + MIGRATION UP)
+
+import std/asyncdispatch
+import std/json
+import std/options
+import std/strformat
+import std/strutils
+import std/tables
+import std/times
+
+import ./env
+import ./query_builder/libs/database_url
+import ./query_builder/models/baradb/baradb_types
+import ./query_builder/models/baradb/baradb_query
+import ./query_builder/models/baradb/baradb_exec
+
+when isExistsPostgres:
+  import ./query_builder/models/postgres/postgres_types
+  import ./query_builder/models/postgres/postgres_query
+  import ./query_builder/models/postgres/postgres_exec
+  import ./query_builder/models/postgres/postgres_open
+
+when isExistsMysql:
+  import ./query_builder/models/mysql/mysql_types
+  import ./query_builder/models/mysql/mysql_query
+  import ./query_builder/models/mysql/mysql_exec
+  import ./query_builder/models/mysql/mysql_open
+
+when isExistsMariadb:
+  import ./query_builder/models/mariadb/mariadb_types
+  import ./query_builder/models/mariadb/mariadb_query
+  import ./query_builder/models/mariadb/mariadb_exec
+  import ./query_builder/models/mariadb/mariadb_open
+
+when isExistsSqlite:
+  import ./query_builder/models/sqlite/sqlite_types
+  import ./query_builder/models/sqlite/sqlite_query
+  import ./query_builder/models/sqlite/sqlite_exec
+  import ./query_builder/models/sqlite/sqlite_open
+
+when isExistsSurrealdb:
+  import ./query_builder/models/surreal/surreal_types
+  import ./query_builder/models/surreal/surreal_query
+  import ./query_builder/models/surreal/surreal_exec
+  import ./query_builder/models/surreal/surreal_open
+
+# ==============================================================================
+# Types
+# ==============================================================================
+
+type
+  ColumnInfo* = tuple[name: string, typ: string, isPk: bool, isNullable: bool, defaultVal: string]
+
+  TableInfo* = object
+    name*: string
+    columns*: seq[ColumnInfo]
+
+  MigrationProgress* = object
+    tableName*: string
+    totalRows*: int
+    transferredRows*: int
+    status*: string  # "pending", "in_progress", "done", "failed"
+    error*: string
+
+  MigrationReport* = object
+    sourceDb*: string
+    targetDb*: string
+    tablesTotal*: int
+    tablesDone*: int
+    rowsTotal*: int
+    rowsTransferred*: int
+    startedAt*: float
+    completedAt*: float
+    errors*: seq[string]
+
+# ==============================================================================
+# Type Mapping: source DB → BaraDB
+# ==============================================================================
+
+const BARADB_TYPE_MAP = {
+  # PostgreSQL
+  "smallint": "SMALLINT",
+  "integer": "INTEGER",
+  "bigint": "BIGINT",
+  "serial": "SERIAL",
+  "bigserial": "BIGSERIAL",
+  "real": "REAL",
+  "double precision": "DOUBLE PRECISION",
+  "numeric": "DECIMAL",
+  "decimal": "DECIMAL",
+  "character": "VARCHAR(255)",
+  "character varying": "VARCHAR",
+  "varchar": "VARCHAR",
+  "text": "TEXT",
+  "boolean": "BOOLEAN",
+  "bool": "BOOLEAN",
+  "date": "DATE",
+  "timestamp": "TIMESTAMP",
+  "timestamp without time zone": "TIMESTAMP",
+  "timestamp with time zone": "TIMESTAMPTZ",
+  "time": "TIME",
+  "time without time zone": "TIME",
+  "bytea": "BYTEA",
+  "uuid": "UUID",
+  "json": "JSON",
+  "jsonb": "JSON",
+  # MySQL/MariaDB
+  "tinyint": "SMALLINT",
+  "mediumint": "INTEGER",
+  "int": "INTEGER",
+  "float": "REAL",
+  "double": "DOUBLE PRECISION",
+  "char": "VARCHAR(255)",
+  "longtext": "TEXT",
+  "mediumtext": "TEXT",
+  "tinytext": "TEXT",
+  "blob": "BYTEA",
+  "longblob": "BYTEA",
+  "mediumblob": "BYTEA",
+  "tinyblob": "BYTEA",
+  "datetime": "TIMESTAMP",
+  "enum": "VARCHAR(255)",
+  "set": "VARCHAR(255)",
+  "year": "SMALLINT",
+  # SQLite
+  "int": "INTEGER",
+  "integer": "INTEGER",
+  "real": "REAL",
+  "blob": "BYTEA",
+  # SurrealDB
+  "string": "VARCHAR(255)",
+  "number": "DOUBLE PRECISION",
+  "object": "JSON",
+  "array": "JSON",
+}.toTable()
+
+proc mapType*(sourceType: string): string =
+  ## Map a source database column type to the closest BaraDB type.
+  let lower = sourceType.toLower().split("(")[0].strip()
+  if lower in BARADB_TYPE_MAP:
+    result = BARADB_TYPE_MAP[lower]
+  else:
+    result = "VARCHAR(255)"
+
+# ==============================================================================
+# Schema Extraction
+# ==============================================================================
+
+# --- PostgreSQL ---
+when isExistsPostgres:
+  proc extractSchema*(rdb: PostgresConnections): Future[seq[TableInfo]] {.async.} =
+    result = @[]
+    let tables = await rdb.raw(
+      "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
+    ).get()
+    for table in tables:
+      let tableName = table["table_name"].getStr()
+      if tableName == "_allographer_migrations": continue
+      var cols: seq[ColumnInfo] = @[]
+      let columns = await rdb.raw(
+        """SELECT c.column_name, c.data_type, c.is_nullable, c.column_default,
+           CASE WHEN pk.column_name IS NOT NULL THEN true ELSE false END as is_pk
+           FROM information_schema.columns c
+           LEFT JOIN (
+             SELECT ku.column_name FROM information_schema.table_constraints tc
+             JOIN information_schema.key_column_usage ku
+               ON tc.constraint_name = ku.constraint_name
+             WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_name = ?
+           ) pk ON c.column_name = pk.column_name
+           WHERE c.table_name = ?
+           ORDER BY c.ordinal_position""",
+        %*[tableName, tableName]
+      ).get()
+      for col in columns:
+        cols.add((
+          name: col["column_name"].getStr(),
+          typ: col["data_type"].getStr(),
+          isPk: col["is_pk"].getStr() == "true",
+          isNullable: col["is_nullable"].getStr() == "YES",
+          defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
+                        col["column_default"].getStr() else: ""
+        ))
+      result.add(TableInfo(name: tableName, columns: cols))
+
+# --- MySQL ---
+when isExistsMysql:
+  proc extractSchema*(rdb: MysqlConnections): Future[seq[TableInfo]] {.async.} =
+    result = @[]
+    let tables = await rdb.raw(
+      "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
+    ).get()
+    for table in tables:
+      let tableName = table["table_name"].getStr()
+      if tableName == "_allographer_migrations": continue
+      var cols: seq[ColumnInfo] = @[]
+      let columns = await rdb.raw(
+        """SELECT column_name, data_type, is_nullable, column_default,
+           column_key FROM information_schema.columns
+           WHERE table_name = ? AND table_schema = DATABASE()
+           ORDER BY ordinal_position""",
+        %*[tableName]
+      ).get()
+      for col in columns:
+        cols.add((
+          name: col["column_name"].getStr(),
+          typ: col["data_type"].getStr(),
+          isPk: col["column_key"].getStr() == "PRI",
+          isNullable: col["is_nullable"].getStr() == "YES",
+          defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
+                        col["column_default"].getStr() else: ""
+        ))
+      result.add(TableInfo(name: tableName, columns: cols))
+
+# --- MariaDB ---
+when isExistsMariadb:
+  proc extractSchema*(rdb: MariadbConnections): Future[seq[TableInfo]] {.async.} =
+    result = @[]
+    let tables = await rdb.raw(
+      "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()"
+    ).get()
+    for table in tables:
+      let tableName = table["table_name"].getStr()
+      if tableName == "_allographer_migrations": continue
+      var cols: seq[ColumnInfo] = @[]
+      let columns = await rdb.raw(
+        """SELECT column_name, data_type, is_nullable, column_default,
+           column_key FROM information_schema.columns
+           WHERE table_name = ? AND table_schema = DATABASE()
+           ORDER BY ordinal_position""",
+        %*[tableName]
+      ).get()
+      for col in columns:
+        cols.add((
+          name: col["column_name"].getStr(),
+          typ: col["data_type"].getStr(),
+          isPk: col["column_key"].getStr() == "PRI",
+          isNullable: col["is_nullable"].getStr() == "YES",
+          defaultVal: if col.hasKey("column_default") and not col["column_default"].isNull:
+                        col["column_default"].getStr() else: ""
+        ))
+      result.add(TableInfo(name: tableName, columns: cols))
+
+# --- SQLite ---
+when isExistsSqlite:
+  proc extractSchema*(rdb: SqliteConnections): Future[seq[TableInfo]] {.async.} =
+    result = @[]
+    let tables = await rdb.raw(
+      "SELECT name as table_name FROM sqlite_master WHERE type = 'table'"
+    ).get()
+    for table in tables:
+      let tableName = table["table_name"].getStr()
+      if tableName == "_allographer_migrations": continue
+      if tableName == "sqlite_sequence": continue
+      var cols: seq[ColumnInfo] = @[]
+      let columns = await rdb.raw("PRAGMA table_info(?)", %*[tableName]).get()
+      for col in columns:
+        cols.add((
+          name: col["name"].getStr(),
+          typ: col["type"].getStr(),
+          isPk: col["pk"].getStr() == "1",
+          isNullable: col["notnull"].getStr() == "0",
+          defaultVal: if col.hasKey("dflt_value") and not col["dflt_value"].isNull:
+                        col["dflt_value"].getStr() else: ""
+        ))
+      result.add(TableInfo(name: tableName, columns: cols))
+
+# --- SurrealDB ---
+when isExistsSurrealdb:
+  proc extractSchema*(rdb: SurrealConnections): Future[seq[TableInfo]] {.async.} =
+    result = @[]
+    let dbResponse = await rdb.raw("INFO FOR DB").get()
+    if dbResponse.len == 0: return
+    let tables = dbResponse[0]["result"]["tables"]
+    for tableName, _ in tables.getFields().pairs:
+      if tableName == "_allographer_migrations": continue
+      if tableName == "_autoincrement_sequences": continue
+      var cols: seq[ColumnInfo] = @[]
+      let tableInfo = await rdb.raw(&"INFO FOR TABLE {tableName}").get()
+      if tableInfo.len > 0:
+        let fields = tableInfo[0]["result"]["fields"]
+        for fieldName, _ in fields.getFields().pairs:
+          cols.add((
+            name: fieldName,
+            typ: "string",  # SurrealDB is schemaless
+            isPk: fieldName == "id",
+            isNullable: fieldName != "id",
+            defaultVal: ""
+          ))
+      result.add(TableInfo(name: tableName, columns: cols))
+
+# ==============================================================================
+# DDL Generation
+# ==============================================================================
+
+proc generateBaraDBDDL*(table: TableInfo): tuple[upSql: string, downSql: string] =
+  ## Generate CREATE TABLE (UP) and DROP TABLE (DOWN) DDL for BaraDB.
+  var colDefs: seq[string] = @[]
+  for col in table.columns:
+    var def = &"`{col.name}` {mapType(col.typ)}"
+    if col.isPk:
+      def &= " PRIMARY KEY"
+    if not col.isNullable:
+      def &= " NOT NULL"
+    if col.defaultVal.len > 0:
+      def &= " DEFAULT " & col.defaultVal
+    colDefs.add(def)
+  let upSql = &"CREATE TABLE `{table.name}` ({colDefs.join(\", \")})"
+  let downSql = &"DROP TABLE IF EXISTS `{table.name}`"
+  return (upSql, downSql)
+
+proc generateMigrationName*(tableName: string): string =
+  let timestamp = getTime().toUnix()
+  return &"migrate_{tableName}_{timestamp}"
+
+# ==============================================================================
+# Data Transfer
+# ==============================================================================
+
+proc transferTable*(sourceConn: BaradbConnections, targetConn: BaradbConnections,
+                    tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+  ## Generic table transfer for BaraDB→BaraDB (used internally).
+  ## For cross-DB, use the typed overloads below.
+  result = MigrationProgress(tableName: tableName, totalRows: 0,
+                              transferredRows: 0, status: "in_progress")
+  try:
+    # Count rows
+    let countVal = await sourceConn.table(tableName).count()
+    result.totalRows = countVal
+
+    # Transfer in batches
+    var offset = 0
+    while offset < result.totalRows:
+      let rows = await sourceConn.table(tableName)
+        .limit(batchSize)
+        .offset(offset)
+        .get()
+      if rows.len == 0: break
+      await targetConn.table(tableName).insert(rows)
+      result.transferredRows += rows.len
+      offset += batchSize
+
+    result.status = "done"
+  except CatchableError as e:
+    result.status = "failed"
+    result.error = e.msg
+
+# Typed overloads for each source DB type
+when isExistsPostgres:
+  proc transferTable*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
+                      tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+    result = MigrationProgress(tableName: tableName, totalRows: 0,
+                                transferredRows: 0, status: "in_progress")
+    try:
+      let countVal = await sourceConn.table(tableName).count()
+      result.totalRows = countVal
+      var offset = 0
+      while offset < result.totalRows:
+        let rows = await sourceConn.table(tableName)
+          .limit(batchSize)
+          .offset(offset)
+          .get()
+        if rows.len == 0: break
+        await targetConn.table(tableName).insert(rows)
+        result.transferredRows += rows.len
+        offset += batchSize
+      result.status = "done"
+    except CatchableError as e:
+      result.status = "failed"
+      result.error = e.msg
+
+when isExistsMysql:
+  proc transferTable*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
+                      tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+    result = MigrationProgress(tableName: tableName, totalRows: 0,
+                                transferredRows: 0, status: "in_progress")
+    try:
+      let countVal = await sourceConn.table(tableName).count()
+      result.totalRows = countVal
+      var offset = 0
+      while offset < result.totalRows:
+        let rows = await sourceConn.table(tableName)
+          .limit(batchSize)
+          .offset(offset)
+          .get()
+        if rows.len == 0: break
+        await targetConn.table(tableName).insert(rows)
+        result.transferredRows += rows.len
+        offset += batchSize
+      result.status = "done"
+    except CatchableError as e:
+      result.status = "failed"
+      result.error = e.msg
+
+when isExistsMariadb:
+  proc transferTable*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
+                      tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+    result = MigrationProgress(tableName: tableName, totalRows: 0,
+                                transferredRows: 0, status: "in_progress")
+    try:
+      let countVal = await sourceConn.table(tableName).count()
+      result.totalRows = countVal
+      var offset = 0
+      while offset < result.totalRows:
+        let rows = await sourceConn.table(tableName)
+          .limit(batchSize)
+          .offset(offset)
+          .get()
+        if rows.len == 0: break
+        await targetConn.table(tableName).insert(rows)
+        result.transferredRows += rows.len
+        offset += batchSize
+      result.status = "done"
+    except CatchableError as e:
+      result.status = "failed"
+      result.error = e.msg
+
+when isExistsSqlite:
+  proc transferTable*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
+                      tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+    result = MigrationProgress(tableName: tableName, totalRows: 0,
+                                transferredRows: 0, status: "in_progress")
+    try:
+      let countVal = await sourceConn.table(tableName).count()
+      result.totalRows = countVal
+      var offset = 0
+      while offset < result.totalRows:
+        let rows = await sourceConn.table(tableName)
+          .limit(batchSize)
+          .offset(offset)
+          .get()
+        if rows.len == 0: break
+        await targetConn.table(tableName).insert(rows)
+        result.transferredRows += rows.len
+        offset += batchSize
+      result.status = "done"
+    except CatchableError as e:
+      result.status = "failed"
+      result.error = e.msg
+
+when isExistsSurrealdb:
+  proc transferTable*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
+                      tableName: string, batchSize: int = 1000): Future[MigrationProgress] {.async.} =
+    result = MigrationProgress(tableName: tableName, totalRows: 0,
+                                transferredRows: 0, status: "in_progress")
+    try:
+      let countVal = await sourceConn.table(tableName).count()
+      result.totalRows = countVal
+      var offset = 0
+      while offset < result.totalRows:
+        let rows = await sourceConn.table(tableName)
+          .limit(batchSize)
+          .offset(offset)
+          .get()
+        if rows.len == 0: break
+        await targetConn.table(tableName).insert(rows)
+        result.transferredRows += rows.len
+        offset += batchSize
+      result.status = "done"
+    except CatchableError as e:
+      result.status = "failed"
+      result.error = e.msg
+
+# ==============================================================================
+# Full Migration Orchestrator
+# ==============================================================================
+
+when isExistsPostgres:
+  proc migrate*(sourceConn: PostgresConnections, targetConn: BaradbConnections,
+                tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
+    result = MigrationReport(sourceDb: "PostgreSQL", targetDb: "BaraDB",
+                              startedAt: epochTime())
+    try:
+      let schema = await extractSchema(sourceConn)
+      var tablesToMigrate: seq[TableInfo]
+      if tables.len > 0:
+        for t in schema:
+          if t.name in tables: tablesToMigrate.add(t)
+      else:
+        tablesToMigrate = schema
+
+      result.tablesTotal = tablesToMigrate.len
+      for table in tablesToMigrate:
+        let (upSql, downSql) = generateBaraDBDDL(table)
+        let migName = generateMigrationName(table.name)
+        discard await targetConn.createMigration(migName, upSql, downSql)
+        discard await targetConn.applyMigration(migName)
+        let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
+        if progress.status == "done":
+          result.tablesDone += 1
+          result.rowsTransferred += progress.transferredRows
+        else:
+          result.errors.add(&"{table.name}: {progress.error}")
+    except CatchableError as e:
+      result.errors.add(e.msg)
+    result.completedAt = epochTime()
+
+when isExistsMysql:
+  proc migrate*(sourceConn: MysqlConnections, targetConn: BaradbConnections,
+                tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
+    result = MigrationReport(sourceDb: "MySQL", targetDb: "BaraDB",
+                              startedAt: epochTime())
+    try:
+      let schema = await extractSchema(sourceConn)
+      var tablesToMigrate: seq[TableInfo]
+      if tables.len > 0:
+        for t in schema:
+          if t.name in tables: tablesToMigrate.add(t)
+      else:
+        tablesToMigrate = schema
+      result.tablesTotal = tablesToMigrate.len
+      for table in tablesToMigrate:
+        let (upSql, downSql) = generateBaraDBDDL(table)
+        let migName = generateMigrationName(table.name)
+        discard await targetConn.createMigration(migName, upSql, downSql)
+        discard await targetConn.applyMigration(migName)
+        let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
+        if progress.status == "done":
+          result.tablesDone += 1
+          result.rowsTransferred += progress.transferredRows
+        else:
+          result.errors.add(&"{table.name}: {progress.error}")
+    except CatchableError as e:
+      result.errors.add(e.msg)
+    result.completedAt = epochTime()
+
+when isExistsMariadb:
+  proc migrate*(sourceConn: MariadbConnections, targetConn: BaradbConnections,
+                tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
+    result = MigrationReport(sourceDb: "MariaDB", targetDb: "BaraDB",
+                              startedAt: epochTime())
+    try:
+      let schema = await extractSchema(sourceConn)
+      var tablesToMigrate: seq[TableInfo]
+      if tables.len > 0:
+        for t in schema:
+          if t.name in tables: tablesToMigrate.add(t)
+      else:
+        tablesToMigrate = schema
+      result.tablesTotal = tablesToMigrate.len
+      for table in tablesToMigrate:
+        let (upSql, downSql) = generateBaraDBDDL(table)
+        let migName = generateMigrationName(table.name)
+        discard await targetConn.createMigration(migName, upSql, downSql)
+        discard await targetConn.applyMigration(migName)
+        let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
+        if progress.status == "done":
+          result.tablesDone += 1
+          result.rowsTransferred += progress.transferredRows
+        else:
+          result.errors.add(&"{table.name}: {progress.error}")
+    except CatchableError as e:
+      result.errors.add(e.msg)
+    result.completedAt = epochTime()
+
+when isExistsSqlite:
+  proc migrate*(sourceConn: SqliteConnections, targetConn: BaradbConnections,
+                tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
+    result = MigrationReport(sourceDb: "SQLite", targetDb: "BaraDB",
+                              startedAt: epochTime())
+    try:
+      let schema = await extractSchema(sourceConn)
+      var tablesToMigrate: seq[TableInfo]
+      if tables.len > 0:
+        for t in schema:
+          if t.name in tables: tablesToMigrate.add(t)
+      else:
+        tablesToMigrate = schema
+      result.tablesTotal = tablesToMigrate.len
+      for table in tablesToMigrate:
+        let (upSql, downSql) = generateBaraDBDDL(table)
+        let migName = generateMigrationName(table.name)
+        discard await targetConn.createMigration(migName, upSql, downSql)
+        discard await targetConn.applyMigration(migName)
+        let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
+        if progress.status == "done":
+          result.tablesDone += 1
+          result.rowsTransferred += progress.transferredRows
+        else:
+          result.errors.add(&"{table.name}: {progress.error}")
+    except CatchableError as e:
+      result.errors.add(e.msg)
+    result.completedAt = epochTime()
+
+when isExistsSurrealdb:
+  proc migrate*(sourceConn: SurrealConnections, targetConn: BaradbConnections,
+                tables: seq[string] = @[], batchSize: int = 1000): Future[MigrationReport] {.async.} =
+    result = MigrationReport(sourceDb: "SurrealDB", targetDb: "BaraDB",
+                              startedAt: epochTime())
+    try:
+      let schema = await extractSchema(sourceConn)
+      var tablesToMigrate: seq[TableInfo]
+      if tables.len > 0:
+        for t in schema:
+          if t.name in tables: tablesToMigrate.add(t)
+      else:
+        tablesToMigrate = schema
+      result.tablesTotal = tablesToMigrate.len
+      for table in tablesToMigrate:
+        let (upSql, downSql) = generateBaraDBDDL(table)
+        let migName = generateMigrationName(table.name)
+        discard await targetConn.createMigration(migName, upSql, downSql)
+        discard await targetConn.applyMigration(migName)
+        let progress = await transferTable(sourceConn, targetConn, table.name, batchSize)
+        if progress.status == "done":
+          result.tablesDone += 1
+          result.rowsTransferred += progress.transferredRows
+        else:
+          result.errors.add(&"{table.name}: {progress.error}")
+    except CatchableError as e:
+      result.errors.add(e.msg)
+    result.completedAt = epochTime()
+
+# ==============================================================================
+# URL-based migration
+# ==============================================================================
+
+proc `$`*(report: MigrationReport): string =
+  let duration = report.completedAt - report.startedAt
+  result = &"Migration: {report.sourceDb} → {report.targetDb}\n"
+  result &= &"  Tables: {report.tablesDone}/{report.tablesTotal}\n"
+  result &= &"  Rows:   {report.rowsTransferred}\n"
+  result &= &"  Time:   {duration:.1f}s\n"
+  if report.errors.len > 0:
+    result &= "  Errors:\n"
+    for err in report.errors:
+      result &= &"    - {err}\n"

clients/nim-allographer/src/allographer/query_builder/libs/baradb/baradb_client.nim

@@ -410,6 +410,35 @@ proc exec*(client: BaraClient, sql: string): Future[int] {.async.} =
   let qr = await client.query(sql)
   return qr.affectedRows
 
+# === Migration API (BaraQL native) ===
+
+proc createMigration*(client: BaraClient, name: string, upBody: string,
+                      downBody: string = ""): Future[QueryResult] {.async.} =
+  ## Send CREATE MIGRATION via BaraQL. Server handles checksums, locking, rollback.
+  var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
+  if downBody.len > 0:
+    sql &= " DOWN: " & downBody & ";"
+  sql &= " }"
+  return await client.query(sql)
+
+proc applyMigration*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
+  return await client.query("APPLY MIGRATION " & name)
+
+proc migrateUp*(client: BaraClient, count: int = 0): Future[QueryResult] {.async.} =
+  var sql = "MIGRATION UP"
+  if count > 0:
+    sql &= " " & $count
+  return await client.query(sql)
+
+proc migrateDown*(client: BaraClient, count: int = 1): Future[QueryResult] {.async.} =
+  return await client.query("MIGRATION DOWN " & $count)
+
+proc migrationStatus*(client: BaraClient): Future[QueryResult] {.async.} =
+  return await client.query("MIGRATION STATUS")
+
+proc migrationDryRun*(client: BaraClient, name: string): Future[QueryResult] {.async.} =
+  return await client.query("MIGRATION DRY RUN " & name)
+
 proc auth*(client: BaraClient, token: string) {.async.} =
   if not client.connected:
     raise newException(IOError, "Not connected")
@@ -652,6 +681,34 @@ proc exec*(client: SyncClient, sql: string): int =
   let qr = client.query(sql)
   return qr.affectedRows
 
+# === Migration API (SyncClient, blocking) ===
+
+proc createMigration*(client: SyncClient, name: string, upBody: string,
+                      downBody: string = ""): QueryResult =
+  var sql = "CREATE MIGRATION " & name & " { UP: " & upBody & ";"
+  if downBody.len > 0:
+    sql &= " DOWN: " & downBody & ";"
+  sql &= " }"
+  return client.query(sql)
+
+proc applyMigration*(client: SyncClient, name: string): QueryResult =
+  return client.query("APPLY MIGRATION " & name)
+
+proc migrateUp*(client: SyncClient, count: int = 0): QueryResult =
+  var sql = "MIGRATION UP"
+  if count > 0:
+    sql &= " " & $count
+  return client.query(sql)
+
+proc migrateDown*(client: SyncClient, count: int = 1): QueryResult =
+  return client.query("MIGRATION DOWN " & $count)
+
+proc migrationStatus*(client: SyncClient): QueryResult =
+  return client.query("MIGRATION STATUS")
+
+proc migrationDryRun*(client: SyncClient, name: string): QueryResult =
+  return client.query("MIGRATION DRY RUN " & name)
+
 proc auth*(client: SyncClient, token: string) =
   acquire(client.lock)
   try:

clients/nim-allographer/src/allographer/query_builder/models/baradb/baradb_exec.nim

@@ -699,6 +699,183 @@ proc firstPlain*(self: RawBaradbQuery): Future[seq[string]] {.async.} =
   return await self.getRowPlain(self.queryString, self.placeHolder)
 
 
+# ================================================================================
+# Pagination (cursor-based and offset-based)
+# ================================================================================
+
+type
+  PaginateResult* = object
+    rows*: seq[JsonNode]
+    total*: int
+    page*: int
+    perPage*: int
+    hasMore*: bool
+
+proc paginate*(self: BaradbQuery, page: int = 1, perPage: int = 20): Future[PaginateResult] {.async.} =
+  ## Offset-based pagination. Returns rows for the given page plus metadata.
+  let total = await self.count()
+  let offset = (page - 1) * perPage
+  self.limit(perPage)
+  self.offset(offset)
+  let rows = await self.get()
+  let qr = PaginateResult(
+    rows: rows,
+    total: total,
+    page: page,
+    perPage: perPage,
+    hasMore: offset + perPage < total
+  )
+  return qr
+
+proc fastPaginate*(self: BaradbQuery, cursorColumn: string, perPage: int = 20,
+                    afterId: string = ""): Future[PaginateResult] {.async.} =
+  ## Cursor-based pagination (keyset). More efficient than offset for large tables.
+  ## Requires a unique, ordered column (usually the primary key).
+  let total = await self.count()
+  self.limit(perPage)
+  self.orderBy(cursorColumn, Asc)
+  if afterId.len > 0:
+    self.where(cursorColumn, ">", afterId)
+  let rows = await self.get()
+  var hasMore = rows.len == perPage
+  let qr = PaginateResult(
+    rows: rows,
+    total: total,
+    page: 0,  # cursor-based has no page number
+    perPage: perPage,
+    hasMore: hasMore
+  )
+  return qr
+
+# ================================================================================
+# Migration API (BaraQL native — server handles checksums, locks, rollback)
+# ================================================================================
+
+proc createMigration*(self: BaradbConnections, name: string, upBody: string,
+                      downBody: string = ""): Future[QueryResult] {.async.} =
+  ## Register a migration on the server. The server computes checksums, stores
+  ## the UP/DOWN bodies, and manages the migration lifecycle.
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  return await self.pools.conns[connI].client.createMigration(name, upBody, downBody)
+
+proc applyMigration*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  return await self.pools.conns[connI].client.applyMigration(name)
+
+proc migrateUp*(self: BaradbConnections, count: int = 0): Future[QueryResult] {.async.} =
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  return await self.pools.conns[connI].client.migrateUp(count)
+
+proc migrateDown*(self: BaradbConnections, count: int = 1): Future[QueryResult] {.async.} =
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  return await self.pools.conns[connI].client.migrateDown(count)
+
+proc migrationStatus*(self: BaradbConnections): Future[seq[JsonNode]] {.async.} =
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  let qr = await self.pools.conns[connI].client.migrationStatus()
+  return toJson(qr)
+
+proc migrationDryRun*(self: BaradbConnections, name: string): Future[QueryResult] {.async.} =
+  var connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  return await self.pools.conns[connI].client.migrationDryRun(name)
+
+proc isMigrationApplied*(self: BaradbConnections, name: string): Future[bool] {.async.} =
+  let status = await self.migrationStatus()
+  for row in status:
+    if row["name"].getStr() == name and row["status"].getStr() == "applied":
+      return true
+  return false
+
+# ================================================================================
+# Prepared Statements (server-side parameterized queries via mkQueryParams)
+# ================================================================================
+
+proc sendPrepared(client: BaraClient, sql: string, params: seq[WireValue]): Future[QueryResult] {.async.} =
+  if not client.connected:
+    raise newException(IOError, "Not connected")
+  let msg = makeQueryParamsMessage(client.nextId(), sql, params)
+  let msgStr = toString(msg)
+  await client.socket.send(msgStr)
+  return await client.readQueryResponse()
+
+proc prepare*(self: BaradbConnections, sql: string, nArgs: int = 0): Future[BaradbPreparedStatement] {.async.} =
+  ## Create a server-side prepared statement. Subsequent calls reuse the cached entry.
+  let entryKey = sql & ":" & $nArgs
+  if entryKey in self.pools.preparedCache:
+    let entry = self.pools.preparedCache[entryKey]
+    inc entry.refCount
+    entry.lastUsedAt = nowUnix()
+    return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
+                                    nArgs: nArgs, isClosed: false)
+  let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 1,
+                                   lastUsedAt: nowUnix())
+  self.pools.preparedCache[entryKey] = entry
+  return BaradbPreparedStatement(owner: self, entry: entry, sql: sql,
+                                  nArgs: nArgs, isClosed: false)
+
+proc ensureStmt*(self: BaradbConnections, sql: string, nArgs: int): BaradbPreparedEntry =
+  let entryKey = sql & ":" & $nArgs
+  if entryKey in self.pools.preparedCache:
+    let entry = self.pools.preparedCache[entryKey]
+    entry.lastUsedAt = nowUnix()
+    return entry
+  let entry = BaradbPreparedEntry(sql: sql, nArgs: nArgs, refCount: 0,
+                                   lastUsedAt: nowUnix())
+  self.pools.preparedCache[entryKey] = entry
+  return entry
+
+proc preparedGet*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[seq[JsonNode]] {.async.} =
+  if stmt.isClosed:
+    raise newException(IOError, "Prepared statement is closed")
+  var connI = getFreeConn(stmt.owner).await
+  defer: stmt.owner.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(stmt.owner)
+  let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
+  return toJson(qr)
+
+proc preparedExec*(stmt: BaradbPreparedStatement, args: seq[WireValue]): Future[int] {.async.} =
+  if stmt.isClosed:
+    raise newException(IOError, "Prepared statement is closed")
+  var connI = getFreeConn(stmt.owner).await
+  defer: stmt.owner.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(stmt.owner)
+  let qr = await sendPrepared(stmt.owner.pools.conns[connI].client, stmt.sql, args)
+  return qr.affectedRows
+
+proc flushStmt*(stmt: BaradbPreparedStatement) =
+  stmt.isClosed = true
+  dec stmt.entry.refCount
+
+proc clearStmtCache*(self: BaradbConnections) =
+  self.pools.preparedCache.clear()
+
+proc withConn*(self: BaradbConnections, callback: proc(connI: int): Future[void] {.gcsafe.}): Future[void] {.async.} =
+  let connI = getFreeConn(self).await
+  defer: self.returnConn(connI).await
+  if connI == errorConnectionNum:
+    raisePoolTimeout(self)
+  await callback(connI)
+
 # seeder templates
 template seeder*(rdb: BaradbConnections, tableName: string, body: untyped): untyped =
   block:

clients/nim-allographer/src/allographer/query_builder/models/baradb/baradb_open.nim

@@ -4,6 +4,7 @@ import std/json
 import std/tables
 import std/times
 import ../../libs/baradb/baradb_client
+import ../../libs/database_url
 import ../../log
 import ./baradb_types
 
@@ -50,3 +51,18 @@ proc dbOpen*(_: type Baradb, database: string, user: string, password: string,
     pools: pools,
     log: LogSetting(shouldDisplayLog: shouldDisplayLog, shouldOutputLogFile: shouldOutputLogFile, logDir: logDir)
   )
+
+proc dbOpen*(_: type Baradb, databaseUrl: DatabaseUrl,
+              maxConnections = 1, timeout = 30,
+              shouldDisplayLog = false, shouldOutputLogFile = false, logDir = "",
+              maxConnectionLifetime = DEFAULT_CONN_MAX_LIFETIME_SECONDS,
+              maxConnectionIdleTime = DEFAULT_CONN_MAX_IDLE_SECONDS): BaradbConnections =
+  ## Open a BaraDB connection using a URL: baradb://user:pass@host:port/database
+  let parsed = parseDatabaseUrl(databaseUrl)
+  requireDatabaseUrlScheme(parsed, ["baradb"], "BaraDB")
+  let database = parsed.path.stripLeadingSlash()
+  let host = if parsed.hostname.len > 0: parsed.hostname else: "127.0.0.1"
+  let port = if parsed.hasPort: parsed.port else: 9472
+  return Baradb.dbOpen(database, parsed.username, parsed.password, host, port,
+                        maxConnections, timeout, shouldDisplayLog, shouldOutputLogFile,
+                        logDir, maxConnectionLifetime, maxConnectionIdleTime)

clients/nim-allographer/src/allographer/schema_builder/queries/baradb/create_migration_table.nim

@@ -3,10 +3,6 @@ import ../../models/table
 import ./baradb_query_type
 
 proc createMigrationTable*(self: BaradbSchema) =
-  let sql = """CREATE TABLE IF NOT EXISTS `schema_migrations` (
-    `id` SERIAL PRIMARY KEY,
-    `name` VARCHAR(255) NOT NULL,
-    `checksum` VARCHAR(64) NOT NULL,
-    `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-  )"""
-  discard waitFor self.rdb.raw(sql).exec()
+  ## BaraDB tracks migrations natively via BaraQL (CREATE MIGRATION, MIGRATION STATUS).
+  ## No client-side migration table needed — server stores state in LSM-Tree.
+  discard

clients/nim-allographer/src/allographer/schema_builder/queries/baradb/schema_utils.nim

@@ -1,12 +1,47 @@
+import std/asyncdispatch
 import std/strformat
 import ../../../query_builder/models/baradb/baradb_types
+import ../../../query_builder/models/baradb/baradb_exec
+import ../../../query_builder/error
 import ../../models/table
 
-proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
+proc exec*(rdb: BaradbConnections, queries: seq[string]) =
   for sql in queries:
     discard waitFor rdb.raw(sql).exec()
-  let hist = &"INSERT INTO `schema_migrations` (`name`, `checksum`) VALUES ('{name}', '{checksum}')"
-  discard waitFor rdb.raw(hist).exec()
+
+proc execThenSaveHistory*(rdb: BaradbConnections, name: string, queries: seq[string], checksum: string) =
+  ## Register and apply a migration via BaraQL native migration system.
+  ## The server handles checksums, locking, rollback, and status tracking.
+  let upBody = queries.join("; ")
+  var downBody = ""
+  # Generate DOWN script: DROP TABLE for CREATE TABLE
+  for sql in queries:
+    if sql.toLower().startsWith("create table"):
+      let parts = sql.split(" ")
+      if parts.len >= 3:
+        let tableName = parts[2]  # CREATE TABLE `name` ...
+        downBody &= &"DROP TABLE IF EXISTS {tableName}; "
+
+  # Register migration on server
+  var qr = waitFor rdb.createMigration(name, upBody, downBody)
+  if qr.affectedRows < 0:
+    raise newException(DbError, "Failed to create migration: " & name)
+
+  # Apply migration
+  qr = waitFor rdb.applyMigration(name)
+  if qr.affectedRows < 0:
+    raise newException(DbError, "Failed to apply migration: " & name)
+
+proc execThenSaveHistory*(rdb: BaradbConnections, name: string, query: string, checksum: string) =
+  execThenSaveHistory(rdb, name, @[query], checksum)
 
 proc shouldRun*(rdb: BaradbConnections, table: Table, checksum: string, isReset: bool): bool =
-  return true
+  ## Check with the server whether this migration has already been applied.
+  if isReset:
+    return true
+  try:
+    let applied = waitFor rdb.isMigrationApplied(table.name)
+    return not applied
+  except CatchableError:
+    # If server is unavailable or migration doesn't exist, run it
+    return true