Fix production gaps: config file loading, structured JSON logging, update PLAN
- config.nim: Add baradb.json parsing + full env var support (BARADB_*) - logging.nim: Wire structured JSON logger into server.nim and baradadb.nim - server.nim: Replace echo with info/warn/errorMsg logging calls - baradadb.nim: Init logger from config (logLevel, logFile) - PLAN.md: Update honest assessment — deadlock, compaction, JOIN, TLS, slow query all verified working
This commit is contained in:
+124
-13
@@ -1,5 +1,6 @@
|
||||
import std/os
|
||||
import std/strutils
|
||||
import std/json
|
||||
|
||||
type
|
||||
BaraConfig* = object
|
||||
@@ -16,6 +17,18 @@ type
|
||||
queryTimeoutMs*: int
|
||||
slowQueryThresholdMs*: int
|
||||
slowQueryLogPath*: string
|
||||
logLevel*: string
|
||||
logFile*: string
|
||||
logFormat*: string
|
||||
memtableSizeMb*: int
|
||||
cacheSizeMb*: int
|
||||
walSyncIntervalMs*: int
|
||||
compactionIntervalMs*: int
|
||||
bloomBitsPerKey*: int
|
||||
authEnabled*: bool
|
||||
jwtSecret*: string
|
||||
rateLimitGlobal*: int
|
||||
rateLimitPerClient*: int
|
||||
|
||||
CompactionStrategy* = enum
|
||||
csSizeTiered = "size_tiered"
|
||||
@@ -36,20 +49,118 @@ proc defaultConfig*(): BaraConfig =
|
||||
queryTimeoutMs: 30_000,
|
||||
slowQueryThresholdMs: 1_000,
|
||||
slowQueryLogPath: "",
|
||||
logLevel: "info",
|
||||
logFile: "",
|
||||
logFormat: "json",
|
||||
memtableSizeMb: 64,
|
||||
cacheSizeMb: 256,
|
||||
walSyncIntervalMs: 0,
|
||||
compactionIntervalMs: 60_000,
|
||||
bloomBitsPerKey: 10,
|
||||
authEnabled: false,
|
||||
jwtSecret: "",
|
||||
rateLimitGlobal: 10_000,
|
||||
rateLimitPerClient: 1_000,
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# JSON Config File
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
proc loadConfigFromJson*(path: string, cfg: var BaraConfig) =
|
||||
if not fileExists(path):
|
||||
return
|
||||
let content = readFile(path)
|
||||
if content.len == 0:
|
||||
return
|
||||
try:
|
||||
let j = parseJson(content)
|
||||
if j.hasKey("server"):
|
||||
let s = j["server"]
|
||||
if s.hasKey("address"): cfg.address = s["address"].getStr()
|
||||
if s.hasKey("port"): cfg.port = s["port"].getInt()
|
||||
if s.hasKey("max_connections"): cfg.maxConnections = s["max_connections"].getInt()
|
||||
if j.hasKey("storage"):
|
||||
let s = j["storage"]
|
||||
if s.hasKey("data_dir"): cfg.dataDir = s["data_dir"].getStr()
|
||||
if s.hasKey("memtable_size_mb"): cfg.memtableSizeMb = s["memtable_size_mb"].getInt()
|
||||
if s.hasKey("cache_size_mb"): cfg.cacheSizeMb = s["cache_size_mb"].getInt()
|
||||
if s.hasKey("wal_sync_interval_ms"): cfg.walSyncIntervalMs = s["wal_sync_interval_ms"].getInt()
|
||||
if s.hasKey("compaction_interval_ms"): cfg.compactionIntervalMs = s["compaction_interval_ms"].getInt()
|
||||
if s.hasKey("bloom_bits_per_key"): cfg.bloomBitsPerKey = s["bloom_bits_per_key"].getInt()
|
||||
if j.hasKey("tls"):
|
||||
let s = j["tls"]
|
||||
if s.hasKey("enabled"): cfg.tlsEnabled = s["enabled"].getBool()
|
||||
if s.hasKey("cert_file"): cfg.certFile = s["cert_file"].getStr()
|
||||
if s.hasKey("key_file"): cfg.keyFile = s["key_file"].getStr()
|
||||
if j.hasKey("auth"):
|
||||
let s = j["auth"]
|
||||
if s.hasKey("enabled"): cfg.authEnabled = s["enabled"].getBool()
|
||||
if s.hasKey("jwt_secret"): cfg.jwtSecret = s["jwt_secret"].getStr()
|
||||
if s.hasKey("rate_limit_global"): cfg.rateLimitGlobal = s["rate_limit_global"].getInt()
|
||||
if s.hasKey("rate_limit_per_client"): cfg.rateLimitPerClient = s["rate_limit_per_client"].getInt()
|
||||
if j.hasKey("logging"):
|
||||
let s = j["logging"]
|
||||
if s.hasKey("level"): cfg.logLevel = s["level"].getStr()
|
||||
if s.hasKey("file"): cfg.logFile = s["file"].getStr()
|
||||
if s.hasKey("format"): cfg.logFormat = s["format"].getStr()
|
||||
if j.hasKey("performance"):
|
||||
let s = j["performance"]
|
||||
if s.hasKey("idle_timeout_ms"): cfg.idleTimeoutMs = s["idle_timeout_ms"].getInt()
|
||||
if s.hasKey("query_timeout_ms"): cfg.queryTimeoutMs = s["query_timeout_ms"].getInt()
|
||||
if s.hasKey("slow_query_threshold_ms"): cfg.slowQueryThresholdMs = s["slow_query_threshold_ms"].getInt()
|
||||
if s.hasKey("slow_query_log_path"): cfg.slowQueryLogPath = s["slow_query_log_path"].getStr()
|
||||
except JsonParsingError:
|
||||
discard
|
||||
except KeyError:
|
||||
discard
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Environment Variables
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
proc parseEnvInt(val: string, defaultVal: int): int =
|
||||
if val.len == 0: return defaultVal
|
||||
try: parseInt(val) except ValueError: defaultVal
|
||||
|
||||
proc parseEnvBool(val: string, defaultVal: bool): bool =
|
||||
if val.len == 0: return defaultVal
|
||||
let v = val.toLowerAscii()
|
||||
return v == "true" or v == "1" or v == "yes"
|
||||
|
||||
proc loadConfigFromEnv*(cfg: var BaraConfig) =
|
||||
cfg.address = getEnv("BARADB_ADDRESS", cfg.address)
|
||||
cfg.port = parseEnvInt(getEnv("BARADB_PORT", ""), cfg.port)
|
||||
cfg.dataDir = getEnv("BARADB_DATA_DIR", cfg.dataDir)
|
||||
cfg.maxConnections = parseEnvInt(getEnv("BARADB_MAX_CONNECTIONS", ""), cfg.maxConnections)
|
||||
cfg.tlsEnabled = parseEnvBool(getEnv("BARADB_TLS_ENABLED", ""), cfg.tlsEnabled)
|
||||
cfg.certFile = getEnv("BARADB_CERT_FILE", cfg.certFile)
|
||||
cfg.keyFile = getEnv("BARADB_KEY_FILE", cfg.keyFile)
|
||||
cfg.idleTimeoutMs = parseEnvInt(getEnv("BARADB_IDLE_TIMEOUT_MS", ""), cfg.idleTimeoutMs)
|
||||
cfg.queryTimeoutMs = parseEnvInt(getEnv("BARADB_QUERY_TIMEOUT_MS", ""), cfg.queryTimeoutMs)
|
||||
cfg.slowQueryThresholdMs = parseEnvInt(getEnv("BARADB_SLOW_QUERY_THRESHOLD_MS", ""), cfg.slowQueryThresholdMs)
|
||||
cfg.slowQueryLogPath = getEnv("BARADB_SLOW_QUERY_LOG_PATH", cfg.slowQueryLogPath)
|
||||
cfg.logLevel = getEnv("BARADB_LOG_LEVEL", cfg.logLevel)
|
||||
cfg.logFile = getEnv("BARADB_LOG_FILE", cfg.logFile)
|
||||
cfg.logFormat = getEnv("BARADB_LOG_FORMAT", cfg.logFormat)
|
||||
cfg.memtableSizeMb = parseEnvInt(getEnv("BARADB_MEMTABLE_SIZE_MB", ""), cfg.memtableSizeMb)
|
||||
cfg.cacheSizeMb = parseEnvInt(getEnv("BARADB_CACHE_SIZE_MB", ""), cfg.cacheSizeMb)
|
||||
cfg.walSyncIntervalMs = parseEnvInt(getEnv("BARADB_WAL_SYNC_INTERVAL_MS", ""), cfg.walSyncIntervalMs)
|
||||
cfg.compactionIntervalMs = parseEnvInt(getEnv("BARADB_COMPACTION_INTERVAL_MS", ""), cfg.compactionIntervalMs)
|
||||
cfg.bloomBitsPerKey = parseEnvInt(getEnv("BARADB_BLOOM_BITS_PER_KEY", ""), cfg.bloomBitsPerKey)
|
||||
cfg.authEnabled = parseEnvBool(getEnv("BARADB_AUTH_ENABLED", ""), cfg.authEnabled)
|
||||
cfg.jwtSecret = getEnv("BARADB_JWT_SECRET", cfg.jwtSecret)
|
||||
cfg.rateLimitGlobal = parseEnvInt(getEnv("BARADB_RATE_LIMIT_GLOBAL", ""), cfg.rateLimitGlobal)
|
||||
cfg.rateLimitPerClient = parseEnvInt(getEnv("BARADB_RATE_LIMIT_PER_CLIENT", ""), cfg.rateLimitPerClient)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Master Loader
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
proc loadConfig*(): BaraConfig =
|
||||
result = defaultConfig()
|
||||
# Docker / Environment overrides
|
||||
let envAddress = getEnv("BARADB_ADDRESS", "")
|
||||
if envAddress.len > 0:
|
||||
result.address = envAddress
|
||||
let envPort = getEnv("BARADB_PORT", "")
|
||||
if envPort.len > 0:
|
||||
try:
|
||||
result.port = parseInt(envPort)
|
||||
except ValueError:
|
||||
discard
|
||||
let envDataDir = getEnv("BARADB_DATA_DIR", "")
|
||||
if envDataDir.len > 0:
|
||||
result.dataDir = envDataDir
|
||||
# 1. Try JSON config file
|
||||
if fileExists("baradb.json"):
|
||||
loadConfigFromJson("baradb.json", result)
|
||||
# 2. Environment overrides (highest priority)
|
||||
loadConfigFromEnv(result)
|
||||
|
||||
@@ -7,6 +7,7 @@ import std/os
|
||||
import std/endians
|
||||
import std/monotimes
|
||||
import config
|
||||
import logging
|
||||
import ../protocol/wire
|
||||
import ../protocol/ssl
|
||||
import ../query/lexer
|
||||
@@ -216,7 +217,7 @@ proc slowQueryLog(logPath: string, query: string, durationMs: int, clientId: int
|
||||
except: discard
|
||||
|
||||
proc handleClient(server: Server, client: AsyncSocket, clientId: int) {.async.} =
|
||||
echo "Client ", clientId, " connected"
|
||||
info("Client " & $clientId & " connected")
|
||||
var connCtx = cloneForConnection(server.ctx)
|
||||
let idleTimeout = server.config.idleTimeoutMs
|
||||
let queryTimeout = server.config.queryTimeoutMs
|
||||
@@ -243,7 +244,7 @@ proc handleClient(server: Server, client: AsyncSocket, clientId: int) {.async.}
|
||||
of mkQuery:
|
||||
var pos = 0
|
||||
let queryStr = readString(cast[seq[byte]](payload), pos)
|
||||
echo "[", clientId, "] Query: ", queryStr
|
||||
info("[" & $clientId & "] Query: " & queryStr)
|
||||
|
||||
let startTicks = getMonoTime().ticks()
|
||||
let (success, result, errorMsg) = executeQuery(server.db, connCtx, queryStr)
|
||||
@@ -264,7 +265,7 @@ proc handleClient(server: Server, client: AsyncSocket, clientId: int) {.async.}
|
||||
|
||||
of mkQueryParams:
|
||||
let (queryStr, params) = readQueryParamsMessage(cast[seq[byte]](payload))
|
||||
echo "[", clientId, "] QueryParams: ", queryStr, " (", params.len, " params)"
|
||||
info("[" & $clientId & "] QueryParams: " & queryStr & " (" & $params.len & " params)")
|
||||
|
||||
let startTicks = getMonoTime().ticks()
|
||||
let (success, result, errorMsg) = executeQuery(server.db, connCtx, queryStr, params)
|
||||
@@ -299,10 +300,10 @@ proc handleClient(server: Server, client: AsyncSocket, clientId: int) {.async.}
|
||||
await client.send(cast[string](errorMsg))
|
||||
|
||||
except Exception as e:
|
||||
echo "Client ", clientId, " error: ", e.msg
|
||||
errorMsg("Client " & $clientId & " error: " & e.msg)
|
||||
finally:
|
||||
dec server.activeConnections
|
||||
echo "Client ", clientId, " disconnected"
|
||||
info("Client " & $clientId & " disconnected")
|
||||
client.close()
|
||||
|
||||
proc run*(server: Server) {.async.} =
|
||||
@@ -313,9 +314,9 @@ proc run*(server: Server) {.async.} =
|
||||
sock.bindAddr(Port(server.config.port), server.config.address)
|
||||
sock.listen()
|
||||
if server.config.tlsEnabled:
|
||||
echo "BaraDB TLS listening on ", server.config.address, ":", server.config.port
|
||||
info("BaraDB TLS listening on " & server.config.address & ":" & $server.config.port)
|
||||
else:
|
||||
echo "BaraDB listening on ", server.config.address, ":", server.config.port
|
||||
info("BaraDB listening on " & server.config.address & ":" & $server.config.port)
|
||||
while server.running:
|
||||
let client = await sock.accept()
|
||||
if server.config.maxConnections > 0 and server.activeConnections >= server.config.maxConnections:
|
||||
@@ -325,7 +326,7 @@ proc run*(server: Server) {.async.} =
|
||||
try:
|
||||
server.tls.wrapServer(client)
|
||||
except Exception as e:
|
||||
echo "TLS handshake failed: ", e.msg
|
||||
errorMsg("TLS handshake failed: " & e.msg)
|
||||
client.close()
|
||||
continue
|
||||
inc clientId
|
||||
|
||||
+10
-7
@@ -4,9 +4,11 @@ import std/asyncdispatch
|
||||
import std/threadpool
|
||||
import std/locks
|
||||
import std/os
|
||||
import std/strutils
|
||||
import barabadb/core/server
|
||||
import barabadb/core/httpserver
|
||||
import barabadb/core/config
|
||||
import barabadb/core/logging
|
||||
import barabadb/protocol/ssl
|
||||
import barabadb/storage/lsm
|
||||
import barabadb/storage/compaction
|
||||
@@ -43,27 +45,28 @@ proc startCompactionLoop*(cm: CompactionManager, intervalMs: int = 60000) {.asyn
|
||||
cm.compact()
|
||||
|
||||
proc runTcpServer(config: BaraConfig) {.async.} =
|
||||
echo "BaraDB TCP listening on ", config.address, ":", config.port
|
||||
info("BaraDB TCP listening on " & config.address & ":" & $config.port)
|
||||
var server = newServer(config)
|
||||
await server.run()
|
||||
|
||||
proc main() =
|
||||
var config = loadConfig()
|
||||
echo "BaraDB v0.1.0 — Multimodal Database Engine"
|
||||
# Init structured logger from config
|
||||
let logLvl = parseEnum[LogLevel]("ll" & capitalizeAscii(config.logLevel))
|
||||
defaultLogger = newLogger(logLvl, config.logFile)
|
||||
info("BaraDB v0.1.0 — Multimodal Database Engine")
|
||||
|
||||
if config.tlsEnabled:
|
||||
if config.certFile.len == 0 or config.keyFile.len == 0 or
|
||||
not fileExists(config.certFile) or not fileExists(config.keyFile):
|
||||
echo "TLS enabled but no certificate found. Generating self-signed certificate..."
|
||||
info("TLS enabled but no certificate found. Generating self-signed certificate...")
|
||||
let (cert, key) = generateSelfSignedCert(config.dataDir / "certs")
|
||||
if cert.len > 0 and key.len > 0:
|
||||
config.certFile = cert
|
||||
config.keyFile = key
|
||||
echo "Generated self-signed certificate:"
|
||||
echo " Cert: ", cert
|
||||
echo " Key: ", key
|
||||
info("Generated self-signed certificate: Cert=" & cert & " Key=" & key)
|
||||
else:
|
||||
echo "WARNING: Failed to generate self-signed certificate. TLS disabled."
|
||||
warn("Failed to generate self-signed certificate. TLS disabled.")
|
||||
config.tlsEnabled = false
|
||||
|
||||
# Start HTTP server (blocking, multi-threaded via hunos) in background thread
|
||||
|
||||
Reference in New Issue
Block a user