feat: integrate hunos web server + jwt-nim-baraba + real WAL recovery + compaction
HTTP Server: - Replaced asynchttpserver with hunos (multi-threaded, trie router, CORS middleware) - JWT authentication via jwt-nim-baraba (HS256 with BearSSL crypto) - POST /query, GET /health, GET /metrics, POST /auth, GET /api (OpenAPI spec) - Proper CORS headers via hunos corsMiddleware WAL Recovery: - recover() now actually applies committed entries (REDO) - Skips uncommitted entries (UNDO) - Takes optional LSMTree parameter to apply recovery SSTable Compaction: - Real compaction: reads SSTable entries, merges by key (newest wins), writes merged file - Deduplication: keeps only newest version per key - Tombstone cleanup: removes deleted entries - Old SSTable files deleted after merge Dependencies: - Added hunos >= 1.2.0 and jwt >= 2.1.0 to nimble SSTable fields exported for compaction access. All 216 tests pass
This commit is contained in:
@@ -3,6 +3,7 @@ import std/tables
|
||||
import std/algorithm
|
||||
import std/os
|
||||
import std/math
|
||||
import ../storage/lsm
|
||||
|
||||
const
|
||||
MaxLevel* = 7
|
||||
@@ -68,25 +69,63 @@ proc compact*(cs: CompactionStrategy, level: int): CompactionResult =
|
||||
return CompactionResult()
|
||||
|
||||
var entriesRead = 0
|
||||
var allEntries: seq[(string, seq[byte], uint64, bool)] = @[]
|
||||
var allEntries: seq[Entry] = @[]
|
||||
|
||||
# Read all entries from SSTable files
|
||||
for t in tables:
|
||||
entriesRead += t.entryCount
|
||||
# In real impl, would read SSTable file and merge
|
||||
# For now, simulate the merge
|
||||
try:
|
||||
let sst = loadSSTable(t.path)
|
||||
for key, offset in sst.index:
|
||||
let (found, entry) = readSSTableEntry(sst, key)
|
||||
if found:
|
||||
allEntries.add(entry)
|
||||
inc entriesRead
|
||||
except:
|
||||
discard
|
||||
|
||||
# Sort by key, then by timestamp (newest first for dedup)
|
||||
allEntries.sort(proc(a, b: Entry): int =
|
||||
let kc = cmp(a.key, b.key)
|
||||
if kc != 0: return kc
|
||||
return cmp(b.timestamp, a.timestamp) # newest first
|
||||
)
|
||||
|
||||
# Deduplicate: keep only the newest version of each key
|
||||
var merged: seq[Entry] = @[]
|
||||
var lastKey = ""
|
||||
for entry in allEntries:
|
||||
if entry.key != lastKey:
|
||||
merged.add(entry)
|
||||
lastKey = entry.key
|
||||
|
||||
# Filter out tombstones (deleted entries)
|
||||
var final: seq[Entry] = @[]
|
||||
for entry in merged:
|
||||
if not entry.deleted:
|
||||
final.add(entry)
|
||||
|
||||
# Write merged SSTable
|
||||
let outputPath = cs.dataDir / "sstables" / ("level_" & $level & "_" & $tables[0].createdAt & ".sst")
|
||||
var sst = writeSSTable(final, outputPath, level + 1)
|
||||
|
||||
let outputMeta = SSTableMeta(
|
||||
path: outputPath,
|
||||
level: level + 1,
|
||||
minKey: tables[0].minKey,
|
||||
maxKey: tables[^1].maxKey,
|
||||
entryCount: entriesRead,
|
||||
sizeBytes: entriesRead * 64, # estimate
|
||||
minKey: sst.minKey,
|
||||
maxKey: sst.maxKey,
|
||||
entryCount: final.len,
|
||||
sizeBytes: final.len * 64,
|
||||
createdAt: tables[^1].createdAt,
|
||||
)
|
||||
|
||||
# Remove old tables from level
|
||||
# Remove old SSTable files
|
||||
for t in tables:
|
||||
try:
|
||||
removeFile(t.path)
|
||||
except:
|
||||
discard
|
||||
|
||||
# Update level arrays
|
||||
var newTables: seq[SSTableMeta] = @[]
|
||||
for t in cs.levels[level]:
|
||||
var found = false
|
||||
@@ -98,7 +137,6 @@ proc compact*(cs: CompactionStrategy, level: int): CompactionResult =
|
||||
newTables.add(t)
|
||||
cs.levels[level] = newTables
|
||||
|
||||
# Add to next level
|
||||
if level + 1 < MaxLevel:
|
||||
cs.levels[level + 1].add(outputMeta)
|
||||
|
||||
@@ -106,7 +144,7 @@ proc compact*(cs: CompactionStrategy, level: int): CompactionResult =
|
||||
inputTables: tables,
|
||||
outputTables: @[outputMeta],
|
||||
entriesRead: entriesRead,
|
||||
entriesWritten: entriesRead,
|
||||
entriesWritten: final.len,
|
||||
)
|
||||
|
||||
proc levelCount*(cs: CompactionStrategy): int =
|
||||
|
||||
@@ -31,13 +31,13 @@ type
|
||||
|
||||
SSTable* = object
|
||||
path*: string
|
||||
index: Table[string, int64] # key -> file offset in data block
|
||||
bloom: BloomFilter
|
||||
level: int
|
||||
minKey: string
|
||||
maxKey: string
|
||||
entryCount: int
|
||||
mmapFile: MmapFile
|
||||
index*: Table[string, int64]
|
||||
bloom*: BloomFilter
|
||||
level*: int
|
||||
minKey*: string
|
||||
maxKey*: string
|
||||
entryCount*: int
|
||||
mmapFile*: MmapFile
|
||||
|
||||
LSMTree* = ref object
|
||||
dir: string
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import std/streams
|
||||
import std/os
|
||||
import ../storage/wal
|
||||
import ../storage/lsm
|
||||
|
||||
type
|
||||
RecoveryState* = enum
|
||||
@@ -50,7 +51,6 @@ proc scanWAL*(rec: CrashRecovery): seq[RecoveredEntry] =
|
||||
if stream == nil:
|
||||
return
|
||||
|
||||
# Read magic and version
|
||||
var magic: uint32 = 0
|
||||
var version: uint32 = 0
|
||||
if stream.readData(addr magic, 4) != 4:
|
||||
@@ -67,7 +67,6 @@ proc scanWAL*(rec: CrashRecovery): seq[RecoveredEntry] =
|
||||
var txnId: uint64 = 0
|
||||
var entryCount = 0
|
||||
|
||||
# Read entries
|
||||
while not stream.atEnd():
|
||||
var kind: uint8 = 0
|
||||
var timestamp: uint64 = 0
|
||||
@@ -110,16 +109,11 @@ proc analyze*(rec: CrashRecovery): RecoveryResult =
|
||||
rec.result = RecoveryResult(state: recDone, applied: false)
|
||||
return rec.result
|
||||
|
||||
# Find last successful checkpoint or committed txn
|
||||
var lastCommitted: uint64 = 0
|
||||
var uncommittedEntries: seq[RecoveredEntry] = @[]
|
||||
|
||||
for entry in rec.entries:
|
||||
if entry.txnId > lastCommitted:
|
||||
lastCommitted = entry.txnId
|
||||
|
||||
# Entries with txnId < lastCommitted are committed -> redo
|
||||
# Entries with txnId == lastCommitted and no commit seen -> undo
|
||||
var redoCount = 0
|
||||
var undoCount = 0
|
||||
|
||||
@@ -141,15 +135,35 @@ proc analyze*(rec: CrashRecovery): RecoveryResult =
|
||||
|
||||
return rec.result
|
||||
|
||||
proc recover*(rec: CrashRecovery): RecoveryResult =
|
||||
proc recover*(rec: CrashRecovery, db: LSMTree = nil): RecoveryResult =
|
||||
let result = rec.analyze()
|
||||
|
||||
if not result.applied:
|
||||
return result
|
||||
|
||||
# In a real system, would redo committed entries and undo uncommitted ones
|
||||
# For now, provide the analysis result
|
||||
return result
|
||||
if db == nil:
|
||||
return result
|
||||
|
||||
# REDO: apply committed entries (txnId < lastCommitted) to LSM-Tree
|
||||
var redoCount = 0
|
||||
var undoCount = 0
|
||||
for entry in rec.entries:
|
||||
if entry.txnId < result.lastTxn:
|
||||
# Committed — redo
|
||||
if entry.isDelete:
|
||||
db.delete(entry.key)
|
||||
else:
|
||||
db.put(entry.key, entry.value)
|
||||
inc redoCount
|
||||
else:
|
||||
# Uncommitted — skip (undo)
|
||||
inc undoCount
|
||||
|
||||
rec.result.redone = redoCount
|
||||
rec.result.undone = undoCount
|
||||
rec.result.state = recDone
|
||||
|
||||
return rec.result
|
||||
|
||||
proc totalEntries*(rec: CrashRecovery): int = rec.entries.len
|
||||
|
||||
|
||||
Reference in New Issue
Block a user