Phase 5: Storage fuzz tests + WAL entryCount fix + SSTable overflow checks + compaction sizeBytes accuracy
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled

- WAL: entryCount is now accurate after restart by counting existing entries
- LSM writeSSTable: adds uint32 overflow checks for key/value/entry counts
- Compaction: sizeBytes now uses actual file size instead of rough guess
- Fuzz tests: 14 new tests covering WAL truncation, LSM put/get/delete,
  recovery, scan, overwrites, SSTable roundtrip, corruption, tombstones
This commit is contained in:
2026-05-18 12:34:26 +03:00
parent 3f29c519d4
commit 132e8c7a31
4 changed files with 326 additions and 5 deletions
+4 -1
View File
@@ -107,13 +107,16 @@ proc compact*(cs: CompactionStrategy, level: int): CompactionResult =
let outputPath = cs.dataDir / "sstables" / ("level_" & $level & "_" & $tables[0].createdAt & ".sst")
var sst = writeSSTable(final, outputPath, level + 1)
# Use actual file size instead of rough guess
let actualSize = try: getFileSize(outputPath) except: final.len * 64
let outputMeta = SSTableMeta(
path: outputPath,
level: level + 1,
minKey: sst.minKey,
maxKey: sst.maxKey,
entryCount: final.len,
sizeBytes: final.len * 64,
sizeBytes: actualSize,
createdAt: tables[^1].createdAt,
)
+6
View File
@@ -135,6 +135,8 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
# Write header
s.write(SSTableMagic)
s.write(SSTableVersion)
if entries.len > high(uint32).int:
raise newException(ValueError, "SSTable entry count exceeds uint32 limit")
s.write(uint32(entries.len))
s.write(uint32(level))
let indexOffsetPos = s.getPosition()
@@ -146,8 +148,12 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
var offsets = newSeq[(string, int64)](entries.len)
for i, entry in entries:
offsets[i] = (entry.key, int64(s.getPosition()))
if entry.key.len > high(uint32).int:
raise newException(ValueError, "SSTable key length exceeds uint32 limit")
s.write(uint32(entry.key.len))
s.write(entry.key)
if entry.value.len > high(uint32).int:
raise newException(ValueError, "SSTable value length exceeds uint32 limit")
s.write(uint32(entry.value.len))
if entry.value.len > 0:
s.writeData(addr entry.value[0], entry.value.len)
+9 -3
View File
@@ -26,6 +26,8 @@ type
entryCount: uint64
syncOnWrite: bool
proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry]
proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
createDir(dir)
let path = dir / "wal.log"
@@ -38,9 +40,13 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
stream.write(WALMagic)
stream.write(WALVersion)
stream.flush()
# NOTE: entryCount is reset to 0 when appending to existing WAL.
# For an accurate count, call readEntries() after construction.
WriteAheadLog(path: path, stream: stream, entryCount: 0, syncOnWrite: syncOnWrite)
# Count existing entries when appending to an existing WAL so
# entryCount() remains accurate across restarts.
var count: uint64 = 0
if exists and not isEmpty:
for e in readEntries(path):
inc count
WriteAheadLog(path: path, stream: stream, entryCount: count, syncOnWrite: syncOnWrite)
proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
wal.stream.write(uint8(entry.kind))