Phase 5: Storage fuzz tests + WAL entryCount fix + SSTable overflow checks + compaction sizeBytes accuracy
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
CI / test (push) Has been cancelled
CI / verify (push) Has been cancelled
Clients CI / build-server (push) Has been cancelled
Clients CI / test-python (push) Has been cancelled
Clients CI / test-javascript (push) Has been cancelled
Clients CI / test-nim (push) Has been cancelled
Clients CI / test-rust (push) Has been cancelled
- WAL: entryCount is now accurate after restart by counting existing entries - LSM writeSSTable: adds uint32 overflow checks for key/value/entry counts - Compaction: sizeBytes now uses actual file size instead of rough guess - Fuzz tests: 14 new tests covering WAL truncation, LSM put/get/delete, recovery, scan, overwrites, SSTable roundtrip, corruption, tombstones
This commit is contained in:
@@ -107,13 +107,16 @@ proc compact*(cs: CompactionStrategy, level: int): CompactionResult =
|
||||
let outputPath = cs.dataDir / "sstables" / ("level_" & $level & "_" & $tables[0].createdAt & ".sst")
|
||||
var sst = writeSSTable(final, outputPath, level + 1)
|
||||
|
||||
# Use actual file size instead of rough guess
|
||||
let actualSize = try: getFileSize(outputPath) except: final.len * 64
|
||||
|
||||
let outputMeta = SSTableMeta(
|
||||
path: outputPath,
|
||||
level: level + 1,
|
||||
minKey: sst.minKey,
|
||||
maxKey: sst.maxKey,
|
||||
entryCount: final.len,
|
||||
sizeBytes: final.len * 64,
|
||||
sizeBytes: actualSize,
|
||||
createdAt: tables[^1].createdAt,
|
||||
)
|
||||
|
||||
|
||||
@@ -135,6 +135,8 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
|
||||
# Write header
|
||||
s.write(SSTableMagic)
|
||||
s.write(SSTableVersion)
|
||||
if entries.len > high(uint32).int:
|
||||
raise newException(ValueError, "SSTable entry count exceeds uint32 limit")
|
||||
s.write(uint32(entries.len))
|
||||
s.write(uint32(level))
|
||||
let indexOffsetPos = s.getPosition()
|
||||
@@ -146,8 +148,12 @@ proc writeSSTable*(entries: seq[Entry], path: string, level: int): SSTable =
|
||||
var offsets = newSeq[(string, int64)](entries.len)
|
||||
for i, entry in entries:
|
||||
offsets[i] = (entry.key, int64(s.getPosition()))
|
||||
if entry.key.len > high(uint32).int:
|
||||
raise newException(ValueError, "SSTable key length exceeds uint32 limit")
|
||||
s.write(uint32(entry.key.len))
|
||||
s.write(entry.key)
|
||||
if entry.value.len > high(uint32).int:
|
||||
raise newException(ValueError, "SSTable value length exceeds uint32 limit")
|
||||
s.write(uint32(entry.value.len))
|
||||
if entry.value.len > 0:
|
||||
s.writeData(addr entry.value[0], entry.value.len)
|
||||
|
||||
@@ -26,6 +26,8 @@ type
|
||||
entryCount: uint64
|
||||
syncOnWrite: bool
|
||||
|
||||
proc readEntries*(walPath: string, untilTimestamp: uint64 = 0): seq[WalEntry]
|
||||
|
||||
proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
|
||||
createDir(dir)
|
||||
let path = dir / "wal.log"
|
||||
@@ -38,9 +40,13 @@ proc newWriteAheadLog*(dir: string, syncOnWrite: bool = true): WriteAheadLog =
|
||||
stream.write(WALMagic)
|
||||
stream.write(WALVersion)
|
||||
stream.flush()
|
||||
# NOTE: entryCount is reset to 0 when appending to existing WAL.
|
||||
# For an accurate count, call readEntries() after construction.
|
||||
WriteAheadLog(path: path, stream: stream, entryCount: 0, syncOnWrite: syncOnWrite)
|
||||
# Count existing entries when appending to an existing WAL so
|
||||
# entryCount() remains accurate across restarts.
|
||||
var count: uint64 = 0
|
||||
if exists and not isEmpty:
|
||||
for e in readEntries(path):
|
||||
inc count
|
||||
WriteAheadLog(path: path, stream: stream, entryCount: count, syncOnWrite: syncOnWrite)
|
||||
|
||||
proc writeEntry*(wal: var WriteAheadLog, entry: WalEntry) =
|
||||
wal.stream.write(uint8(entry.kind))
|
||||
|
||||
Reference in New Issue
Block a user