baraba/Baradb
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(raft): cold-node rejoin and wiped-node join e2e

Browse Source 
Includes two raft fixes the e2e exposed:
- heartbeatLoop drops cached peer sockets on leadership acquisition
  (half-dead sockets to a restarted peer never errored, so no redial
  ever happened and the cluster livelocked without heartbeats)
- restoreSnapshot repoints tcpServer ctx/db at the reopened database
  (queries kept reading the closed pre-restore LSM, serving 0 rows)
This commit is contained in:
dimgigov
2026-07-31 03:53:48 +03:00
parent 9ff9c2f6be
commit ad90ebcd5e
6 changed files with 513 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/raft_coldnode_e2e_test.nim
+480
View File
@@ -0,0 +1,480 @@
## Raft cold-node E2E — real 3-node cluster; end-to-end proof of the
## InstallSnapshot work (compaction, snapshot build/send, follower restore).
## Starts three actual build/baradadb processes with a tiny raft log
## (BARADB_RAFT_LOG_MAX_ENTRIES=16) and a short stale window
## (BARADB_RAFT_PEER_STALE_MS=3000) so the leader compacts quickly past a
## downed peer's matchIndex.
##
## Scenario A: a node is killed, 100 rows are written through the leader
## (forcing compaction past its matchIndex), the node restarts with its
## intact data dir and must catch up via InstallSnapshot within 15 s.
## Scenario B: the node is stopped, its data dir is WIPED, it rejoins with
## the same node id and must serve the full row set within 20 s.
##
## Process-management conventions follow tests/raft_writes_e2e_test.nim
## (copying is the repo's e2e convention — do not factor out).
import std/unittest
import std/osproc
import std/os
import std/strtabs
import std/strutils
import std/times
import std/net
import std/posix
import std/httpclient
import ../adaptors/nim/baradb_sqlite as sqlite
const
BinaryPath = "./build/baradadb"
LeaderMarker = "became leader"
SnapshotMarker = "Installing snapshot"
TableName = "cold_test"
RowCount = 100
type
NodeProc = object
id: string
clientPort: int
raftPort: int
peers: string
clientPeers: string
p: Process
dataDir: string
output: string
alive: bool
proc drainOutput(n: var NodeProc) =
## Reads whatever the child has written so far. The pipe was set O_NONBLOCK
## at start, so this never blocks — a hung read is impossible here.
var tmp: array[8192, char]
while true:
let count = posix.read(n.p.outputHandle.cint, tmp[0].addr, tmp.len)
if count <= 0: break
for i in 0 ..< count: n.output.add tmp[i]
proc drainAll(nodes: var seq[NodeProc]) =
for n in nodes.mitems:
if n.p != nil: n.drainOutput()
proc dumpAll(nodes: var seq[NodeProc]) =
## Debuggability: on failure, everything the nodes said.
nodes.drainAll()
for n in nodes:
echo "===== output of ", n.id, " (port ", n.clientPort, ") ====="
echo n.output
proc portOpen(port: int): bool =
var s: Socket
try:
s = newSocket()
s.connect("127.0.0.1", Port(port), timeout = 250)
s.close()
result = true
except CatchableError:
if s != nil: s.close()
result = false
proc killNode(n: var NodeProc) =
if n.p != nil and n.alive:
try:
n.p.terminate()
discard n.p.waitForExit()
except CatchableError:
discard
n.alive = false
proc leaderTerms(output: string): seq[int] =
## All terms this node logged leadership for ("became leader for term T").
var pos = 0
while true:
let idx = output.find(LeaderMarker, pos)
if idx < 0: break
let tIdx = output.find("term ", idx)
if tIdx < 0: break
let numStart = tIdx + 5
var numEnd = numStart
while numEnd < output.len and output[numEnd] in Digits: inc numEnd
if numEnd > numStart:
result.add(parseInt(output[numStart ..< numEnd]))
pos = numEnd
proc maxLeader(nodes: seq[NodeProc]): tuple[idx, term: int] =
## Node that logged leadership for the highest term seen so far.
result = (-1, 0)
for i in 0 ..< nodes.len:
for t in leaderTerms(nodes[i].output):
if t > result.term: result = (i, t)
proc drainFor(nodes: var seq[NodeProc], ms: int) =
let start = getTime()
while getTime() - start < initDuration(milliseconds = ms):
nodes.drainAll()
sleep(50)
proc openClient(port: int): DbConn =
## Connect with retries — the port may accept TCP before the DB is usable.
for i in 0 ..< 50:
try:
return open("127.0.0.1:" & $port, "", "", "default")
except CatchableError:
sleep(100)
raise newException(IOError, "cannot connect to port " & $port)
proc rowCountOn(port: int): int =
## SELECT COUNT(*) — raises while the table is not there yet (e.g. before
## the snapshot restore has landed); callers poll and tolerate that.
let db = openClient(port)
defer: db.close()
parseInt(db.getValue(sql("SELECT COUNT(*) FROM " & TableName)))
proc fetchMetric(port: int, name: string): int =
## GET /metrics on the node's HTTP port (clientPort + 440) and parse
## `name{...} <value>`. Returns -1 on any error or when absent.
let client = newHttpClient(timeout = 1500)
defer: client.close()
try:
let body = client.getContent("http://127.0.0.1:" & $(port + 440) & "/metrics")
for line in body.splitLines():
if line.startsWith(name & "{"):
let parts = line.splitWhitespace()
if parts.len >= 2:
return parseInt(parts[^1])
except CatchableError:
discard
return -1
proc startNode(id, dataDir: string, clientPort, raftPort: int,
peers, clientPeers: string): NodeProc =
## Boots one build/baradadb process. Takes the data dir explicitly so a
## node can be restarted with the same dir (scenario A) or a wiped dir
## (scenario B). Compaction/stale-window env goes on EVERY node.
createDir(dataDir)
var env = newStringTable()
for key, val in envPairs():
env[key] = val
env["BARADB_PORT"] = $clientPort
env["BARADB_RAFT_ENABLED"] = "true"
env["BARADB_RAFT_PORT"] = $raftPort
env["BARADB_RAFT_NODE_ID"] = id
env["BARADB_RAFT_PEERS"] = peers
env["BARADB_RAFT_CLIENT_PEERS"] = clientPeers
env["BARADB_RAFT_LOG_MAX_ENTRIES"] = "16"
env["BARADB_RAFT_PEER_STALE_MS"] = "3000"
env["BARADB_DATA_DIR"] = dataDir
env["BARADB_LOG_LEVEL"] = "info"
let p = startProcess(BinaryPath, env = env,
options = {poStdErrToStdOut, poDaemon})
discard fcntl(p.outputHandle.cint, F_SETFL,
fcntl(p.outputHandle.cint, F_GETFL) or O_NONBLOCK)
NodeProc(id: id, clientPort: clientPort, raftPort: raftPort,
peers: peers, clientPeers: clientPeers, p: p,
dataDir: dataDir, alive: true)
proc restartNode(n: var NodeProc, wipe: bool) =
## Kill, then boot the same node id again. wipe=false keeps the data dir
## (cold-node return); wipe=true deletes it first (fresh node rejoin).
n.killNode()
if n.p != nil: n.p.close()
if wipe:
removeDir(n.dataDir)
let fresh = startNode(n.id, n.dataDir, n.clientPort, n.raftPort,
n.peers, n.clientPeers)
n.p = fresh.p
n.alive = true
proc waitReady(n: NodeProc, deadlineSec: int): bool =
let start = getTime()
while getTime() - start < initDuration(seconds = deadlineSec):
if portOpen(n.clientPort):
return true
sleep(100)
return false
proc runColdNodeScenario() =
## Fatal phase failures dump all captured node output, record a test
## failure, and return; cleanup happens in the finally below either way.
let tstamp = getTime().toUnix.int
# Port base per the T11 brief: distinct from raft_writes_e2e_test
# (46000+mod4000) and raft_tls_e2e_test (54000+mod4000). Client ports are
# spaced by 10 because the server derives HTTP (port+440), WS (port+441)
# and gossip (raftPort+100) ports — consecutive client ports collide.
let cbase = 58000 + (tstamp mod 4000)
let rbase = cbase + 100
let peers = "n1@127.0.0.1:" & $(rbase + 1) &
",n2@127.0.0.1:" & $(rbase + 2) &
",n3@127.0.0.1:" & $(rbase + 3)
# SQL client ports for transparent leader write forwarding.
let clientPeers = "n1@127.0.0.1:" & $(cbase + 10) &
",n2@127.0.0.1:" & $(cbase + 20) &
",n3@127.0.0.1:" & $(cbase + 30)
var nodes: seq[NodeProc]
try:
# Node starts live inside the try so a raise from start #2/#3 still
# reaches the cleanup in the finally below.
for i in 1 .. 3:
let id = "n" & $i
let dataDir = getTempDir() / "baradb_raft_coldnode_e2e_" & $tstamp & "_" & id
nodes.add startNode(id, dataDir, cbase + i * 10, rbase + i,
peers, clientPeers)
# Readiness: all three client ports accept TCP connections (10s each).
for i in 0 ..< nodes.len:
if not waitReady(nodes[i], 10):
echo "node ", nodes[i].id, " never became ready"
dumpAll(nodes)
fail()
return
# Election: timeouts are 150-300ms, heartbeat 50ms — a leader should
# emerge within ~2s; 10s deadline for margin.
var elected = false
let electStart = getTime()
while getTime() - electStart < initDuration(seconds = 10):
nodes.drainAll()
if maxLeader(nodes).idx >= 0:
elected = true
break
sleep(50)
if not elected:
echo "no leader elected within 10s"
dumpAll(nodes)
fail()
return
# Settle and require stability, same as raft_writes_e2e_test.
nodes.drainFor(2000)
let (leaderIdx, leaderTerm) = maxLeader(nodes)
nodes.drainFor(1000)
let (stableIdx, stableTerm) = maxLeader(nodes)
if stableIdx != leaderIdx or stableTerm != leaderTerm:
echo "cluster unstable: leadership moved from ", nodes[leaderIdx].id,
" (term ", leaderTerm, ") to ", nodes[stableIdx].id,
" (term ", stableTerm, ")"
dumpAll(nodes)
fail()
return
echo "leader elected: ", nodes[leaderIdx].id, " (term ", leaderTerm, ")"
# The cold node: n3 unless n3 is the leader (then n1) — it must be a
# follower so killing it never forces a re-election.
let coldIdx = (if leaderIdx == 2: 0 else: 2)
let otherIdx = 3 - leaderIdx - coldIdx
echo "cold node: ", nodes[coldIdx].id, "; surviving follower: ",
nodes[otherIdx].id
# Schema: CREATE TABLE goes through the raft "ddl" log (C3c). Create on
# the leader; wait until the cold node has applied it before killing it.
block:
let db = openClient(nodes[leaderIdx].clientPort)
try:
db.exec(sql("CREATE TABLE " & TableName & " (id INT PRIMARY KEY, name STRING)"))
except CatchableError as e:
echo "leader CREATE TABLE failed: ", e.msg
dumpAll(nodes)
fail()
return
db.close()
block:
let start = getTime()
var ready = false
while getTime() - start < initDuration(seconds = 5):
try:
discard rowCountOn(nodes[coldIdx].clientPort)
ready = true
break
except CatchableError:
sleep(100)
if not ready:
echo "cold node ", nodes[coldIdx].id,
" never applied CREATE TABLE within 5s"
dumpAll(nodes)
fail()
return
echo "schema replicated to cold node ", nodes[coldIdx].id
# ---- Kill the cold node; write 100 rows through the leader. ----
# With logMaxEntries=16 and peerStaleMs=3000 the leader compacts past the
# dead node's matchIndex partway through the writes, so the node can only
# catch up via InstallSnapshot on return.
killNode(nodes[coldIdx])
echo "cold node ", nodes[coldIdx].id, " killed; writing ", RowCount, " rows"
block:
let db = openClient(nodes[leaderIdx].clientPort)
var atRow = 0
try:
for i in 1 .. RowCount:
atRow = i
db.exec(sql("INSERT INTO " & TableName & " (id, name) VALUES (" &
$i & ", 'row-" & $i & "')"))
except CatchableError as e:
echo "leader INSERT failed at row ", atRow, ": ", e.msg
dumpAll(nodes)
fail()
return
db.close()
echo RowCount, " rows committed with ", nodes[coldIdx].id, " down"
# Compaction evidence on the leader: the in-memory log stayed bounded
# (<= 64, far below the 100 entries written) and the snapshot base moved.
nodes.drainAll()
let leaderLogLen = fetchMetric(nodes[leaderIdx].clientPort,
"baradb_raft_log_entries")
let leaderSnapIdx = fetchMetric(nodes[leaderIdx].clientPort,
"baradb_raft_snapshot_index")
echo "leader after writes: log_entries=", leaderLogLen,
" snapshot_index=", leaderSnapIdx
if leaderLogLen < 0 or leaderLogLen > 64:
echo "leader raft log not bounded: baradb_raft_log_entries=",
leaderLogLen, " (want <= 64)"
dumpAll(nodes)
fail()
return
if leaderSnapIdx <= 0:
echo "leader never compacted: baradb_raft_snapshot_index=",
leaderSnapIdx, " (want > 0)"
dumpAll(nodes)
fail()
return
echo "leader log stayed bounded and compaction advanced"
# Sanity: leader and surviving follower agree on the row count.
let wantCount = rowCountOn(nodes[leaderIdx].clientPort)
if wantCount != RowCount:
echo "leader count=", wantCount, " want ", RowCount
dumpAll(nodes)
fail()
return
# ---- Scenario A: cold node returns with its intact data dir. ----
nodes[coldIdx].output.setLen(0) # fresh log for the snapshot marker scan
restartNode(nodes[coldIdx], wipe = false)
if not waitReady(nodes[coldIdx], 10):
echo "cold node ", nodes[coldIdx].id, " never became ready after restart"
dumpAll(nodes)
fail()
return
echo "scenario A: ", nodes[coldIdx].id, " restarted with intact data dir"
# Within 15s: snapshot evidence (metric on the cold node, or its restore
# log line) AND data convergence with the leader.
block:
let start = getTime()
var snapSeen = false
var converged = false
while getTime() - start < initDuration(seconds = 15):
nodes.drainAll()
if not snapSeen:
snapSeen = fetchMetric(nodes[coldIdx].clientPort,
"baradb_raft_snapshot_index") > 0 or
nodes[coldIdx].output.contains(SnapshotMarker)
if not converged:
try:
converged = rowCountOn(nodes[coldIdx].clientPort) == wantCount
except CatchableError:
discard
if snapSeen and converged: break
sleep(200)
if not snapSeen:
echo "scenario A: no snapshot evidence on ", nodes[coldIdx].id,
" (baradb_raft_snapshot_index stayed 0, no restore log line)"
dumpAll(nodes)
fail()
return
if not converged:
echo "scenario A: ", nodes[coldIdx].id,
" count did not converge to ", wantCount, " within 15s"
dumpAll(nodes)
fail()
return
echo "scenario A: snapshot installed, count converged to ", wantCount
# Spot-check a few ids survived the snapshot round-trip.
block:
let db = openClient(nodes[coldIdx].clientPort)
defer: db.close()
for i in [1, 42, RowCount]:
let v = db.getValue(sql("SELECT name FROM " & TableName &
" WHERE id = " & $i))
if v != "row-" & $i:
echo "scenario A: spot-check id=", i, " got '", v,
"' want 'row-", i, "'"
dumpAll(nodes)
fail()
return
echo "scenario A: spot-checks passed"
# ---- Scenario B: wiped node rejoins with the same node id. ----
nodes[coldIdx].output.setLen(0)
restartNode(nodes[coldIdx], wipe = true)
if not waitReady(nodes[coldIdx], 10):
echo "wiped node ", nodes[coldIdx].id, " never became ready after restart"
dumpAll(nodes)
fail()
return
echo "scenario B: ", nodes[coldIdx].id, " restarted with a wiped data dir"
# Within 20s: snapshot → catch-up, then the full row set.
block:
let start = getTime()
var snapSeen = false
var converged = false
while getTime() - start < initDuration(seconds = 20):
nodes.drainAll()
if not snapSeen:
snapSeen = fetchMetric(nodes[coldIdx].clientPort,
"baradb_raft_snapshot_index") > 0 or
nodes[coldIdx].output.contains(SnapshotMarker)
if not converged:
try:
converged = rowCountOn(nodes[coldIdx].clientPort) == wantCount
except CatchableError:
discard
if snapSeen and converged: break
sleep(200)
if not snapSeen:
echo "scenario B: no snapshot evidence on wiped ", nodes[coldIdx].id
dumpAll(nodes)
fail()
return
if not converged:
echo "scenario B: wiped ", nodes[coldIdx].id,
" count did not converge to ", wantCount, " within 20s"
dumpAll(nodes)
fail()
return
echo "scenario B: wiped node converged to ", wantCount, " rows"
# Spot-check again on the wiped node.
block:
let db = openClient(nodes[coldIdx].clientPort)
defer: db.close()
for i in [1, 42, RowCount]:
let v = db.getValue(sql("SELECT name FROM " & TableName &
" WHERE id = " & $i))
if v != "row-" & $i:
echo "scenario B: spot-check id=", i, " got '", v,
"' want 'row-", i, "'"
dumpAll(nodes)
fail()
return
echo "scenario B: spot-checks passed"
finally:
for n in nodes.mitems:
n.killNode()
if n.p != nil: n.p.close()
removeDir(n.dataDir)
suite "Raft cold-node E2E":
test "compacted-away node and wiped node rejoin and converge":
if not fileExists(BinaryPath):
if getEnv("CI").len > 0:
echo "[FAIL] ", BinaryPath, " missing under CI — build step broken?"
fail()
else:
echo "[SKIP] ", BinaryPath, " missing — run `nimble test` (builds the server first)"
skip()
else:
runColdNodeScenario()