From f2b7ed1ce2e51a1fe0790ad219cdae08d9632ec5 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Fri, 31 Jul 2026 01:55:43 +0300 Subject: [PATCH] test(raft): 3-node TLS cluster e2e with plaintext rejection --- .github/workflows/ci.yml | 1 + .gitignore | 1 + baradadb.nimble | 2 +- tests/raft_tls_e2e_test.nim | 389 ++++++++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 tests/raft_tls_e2e_test.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 089609b..f5fd35e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,7 @@ jobs: nim c -d:ssl --threads:on --path:src -r tests/raft_e2e_test.nim nim c -d:ssl --threads:on --path:src -r tests/raft_writes_e2e_test.nim nim c -d:ssl --threads:on --path:src -r tests/raft_failover_load_e2e_test.nim + nim c -d:ssl --threads:on --path:src -r tests/raft_tls_e2e_test.nim verify: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 94e3ac1..4eca121 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ tests/nimforum_smoke_test tests/raft_e2e_test tests/raft_writes_e2e_test tests/raft_failover_load_e2e_test +tests/raft_tls_e2e_test benchmarks/bench_all benchmarks/compare clients/nim/tests/test_client diff --git a/baradadb.nimble b/baradadb.nimble index 644867f..9cb6d27 100644 --- a/baradadb.nimble +++ b/baradadb.nimble @@ -30,7 +30,7 @@ task test, "Run all tests": for t in ["test_minimal", "test_all", "bugfix_test", "join_tests", "test_lock", "test_schema_persist", "test_storage_hardening", "tla_faithfulness", "nimforum_smoke_test", "raft_e2e_test", "raft_writes_e2e_test", - "raft_failover_load_e2e_test", + "raft_failover_load_e2e_test", "raft_tls_e2e_test", "fuzz_test", "prop_test", "test_wire_insert_stress", "stress_test"]: exec "nim c -r tests/" & t & ".nim" diff --git a/tests/raft_tls_e2e_test.nim b/tests/raft_tls_e2e_test.nim new file mode 100644 index 0000000..10c93e3 --- /dev/null +++ b/tests/raft_tls_e2e_test.nim @@ -0,0 +1,389 @@ +## Raft TLS E2E — real 3-node cluster with TLS on the raft transport. +## Starts three actual build/baradadb processes with per-node self-signed +## certs (BARADB_RAFT_TLS_ENABLED + CERT/KEY_FILE), asserts election and +## replicated writes work over the encrypted transport, then starts a 4th +## plaintext node pointed at the same peers and asserts it never becomes +## leader (its frames are undecryptable) while the TLS cluster keeps +## operating among its 3 members. +## Process-management conventions follow tests/raft_writes_e2e_test.nim +## (copying is the repo's e2e convention — do not factor out). +## +## NOTE: only the raft port is TLS here. The SQL client port stays +## plaintext (BARADB_TLS_ENABLED is the client wire port — a different +## feature), so the baradb_sqlite adaptor connects as usual. +import std/unittest +import std/osproc +import std/os +import std/strtabs +import std/strutils +import std/times +import std/net +import std/posix + +import ../adaptors/nim/baradb_sqlite as sqlite +import barabadb/protocol/ssl + +const + BinaryPath = "./build/baradadb" + LeaderMarker = "became leader" + +type + NodeProc = object + id: string + clientPort: int + p: Process + dataDir: string + output: string + alive: bool + +proc drainOutput(n: var NodeProc) = + ## Reads whatever the child has written so far. The pipe was set O_NONBLOCK + ## at start, so this never blocks — a hung read is impossible here. + var tmp: array[8192, char] + while true: + let count = posix.read(n.p.outputHandle.cint, tmp[0].addr, tmp.len) + if count <= 0: break + for i in 0 ..< count: n.output.add tmp[i] + +proc drainAll(nodes: var seq[NodeProc]) = + for n in nodes.mitems: + if n.p != nil: n.drainOutput() + +proc dumpAll(nodes: var seq[NodeProc]) = + ## Debuggability: on failure, everything the nodes said. + nodes.drainAll() + for n in nodes: + echo "===== output of ", n.id, " (port ", n.clientPort, ") =====" + echo n.output + +proc portOpen(port: int): bool = + var s: Socket + try: + s = newSocket() + s.connect("127.0.0.1", Port(port), timeout = 250) + s.close() + result = true + except CatchableError: + if s != nil: s.close() + result = false + +proc killNode(n: var NodeProc) = + if n.p != nil and n.alive: + try: + n.p.terminate() + discard n.p.waitForExit() + except CatchableError: + discard + n.alive = false + +proc leaderTerms(output: string): seq[int] = + ## All terms this node logged leadership for ("became leader for term T"). + var pos = 0 + while true: + let idx = output.find(LeaderMarker, pos) + if idx < 0: break + let tIdx = output.find("term ", idx) + if tIdx < 0: break + let numStart = tIdx + 5 + var numEnd = numStart + while numEnd < output.len and output[numEnd] in Digits: inc numEnd + if numEnd > numStart: + result.add(parseInt(output[numStart ..< numEnd])) + pos = numEnd + +proc maxLeader(nodes: seq[NodeProc]): tuple[idx, term: int] = + ## Node that logged leadership for the highest term seen so far. + result = (-1, 0) + for i in 0 ..< nodes.len: + for t in leaderTerms(nodes[i].output): + if t > result.term: result = (i, t) + +proc drainFor(nodes: var seq[NodeProc], ms: int) = + let start = getTime() + while getTime() - start < initDuration(milliseconds = ms): + nodes.drainAll() + sleep(50) + +proc openClient(port: int): DbConn = + ## Connect with retries — the port may accept TCP before the DB is usable. + for i in 0 ..< 50: + try: + return open("127.0.0.1:" & $port, "", "", "default") + except CatchableError: + sleep(100) + raise newException(IOError, "cannot connect to port " & $port) + +proc waitForRow(port: int, table, name: string, deadlineSec: int): bool = + ## Poll SELECT on `port` until a row with `name` appears. Tolerates errors + ## (e.g. "unknown table" while schema has not been created yet) by retrying. + let db = openClient(port) + defer: db.close() + let start = getTime() + while getTime() - start < initDuration(seconds = deadlineSec): + try: + let rows = db.getAllRows(sql("SELECT * FROM " & table)) + for row in rows: + if row.len >= 2 and row[1] == name: + return true + except CatchableError: + discard + sleep(100) + return false + +proc startNode(id: string, clientPort, raftPort: int, peers, clientPeers: string, + tlsEnabled: bool): NodeProc = + ## Boots one build/baradadb process. When tlsEnabled, a self-signed cert + ## (CN = node id) is generated into the node's data dir first. + let tstamp = getTime().toUnix.int + let dataDir = getTempDir() / "baradb_raft_tls_e2e_" & $tstamp & "_" & id + createDir(dataDir) + var env = newStringTable() + for key, val in envPairs(): + env[key] = val + env["BARADB_PORT"] = $clientPort + env["BARADB_RAFT_ENABLED"] = "true" + env["BARADB_RAFT_PORT"] = $raftPort + env["BARADB_RAFT_NODE_ID"] = id + env["BARADB_RAFT_PEERS"] = peers + env["BARADB_RAFT_CLIENT_PEERS"] = clientPeers + env["BARADB_DATA_DIR"] = dataDir + env["BARADB_LOG_LEVEL"] = "info" + if tlsEnabled: + let (certFile, keyFile) = generateSelfSignedCert(dataDir, id) + doAssert certFile.len > 0 and keyFile.len > 0, + "openssl cert generation failed for " & id + env["BARADB_RAFT_TLS_ENABLED"] = "true" + env["BARADB_RAFT_TLS_CERT_FILE"] = certFile + env["BARADB_RAFT_TLS_KEY_FILE"] = keyFile + let p = startProcess(BinaryPath, env = env, + options = {poStdErrToStdOut, poDaemon}) + discard fcntl(p.outputHandle.cint, F_SETFL, + fcntl(p.outputHandle.cint, F_GETFL) or O_NONBLOCK) + NodeProc(id: id, clientPort: clientPort, p: p, dataDir: dataDir, alive: true) + +proc runTlsScenario() = + ## Fatal phase failures dump all captured node output, record a test + ## failure, and return; cleanup happens in the finally below either way. + let tstamp = getTime().toUnix.int + # Port base per the T6 brief: distinct from raft_writes_e2e_test + # (46000+mod4000). Client ports are spaced by 10 because the server + # derives HTTP (port+440), WS (port+441) and gossip (raftPort+100) + # ports — consecutive client ports collide. + let cbase = 54000 + (tstamp mod 4000) + let rbase = cbase + 100 + let peers = "n1@127.0.0.1:" & $(rbase + 1) & + ",n2@127.0.0.1:" & $(rbase + 2) & + ",n3@127.0.0.1:" & $(rbase + 3) + # SQL client ports for transparent leader write forwarding. + let clientPeers = "n1@127.0.0.1:" & $(cbase + 10) & + ",n2@127.0.0.1:" & $(cbase + 20) & + ",n3@127.0.0.1:" & $(cbase + 30) + + var nodes: seq[NodeProc] + for i in 1 .. 3: + nodes.add startNode("n" & $i, cbase + i * 10, rbase + i, + peers, clientPeers, tlsEnabled = true) + + # Negative-case node: same peers, raft TLS DISABLED, own dir and ports + # (4th port in each base range). Started later, after the TLS cluster is + # up, so the positive assertions are not polluted by its noise. + var rogue: NodeProc + + try: + # Readiness: all three client ports accept TCP connections (10s each). + for i in 0 ..< nodes.len: + let readyStart = getTime() + var ok = false + while getTime() - readyStart < initDuration(seconds = 10): + if portOpen(nodes[i].clientPort): + ok = true + break + sleep(100) + if not ok: + echo "node ", nodes[i].id, " never became ready" + dumpAll(nodes) + fail() + return + + # Election over TLS: timeouts are 150-300ms, heartbeat 50ms — a leader + # should emerge within ~2s; 10s deadline for margin. + var elected = false + let electStart = getTime() + while getTime() - electStart < initDuration(seconds = 10): + nodes.drainAll() + if maxLeader(nodes).idx >= 0: + elected = true + break + sleep(50) + if not elected: + echo "no leader elected within 10s over TLS" + dumpAll(nodes) + fail() + return + + # Settle and require stability, same as raft_writes_e2e_test. + nodes.drainFor(2000) + let (leaderIdx, leaderTerm) = maxLeader(nodes) + nodes.drainFor(1000) + let (stableIdx, stableTerm) = maxLeader(nodes) + if stableIdx != leaderIdx or stableTerm != leaderTerm: + echo "TLS cluster unstable: leadership moved from ", nodes[leaderIdx].id, + " (term ", leaderTerm, ") to ", nodes[stableIdx].id, + " (term ", stableTerm, ")" + dumpAll(nodes) + fail() + return + echo "TLS leader elected: ", nodes[leaderIdx].id, " (term ", leaderTerm, ")" + + let followerIdx = (if leaderIdx == 0: 1 else: 0) + + # Schema: CREATE TABLE goes through the raft "ddl" log (C3c) — this + # proves raft DDL replication works over the TLS transport. + block: + let db = openClient(nodes[leaderIdx].clientPort) + try: + db.exec(sql"CREATE TABLE tls_test (id INT PRIMARY KEY, name STRING)") + except CatchableError as e: + echo "leader CREATE TABLE failed: ", e.msg + dumpAll(nodes) + fail() + return + db.close() + echo "leader schema committed via raft ddl over TLS" + + # Wait until the follower has applied CREATE TABLE (SELECT no longer + # errors with unknown table). Deadline 5s. + block: + let db = openClient(nodes[followerIdx].clientPort) + defer: db.close() + let start = getTime() + var ready = false + while getTime() - start < initDuration(seconds = 5): + try: + discard db.getAllRows(sql"SELECT * FROM tls_test") + ready = true + break + except CatchableError: + sleep(100) + if not ready: + echo "follower ", nodes[followerIdx].id, + " never applied CREATE TABLE within 5s" + dumpAll(nodes) + fail() + return + echo "schema replicated to follower ", nodes[followerIdx].id, " over TLS" + + # Leader write: INSERT goes through the raft log and waits for majority + # commit before responding — expect success over TLS. + block: + let db = openClient(nodes[leaderIdx].clientPort) + try: + db.exec(sql"INSERT INTO tls_test (id, name) VALUES (1, 'tls-row')") + except CatchableError as e: + echo "leader INSERT failed: ", e.msg + dumpAll(nodes) + fail() + return + db.close() + echo "leader INSERT committed over TLS" + + # Follower visibility: poll until the row shows up (5s deadline). + if not waitForRow(nodes[followerIdx].clientPort, "tls_test", "tls-row", 5): + echo "follower ", nodes[followerIdx].id, + " never saw the replicated row within 5s" + dumpAll(nodes) + fail() + return + echo "row replicated to follower ", nodes[followerIdx].id, " over TLS" + + # Negative scenario: start the plaintext node pointed at the same peers. + # Its TLS-less frames are undecryptable to the cluster, so it can never + # collect votes and must never become leader. + rogue = startNode("n4", cbase + 40, rbase + 4, peers, clientPeers, + tlsEnabled = false) + block: + let readyStart = getTime() + var ok = false + while getTime() - readyStart < initDuration(seconds = 10): + if portOpen(rogue.clientPort): + ok = true + break + sleep(100) + if not ok: + echo "plaintext node never became ready" + rogue.drainOutput() + echo "===== output of ", rogue.id, " (port ", rogue.clientPort, ") =====" + echo rogue.output + dumpAll(nodes) + fail() + return + echo "plaintext node n4 started against the TLS peers" + + # Give n4 many election cycles (timeouts 150-300ms) to try its luck. + # The TLS cluster must keep operating among its 3 members meanwhile. + nodes.drainFor(6000) + + # Cluster still elects/operates: leader among the 3 TLS nodes accepts a + # write and it replicates to a follower, with n4 running. + nodes.drainAll() + let (leaderIdx2, leaderTerm2) = maxLeader(nodes) + if leaderIdx2 < 0: + echo "TLS cluster lost its leader after plaintext node joined" + dumpAll(nodes) + fail() + return + let followerIdx2 = (if leaderIdx2 == 0: 1 else: 0) + block: + let db = openClient(nodes[leaderIdx2].clientPort) + try: + db.exec(sql"INSERT INTO tls_test (id, name) VALUES (2, 'still-tls')") + except CatchableError as e: + echo "post-plaintext INSERT failed on ", nodes[leaderIdx2].id, + " (term ", leaderTerm2, "): ", e.msg + rogue.drainOutput() + echo "===== output of ", rogue.id, " (port ", rogue.clientPort, ") =====" + echo rogue.output + dumpAll(nodes) + fail() + return + db.close() + if not waitForRow(nodes[followerIdx2].clientPort, "tls_test", "still-tls", 5): + echo "post-plaintext row never replicated to ", nodes[followerIdx2].id + dumpAll(nodes) + fail() + return + echo "TLS cluster kept operating with plaintext node present" + + # Final negative assertion: n4 never won an election. + rogue.drainOutput() + if rogue.output.contains(LeaderMarker): + echo "plaintext node became leader — TLS isolation broken" + echo "===== output of ", rogue.id, " (port ", rogue.clientPort, ") =====" + echo rogue.output + dumpAll(nodes) + fail() + return + echo "plaintext node never became leader (TLS rejection confirmed)" + + check leaderIdx2 >= 0 + finally: + for n in nodes.mitems: + n.killNode() + if n.p != nil: n.p.close() + removeDir(n.dataDir) + if rogue.p != nil: + rogue.killNode() + rogue.p.close() + removeDir(rogue.dataDir) + +suite "Raft TLS E2E": + test "3-node TLS cluster elects and replicates; plaintext node rejected": + if not fileExists(BinaryPath): + if getEnv("CI").len > 0: + echo "[FAIL] ", BinaryPath, " missing under CI — build step broken?" + fail() + else: + echo "[SKIP] ", BinaryPath, " missing — run `nimble test` (builds the server first)" + skip() + else: + runTlsScenario()