fix: audit batches 3–4 — TLS verify, WS, OFFSET, B-tree, NULL equality

Close the remaining 2026-08 findings: peer TLS on leader forward, disttxn
SO_ERROR, compaction catalog order, OFFSET without LIMIT, window aggregates,
WebSocket mask/size/auth, SCRAM timing and cbind, B-tree leaf left-max
separators, and SQL three-valued NULL comparisons.

tests/bugfix_test.nim

@@ -8,7 +8,15 @@ import ../src/barabadb/query/exec/dml
 import ../src/barabadb/core/types
 import ../src/barabadb/core/config
 import ../src/barabadb/core/replication
+import ../src/barabadb/core/disttxn
+import ../src/barabadb/core/websocket
+import ../src/barabadb/protocol/auth
+import ../src/barabadb/protocol/scram
 import ../src/barabadb/storage/lsm
+import ../src/barabadb/storage/compaction
+import ../src/barabadb/storage/btree
+import std/random
+import std/sets
 
 const testDir = "/tmp/baradb_bugfix_test"
 
@@ -721,3 +729,220 @@ suite "Query correctness — audit batch 2":
     rm.connectReplica("r1")
     let lsn = rm.writeLsn(@[1'u8, 2, 3])
     check lsn == 0
+
+
+suite "Audit batch 3 — remaining 2026-08 findings":
+
+  test "TLS CA file auto-enables peer verify":
+    putEnv("BARADB_TLS_CA_FILE", "/tmp/ca.crt")
+    defer: delEnv("BARADB_TLS_CA_FILE")
+    var cfg = defaultConfig()
+    loadConfigFromEnv(cfg)
+    check cfg.tlsCaFile == "/tmp/ca.crt"
+    check cfg.tlsVerifyPeer == true
+
+  test "explicit BARADB_TLS_VERIFY_PEER=false wins over CA auto-enable":
+    putEnv("BARADB_TLS_CA_FILE", "/tmp/ca.crt")
+    putEnv("BARADB_TLS_VERIFY_PEER", "false")
+    defer:
+      delEnv("BARADB_TLS_CA_FILE")
+      delEnv("BARADB_TLS_VERIFY_PEER")
+    var cfg = defaultConfig()
+    loadConfigFromEnv(cfg)
+    check cfg.tlsVerifyPeer == false
+
+  test "production TLS without verify is rejected":
+    putEnv("BARADB_ENV", "production")
+    defer: delEnv("BARADB_ENV")
+    var cfg = defaultConfig()
+    cfg.authEnabled = true
+    cfg.jwtSecret = "a".repeat(32)
+    cfg.tlsEnabled = true
+    var msg = ""
+    try:
+      validateProductionConfig(cfg)
+    except ValueError as e:
+      msg = e.msg
+    check "TLS" in msg or "verify" in msg.toLower()
+
+  test "OFFSET without LIMIT returns remaining rows":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, 'a')"))
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (2, 'b')"))
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (3, 'c')"))
+    let r = executeQuery(ctx, parse("SELECT id FROM users ORDER BY id OFFSET 1"))
+    check r.success
+    check r.rows.len == 2
+    check valueToString(r.rows[0]["id"]) == "2"
+    check valueToString(r.rows[1]["id"]) == "3"
+
+  test "LIMIT 0 returns no rows":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, 'a')"))
+    let r = executeQuery(ctx, parse("SELECT id FROM users LIMIT 0"))
+    check r.success
+    check r.rows.len == 0
+
+  test "negative LIMIT is clamped to empty, not IndexDefect":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, 'a')"))
+    let r = executeQuery(ctx, parse("SELECT id FROM users LIMIT -5"))
+    check r.success
+    check r.rows.len == 0
+
+  test "WebSocket decode rejects unmasked client frames":
+    let raw = encodeFrame(0x1, "SUBSCRIBE t", masked = false)
+    let (_, n) = decodeFrame(raw)
+    check n < 0
+
+  test "WebSocket decode accepts masked client frames":
+    let raw = encodeFrame(0x1, "SUBSCRIBE t", masked = true,
+                          maskKey = [1'u8, 2, 3, 4])
+    let (frame, n) = decodeFrame(raw)
+    check n > 0
+    check frame.masked
+    check frame.payload == "SUBSCRIBE t"
+
+  test "WebSocket decode rejects oversized frame length":
+    var raw = newString(10)
+    raw[0] = char(0x81)  # FIN + text
+    raw[1] = char(0xFF)  # MASK + 127
+    # 8-byte length = 2 GiB
+    raw[2] = char(0)
+    raw[3] = char(0)
+    raw[4] = char(0)
+    raw[5] = char(0)
+    raw[6] = char(0x80)
+    raw[7] = char(0)
+    raw[8] = char(0)
+    raw[9] = char(0)
+    let (_, n) = decodeFrame(raw)
+    check n < 0
+
+  test "disttxn prepare against refused port fails closed":
+    var tm = newDistTxnManager()
+    let txn = tm.beginTransaction("coord")
+    txn.addParticipant("n1", "127.0.0.1", 1)
+    check txn.prepare() == false
+    check txn.isAborted
+
+  test "compact leaves input files on disk for catalog apply":
+    let testDir = "/tmp/baradb_bugfix_compact_order"
+    removeDir(testDir)
+    var db = newLSMTree(testDir, 128)
+    defer:
+      db.close()
+      removeDir(testDir)
+    for round in 0 ..< L0CompactionTrigger:
+      db.put("r" & $round, cast[seq[byte]]("v" & $round))
+      db.flush()
+    var cs = newCompactionStrategy(testDir)
+    cs.rebuildFromLSM(db)
+    let cr = cs.compact(0)
+    check cr.outputTables.len == 1
+    check fileExists(cr.outputTables[0].path)
+    for t in cr.inputTables:
+      check fileExists(t.path)
+
+  test "SCRAM unknown user fails without leaking existence":
+    var am = newAuthManager()
+    am.registerScramUser("alice", "wonderland")
+    var msg = ""
+    try:
+      discard am.startScram("n,,n=eve,r=abcnonceabcnonceabcn")
+    except ValueError as e:
+      msg = e.msg
+    check msg == "Authentication failed"
+
+  test "SCRAM rejects mismatched channel binding":
+    var am = newAuthManager()
+    am.registerScramUser("alice", "wonderland")
+    let clientNonce = generateNonce()
+    let clientFirst = "n,,n=alice,r=" & clientNonce
+    let serverFirst = am.startScram(clientFirst)
+    var combinedNonce = ""
+    for part in serverFirst.split(","):
+      if part.startsWith("r="): combinedNonce = part[2..^1]
+    let (ok, err) = am.finishScram("c=AAAA,r=" & combinedNonce & ",p=AA")
+    check ok == false
+    check err == "e=channel-bindings-dont-match"
+
+  test "SCRAM expected cbind for gs2 n is biws":
+    check expectedChannelBinding("n") == "biws"
+
+
+suite "Audit batch 4 — B-tree separator (H10) and NULL equality (L4)":
+
+  test "B-tree separators stay valid after interleaved insert/remove":
+    ## Search routes with `key > sep → right`. After delete, removeRec used to
+    ## copy the right child's first key into the separator (right-min), which
+    ## breaks max(left) <= sep < min(right).
+    var rng = initRand(20260828)
+    var btree = newBTreeIndex[int, string](order = 5)
+    var tracker = initTable[int, seq[string]]()
+    for i in 0..<400:
+      let k = rng.rand(0..80)
+      if rng.rand(0..2) < 2:
+        let v = "v" & $i
+        btree.insert(k, v)
+        if k notin tracker: tracker[k] = @[]
+        tracker[k].add(v)
+      else:
+        if k in tracker and tracker[k].len > 0:
+          let v = tracker[k][0]
+          btree.remove(k, v)
+          tracker[k].del(0)
+          if tracker[k].len == 0: tracker.del(k)
+      if i mod 50 == 49:
+        check btree.checkInvariants().len == 0
+    for k, vals in tracker:
+      check btree.get(k).toHashSet == vals.toHashSet
+
+  test "B-tree sequential fill + prefix delete keeps separators":
+    var btree = newBTreeIndex[int, string](order = 4)
+    for i in 0..<60:
+      btree.insert(i, "v" & $i)
+    for i in 0..<30:
+      btree.remove(i, "v" & $i)
+    check btree.checkInvariants().len == 0
+    for i in 30..<60:
+      check btree.get(i) == @["v" & $i]
+    for i in 0..<30:
+      check btree.get(i).len == 0
+
+  test "NULL = NULL is unknown, not true":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, NULL)"))
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (2, 'bob')"))
+    let eqNull = executeQuery(ctx, parse("SELECT id FROM users WHERE name = NULL"))
+    check eqNull.success
+    check eqNull.rows.len == 0
+    let isNull = executeQuery(ctx, parse("SELECT id FROM users WHERE name IS NULL"))
+    check isNull.success
+    check isNull.rows.len == 1
+    check valueToString(isNull.rows[0]["id"]) == "1"
+
+  test "NULL != value is unknown so WHERE excludes the row":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, NULL)"))
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (2, 'bob')"))
+    let r = executeQuery(ctx, parse("SELECT id FROM users WHERE name != 'bob'"))
+    check r.success
+    check r.rows.len == 0
+
+  test "SELECT NULL = NULL yields NULL, IS NULL is true":
+    var ctx = setupCtx()
+    defer: teardown(ctx)
+    discard executeQuery(ctx, parse("INSERT INTO users (id, name) VALUES (1, 'a')"))
+    let eq = executeQuery(ctx, parse("SELECT (NULL = NULL) AS x FROM users"))
+    check eq.success
+    check eq.rows.len == 1
+    check valueToString(eq.rows[0]["x"]) == "\\N"
+    let isn = executeQuery(ctx, parse("SELECT (NULL IS NULL) AS x FROM users"))
+    check isn.success
+    check valueToString(isn.rows[0]["x"]) == "true"

tests/test_all.nim

@@ -4298,6 +4298,21 @@ suite "Window Functions":
       if row["name"] == "Bob":
         check row["last_sal"] == "90000"
 
+  test "SUM/AVG/COUNT window aggregates over a running frame":
+    ## Default frame is UNBOUNDED PRECEDING .. CURRENT ROW (lower.nim).
+    let r = qexec.executeQuery(ctx, parse(
+      "SELECT name, salary, SUM(salary) OVER (ORDER BY salary) AS running, COUNT(*) OVER (ORDER BY salary) AS cnt FROM employees"))
+    check r.success
+    check r.rows.len == 5
+    for row in r.rows:
+      if row["name"] == "Charlie":
+        check parseFloat($row["running"]) == 70000.0
+        check $row["cnt"] == "1"
+      if row["name"] == "Eve":
+        # 70000+75000+80000+90000+95000
+        check parseFloat($row["running"]) == 410000.0
+        check $row["cnt"] == "5"
+
 suite "GROUP BY Aggregates":
   var db: LSMTree
   var ctx: qexec.ExecutionContext