diff --git a/BARADB_DEFICIENCIES.md b/BARADB_DEFICIENCIES.md index 727448d..82f5c50 100644 --- a/BARADB_DEFICIENCIES.md +++ b/BARADB_DEFICIENCIES.md @@ -161,6 +161,56 @@ All DB operations (`query`, `getRow`, `exec`, etc.) are wrapped in this lock. --- +## 9. `key` is a Reserved SQL Keyword + +**Problem:** BaraDB's SQL parser treats `key` as a reserved keyword. Using it as a column or table name causes parse errors: + +``` +Expected tkIdent but got tkKey +``` + +**Result:** The `settings` table could not use `key varchar(100) primary key`. + +**Fix:** Added backtick-quoted identifier support in `query/lexer.nim`. Users can now escape reserved keywords using backticks: + +```sql +CREATE TABLE settings( + `key` varchar(100) primary key, + value varchar(500) default '' +); +``` + +Changes made: +- Added `readBacktickIdent()` proc in `lexer.nim` +- Added backtick case in `nextToken()` to recognize `` `identifier` `` as `tkIdent` + +--- + +## 10. Empty String (`""`) Treated as NULL + +**Problem:** BaraDB normalized empty strings to `NULL` internally. A column defined as `NOT NULL` rejected empty string inserts, even when the application explicitly passed `''`. + +**Result:** SMTP settings saved via the admin panel failed with constraint violations when fields were left blank: + +```sql +INSERT INTO settings (skey, value) VALUES ('smtpUser', ''); +-- ERROR: NOT NULL constraint violation +``` + +**Fix:** Changed internal NULL representation from empty string `""` to `\N` in `query/executor.nim`: + +1. `isNull()` now checks for `\N` instead of `value.len == 0` +2. NULL literals in INSERT/UPDATE store `\N` instead of `""` +3. Missing columns return `\N` instead of `""` +4. `getValue()` returns `\N` for missing fields +5. JOIN padding uses `\N` for unmatched rows +6. Server wire protocol uses `\N` sentinel to send `fkNull` +7. HTTP server sends JSON `null` for NULL values + +This allows empty strings to be stored in NOT NULL columns while still properly supporting NULL values. + +--- + ## Summary Table | # | Issue | Location | Type | @@ -173,7 +223,9 @@ All DB operations (`query`, `getRow`, `exec`, etc.) are wrapped in this lock. | 6 | Inconsistent agg names | N/A (engine behavior) | Naming convention | | 7 | Async client unstable | `forum/src/baradb_client.nim` | Client design | | 8 | No thread safety | `forum/src/baradb_sqlite.nim` | Adapter design | +| 9 | `key` reserved keyword | `query/lexer.nim` | Parser limitation | +| 10 | Empty string = NULL | `query/executor.nim` | Data handling bug | --- -*Document version: 2026-05-15* +*Document version: 2026-05-17* diff --git a/src/barabadb/core/httpserver.nim b/src/barabadb/core/httpserver.nim index b9ea5dd..986b5bf 100644 --- a/src/barabadb/core/httpserver.nim +++ b/src/barabadb/core/httpserver.nim @@ -164,9 +164,10 @@ proc queryHandler(server: HttpServer): RequestHandler = var jsonRow = newJObject() for col in res.columns: let key = col - var val = "" - if key in row: val = row[key] - jsonRow[key] = %val + if key in row and not isNull(row[key]): + jsonRow[key] = %row[key] + else: + jsonRow[key] = newJNull() jsonRows.add(jsonRow) var jsonCols = newJArray() for c in res.columns: diff --git a/src/barabadb/core/server.nim b/src/barabadb/core/server.nim index 0cecde9..231569a 100644 --- a/src/barabadb/core/server.nim +++ b/src/barabadb/core/server.nim @@ -130,7 +130,7 @@ proc typeToFieldKind*(colType: string): FieldKind = return fkString proc valueToWire(val: string, colType: string): WireValue = - if val.len == 0 or val.toLower() == "null": + if val == "\\N" or val.toLower() == "null": return WireValue(kind: fkNull) let t = colType.toUpper() if t.startsWith("INT") or t == "SERIAL" or t == "BIGINT" or t == "SMALLINT" or t == "BIGSERIAL" or t == "SMALLSERIAL": @@ -199,7 +199,7 @@ proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq for row in res.rows: var wireRow: seq[WireValue] = @[] for i, col in res.columns: - let val = if col in row: row[col] else: "" + let val = if col in row: row[col] else: "\\N" let cType = if i < colTypes.len: colTypes[i] else: "" wireRow.add(valueToWire(val, cType)) qr.rows.add(wireRow) diff --git a/src/barabadb/query/executor.nim b/src/barabadb/query/executor.nim index 2fd0e95..620135e 100644 --- a/src/barabadb/query/executor.nim +++ b/src/barabadb/query/executor.nim @@ -428,10 +428,10 @@ proc getValue(values: seq[string], fields: seq[string], colName: string): string for i, f in fields: if f.toLower() == colName.toLower() and i < values.len: return values[i] - return "" + return "\\N" proc isNull*(value: string): bool = - value.len == 0 or value.toLower() == "null" + value == "\\N" or value.toLower() == "null" proc escapeRowVal(v: string): string = v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=") @@ -506,8 +506,8 @@ proc evalExpr*(expr: IRExpr, row: Table[string, string], ctx: ExecutionContext = of vkInt64: return $expr.literal.int64Val of vkFloat64: return $expr.literal.float64Val of vkBool: return $expr.literal.boolVal - of vkNull: return "" - else: return "" + of vkNull: return "\\N" + else: return "\\N" of irekField: if expr.fieldPath.len > 0: # Check full path first for joined columns (e.g. "u.name") @@ -520,12 +520,12 @@ proc evalExpr*(expr: IRExpr, row: Table[string, string], ctx: ExecutionContext = if "$value" in row: let parsed = parseRowData(row["$value"]) if colName in parsed: return parsed[colName] - return "" + return "\\N" of irekStar: return "*" of irekJsonPath: let srcVal = evalExpr(expr.jpExpr, row, ctx) - if srcVal.len == 0: return "" + if srcVal.len == 0 or isNull(srcVal): return "\\N" try: let node = parseJson(srcVal) if node.hasKey(expr.jpKey): @@ -1147,7 +1147,7 @@ proc execDelete*(ctx: ExecutionContext, table: string, key: string, if c in oldRow: oldVals.add(oldRow[c]) else: - oldVals.add("") + oldVals.add("\\N") let oldIdxVal = oldVals.join("|") if oldIdxVal.len > 0 and not isNull(oldIdxVal): ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existingVal))) @@ -1196,11 +1196,11 @@ proc execUpdateRow*(ctx: ExecutionContext, table: string, key: string, sets: Tab if c in oldRow: oldVals.add(oldRow[c]) else: - oldVals.add("") + oldVals.add("\\N") if c in parsed: newVals.add(parsed[c]) else: - newVals.add("") + newVals.add("\\N") let oldIdxVal = oldVals.join("|") if oldIdxVal.len > 0 and not isNull(oldIdxVal): ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existing))) @@ -1220,22 +1220,23 @@ proc execUpdateRow*(ctx: ExecutionContext, table: string, key: string, sets: Tab docId = docId * 31 + uint64(ord(ch)) ftsIdx.removeDocument(docId) let colName = ftsKey[table.len + 1..^1] - let newText = if colName in parsed: parsed[colName] else: "" - if newText.len > 0: + let newText = if colName in parsed: parsed[colName] else: "\\N" + if newText.len > 0 and not isNull(newText): ftsIdx.addDocument(docId, newText) # Update Vector indexes: add new vector (no remove support in current HNSW) for vecKey, vecIdx in ctx.vectorIndexes: if vecKey.startsWith(table & "."): let colName = vecKey[table.len + 1..^1] - let vecStr = if colName in parsed: parsed[colName] else: "" - let vec = parseVectorString(vecStr) - if vec.len > 0: - var docId: uint64 = 0 - for ch in fullKey: - docId = docId * 31 + uint64(ord(ch)) - var meta = initTable[string, string]() - meta["key"] = fullKey - vengine.insert(vecIdx, docId, vec, meta) + let vecStr = if colName in parsed: parsed[colName] else: "\\N" + if not isNull(vecStr): + let vec = parseVectorString(vecStr) + if vec.len > 0: + var docId: uint64 = 0 + for ch in fullKey: + docId = docId * 31 + uint64(ord(ch)) + var meta = initTable[string, string]() + meta["key"] = fullKey + vengine.insert(vecIdx, docId, vec, meta) return 1 # ---------------------------------------------------------------------- @@ -1357,7 +1358,7 @@ proc validateConstraints*(ctx: ExecutionContext, tableName: string, if i < rowVals.len: row[f] = rowVals[i] else: - row[f] = "" + row[f] = "\\N" let checkExpr = lowerExpr(check.checkNode) let checkResult = evalExpr(checkExpr, row, ctx) if checkResult != "true": @@ -1878,9 +1879,9 @@ proc computeWindowValues*(rows: seq[Row], expr: IRExpr, ctx: ExecutionContext = let (_, fEnd) = resolveFrameBounds(pos, sortedIdxs.len, frameStart, frameEnd) result[rowIdx] = evalExpr(expr.wfArgs[0], rows[sortedIdxs[fEnd]], ctx) else: - # Unknown window function — fill with empty + # Unknown window function — fill with null for rowIdx in sortedIdxs: - result[rowIdx] = "" + result[rowIdx] = "\\N" # ---------------------------------------------------------------------- # IR Plan Execution (with actual filter/sort/projection) @@ -2314,14 +2315,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] = if not k.startsWith("$"): padded[k] = v for col in rightCols: - if col notin padded: padded[col] = "" + if col notin padded: padded[col] = "\\N" if leftAlias.len > 0: for k, v in l: if not k.startsWith("$"): padded[leftAlias & "." & k] = v if rightAlias.len > 0: for col in rightCols: - padded[rightAlias & "." & col] = "" + padded[rightAlias & "." & col] = "\\N" result.add(padded) return result @@ -2361,14 +2362,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] = if not k.startsWith("$"): padded[k] = v for col in rightCols: - if col notin padded: padded[col] = "" + if col notin padded: padded[col] = "\\N" if leftAlias.len > 0: for k, v in l: if not k.startsWith("$"): padded[leftAlias & "." & k] = v if rightAlias.len > 0: for col in rightCols: - padded[rightAlias & "." & col] = "" + padded[rightAlias & "." & col] = "\\N" result.add(padded) if plan.joinKind == irjkRight or plan.joinKind == irjkFull: @@ -2385,14 +2386,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] = if not k.startsWith("$"): padded[k] = v for col in leftCols: - if col notin padded: padded[col] = "" + if col notin padded: padded[col] = "\\N" if rightAlias.len > 0: for k, v in r: if not k.startsWith("$"): padded[rightAlias & "." & k] = v if leftAlias.len > 0: for col in leftCols: - padded[leftAlias & "." & col] = "" + padded[leftAlias & "." & col] = "\\N" result.add(padded) return result @@ -2496,7 +2497,7 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] = if col in row: newRow[col] = row[col] newRow[plan.unpivotForCol] = inCol - newRow[plan.unpivotValueCol] = (if inCol in row: row[inCol] else: "") + newRow[plan.unpivotValueCol] = (if inCol in row: row[inCol] else: "\\N") result.add(newRow) return result @@ -3058,14 +3059,14 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] elif v.kind == nkIntLit: row.add($v.intVal) elif v.kind == nkFloatLit: row.add($v.floatVal) elif v.kind == nkBoolLit: row.add($v.boolVal) - elif v.kind == nkNullLit: row.add("") + elif v.kind == nkNullLit: row.add("\\N") else: row.add(evalNodeToString(v)) else: if rowNode.kind == nkStringLit: row.add(rowNode.strVal) elif rowNode.kind == nkIntLit: row.add($rowNode.intVal) elif rowNode.kind == nkFloatLit: row.add($rowNode.floatVal) elif rowNode.kind == nkBoolLit: row.add($rowNode.boolVal) - elif rowNode.kind == nkNullLit: row.add("") + elif rowNode.kind == nkNullLit: row.add("\\N") else: row.add(evalNodeToString(rowNode)) values.add(row) @@ -3173,7 +3174,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] elif s.binRight.kind == nkIntLit: $s.binRight.intVal elif s.binRight.kind == nkFloatLit: $s.binRight.floatVal elif s.binRight.kind == nkBoolLit: $s.binRight.boolVal - elif s.binRight.kind == nkNullLit: "" + elif s.binRight.kind == nkNullLit: "\\N" else: evalNodeToString(s.binRight) sets[s.binLeft.identName] = val @@ -3199,7 +3200,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] elif col.name in row: updValues.add(row[col.name]) else: - updValues.add("") + updValues.add("\\N") let (valid, errMsg) = validateConstraints(ctx, stmt.updTarget, updFields, @[updValues], skipPkCheck = true) if not valid: return errResult(errMsg) # Fire BEFORE UPDATE triggers @@ -3296,7 +3297,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] let valExpr = lowerExpr(v) values.add(evalExpr(valExpr, combinedRow, ctx)) else: - values.add("") + values.add("\\N") if fields.len > 0: var row = initTable[string, string]() for i, f in fields: @@ -3724,7 +3725,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue] if col in row: colVals.add(row[col]) else: - colVals.add("") + colVals.add("\\N") let idxVal = colVals.join("|") if idxVal.len > 0 and not isNull(idxVal): let lsmKey = if "$key" in row: stmt.ciTarget & "." & row["$key"] else: "" diff --git a/src/barabadb/query/lexer.nim b/src/barabadb/query/lexer.nim index 6c30c44..5dcb333 100644 --- a/src/barabadb/query/lexer.nim +++ b/src/barabadb/query/lexer.nim @@ -481,6 +481,16 @@ proc readString(l: var Lexer, quote: char): string = if l.pos < l.input.len: discard l.advance() # skip closing quote +proc readBacktickIdent(l: var Lexer, startLine, startCol: int): Token = + var ident = "" + discard l.advance() # skip opening backtick + while l.pos < l.input.len and l.input[l.pos] != '`': + ident.add(l.input[l.pos]) + discard l.advance() + if l.pos < l.input.len: + discard l.advance() # skip closing backtick + return Token(kind: tkIdent, value: ident, line: startLine, col: startCol) + proc readNumber(l: var Lexer, startLine, startCol: int): Token = var numStr = "" var isFloat = false @@ -692,6 +702,8 @@ proc nextToken*(l: var Lexer): Token = of '#': l.skipLineComment() return l.nextToken() + of '`': + return l.readBacktickIdent(startLine, startCol) else: if ch in Digits: return l.readNumber(startLine, startCol) diff --git a/tests/join_tests.nim b/tests/join_tests.nim index 601fce6..0e603d2 100644 --- a/tests/join_tests.nim +++ b/tests/join_tests.nim @@ -47,13 +47,13 @@ suite "JOIN execution": check r.rows[0]["name"] == "Alice" check r.rows[1]["name"] == "Alice" check r.rows[2]["name"] == "Bob" - check r.rows[2]["total"] == "" # NULL represented as empty string + check r.rows[2]["total"] == "\\N" # NULL represented as \N test "RIGHT JOIN keeps unmatched right rows": let r = execSql(ctx, "SELECT * FROM users u RIGHT JOIN orders o ON u.id = o.user_id") check r.rows.len == 3 check r.rows[2]["id"] == "30" - check r.rows[2]["name"] == "" # NULL + check r.rows[2]["name"] == "\\N" # NULL check r.rows[2]["total"] == "150.0" test "FULL JOIN keeps all rows": @@ -100,7 +100,7 @@ suite "JOIN execution": var foundBob = false for row in r.rows: if row["u.name"] == "Bob": - check row["x.total"] == "" + check row["x.total"] == "\\N" foundBob = true check foundBob