fix: issue #9 reserved keyword 'key' + issue #10 empty string treated as NULL

Issue 9: Add backtick-quoted identifier support in lexer.nim
- Users can now write `key` to use reserved keywords as column/table names

Issue 10: Use \N as internal NULL marker instead of empty string
- isNull() now checks for \N instead of value.len == 0
- Empty strings can now be stored in NOT NULL columns
- NULL values properly distinguished from empty strings in storage, wire protocol, and HTTP JSON responses

BARADB_DEFICIENCIES.md

@@ -161,6 +161,56 @@ All DB operations (`query`, `getRow`, `exec`, etc.) are wrapped in this lock.
 
 ---
 
+## 9. `key` is a Reserved SQL Keyword
+
+**Problem:** BaraDB's SQL parser treats `key` as a reserved keyword. Using it as a column or table name causes parse errors:
+
+```
+Expected tkIdent but got tkKey
+```
+
+**Result:** The `settings` table could not use `key varchar(100) primary key`.
+
+**Fix:** Added backtick-quoted identifier support in `query/lexer.nim`. Users can now escape reserved keywords using backticks:
+
+```sql
+CREATE TABLE settings(
+  `key` varchar(100) primary key,
+  value varchar(500) default ''
+);
+```
+
+Changes made:
+- Added `readBacktickIdent()` proc in `lexer.nim`
+- Added backtick case in `nextToken()` to recognize `` `identifier` `` as `tkIdent`
+
+---
+
+## 10. Empty String (`""`) Treated as NULL
+
+**Problem:** BaraDB normalized empty strings to `NULL` internally. A column defined as `NOT NULL` rejected empty string inserts, even when the application explicitly passed `''`.
+
+**Result:** SMTP settings saved via the admin panel failed with constraint violations when fields were left blank:
+
+```sql
+INSERT INTO settings (skey, value) VALUES ('smtpUser', '');
+-- ERROR: NOT NULL constraint violation
+```
+
+**Fix:** Changed internal NULL representation from empty string `""` to `\N` in `query/executor.nim`:
+
+1. `isNull()` now checks for `\N` instead of `value.len == 0`
+2. NULL literals in INSERT/UPDATE store `\N` instead of `""`
+3. Missing columns return `\N` instead of `""`
+4. `getValue()` returns `\N` for missing fields
+5. JOIN padding uses `\N` for unmatched rows
+6. Server wire protocol uses `\N` sentinel to send `fkNull`
+7. HTTP server sends JSON `null` for NULL values
+
+This allows empty strings to be stored in NOT NULL columns while still properly supporting NULL values.
+
+---
+
 ## Summary Table
 
 | # | Issue | Location | Type |
@@ -173,7 +223,9 @@ All DB operations (`query`, `getRow`, `exec`, etc.) are wrapped in this lock.
 | 6 | Inconsistent agg names | N/A (engine behavior) | Naming convention |
 | 7 | Async client unstable | `forum/src/baradb_client.nim` | Client design |
 | 8 | No thread safety | `forum/src/baradb_sqlite.nim` | Adapter design |
+| 9 | `key` reserved keyword | `query/lexer.nim` | Parser limitation |
+| 10 | Empty string = NULL | `query/executor.nim` | Data handling bug |
 
 ---
 
-*Document version: 2026-05-15*
+*Document version: 2026-05-17*

src/barabadb/core/httpserver.nim

@@ -164,9 +164,10 @@ proc queryHandler(server: HttpServer): RequestHandler =
           var jsonRow = newJObject()
           for col in res.columns:
             let key = col
-            var val = ""
+            if key in row and not isNull(row[key]):
-            if key in row: val = row[key]
+              jsonRow[key] = %row[key]
-            jsonRow[key] = %val
+            else:
+              jsonRow[key] = newJNull()
           jsonRows.add(jsonRow)
         var jsonCols = newJArray()
         for c in res.columns:

src/barabadb/core/server.nim

@@ -130,7 +130,7 @@ proc typeToFieldKind*(colType: string): FieldKind =
     return fkString
 
 proc valueToWire(val: string, colType: string): WireValue =
-  if val.len == 0 or val.toLower() == "null":
+  if val == "\\N" or val.toLower() == "null":
     return WireValue(kind: fkNull)
   let t = colType.toUpper()
   if t.startsWith("INT") or t == "SERIAL" or t == "BIGINT" or t == "SMALLINT" or t == "BIGSERIAL" or t == "SMALLSERIAL":
@@ -199,7 +199,7 @@ proc executeQuery(db: LSMTree, ctx: ExecutionContext, query: string, params: seq
       for row in res.rows:
         var wireRow: seq[WireValue] = @[]
         for i, col in res.columns:
-          let val = if col in row: row[col] else: ""
+          let val = if col in row: row[col] else: "\\N"
           let cType = if i < colTypes.len: colTypes[i] else: ""
           wireRow.add(valueToWire(val, cType))
         qr.rows.add(wireRow)

src/barabadb/query/executor.nim

@@ -428,10 +428,10 @@ proc getValue(values: seq[string], fields: seq[string], colName: string): string
   for i, f in fields:
     if f.toLower() == colName.toLower() and i < values.len:
       return values[i]
-  return ""
+  return "\\N"
 
 proc isNull*(value: string): bool =
-  value.len == 0 or value.toLower() == "null"
+  value == "\\N" or value.toLower() == "null"
 
 proc escapeRowVal(v: string): string =
   v.replace("\\", "\\\\").replace(",", "\\,").replace("=", "\\=")
@@ -506,8 +506,8 @@ proc evalExpr*(expr: IRExpr, row: Table[string, string], ctx: ExecutionContext =
     of vkInt64: return $expr.literal.int64Val
     of vkFloat64: return $expr.literal.float64Val
     of vkBool: return $expr.literal.boolVal
-    of vkNull: return ""
+    of vkNull: return "\\N"
-    else: return ""
+    else: return "\\N"
   of irekField:
     if expr.fieldPath.len > 0:
       # Check full path first for joined columns (e.g. "u.name")
@@ -520,12 +520,12 @@ proc evalExpr*(expr: IRExpr, row: Table[string, string], ctx: ExecutionContext =
       if "$value" in row:
         let parsed = parseRowData(row["$value"])
         if colName in parsed: return parsed[colName]
-    return ""
+    return "\\N"
   of irekStar:
     return "*"
   of irekJsonPath:
     let srcVal = evalExpr(expr.jpExpr, row, ctx)
-    if srcVal.len == 0: return ""
+    if srcVal.len == 0 or isNull(srcVal): return "\\N"
     try:
       let node = parseJson(srcVal)
       if node.hasKey(expr.jpKey):
@@ -1147,7 +1147,7 @@ proc execDelete*(ctx: ExecutionContext, table: string, key: string,
           if c in oldRow:
             oldVals.add(oldRow[c])
           else:
-            oldVals.add("")
+            oldVals.add("\\N")
         let oldIdxVal = oldVals.join("|")
         if oldIdxVal.len > 0 and not isNull(oldIdxVal):
           ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existingVal)))
@@ -1196,11 +1196,11 @@ proc execUpdateRow*(ctx: ExecutionContext, table: string, key: string, sets: Tab
         if c in oldRow:
           oldVals.add(oldRow[c])
         else:
-          oldVals.add("")
+          oldVals.add("\\N")
         if c in parsed:
           newVals.add(parsed[c])
         else:
-          newVals.add("")
+          newVals.add("\\N")
       let oldIdxVal = oldVals.join("|")
       if oldIdxVal.len > 0 and not isNull(oldIdxVal):
         ctx.btrees[colName].remove(oldIdxVal, IndexEntry(lsmKey: fullKey, rowValue: cast[string](existing)))
@@ -1220,14 +1220,15 @@ proc execUpdateRow*(ctx: ExecutionContext, table: string, key: string, sets: Tab
         docId = docId * 31 + uint64(ord(ch))
       ftsIdx.removeDocument(docId)
       let colName = ftsKey[table.len + 1..^1]
-      let newText = if colName in parsed: parsed[colName] else: ""
+      let newText = if colName in parsed: parsed[colName] else: "\\N"
-      if newText.len > 0:
+      if newText.len > 0 and not isNull(newText):
         ftsIdx.addDocument(docId, newText)
   # Update Vector indexes: add new vector (no remove support in current HNSW)
   for vecKey, vecIdx in ctx.vectorIndexes:
     if vecKey.startsWith(table & "."):
       let colName = vecKey[table.len + 1..^1]
-      let vecStr = if colName in parsed: parsed[colName] else: ""
+      let vecStr = if colName in parsed: parsed[colName] else: "\\N"
+      if not isNull(vecStr):
         let vec = parseVectorString(vecStr)
         if vec.len > 0:
           var docId: uint64 = 0
@@ -1357,7 +1358,7 @@ proc validateConstraints*(ctx: ExecutionContext, tableName: string,
           if i < rowVals.len:
             row[f] = rowVals[i]
           else:
-            row[f] = ""
+            row[f] = "\\N"
         let checkExpr = lowerExpr(check.checkNode)
         let checkResult = evalExpr(checkExpr, row, ctx)
         if checkResult != "true":
@@ -1878,9 +1879,9 @@ proc computeWindowValues*(rows: seq[Row], expr: IRExpr, ctx: ExecutionContext =
         let (_, fEnd) = resolveFrameBounds(pos, sortedIdxs.len, frameStart, frameEnd)
         result[rowIdx] = evalExpr(expr.wfArgs[0], rows[sortedIdxs[fEnd]], ctx)
     else:
-      # Unknown window function — fill with empty
+      # Unknown window function — fill with null
       for rowIdx in sortedIdxs:
-        result[rowIdx] = ""
+        result[rowIdx] = "\\N"
 
 # ----------------------------------------------------------------------
 # IR Plan Execution (with actual filter/sort/projection)
@@ -2314,14 +2315,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
             if not k.startsWith("$"):
               padded[k] = v
           for col in rightCols:
-            if col notin padded: padded[col] = ""
+            if col notin padded: padded[col] = "\\N"
           if leftAlias.len > 0:
             for k, v in l:
               if not k.startsWith("$"):
                 padded[leftAlias & "." & k] = v
           if rightAlias.len > 0:
             for col in rightCols:
-              padded[rightAlias & "." & col] = ""
+              padded[rightAlias & "." & col] = "\\N"
           result.add(padded)
       return result
 
@@ -2361,14 +2362,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
           if not k.startsWith("$"):
             padded[k] = v
         for col in rightCols:
-          if col notin padded: padded[col] = ""
+          if col notin padded: padded[col] = "\\N"
         if leftAlias.len > 0:
           for k, v in l:
             if not k.startsWith("$"):
               padded[leftAlias & "." & k] = v
         if rightAlias.len > 0:
           for col in rightCols:
-            padded[rightAlias & "." & col] = ""
+            padded[rightAlias & "." & col] = "\\N"
         result.add(padded)
 
     if plan.joinKind == irjkRight or plan.joinKind == irjkFull:
@@ -2385,14 +2386,14 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
             if not k.startsWith("$"):
               padded[k] = v
           for col in leftCols:
-            if col notin padded: padded[col] = ""
+            if col notin padded: padded[col] = "\\N"
           if rightAlias.len > 0:
             for k, v in r:
               if not k.startsWith("$"):
                 padded[rightAlias & "." & k] = v
           if leftAlias.len > 0:
             for col in leftCols:
-              padded[leftAlias & "." & col] = ""
+              padded[leftAlias & "." & col] = "\\N"
           result.add(padded)
 
     return result
@@ -2496,7 +2497,7 @@ proc executePlan*(ctx: ExecutionContext, plan: IRPlan): seq[Row] =
           if col in row:
             newRow[col] = row[col]
         newRow[plan.unpivotForCol] = inCol
-        newRow[plan.unpivotValueCol] = (if inCol in row: row[inCol] else: "")
+        newRow[plan.unpivotValueCol] = (if inCol in row: row[inCol] else: "\\N")
         result.add(newRow)
     return result
 
@@ -3058,14 +3059,14 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue]
           elif v.kind == nkIntLit: row.add($v.intVal)
           elif v.kind == nkFloatLit: row.add($v.floatVal)
           elif v.kind == nkBoolLit: row.add($v.boolVal)
-          elif v.kind == nkNullLit: row.add("")
+          elif v.kind == nkNullLit: row.add("\\N")
           else: row.add(evalNodeToString(v))
       else:
         if rowNode.kind == nkStringLit: row.add(rowNode.strVal)
         elif rowNode.kind == nkIntLit: row.add($rowNode.intVal)
         elif rowNode.kind == nkFloatLit: row.add($rowNode.floatVal)
         elif rowNode.kind == nkBoolLit: row.add($rowNode.boolVal)
-        elif rowNode.kind == nkNullLit: row.add("")
+        elif rowNode.kind == nkNullLit: row.add("\\N")
         else: row.add(evalNodeToString(rowNode))
       values.add(row)
 
@@ -3173,7 +3174,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue]
                     elif s.binRight.kind == nkIntLit: $s.binRight.intVal
                     elif s.binRight.kind == nkFloatLit: $s.binRight.floatVal
                     elif s.binRight.kind == nkBoolLit: $s.binRight.boolVal
-                    elif s.binRight.kind == nkNullLit: ""
+                    elif s.binRight.kind == nkNullLit: "\\N"
                     else: evalNodeToString(s.binRight)
           sets[s.binLeft.identName] = val
 
@@ -3199,7 +3200,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue]
           elif col.name in row:
             updValues.add(row[col.name])
           else:
-            updValues.add("")
+            updValues.add("\\N")
         let (valid, errMsg) = validateConstraints(ctx, stmt.updTarget, updFields, @[updValues], skipPkCheck = true)
         if not valid: return errResult(errMsg)
         # Fire BEFORE UPDATE triggers
@@ -3296,7 +3297,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue]
               let valExpr = lowerExpr(v)
               values.add(evalExpr(valExpr, combinedRow, ctx))
             else:
-              values.add("")
+              values.add("\\N")
         if fields.len > 0:
           var row = initTable[string, string]()
           for i, f in fields:
@@ -3724,7 +3725,7 @@ proc executeQuery*(ctx: ExecutionContext, astNode: Node, params: seq[WireValue]
         if col in row:
           colVals.add(row[col])
         else:
-          colVals.add("")
+          colVals.add("\\N")
       let idxVal = colVals.join("|")
       if idxVal.len > 0 and not isNull(idxVal):
         let lsmKey = if "$key" in row: stmt.ciTarget & "." & row["$key"] else: ""

src/barabadb/query/lexer.nim

@@ -481,6 +481,16 @@ proc readString(l: var Lexer, quote: char): string =
   if l.pos < l.input.len:
     discard l.advance()  # skip closing quote
 
+proc readBacktickIdent(l: var Lexer, startLine, startCol: int): Token =
+  var ident = ""
+  discard l.advance()  # skip opening backtick
+  while l.pos < l.input.len and l.input[l.pos] != '`':
+    ident.add(l.input[l.pos])
+    discard l.advance()
+  if l.pos < l.input.len:
+    discard l.advance()  # skip closing backtick
+  return Token(kind: tkIdent, value: ident, line: startLine, col: startCol)
+
 proc readNumber(l: var Lexer, startLine, startCol: int): Token =
   var numStr = ""
   var isFloat = false
@@ -692,6 +702,8 @@ proc nextToken*(l: var Lexer): Token =
   of '#':
     l.skipLineComment()
     return l.nextToken()
+  of '`':
+    return l.readBacktickIdent(startLine, startCol)
   else:
     if ch in Digits:
       return l.readNumber(startLine, startCol)

tests/join_tests.nim

@@ -47,13 +47,13 @@ suite "JOIN execution":
     check r.rows[0]["name"] == "Alice"
     check r.rows[1]["name"] == "Alice"
     check r.rows[2]["name"] == "Bob"
-    check r.rows[2]["total"] == ""  # NULL represented as empty string
+    check r.rows[2]["total"] == "\\N"  # NULL represented as \N
 
   test "RIGHT JOIN keeps unmatched right rows":
     let r = execSql(ctx, "SELECT * FROM users u RIGHT JOIN orders o ON u.id = o.user_id")
     check r.rows.len == 3
     check r.rows[2]["id"] == "30"
-    check r.rows[2]["name"] == ""  # NULL
+    check r.rows[2]["name"] == "\\N"  # NULL
     check r.rows[2]["total"] == "150.0"
 
   test "FULL JOIN keeps all rows":
@@ -100,7 +100,7 @@ suite "JOIN execution":
     var foundBob = false
     for row in r.rows:
       if row["u.name"] == "Bob":
-        check row["x.total"] == ""
+        check row["x.total"] == "\\N"
         foundBob = true
     check foundBob