Baradb/src/barabadb/query/lexer.nim

## BaraQL Lexer — tokenization
import std/tables
import std/strutils
import std/unicode

type
  TokenKind* = enum
    # Literals
    tkIntLit
    tkFloatLit
    tkStringLit
    tkBoolLit

    # Identifiers
    tkIdent

    # Keywords
    tkSelect
    tkInsert
    tkUpdate
    tkDelete
    tkFrom
    tkWhere
    tkAnd
    tkOr
    tkNot
    tkIn
    tkIs
    tkAs
    tkOn
    tkJoin
    tkLeft
    tkRight
    tkInner
    tkOuter
    tkFull
    tkCross
    tkLateral
    tkOrder
    tkBy
    tkAsc
    tkDesc
    tkGroup
    tkHaving
    tkLimit
    tkOffset
    tkSet
    tkInto
    tkValues
    tkCreate
    tkDrop
    tkAlter
    tkTable
    tkIndex
    tkType
    tkLink
    tkProperty
    tkRequired
    tkMulti
    tkSingle
    tkTrue
    tkFalse
    tkNull
    tkIf
    tkThen
    tkElse
    tkEnd
    tkCase
    tkWhen
    tkWith
    tkRecursive
    tkDistinct
    tkUnion
    tkIntersect
    tkExcept
    tkAll
    tkExists
    tkBetween
    tkLike
    tkILike
    tkFilter
    tkReturning
    tkPrimary
    tkKey
    tkForeign
    tkReferences
    tkCascade
    tkRestrict
    tkUnique
    tkCheck
    tkDefault
    tkAdd
    tkColumn
    tkRename
    tkBegin
    tkCommit
    tkRollback
    tkExplain
    tkView
    tkTrigger
    tkBefore
    tkAfter
    tkInstead
    tkOf
    tkImport
    tkExport
    tkFormat
    tkDelimiter
    tkHeader
    tkBatch
    tkCsv
    tkJsonFmt
    tkNdjson
    tkMigration
    tkApply
    tkStatus
    tkUp
    tkDown
    tkDryRun
    tkUser
    tkPolicy
    tkEnable
    tkDisable
    tkRecover
    tkTimestamp
    tkFor
    tkUsing
    tkGrant
    tkRevoke
    tkCount
    tkSum
    tkAvg
    tkMin
    tkMax
    tkArrayAgg
    tkStringAgg
    tkGrouping
    tkSets
    tkRollup
    tkCube
    tkPivot
    tkUnpivot
    tkVertex
    tkEdge
    tkLabels
    tkGraphTable
    tkCreateGraph
    tkDropGraph
    tkMatch
    tkColumns
    tkSrc
    tkDst
    tkMerge
    tkMatched
    tkArray
    tkVector
    tkGraph
    tkDocument
    tkArrowR       # ->
    tkArrowRR      # ->>
    tkFtsMatch     # @@
    tkJsonContains # @>
    tkJsonContainedBy # <@
    tkJsonHasAny   # ?|
    tkJsonHasAll   # ?&
    tkSimilar
    tkNearest
    tkTo
    tkBfs
    tkDfs
    tkPath
    tkDatabase
    tkDatabases
    tkUse
    tkShow
    tkDo
    tkNothing

    tkAutoIncrement
    tkSequence
    tkNextval
    tkCurrval
    tkUuid

    # Window functions
    tkOver
    tkPartition
    tkRow
    tkRows
    tkRange
    tkUnbounded
    tkPreceding
    tkFollowing
    tkCurrent
    tkCurrentUser
    tkCurrentRole
    tkRowNumber
    tkRank
    tkDenseRank
    tkLead
    tkLag
    tkFirstValue
    tkLastValue
    tkNtile

    # Operators
    tkPlus
    tkMinus
    tkStar
    tkSlash
    tkPercent
    tkPower
    tkEq
    tkNotEq
    tkLt
    tkLtEq
    tkGt
    tkGtEq
    tkAssign
    tkArrow
    tkDoubleColon
    tkColon
    tkDot
    tkComma
    tkSemicolon
    tkLParen
    tkRParen
    tkLBrace
    tkRbrace
    tkLBracket
    tkRBracket
    tkAmp
    tkPipe
    tkTilde
    tkConcat
    tkCoalesce
    tkFloorDiv
    tkDistanceOp   # <->
    tkPlaceholder

    # Special
    tkEof
    tkNewline
    tkInvalid

  Token* = object
    kind*: TokenKind
    value*: string
    line*: int
    col*: int

  Lexer* = object
    input: string
    pos: int
    line: int
    col: int

const keywords*: Table[string, TokenKind] = {
  "select": tkSelect,
  "insert": tkInsert,
  "update": tkUpdate,
  "delete": tkDelete,
  "from": tkFrom,
  "where": tkWhere,
  "and": tkAnd,
  "or": tkOr,
  "not": tkNot,
  "in": tkIn,
  "is": tkIs,
  "as": tkAs,
  "on": tkOn,
  "join": tkJoin,
  "left": tkLeft,
  "right": tkRight,
  "inner": tkInner,
  "outer": tkOuter,
  "full": tkFull,
  "cross": tkCross,
  "lateral": tkLateral,
  "order": tkOrder,
  "by": tkBy,
  "asc": tkAsc,
  "desc": tkDesc,
  "group": tkGroup,
  "having": tkHaving,
  "limit": tkLimit,
  "offset": tkOffset,
  "set": tkSet,
  "into": tkInto,
  "values": tkValues,
  "create": tkCreate,
  "drop": tkDrop,
  "alter": tkAlter,
  "table": tkTable,
  "index": tkIndex,
  "type": tkType,
  "link": tkLink,
  "property": tkProperty,
  "required": tkRequired,
  "multi": tkMulti,
  "single": tkSingle,
  "true": tkTrue,
  "false": tkFalse,
  "null": tkNull,
  "if": tkIf,
  "then": tkThen,
  "else": tkElse,
  "end": tkEnd,
  "case": tkCase,
  "when": tkWhen,
  "with": tkWith,
  "recursive": tkRecursive,
  "distinct": tkDistinct,
  "union": tkUnion,
  "intersect": tkIntersect,
  "except": tkExcept,
  "all": tkAll,
  "exists": tkExists,
  "between": tkBetween,
  "like": tkLike,
  "ilike": tkILike,
  "filter": tkFilter,
  "returning": tkReturning,
  "primary": tkPrimary,
  "key": tkKey,
  "foreign": tkForeign,
  "references": tkReferences,
  "cascade": tkCascade,
  "restrict": tkRestrict,
  "unique": tkUnique,
  "check": tkCheck,
  "default": tkDefault,
  "add": tkAdd,
  "column": tkColumn,
  "rename": tkRename,
  "begin": tkBegin,
  "commit": tkCommit,
  "rollback": tkRollback,
  "explain": tkExplain,
  "view": tkView,
  "trigger": tkTrigger,
  "before": tkBefore,
  "after": tkAfter,
  "instead": tkInstead,
  "of": tkOf,
  "import": tkImport,
  "export": tkExport,
  "format": tkFormat,
  "delimiter": tkDelimiter,
  "header": tkHeader,
  "batch": tkBatch,
  "csv": tkCsv,
  "json": tkJsonFmt,
  "ndjson": tkNdjson,
  "migration": tkMigration,
  "apply": tkApply,
  "status": tkStatus,
  "up": tkUp,
  "down": tkDown,
  "dryrun": tkDryRun,
  "user": tkUser,
  "policy": tkPolicy,
  "enable": tkEnable,
  "disable": tkDisable,
  "for": tkFor,
  "using": tkUsing,
  "recover": tkRecover,
  "timestamp": tkTimestamp,
  "grant": tkGrant,
  "revoke": tkRevoke,
  "count": tkCount,
  "sum": tkSum,
  "avg": tkAvg,
  "min": tkMin,
  "max": tkMax,
  "array_agg": tkArrayAgg,
  "string_agg": tkStringAgg,
  "grouping": tkGrouping,
  "sets": tkSets,
  "rollup": tkRollup,
  "cube": tkCube,
  "pivot": tkPivot,
  "unpivot": tkUnpivot,
  "vertex": tkVertex,
  "vertices": tkVertex,
  "edge": tkEdge,
  "edges": tkEdge,
  "label": tkLabels,
  "labels": tkLabels,
  "graph_table": tkGraphTable,
  "create_graph": tkCreateGraph,
  "drop_graph": tkDropGraph,
  "graph": tkGraph,
  "match": tkMatch,
  "columns": tkColumns,
  "src": tkSrc,
  "dst": tkDst,
  "merge": tkMerge,
  "matched": tkMatched,
  "do": tkDo,
  "nothing": tkNothing,
  "array": tkArray,
  "vector": tkVector,
  "document": tkDocument,
  "similar": tkSimilar,
  "nearest": tkNearest,
  "to": tkTo,
  "bfs": tkBfs,
  "dfs": tkDfs,
  "path": tkPath,
  "database": tkDatabase,
  "databases": tkDatabases,
  "use": tkUse,
  "show": tkShow,
  "over": tkOver,
  "partition": tkPartition,
  "row": tkRow,
  "rows": tkRows,
  "range": tkRange,
  "unbounded": tkUnbounded,
  "preceding": tkPreceding,
  "following": tkFollowing,
  "current": tkCurrent,
  "current_user": tkCurrentUser,
  "current_role": tkCurrentRole,
  "row_number": tkRowNumber,
  "rank": tkRank,
  "dense_rank": tkDenseRank,
  "lead": tkLead,
  "lag": tkLag,
  "first_value": tkFirstValue,
  "last_value": tkLastValue,
  "ntile": tkNtile,
  "auto_increment": tkAutoIncrement,
  "sequence": tkSequence,
}.toTable

proc newLexer*(input: string): Lexer =
  Lexer(input: input, pos: 0, line: 1, col: 1)

proc peek(l: Lexer): char =
  if l.pos < l.input.len:
    return l.input[l.pos]
  return '\0'

proc advance(l: var Lexer): char =
  result = l.input[l.pos]
  inc l.pos
  if result == '\n':
    inc l.line
    l.col = 1
  else:
    inc l.col

proc peekRune(l: Lexer): Rune =
  if l.pos < l.input.len:
    var p = l.pos
    var r: Rune
    fastRuneAt(l.input, p, r, true)
    return r
  return Rune(0)

proc advanceRune(l: var Lexer): Rune {.used.} =
  if l.pos < l.input.len:
    var r: Rune
    fastRuneAt(l.input, l.pos, r, true)
    if r == Rune('\n'):
      inc l.line
      l.col = 1
    else:
      inc l.col
    return r
  return Rune(0)

proc isIdentStartRune(r: Rune): bool =
  return isAlpha(r) or r == Rune('_')

proc isIdentPartRune(r: Rune): bool =
  return isAlpha(r) or (r.int >= ord('0') and r.int <= ord('9')) or r == Rune('_')

proc skipWhitespace(l: var Lexer) =
  while l.pos < l.input.len and l.input[l.pos] in {' ', '\t', '\r', '\n'}:
    discard l.advance()

proc skipLineComment(l: var Lexer) =
  while l.pos < l.input.len and l.input[l.pos] != '\n':
    discard l.advance()

proc skipBlockComment(l: var Lexer) =
  discard l.advance()  # skip *
  discard l.advance()  # skip *
  while l.pos < l.input.len - 1:
    if l.input[l.pos] == '*' and l.input[l.pos + 1] == '/':
      discard l.advance()
      discard l.advance()
      return
    discard l.advance()
  raise newException(ValueError, "Unclosed block comment at line " & $l.line & ", col " & $l.col)

proc readString(l: var Lexer, quote: char): string =
  result = ""
  while l.pos < l.input.len and l.input[l.pos] != quote:
    if l.input[l.pos] == '\\':
      discard l.advance()
      if l.pos >= l.input.len: break
      case l.input[l.pos]
      of 'n': result.add('\n')
      of 't': result.add('\t')
      of 'r': result.add('\r')
      of '\\': result.add('\\')
      of '\'': result.add('\'')
      of '"': result.add('"')
      else: result.add(l.input[l.pos])
    else:
      result.add(l.input[l.pos])
    discard l.advance()
  if l.pos < l.input.len:
    discard l.advance()  # skip closing quote

proc readBacktickIdent(l: var Lexer, startLine, startCol: int): Token =
  var ident = ""
  discard l.advance()  # skip opening backtick
  while l.pos < l.input.len and l.input[l.pos] != '`':
    ident.add(l.input[l.pos])
    discard l.advance()
  if l.pos < l.input.len:
    discard l.advance()  # skip closing backtick
  return Token(kind: tkIdent, value: ident, line: startLine, col: startCol)

proc readNumber(l: var Lexer, startLine, startCol: int): Token =
  var numStr = ""
  var isFloat = false
  while l.pos < l.input.len and (l.input[l.pos] in Digits or l.input[l.pos] == '.'):
    if l.input[l.pos] == '.':
      if isFloat:
        break  # second dot ends the number
      isFloat = true
    numStr.add(l.input[l.pos])
    discard l.advance()
  if isFloat:
    if numStr.endsWith("."):
      numStr.add("0")
    Token(kind: tkFloatLit, value: numStr, line: startLine, col: startCol)
  else:
    Token(kind: tkIntLit, value: numStr, line: startLine, col: startCol)

proc readIdent(l: var Lexer, startLine, startCol: int): Token =
  var ident = ""
  while l.pos < l.input.len:
    let r = l.peekRune()
    if isIdentPartRune(r):
      var run: Rune
      fastRuneAt(l.input, l.pos, run, true)
      ident.add($run)
      inc l.col
      discard l.advanceRune()
    else:
      break
  let lowerIdent = ident.toLower()
  if lowerIdent in keywords:
    Token(kind: keywords[lowerIdent], value: ident, line: startLine, col: startCol)
  else:
    Token(kind: tkIdent, value: ident, line: startLine, col: startCol)

proc nextToken*(l: var Lexer): Token =
  l.skipWhitespace()
  if l.pos >= l.input.len:
    return Token(kind: tkEof, line: l.line, col: l.col)

  let startLine = l.line
  let startCol = l.col
  let ch = l.peek()

  case ch
  of '/':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '/':
      l.skipLineComment()
      return l.nextToken()
    elif l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*':
      l.skipBlockComment()
      return l.nextToken()
    else:
      discard l.advance()
      return Token(kind: tkSlash, value: "/", line: startLine, col: startCol)
  of '+':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '+':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkConcat, value: "++", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkPlus, value: "+", line: startLine, col: startCol)
  of '-':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>':
      discard l.advance()
      if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>':
        discard l.advance()
        discard l.advance()
        return Token(kind: tkArrowRR, value: "->>", line: startLine, col: startCol)
      discard l.advance()
      return Token(kind: tkArrowR, value: "->", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkMinus, value: "-", line: startLine, col: startCol)
  of '*':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkPower, value: "**", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkStar, value: "*", line: startLine, col: startCol)
  of '%':
    discard l.advance()
    return Token(kind: tkPercent, value: "%", line: startLine, col: startCol)
  of '=':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkArrow, value: "=>", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkEq, value: "==", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkEq, value: "=", line: startLine, col: startCol)
  of ':':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkAssign, value: ":=", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == ':':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkDoubleColon, value: "::", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkColon, value: ":", line: startLine, col: startCol)
  of '!':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkNotEq, value: "!=", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkInvalid, value: "!", line: startLine, col: startCol)
  of '<':
    if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '-' and l.input[l.pos + 2] == '>':
      discard l.advance()
      discard l.advance()
      discard l.advance()
      return Token(kind: tkDistanceOp, value: "<->", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkLtEq, value: "<=", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkNotEq, value: "<>", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '@':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkJsonContainedBy, value: "<@", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkLt, value: "<", line: startLine, col: startCol)
  of '>':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkGtEq, value: ">=", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkGt, value: ">", line: startLine, col: startCol)
  of '?':
    if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '|':
      discard l.advance()
      discard l.advance()
      discard l.advance()
      return Token(kind: tkJsonHasAny, value: "?|", line: startLine, col: startCol)
    if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '&':
      discard l.advance()
      discard l.advance()
      discard l.advance()
      return Token(kind: tkJsonHasAll, value: "?&", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '?':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkCoalesce, value: "??", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkPlaceholder, value: "?", line: startLine, col: startCol)
  of '@':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '@':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkFtsMatch, value: "@@", line: startLine, col: startCol)
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkJsonContains, value: "@>", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkInvalid, value: "@", line: startLine, col: startCol)
  of '.':
    if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '<':
      discard l.advance()
      discard l.advance()
      return Token(kind: tkInvalid, value: ".<", line: startLine, col: startCol)
    discard l.advance()
    return Token(kind: tkDot, value: ".", line: startLine, col: startCol)
  of ',':
    discard l.advance()
    return Token(kind: tkComma, value: ",", line: startLine, col: startCol)
  of ';':
    discard l.advance()
    return Token(kind: tkSemicolon, value: ";", line: startLine, col: startCol)
  of '(':
    discard l.advance()
    return Token(kind: tkLParen, value: "(", line: startLine, col: startCol)
  of ')':
    discard l.advance()
    return Token(kind: tkRParen, value: ")", line: startLine, col: startCol)
  of '{':
    discard l.advance()
    return Token(kind: tkLBrace, value: "{", line: startLine, col: startCol)
  of '}':
    discard l.advance()
    return Token(kind: tkRbrace, value: "}", line: startLine, col: startCol)
  of '[':
    discard l.advance()
    return Token(kind: tkLBracket, value: "[", line: startLine, col: startCol)
  of ']':
    discard l.advance()
    return Token(kind: tkRBracket, value: "]", line: startLine, col: startCol)
  of '&':
    discard l.advance()
    return Token(kind: tkAmp, value: "&", line: startLine, col: startCol)
  of '|':
    discard l.advance()
    return Token(kind: tkPipe, value: "|", line: startLine, col: startCol)
  of '~':
    discard l.advance()
    return Token(kind: tkTilde, value: "~", line: startLine, col: startCol)
  of '\'', '"':
    discard l.advance()
    let s = l.readString(ch)
    return Token(kind: tkStringLit, value: s, line: startLine, col: startCol)
  of '#':
    l.skipLineComment()
    return l.nextToken()
  of '`':
    return l.readBacktickIdent(startLine, startCol)
  else:
    if ch in Digits:
      return l.readNumber(startLine, startCol)
    elif ch in IdentStartChars or isIdentStartRune(l.peekRune()):
      return l.readIdent(startLine, startCol)
    else:
      discard l.advance()
      return Token(kind: tkInvalid, value: $ch, line: startLine, col: startCol)

proc tokenize*(input: string): seq[Token] =
  var lexer = newLexer(input)
  result = @[]
  while true:
    let tok = lexer.nextToken()
    result.add(tok)
    if tok.kind == tkEof:
      break