## BaraQL Lexer — tokenization import std/tables import std/strutils import std/unicode type TokenKind* = enum # Literals tkIntLit tkFloatLit tkStringLit tkBoolLit # Identifiers tkIdent # Keywords tkSelect tkInsert tkUpdate tkDelete tkFrom tkWhere tkAnd tkOr tkNot tkIn tkIs tkAs tkOn tkJoin tkLeft tkRight tkInner tkOuter tkFull tkCross tkLateral tkOrder tkBy tkAsc tkDesc tkGroup tkHaving tkLimit tkOffset tkSet tkInto tkValues tkCreate tkDrop tkAlter tkTable tkIndex tkType tkLink tkProperty tkRequired tkMulti tkSingle tkTrue tkFalse tkNull tkIf tkThen tkElse tkEnd tkCase tkWhen tkWith tkRecursive tkDistinct tkUnion tkIntersect tkExcept tkAll tkExists tkBetween tkLike tkILike tkFilter tkReturning tkPrimary tkKey tkForeign tkReferences tkCascade tkUnique tkCheck tkDefault tkAdd tkColumn tkRename tkBegin tkCommit tkRollback tkExplain tkView tkTrigger tkBefore tkAfter tkInstead tkOf tkMigration tkApply tkStatus tkUp tkDown tkDryRun tkUser tkPolicy tkEnable tkDisable tkRecover tkTimestamp tkFor tkUsing tkGrant tkRevoke tkCount tkSum tkAvg tkMin tkMax tkArrayAgg tkStringAgg tkGrouping tkSets tkRollup tkCube tkPivot tkUnpivot tkVertex tkEdge tkLabels tkGraphTable tkMatch tkColumns tkSrc tkDst tkMerge tkMatched tkArray tkVector tkGraph tkDocument tkArrowR # -> tkArrowRR # ->> tkFtsMatch # @@ tkJsonContains # @> tkJsonContainedBy # <@ tkJsonHasAny # ?| tkJsonHasAll # ?& tkSimilar tkNearest tkTo tkBfs tkDfs tkPath # Window functions tkOver tkPartition tkRow tkRows tkRange tkUnbounded tkPreceding tkFollowing tkCurrent tkCurrentUser tkCurrentRole tkRowNumber tkRank tkDenseRank tkLead tkLag tkFirstValue tkLastValue tkNtile # Operators tkPlus tkMinus tkStar tkSlash tkPercent tkPower tkEq tkNotEq tkLt tkLtEq tkGt tkGtEq tkAssign tkArrow tkDoubleColon tkColon tkDot tkComma tkSemicolon tkLParen tkRParen tkLBrace tkRbrace tkLBracket tkRBracket tkAmp tkPipe tkTilde tkConcat tkCoalesce tkFloorDiv tkDistanceOp # <-> tkPlaceholder # Special tkEof tkNewline tkInvalid Token* = object kind*: TokenKind value*: string line*: int col*: int Lexer* = object input: string pos: int line: int col: int const keywords*: Table[string, TokenKind] = { "select": tkSelect, "insert": tkInsert, "update": tkUpdate, "delete": tkDelete, "from": tkFrom, "where": tkWhere, "and": tkAnd, "or": tkOr, "not": tkNot, "in": tkIn, "is": tkIs, "as": tkAs, "on": tkOn, "join": tkJoin, "left": tkLeft, "right": tkRight, "inner": tkInner, "outer": tkOuter, "full": tkFull, "cross": tkCross, "lateral": tkLateral, "order": tkOrder, "by": tkBy, "asc": tkAsc, "desc": tkDesc, "group": tkGroup, "having": tkHaving, "limit": tkLimit, "offset": tkOffset, "set": tkSet, "into": tkInto, "values": tkValues, "create": tkCreate, "drop": tkDrop, "alter": tkAlter, "table": tkTable, "index": tkIndex, "type": tkType, "link": tkLink, "property": tkProperty, "required": tkRequired, "multi": tkMulti, "single": tkSingle, "true": tkTrue, "false": tkFalse, "null": tkNull, "if": tkIf, "then": tkThen, "else": tkElse, "end": tkEnd, "case": tkCase, "when": tkWhen, "with": tkWith, "recursive": tkRecursive, "distinct": tkDistinct, "union": tkUnion, "intersect": tkIntersect, "except": tkExcept, "all": tkAll, "exists": tkExists, "between": tkBetween, "like": tkLike, "ilike": tkILike, "filter": tkFilter, "returning": tkReturning, "primary": tkPrimary, "key": tkKey, "foreign": tkForeign, "references": tkReferences, "cascade": tkCascade, "unique": tkUnique, "check": tkCheck, "default": tkDefault, "add": tkAdd, "column": tkColumn, "rename": tkRename, "begin": tkBegin, "commit": tkCommit, "rollback": tkRollback, "explain": tkExplain, "view": tkView, "trigger": tkTrigger, "before": tkBefore, "after": tkAfter, "instead": tkInstead, "of": tkOf, "migration": tkMigration, "apply": tkApply, "status": tkStatus, "up": tkUp, "down": tkDown, "dryrun": tkDryRun, "user": tkUser, "policy": tkPolicy, "enable": tkEnable, "disable": tkDisable, "for": tkFor, "using": tkUsing, "recover": tkRecover, "timestamp": tkTimestamp, "grant": tkGrant, "revoke": tkRevoke, "count": tkCount, "sum": tkSum, "avg": tkAvg, "min": tkMin, "max": tkMax, "array_agg": tkArrayAgg, "string_agg": tkStringAgg, "grouping": tkGrouping, "sets": tkSets, "rollup": tkRollup, "cube": tkCube, "pivot": tkPivot, "unpivot": tkUnpivot, "vertex": tkVertex, "vertices": tkVertex, "edge": tkEdge, "edges": tkEdge, "label": tkLabels, "labels": tkLabels, "graph_table": tkGraphTable, "match": tkMatch, "columns": tkColumns, "src": tkSrc, "dst": tkDst, "merge": tkMerge, "matched": tkMatched, "array": tkArray, "vector": tkVector, "graph": tkGraph, "document": tkDocument, "similar": tkSimilar, "nearest": tkNearest, "to": tkTo, "bfs": tkBfs, "dfs": tkDfs, "path": tkPath, "over": tkOver, "partition": tkPartition, "row": tkRow, "rows": tkRows, "range": tkRange, "unbounded": tkUnbounded, "preceding": tkPreceding, "following": tkFollowing, "current": tkCurrent, "current_user": tkCurrentUser, "current_role": tkCurrentRole, "row_number": tkRowNumber, "rank": tkRank, "dense_rank": tkDenseRank, "lead": tkLead, "lag": tkLag, "first_value": tkFirstValue, "last_value": tkLastValue, "ntile": tkNtile, }.toTable proc newLexer*(input: string): Lexer = Lexer(input: input, pos: 0, line: 1, col: 1) proc peek(l: Lexer): char = if l.pos < l.input.len: return l.input[l.pos] return '\0' proc advance(l: var Lexer): char = result = l.input[l.pos] inc l.pos if result == '\n': inc l.line l.col = 1 else: inc l.col proc peekRune(l: Lexer): Rune = if l.pos < l.input.len: var p = l.pos var r: Rune fastRuneAt(l.input, p, r, true) return r return Rune(0) proc advanceRune(l: var Lexer): Rune = if l.pos < l.input.len: var r: Rune fastRuneAt(l.input, l.pos, r, true) if r == Rune('\n'): inc l.line l.col = 1 else: inc l.col return r return Rune(0) proc isIdentStartRune(r: Rune): bool = return isAlpha(r) or r == Rune('_') proc isIdentPartRune(r: Rune): bool = return isAlpha(r) or (r.int >= ord('0') and r.int <= ord('9')) or r == Rune('_') proc skipWhitespace(l: var Lexer) = while l.pos < l.input.len and l.input[l.pos] in {' ', '\t', '\r', '\n'}: discard l.advance() proc skipLineComment(l: var Lexer) = while l.pos < l.input.len and l.input[l.pos] != '\n': discard l.advance() proc skipBlockComment(l: var Lexer) = discard l.advance() # skip * discard l.advance() # skip * while l.pos < l.input.len - 1: if l.input[l.pos] == '*' and l.input[l.pos + 1] == '/': discard l.advance() discard l.advance() return discard l.advance() raise newException(ValueError, "Unclosed block comment at line " & $l.line & ", col " & $l.col) proc readString(l: var Lexer, quote: char): string = result = "" while l.pos < l.input.len and l.input[l.pos] != quote: if l.input[l.pos] == '\\': discard l.advance() if l.pos >= l.input.len: break case l.input[l.pos] of 'n': result.add('\n') of 't': result.add('\t') of 'r': result.add('\r') of '\\': result.add('\\') of '\'': result.add('\'') of '"': result.add('"') else: result.add(l.input[l.pos]) else: result.add(l.input[l.pos]) discard l.advance() if l.pos < l.input.len: discard l.advance() # skip closing quote proc readNumber(l: var Lexer, startLine, startCol: int): Token = var numStr = "" var isFloat = false while l.pos < l.input.len and (l.input[l.pos] in Digits or l.input[l.pos] == '.'): if l.input[l.pos] == '.': if isFloat: break # second dot ends the number isFloat = true numStr.add(l.input[l.pos]) discard l.advance() if isFloat: Token(kind: tkFloatLit, value: numStr, line: startLine, col: startCol) else: Token(kind: tkIntLit, value: numStr, line: startLine, col: startCol) proc readIdent(l: var Lexer, startLine, startCol: int): Token = var ident = "" while l.pos < l.input.len: let r = l.peekRune() if isIdentPartRune(r): var run: Rune fastRuneAt(l.input, l.pos, run, true) ident.add($run) inc l.col else: break let lowerIdent = ident.toLower() if lowerIdent in keywords: Token(kind: keywords[lowerIdent], value: ident, line: startLine, col: startCol) else: Token(kind: tkIdent, value: ident, line: startLine, col: startCol) proc nextToken*(l: var Lexer): Token = l.skipWhitespace() if l.pos >= l.input.len: return Token(kind: tkEof, line: l.line, col: l.col) let startLine = l.line let startCol = l.col let ch = l.peek() case ch of '/': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '/': l.skipLineComment() return l.nextToken() elif l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*': l.skipBlockComment() return l.nextToken() else: discard l.advance() return Token(kind: tkSlash, value: "/", line: startLine, col: startCol) of '+': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '+': discard l.advance() discard l.advance() return Token(kind: tkConcat, value: "++", line: startLine, col: startCol) discard l.advance() return Token(kind: tkPlus, value: "+", line: startLine, col: startCol) of '-': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': discard l.advance() if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': discard l.advance() discard l.advance() return Token(kind: tkArrowRR, value: "->>", line: startLine, col: startCol) discard l.advance() return Token(kind: tkArrowR, value: "->", line: startLine, col: startCol) discard l.advance() return Token(kind: tkMinus, value: "-", line: startLine, col: startCol) of '*': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '*': discard l.advance() discard l.advance() return Token(kind: tkPower, value: "**", line: startLine, col: startCol) discard l.advance() return Token(kind: tkStar, value: "*", line: startLine, col: startCol) of '%': discard l.advance() return Token(kind: tkPercent, value: "%", line: startLine, col: startCol) of '=': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': discard l.advance() discard l.advance() return Token(kind: tkArrow, value: "=>", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': discard l.advance() discard l.advance() return Token(kind: tkEq, value: "==", line: startLine, col: startCol) discard l.advance() return Token(kind: tkEq, value: "=", line: startLine, col: startCol) of ':': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': discard l.advance() discard l.advance() return Token(kind: tkAssign, value: ":=", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == ':': discard l.advance() discard l.advance() return Token(kind: tkDoubleColon, value: "::", line: startLine, col: startCol) discard l.advance() return Token(kind: tkColon, value: ":", line: startLine, col: startCol) of '!': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': discard l.advance() discard l.advance() return Token(kind: tkNotEq, value: "!=", line: startLine, col: startCol) discard l.advance() return Token(kind: tkInvalid, value: "!", line: startLine, col: startCol) of '<': if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '-' and l.input[l.pos + 2] == '>': discard l.advance() discard l.advance() discard l.advance() return Token(kind: tkDistanceOp, value: "<->", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': discard l.advance() discard l.advance() return Token(kind: tkLtEq, value: "<=", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': discard l.advance() discard l.advance() return Token(kind: tkNotEq, value: "<>", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '@': discard l.advance() discard l.advance() return Token(kind: tkJsonContainedBy, value: "<@", line: startLine, col: startCol) discard l.advance() return Token(kind: tkLt, value: "<", line: startLine, col: startCol) of '>': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '=': discard l.advance() discard l.advance() return Token(kind: tkGtEq, value: ">=", line: startLine, col: startCol) discard l.advance() return Token(kind: tkGt, value: ">", line: startLine, col: startCol) of '?': if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '|': discard l.advance() discard l.advance() discard l.advance() return Token(kind: tkJsonHasAny, value: "?|", line: startLine, col: startCol) if l.pos + 2 < l.input.len and l.input[l.pos + 1] == '&': discard l.advance() discard l.advance() discard l.advance() return Token(kind: tkJsonHasAll, value: "?&", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '?': discard l.advance() discard l.advance() return Token(kind: tkCoalesce, value: "??", line: startLine, col: startCol) discard l.advance() return Token(kind: tkPlaceholder, value: "?", line: startLine, col: startCol) of '@': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '@': discard l.advance() discard l.advance() return Token(kind: tkFtsMatch, value: "@@", line: startLine, col: startCol) if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '>': discard l.advance() discard l.advance() return Token(kind: tkJsonContains, value: "@>", line: startLine, col: startCol) discard l.advance() return Token(kind: tkInvalid, value: "@", line: startLine, col: startCol) of '.': if l.pos + 1 < l.input.len and l.input[l.pos + 1] == '<': discard l.advance() discard l.advance() return Token(kind: tkInvalid, value: ".<", line: startLine, col: startCol) discard l.advance() return Token(kind: tkDot, value: ".", line: startLine, col: startCol) of ',': discard l.advance() return Token(kind: tkComma, value: ",", line: startLine, col: startCol) of ';': discard l.advance() return Token(kind: tkSemicolon, value: ";", line: startLine, col: startCol) of '(': discard l.advance() return Token(kind: tkLParen, value: "(", line: startLine, col: startCol) of ')': discard l.advance() return Token(kind: tkRParen, value: ")", line: startLine, col: startCol) of '{': discard l.advance() return Token(kind: tkLBrace, value: "{", line: startLine, col: startCol) of '}': discard l.advance() return Token(kind: tkRbrace, value: "}", line: startLine, col: startCol) of '[': discard l.advance() return Token(kind: tkLBracket, value: "[", line: startLine, col: startCol) of ']': discard l.advance() return Token(kind: tkRBracket, value: "]", line: startLine, col: startCol) of '&': discard l.advance() return Token(kind: tkAmp, value: "&", line: startLine, col: startCol) of '|': discard l.advance() return Token(kind: tkPipe, value: "|", line: startLine, col: startCol) of '~': discard l.advance() return Token(kind: tkTilde, value: "~", line: startLine, col: startCol) of '\'', '"': discard l.advance() let s = l.readString(ch) return Token(kind: tkStringLit, value: s, line: startLine, col: startCol) of '#': l.skipLineComment() return l.nextToken() else: if ch in Digits: return l.readNumber(startLine, startCol) elif ch in IdentStartChars or isIdentStartRune(l.peekRune()): return l.readIdent(startLine, startCol) else: discard l.advance() return Token(kind: tkInvalid, value: $ch, line: startLine, col: startCol) proc tokenize*(input: string): seq[Token] = var lexer = newLexer(input) result = @[] while true: let tok = lexer.nextToken() result.add(tok) if tok.kind == tkEof: break