feat: parser + AST (Phase 1)

src/ast.nim

@@ -0,0 +1,388 @@
+import source_location, token
+
+type
+  CallingConvention* = enum
+    ccDefault
+    ccWin64
+
+  IntrinsicKind* = enum
+    ikLine
+    ikColumn
+    ikFile
+    ikFunction
+    ikDate
+    ikTime
+    ikModule
+
+  UseKind* = enum
+    ukSingle
+    ukGlob
+    ukMulti
+
+  # ---------------------------------------------------------------------------
+  # Type expressions
+  # ---------------------------------------------------------------------------
+  TypeExprKind* = enum
+    tekNamed
+    tekPath
+    tekSlice
+    tekPointer
+    tekTuple
+    tekSelf
+
+  TypeExpr* = ref object
+    loc*: SourceLocation
+    case kind*: TypeExprKind
+    of tekNamed:
+      typeName*: string
+      typeArgs*: seq[TypeExpr]
+    of tekPath:
+      pathSegments*: seq[string]
+    of tekSlice:
+      sliceElement*: TypeExpr
+      sliceSize*: Expr          ## nil for unsized slices T[]
+    of tekPointer:
+      pointerPointee*: TypeExpr
+    of tekTuple:
+      tupleElements*: seq[TypeExpr]
+    of tekSelf:
+      discard
+
+  # ---------------------------------------------------------------------------
+  # Patterns
+  # ---------------------------------------------------------------------------
+  PatternKind* = enum
+    pkWildcard
+    pkLiteral
+    pkIdent
+    pkRange
+    pkEnum
+    pkStruct
+    pkTuple
+    pkGuarded
+
+  Pattern* = ref object
+    loc*: SourceLocation
+    case kind*: PatternKind
+    of pkWildcard:
+      discard
+    of pkLiteral:
+      patLit*: Token
+    of pkIdent:
+      patIdent*: string
+    of pkRange:
+      patRangeLo*: Pattern
+      patRangeHi*: Pattern
+      patRangeInclusive*: bool
+    of pkEnum:
+      patEnumPath*: seq[string]
+      patEnumArgs*: seq[Pattern]
+      patEnumNamed*: seq[tuple[name: string, pattern: Pattern]]
+    of pkStruct:
+      patStructName*: string
+      patStructFields*: seq[tuple[name: string, pattern: Pattern]]
+    of pkTuple:
+      patTupleElements*: seq[Pattern]
+    of pkGuarded:
+      patGuardedInner*: Pattern
+      patGuardedExpr*: Expr
+
+  # ---------------------------------------------------------------------------
+  # Expressions
+  # ---------------------------------------------------------------------------
+  ExprKind* = enum
+    ekLiteral
+    ekIdent
+    ekSelf
+    ekPath
+    ekSizeOf
+    ekIntrinsic
+    ekUnary
+    ekPostfix
+    ekBinary
+    ekAssign
+    ekTernary
+    ekRange
+    ekCall
+    ekIndex
+    ekField
+    ekStructInit
+    ekSlice
+    ekSpread
+    ekTuple
+    ekCast
+    ekIs
+    ekBlock
+    ekMatch
+
+  MatchArm* = object
+    loc*: SourceLocation
+    pattern*: Pattern
+    body*: Expr
+
+  Expr* = ref object
+    loc*: SourceLocation
+    case kind*: ExprKind
+    of ekLiteral:
+      exprLit*: Token
+    of ekIdent:
+      exprIdent*: string
+    of ekSelf:
+      discard
+    of ekPath:
+      exprPath*: seq[string]
+    of ekSizeOf:
+      exprSizeOfType*: TypeExpr
+    of ekIntrinsic:
+      exprIntrinsic*: IntrinsicKind
+    of ekUnary:
+      exprUnaryOp*: TokenKind
+      exprUnaryOperand*: Expr
+    of ekPostfix:
+      exprPostfixOp*: TokenKind
+      exprPostfixOperand*: Expr
+    of ekBinary:
+      exprBinaryOp*: TokenKind
+      exprBinaryLeft*: Expr
+      exprBinaryRight*: Expr
+    of ekAssign:
+      exprAssignOp*: TokenKind
+      exprAssignTarget*: Expr
+      exprAssignValue*: Expr
+    of ekTernary:
+      exprTernaryCond*: Expr
+      exprTernaryThen*: Expr
+      exprTernaryElse*: Expr
+    of ekRange:
+      exprRangeLo*: Expr
+      exprRangeHi*: Expr
+      exprRangeInclusive*: bool
+    of ekCall:
+      exprCallCallee*: Expr
+      exprCallArgs*: seq[Expr]
+    of ekIndex:
+      exprIndexObj*: Expr
+      exprIndexIdx*: Expr
+    of ekField:
+      exprFieldObj*: Expr
+      exprFieldName*: string
+    of ekStructInit:
+      exprStructInitName*: string
+      exprStructInitTypeArgs*: seq[TypeExpr]
+      exprStructInitFields*: seq[tuple[name: string, value: Expr]]
+    of ekSlice:
+      exprSliceElements*: seq[Expr]
+    of ekSpread:
+      exprSpreadOperand*: Expr
+    of ekTuple:
+      exprTupleElements*: seq[Expr]
+    of ekCast:
+      exprCastOperand*: Expr
+      exprCastType*: TypeExpr
+    of ekIs:
+      exprIsOperand*: Expr
+      exprIsType*: TypeExpr
+    of ekBlock:
+      exprBlock*: Block
+    of ekMatch:
+      exprMatchSubject*: Expr
+      exprMatchArms*: seq[MatchArm]
+
+  # ---------------------------------------------------------------------------
+  # Statements
+  # ---------------------------------------------------------------------------
+  StmtKind* = enum
+    skExpr
+    skLet
+    skIf
+    skWhile
+    skDoWhile
+    skLoop
+    skFor
+    skMatch
+    skReturn
+    skBreak
+    skContinue
+    skDecl
+
+  ElseIf* = object
+    loc*: SourceLocation
+    cond*: Expr
+    blk*: Block
+
+  Block* = ref object
+    loc*: SourceLocation
+    stmts*: seq[Stmt]
+
+  Stmt* = ref object
+    loc*: SourceLocation
+    case kind*: StmtKind
+    of skExpr:
+      stmtExpr*: Expr
+    of skLet:
+      stmtLetMut*: bool
+      stmtLetName*: string
+      stmtLetPattern*: Pattern
+      stmtLetType*: TypeExpr      ## nil if inferred
+      stmtLetInit*: Expr
+    of skIf:
+      stmtIfCond*: Expr
+      stmtIfThen*: Block
+      stmtIfElseIfs*: seq[ElseIf]
+      stmtIfElse*: Block          ## nil if no else
+    of skWhile:
+      stmtWhileLabel*: string
+      stmtWhileCond*: Expr
+      stmtWhileBody*: Block
+    of skDoWhile:
+      stmtDoWhileLabel*: string
+      stmtDoWhileBody*: Block
+      stmtDoWhileCond*: Expr
+    of skLoop:
+      stmtLoopLabel*: string
+      stmtLoopBody*: Block
+    of skFor:
+      stmtForLabel*: string
+      stmtForVar*: string
+      stmtForIter*: Expr
+      stmtForBody*: Block
+    of skMatch:
+      stmtMatchSubject*: Expr
+      stmtMatchArms*: seq[MatchArm]
+    of skReturn:
+      stmtReturnValue*: Expr      ## nil for bare return
+    of skBreak:
+      stmtBreakLabel*: string
+    of skContinue:
+      stmtContinueLabel*: string
+    of skDecl:
+      stmtDecl*: Decl
+
+  # ---------------------------------------------------------------------------
+  # Declarations
+  # ---------------------------------------------------------------------------
+  DeclKind* = enum
+    dkFunc
+    dkStruct
+    dkEnum
+    dkUnion
+    dkInterface
+    dkImpl
+    dkModule
+    dkUse
+    dkConst
+    dkTypeAlias
+    dkExternFunc
+    dkExternVar
+    dkExternBlock
+
+  Param* = object
+    loc*: SourceLocation
+    name*: string
+    ptype*: TypeExpr
+    isVariadic*: bool
+    defaultValue*: Expr
+
+  StructField* = object
+    loc*: SourceLocation
+    isPublic*: bool
+    name*: string
+    ftype*: TypeExpr
+
+  EnumVariant* = object
+    loc*: SourceLocation
+    name*: string
+    fields*: seq[TypeExpr]
+    namedFields*: seq[tuple[name: string, ftype: TypeExpr]]
+    discriminant*: string
+
+  UnionField* = object
+    loc*: SourceLocation
+    name*: string
+    ftype*: TypeExpr
+
+  Decl* = ref object
+    loc*: SourceLocation
+    isPublic*: bool
+    case kind*: DeclKind
+    of dkFunc:
+      declFuncAsm*: bool
+      declFuncCallConv*: CallingConvention
+      declFuncName*: string
+      declFuncTypeParams*: seq[string]
+      declFuncParams*: seq[Param]
+      declFuncReturnType*: TypeExpr  ## nil if void/inferred
+      declFuncBody*: Block           ## nil for signature-only
+    of dkStruct:
+      declStructName*: string
+      declStructTypeParams*: seq[string]
+      declStructFields*: seq[StructField]
+    of dkEnum:
+      declEnumName*: string
+      declEnumBaseType*: TypeExpr
+      declEnumVariants*: seq[EnumVariant]
+    of dkUnion:
+      declUnionName*: string
+      declUnionFields*: seq[UnionField]
+    of dkInterface:
+      declInterfaceName*: string
+      declInterfaceMethods*: seq[Decl]  ## FuncDecl signatures only
+    of dkImpl:
+      declImplTypeName*: string
+      declImplInterface*: string        ## empty if not for interface
+      declImplMethods*: seq[Decl]
+    of dkModule:
+      declModuleName*: string
+      declModuleItems*: seq[Decl]
+    of dkUse:
+      declUsePath*: seq[string]
+      declUseKind*: UseKind
+      declUseNames*: seq[string]       ## for multi-import
+      declUseTargetOs*: string
+    of dkConst:
+      declConstName*: string
+      declConstType*: TypeExpr
+      declConstValue*: Expr
+    of dkTypeAlias:
+      declAliasName*: string
+      declAliasType*: TypeExpr
+    of dkExternFunc:
+      declExtFuncName*: string
+      declExtFuncDll*: string
+      declExtFuncCallConv*: CallingConvention
+      declExtFuncParams*: seq[Param]
+      declExtFuncVariadic*: bool
+      declExtFuncReturnType*: TypeExpr
+    of dkExternVar:
+      declExtVarName*: string
+      declExtVarType*: TypeExpr
+    of dkExternBlock:
+      declExtBlockDll*: string
+      declExtBlockCallConv*: CallingConvention
+      declExtBlockItems*: seq[Decl]
+
+  # ---------------------------------------------------------------------------
+  # Module (AST root)
+  # ---------------------------------------------------------------------------
+  Module* = ref object
+    name*: string
+    items*: seq[Decl]
+
+# Convenience constructors
+proc newModule*(name: string): Module =
+  result = Module(name: name)
+
+proc newBlock*(loc: SourceLocation): Block =
+  result = Block(loc: loc)
+
+proc newLiteralExpr*(tok: Token): Expr =
+  result = Expr(kind: ekLiteral, loc: tok.loc, exprLit: tok)
+
+proc newIdentExpr*(name: string, loc: SourceLocation): Expr =
+  result = Expr(kind: ekIdent, loc: loc, exprIdent: name)
+
+proc newBinaryExpr*(op: TokenKind, left, right: Expr, loc: SourceLocation): Expr =
+  result = Expr(kind: ekBinary, loc: loc, exprBinaryOp: op, exprBinaryLeft: left, exprBinaryRight: right)
+
+proc newUnaryExpr*(op: TokenKind, operand: Expr, loc: SourceLocation): Expr =
+  result = Expr(kind: ekUnary, loc: loc, exprUnaryOp: op, exprUnaryOperand: operand)

src/cli.nim

@@ -1,5 +1,5 @@
 import std/[os, strutils, terminal, strformat]
-import lexer, manifest
+import lexer, parser, manifest
 
 type
   ColorMode* = enum
@@ -158,14 +158,20 @@ proc cmdCheck*(args: seq[string], opts: GlobalOptions): int =
   for kind, path in walkDir(srcDir):
     if kind == pcFile and path.endsWith(".bux"):
       let source = readFile(path)
-      let res = tokenize(source, path)
+      let lexRes = tokenize(source, path)
-      if res.hasErrors:
+      if lexRes.hasErrors:
         printError(&"lex errors in {path}", useColor)
-        for d in res.diagnostics:
+        for d in lexRes.diagnostics:
           echo $d
         return 1
+      let parseRes = parse(lexRes.tokens, path)
+      if parseRes.diagnostics.len > 0:
+        printError(&"parse errors in {path}", useColor)
+        for d in parseRes.diagnostics:
+          echo &"error: {d.message} at {d.loc}"
+        return 1
       if opts.verbose:
-        printInfo(&"lexed {path} ({res.tokens.len} tokens)", useColor)
+        printInfo(&"parsed {path} ({lexRes.tokens.len} tokens, {parseRes.module.items.len} top-level declarations)", useColor)
       if splitFile(path).name == "Main":
         foundMain = true
 
@@ -174,7 +180,7 @@ proc cmdCheck*(args: seq[string], opts: GlobalOptions): int =
     return 1
 
   if not opts.quiet:
-    printInfo("check passed (lexer only)", useColor)
+    printInfo("check passed", useColor)
   return 0
 
 proc cmdBuild*(args: seq[string], opts: GlobalOptions): int =

src/lexer.nim

@@ -142,7 +142,9 @@ proc scanIdent(lex: var Lexer, startLoc: SourceLocation): Token =
   while not lex.isAtEnd() and isIdentChar(lex.peek()):
     discard lex.advance()
   let text = lex.source[startPos ..< lex.pos]
-  let kind = keywordKind(text)
+  var kind = keywordKind(text)
+  if text == "_":
+    kind = tkUnderscore
   result = Token(kind: kind, text: text, loc: startLoc)
 
 # ---------------------------------------------------------------------------

src/token.nim

@@ -11,6 +11,7 @@ type
 
     ##Identifiers
     tkIdent           # foo  Bar  _x
+    tkUnderscore      # _
 
     ##Control flow keywords
     tkIf              # if
@@ -47,6 +48,7 @@ type
     tkNull            # null
     tkSelf            # self
     tkSuper           # super
+    tkSizeOf          # sizeof
 
     ##Punctuation
     tkLParen          # (
@@ -191,6 +193,7 @@ proc keywordKind*(text: string): TokenKind =
   of "null": tkNull
   of "self": tkSelf
   of "super": tkSuper
+  of "sizeof": tkSizeOf
   of "true", "false": tkBoolLiteral
   else: tkIdent
 
@@ -202,6 +205,8 @@ proc tokenKindName*(kind: TokenKind): string =
   of tkCharLiteral: "char literal"
   of tkBoolLiteral: "boolean literal"
   of tkIdent: "identifier"
+  of tkUnderscore: "'_'"
+  of tkSizeOf: "'sizeof'"
   of tkIf: "'if'"
   of tkElse: "'else'"
   of tkWhile: "'while'"

tests/parser_test.nim

@@ -0,0 +1,122 @@
+import std/[unittest, os, strutils]
+import ../src/[lexer, parser, ast, token]
+
+proc parseSource(source: string): ParseResult =
+  let lexRes = tokenize(source, "<test>")
+  check not lexRes.hasErrors
+  result = parse(lexRes.tokens, "<test>")
+
+suite "Parser":
+  test "empty module":
+    let res = parseSource("")
+    check res.diagnostics.len == 0
+    check res.module.items.len == 0
+
+  test "simple function":
+    let res = parseSource("func Main() -> int { return 0; }")
+    check res.diagnostics.len == 0
+    check res.module.items.len == 1
+    check res.module.items[0].kind == dkFunc
+    check res.module.items[0].declFuncName == "Main"
+
+  test "function with parameters":
+    let res = parseSource("func Add(a: int32, b: int32) -> int32 { return a + b; }")
+    check res.diagnostics.len == 0
+    check res.module.items[0].declFuncParams.len == 2
+    check res.module.items[0].declFuncParams[0].name == "a"
+    check res.module.items[0].declFuncParams[1].name == "b"
+
+  test "struct declaration":
+    let res = parseSource("struct Point { x: float64; y: float64; }")
+    check res.diagnostics.len == 0
+    check res.module.items[0].kind == dkStruct
+    check res.module.items[0].declStructName == "Point"
+    check res.module.items[0].declStructFields.len == 2
+
+  test "enum declaration":
+    let res = parseSource("enum Color { Red, Green, Blue }")
+    check res.diagnostics.len == 0
+    check res.module.items[0].kind == dkEnum
+    check res.module.items[0].declEnumName == "Color"
+    check res.module.items[0].declEnumVariants.len == 3
+
+  test "import declaration":
+    let res = parseSource("import Std::Io::PrintLine;")
+    check res.diagnostics.len == 0
+    check res.module.items[0].kind == dkUse
+    check res.module.items[0].declUsePath == @["Std", "Io", "PrintLine"]
+
+  test "const declaration":
+    let res = parseSource("const PI: float64 = 3.14;")
+    check res.diagnostics.len == 0
+    check res.module.items[0].kind == dkConst
+    check res.module.items[0].declConstName == "PI"
+
+  test "type alias":
+    let res = parseSource("type MyInt = int32;")
+    check res.diagnostics.len == 0
+    check res.module.items[0].kind == dkTypeAlias
+    check res.module.items[0].declAliasName == "MyInt"
+
+  test "let statement in function":
+    let res = parseSource("func Main() -> int { let x: int32 = 42; return x; }")
+    check res.diagnostics.len == 0
+    check res.module.items[0].declFuncBody.stmts.len == 2
+    check res.module.items[0].declFuncBody.stmts[0].kind == skLet
+    check res.module.items[0].declFuncBody.stmts[0].stmtLetName == "x"
+
+  test "if statement":
+    let res = parseSource("func Main() -> int { if true { return 1; } else { return 0; } }")
+    check res.diagnostics.len == 0
+    let body = res.module.items[0].declFuncBody
+    check body.stmts[0].kind == skIf
+    check body.stmts[0].stmtIfThen.stmts.len == 1
+    check body.stmts[0].stmtIfElse.stmts.len == 1
+
+  test "while loop":
+    let res = parseSource("func Main() -> int { while true { break; } return 0; }")
+    check res.diagnostics.len == 0
+    let body = res.module.items[0].declFuncBody
+    check body.stmts[0].kind == skWhile
+    check body.stmts[0].stmtWhileBody.stmts[0].kind == skBreak
+
+  test "for loop":
+    let res = parseSource("func Main() -> int { for i in items { continue; } return 0; }")
+    check res.diagnostics.len == 0
+    let body = res.module.items[0].declFuncBody
+    check body.stmts[0].kind == skFor
+    check body.stmts[0].stmtForVar == "i"
+
+  test "match expression":
+    let res = parseSource("func Main() -> int { let x = match 1 { 1 => 10, _ => 20 }; return x; }")
+    check res.diagnostics.len == 0
+    let body = res.module.items[0].declFuncBody
+    check body.stmts[0].kind == skLet
+    check body.stmts[0].stmtLetInit.kind == ekMatch
+
+  test "binary expression":
+    let res = parseSource("func Main() -> int { return 1 + 2 * 3; }")
+    check res.diagnostics.len == 0
+    let ret = res.module.items[0].declFuncBody.stmts[0]
+    check ret.kind == skReturn
+    check ret.stmtReturnValue.kind == ekBinary
+    # 1 + (2 * 3)
+    check ret.stmtReturnValue.exprBinaryOp == tkPlus
+
+  test "call expression":
+    let res = parseSource("func Main() -> int { return Add(10, 20); }")
+    check res.diagnostics.len == 0
+    let ret = res.module.items[0].declFuncBody.stmts[0]
+    check ret.stmtReturnValue.kind == ekCall
+    check ret.stmtReturnValue.exprCallCallee.kind == ekIdent
+
+  test "full sample file":
+    let source = readFile("tests/testdata/sample.bux")
+    let lexRes = tokenize(source, "sample.bux")
+    check not lexRes.hasErrors
+    let parseRes = parse(lexRes.tokens, "sample.bux")
+    check parseRes.diagnostics.len == 0
+    check parseRes.module.items.len == 3
+    check parseRes.module.items[0].kind == dkUse
+    check parseRes.module.items[1].kind == dkFunc
+    check parseRes.module.items[2].kind == dkFunc