// lexer.bux — Lexer state machine (ported from lexer.nim)
// Tokenizes Bux source into a stream of tokens.
module Lexer {

    extern func bux_strlen(s: String) -> uint;

    // ---------------------------------------------------------------------------
    // Character helpers (wrap C ctype)
    // ---------------------------------------------------------------------------

    func Lex_IsDigit(c: uint32) -> bool {
        return c >= 48 && c <= 57; // '0'..'9'
    }

    func Lex_IsHexDigit(c: uint32) -> bool {
        if c >= 48 && c <= 57 { return true; }  // 0-9
        if c >= 65 && c <= 70 { return true; }  // A-F
        if c >= 97 && c <= 102 { return true; } // a-f
        return false;
    }

    func Lex_IsBinDigit(c: uint32) -> bool {
        return c == 48 || c == 49; // '0' or '1'
    }

    func Lex_IsOctDigit(c: uint32) -> bool {
        return c >= 48 && c <= 55; // '0'..'7'
    }

    func Lex_IsIdentStart(c: uint32) -> bool {
        if c >= 97 && c <= 122 { return true; }  // a-z
        if c >= 65 && c <= 90 { return true; }   // A-Z
        return c == 95; // '_'
    }

    func Lex_IsIdentChar(c: uint32) -> bool {
        if Lex_IsIdentStart(c) { return true; }
        return Lex_IsDigit(c);
    }

    // ---------------------------------------------------------------------------
    // Lexer state
    // ---------------------------------------------------------------------------

    const maxTokens: int = 32768;
    const maxDiags: int = 128;

    struct LexerDiag {
        line: uint32;
        column: uint32;
        message: String;
    }

    struct Lexer {
        source: String;
        sourceLen: int;
        pos: int;
        line: uint32;
        column: uint32;
        startLine: uint32;
        startColumn: uint32;
        startPos: int;
        tokenCount: int;
        tokens: *LexToken;
        diagCount: int;
        diags: *LexerDiag;
    }

    struct LexToken {
        kind: int;
        text: String;
        line: uint32;
        column: uint32;
    }

    // ---------------------------------------------------------------------------
    // Core primitives
    // ---------------------------------------------------------------------------

    func lexIsAtEnd(lex: *Lexer) -> bool {
        return lex.pos >= lex.sourceLen;
    }

    func lexPeek(lex: *Lexer, ahead: int) -> uint32 {
        let i: int = lex.pos + ahead;
        if i < lex.sourceLen {
            return (lex.source[i] as uint32) & 255;
        }
        return 0;
    }

    func lexAdvance(lex: *Lexer) -> uint32 {
        let c: uint32 = lexPeek(lex, 0);
        if !lexIsAtEnd(lex) {
            lex.pos = lex.pos + 1;
            if c == 10 { // '\n'
                lex.line = lex.line + 1;
                lex.column = 1;
            } else {
                lex.column = lex.column + 1;
            }
        }
        return c;
    }

    func lexMatch(lex: *Lexer, expected: uint32) -> bool {
        if lexIsAtEnd(lex) { return false; }
        if lexPeek(lex, 0) != expected { return false; }
        discard lexAdvance(lex);
        return true;
    }

    func lexMatchStr(lex: *Lexer, s: String) -> bool {
        let len: uint = bux_strlen(s);
        var i: int = 0;
        while i < (len as int) {
            if lexPeek(lex, i) != (s[i] as uint32) {
                return false;
            }
            i = i + 1;
        }
        i = 0;
        while i < (len as int) {
            discard lexAdvance(lex);
            i = i + 1;
        }
        return true;
    }

    // ---------------------------------------------------------------------------
    // Token emission
    // ---------------------------------------------------------------------------

    func lexMarkStart(lex: *Lexer) {
        lex.startLine = lex.line;
        lex.startColumn = lex.column;
        lex.startPos = lex.pos;
    }

    func lexMakeToken(lex: *Lexer, kind: int) -> LexToken {
        var text: String = "";
        let endPos: int = lex.pos;
        let startPos: int = lex.startPos;
        if endPos > startPos {
            let len: int = endPos - startPos;
            let buf: *char8 = bux_alloc((len + 1) as uint) as *char8;
            var i: int = 0;
            while i < len {
                buf[i] = lex.source[startPos + i] as char8;
                i = i + 1;
            }
            buf[len] = 0 as char8;
            text = buf;
        } else {
            text = "";
        }
        return LexToken {
            kind: kind, text: text,
            line: lex.startLine, column: lex.startColumn
        };
    }

    func lexEmitToken(lex: *Lexer, kind: int) {
        let tok: LexToken = lexMakeToken(lex, kind);
        if lex.tokenCount < maxTokens {
            lex.tokens[lex.tokenCount] = tok;
            lex.tokenCount = lex.tokenCount + 1;
        }
    }

    func lexSetLastTokenText(lex: *Lexer, text: String) {
        if lex.tokenCount > 0 {
            lex.tokens[lex.tokenCount - 1].text = text;
        }
    }

    func lexEmitDiag(lex: *Lexer, msg: String) {
        if lex.diagCount < maxDiags {
            lex.diags[lex.diagCount] = LexerDiag {
                line: lex.line, column: lex.column, message: msg
            };
            lex.diagCount = lex.diagCount + 1;
        }
    }

    // ---------------------------------------------------------------------------
    // Whitespace / comments
    // ---------------------------------------------------------------------------

    func lexSkipLineComment(lex: *Lexer) {
        while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 10 { // '\n'
            discard lexAdvance(lex);
        }
    }

    func lexSkipBlockComment(lex: *Lexer) {
        var depth: int = 1;
        while !lexIsAtEnd(lex) && depth > 0 {
            if lexPeek(lex, 0) == 47 && lexPeek(lex, 1) == 42 { // /*
                discard lexAdvance(lex);
                discard lexAdvance(lex);
                depth = depth + 1;
            } else if lexPeek(lex, 0) == 42 && lexPeek(lex, 1) == 47 { // */
                discard lexAdvance(lex);
                discard lexAdvance(lex);
                depth = depth - 1;
            } else {
                discard lexAdvance(lex);
            }
        }
        if depth > 0 {
            lexEmitDiag(lex, "unterminated block comment");
        }
    }

    func lexSkipWhitespace(lex: *Lexer) {
        while !lexIsAtEnd(lex) {
            let c: uint32 = lexPeek(lex, 0);
            if c == 32 || c == 9 || c == 13 {
                discard lexAdvance(lex);
            } else {
                if c == 47 && lexPeek(lex, 1) == 47 {
                    lexSkipLineComment(lex);
                } else {
                    if c == 47 && lexPeek(lex, 1) == 42 {
                        discard lexAdvance(lex);
                        discard lexAdvance(lex);
                        lexSkipBlockComment(lex);
                    } else {
                        break;
                    }
                }
            }
        }
    }

    // ---------------------------------------------------------------------------
    // Identifiers
    // ---------------------------------------------------------------------------

    func lexKeywordKind(text: String) -> int {
        if String_Eq(text, "true") { return tkBoolLiteral; }
        if String_Eq(text, "false") { return tkBoolLiteral; }
        if String_Eq(text, "func") { return tkFunc; }
        if String_Eq(text, "let") { return tkLet; }
        if String_Eq(text, "var") { return tkVar; }
        if String_Eq(text, "const") { return tkConst; }
        if String_Eq(text, "type") { return tkType; }
        if String_Eq(text, "struct") { return tkStruct; }
        if String_Eq(text, "enum") { return tkEnum; }
        if String_Eq(text, "union") { return tkUnion; }
        if String_Eq(text, "interface") { return tkInterface; }
        if String_Eq(text, "extend") { return tkExtend; }
        if String_Eq(text, "module") { return tkModule; }
        if String_Eq(text, "import") { return tkImport; }
        if String_Eq(text, "pub") { return tkPub; }
        if String_Eq(text, "extern") { return tkExtern; }
        if String_Eq(text, "if") { return tkIf; }
        if String_Eq(text, "else") { return tkElse; }
        if String_Eq(text, "while") { return tkWhile; }
        if String_Eq(text, "do") { return tkDo; }
        if String_Eq(text, "loop") { return tkLoop; }
        if String_Eq(text, "for") { return tkFor; }
        if String_Eq(text, "in") { return tkIn; }
        if String_Eq(text, "break") { return tkBreak; }
        if String_Eq(text, "continue") { return tkContinue; }
        if String_Eq(text, "return") { return tkReturn; }
        if String_Eq(text, "match") { return tkMatch; }
        if String_Eq(text, "as") { return tkAs; }
        if String_Eq(text, "is") { return tkIs; }
        if String_Eq(text, "null") { return tkNull; }
        if String_Eq(text, "self") { return tkSelf; }
        if String_Eq(text, "super") { return tkSuper; }
        if String_Eq(text, "sizeof") { return tkSizeOf; }
        if String_Eq(text, "discard") { return tkDiscard; }
        if String_Eq(text, "defer") { return tkDefer; }
        if String_Eq(text, "switch") { return tkSwitch; }
        if String_Eq(text, "case") { return tkCase; }
        if String_Eq(text, "default") { return tkDefault; }
        if String_Eq(text, "unsafe") { return tkUnsafe; }
        if String_Eq(text, "async") { return tkAsync; }
        if String_Eq(text, "await") { return tkAwait; }
        if String_Eq(text, "spawn") { return tkSpawn; }
        return tkIdent;
    }

    func lexScanIdent(lex: *Lexer) {
        lexMarkStart(lex);
        while !lexIsAtEnd(lex) && Lex_IsIdentChar(lexPeek(lex, 0)) {
            discard lexAdvance(lex);
        }
        let tok: LexToken = lexMakeToken(lex, tkIdent);
        let kind: int = lexKeywordKind(tok.text);
        tok.kind = kind;
        if lex.tokenCount < maxTokens {
            lex.tokens[lex.tokenCount] = tok;
            lex.tokenCount = lex.tokenCount + 1;
        }
    }

    // ---------------------------------------------------------------------------
    // Numbers
    // ---------------------------------------------------------------------------

    func lexScanDigits(lex: *Lexer) {
        while !lexIsAtEnd(lex) && Lex_IsDigit(lexPeek(lex, 0)) {
            discard lexAdvance(lex);
        }
    }

    func lexScanHexDigits(lex: *Lexer) {
        while !lexIsAtEnd(lex) && Lex_IsHexDigit(lexPeek(lex, 0)) {
            discard lexAdvance(lex);
        }
    }

    func lexScanNumber(lex: *Lexer) {
        lexMarkStart(lex);
        var isFloat: bool = false;

        if lexPeek(lex, 0) == 48 { // '0'
            let p1: uint32 = lexPeek(lex, 1);
            if p1 == 120 || p1 == 88 || p1 == 98 || p1 == 66 || p1 == 111 || p1 == 79 { // x X b B o O
                discard lexAdvance(lex); // 0
                discard lexAdvance(lex); // prefix
                if p1 == 120 || p1 == 88 { lexScanHexDigits(lex); }
                else if p1 == 98 || p1 == 66 { 
                    while !lexIsAtEnd(lex) && Lex_IsBinDigit(lexPeek(lex, 0)) {
                        discard lexAdvance(lex);
                    }
                }
                else {
                    while !lexIsAtEnd(lex) && Lex_IsOctDigit(lexPeek(lex, 0)) {
                        discard lexAdvance(lex);
                    }
                }
                lexEmitToken(lex, tkIntLiteral);
                return;
            }
        }

        lexScanDigits(lex);

        if lexPeek(lex, 0) == 46 && Lex_IsDigit(lexPeek(lex, 1)) { // . digit
            isFloat = true;
            discard lexAdvance(lex); // .
            lexScanDigits(lex);
        }

        let e: uint32 = lexPeek(lex, 0);
        if e == 101 || e == 69 { // e E
            isFloat = true;
            discard lexAdvance(lex);
            let sign: uint32 = lexPeek(lex, 0);
            if sign == 43 || sign == 45 { discard lexAdvance(lex); } // + -
            lexScanDigits(lex);
        }

        if isFloat {
            lexEmitToken(lex, tkFloatLiteral);
        } else {
            lexEmitToken(lex, tkIntLiteral);
        }
    }

    // ---------------------------------------------------------------------------
    // Strings and chars
    // ---------------------------------------------------------------------------

    func lexScanBacktickString(lex: *Lexer) {
        lexMarkStart(lex);
        if lexPeek(lex, 0) == 96 { discard lexAdvance(lex); } // opening backtick
        while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 96 {
            discard lexAdvance(lex);
        }
        if lexIsAtEnd(lex) {
            lexEmitDiag(lex, "unterminated backtick string literal");
        } else {
            discard lexAdvance(lex); // closing backtick
        }
        lexEmitToken(lex, tkStringLiteral);
    }

    // Assumes lex.startPos already marked (may include f/c8/… prefix before the quote).
    func lexScanStringFrom(lex: *Lexer) {
        // Collect the prefix (before opening quote) for the token text
        var prefix: String = "";
        var prefixLen: int = 0;
        if lex.pos > lex.startPos {
            let plen: int = lex.pos - lex.startPos;
            let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8;
            var pi: int = 0;
            while pi < plen {
                pbuf[pi] = lex.source[lex.startPos + pi] as char8;
                pi = pi + 1;
            }
            pbuf[plen] = 0 as char8;
            prefix = pbuf;
            prefixLen = plen;
        }
        // Allocate buffer for resolved content (max = remaining source length)
        let maxLen: int = 4096;
        let resolved: *char8 = bux_alloc(maxLen as uint) as *char8;
        var rpos: int = 0;
        if lexPeek(lex, 0) == 34 { discard lexAdvance(lex); } // opening "
        while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 34 { // closing "
            if lexPeek(lex, 0) == 10 { // \n
                lexEmitDiag(lex, "unterminated string literal");
                break;
            }
            if lexPeek(lex, 0) == 92 { // backslash
                discard lexAdvance(lex);
                if !lexIsAtEnd(lex) {
                    let ec: uint32 = lexAdvance(lex);
                    // Preserve \{ and \} as two chars for f"..." brace escaping
                    if ec == 123 || ec == 125 {
                        resolved[rpos] = 92 as char8;
                        rpos = rpos + 1;
                        resolved[rpos] = ec as char8;
                        rpos = rpos + 1;
                    } else {
                        var rc: char8 = ec as char8;
                        if ec == 110 { rc = 10 as char8; }       // \n
                        else if ec == 114 { rc = 13 as char8; }  // \r
                        else if ec == 116 { rc = 9 as char8; }   // \t
                        else if ec == 48 { rc = 0 as char8; }    // \0
                        else if ec == 92 { rc = 92 as char8; }   // \\
                        else if ec == 34 { rc = 34 as char8; }   // \"
                        else if ec == 39 { rc = 39 as char8; }   // \'
                        resolved[rpos] = rc;
                        rpos = rpos + 1;
                    }
                }
            } else {
                let c: uint32 = lexAdvance(lex);
                resolved[rpos] = c as char8;
                rpos = rpos + 1;
            }
        }
        resolved[rpos] = 0 as char8;
        if lexIsAtEnd(lex) {
            lexEmitDiag(lex, "unterminated string literal");
        } else {
            discard lexAdvance(lex); // closing "
        }
        lexEmitToken(lex, tkStringLiteral);
        // Replace token text with resolved version: prefix + " + resolved + "
        let totalLen: int = prefixLen + 1 + rpos + 1;
        let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8;
        var fi: int = 0;
        var pi2: int = 0;
        while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; }
        finalBuf[fi] = 34 as char8; fi = fi + 1; // opening "
        var ri: int = 0;
        while ri < rpos { finalBuf[fi] = resolved[ri]; fi = fi + 1; ri = ri + 1; }
        finalBuf[fi] = 34 as char8; fi = fi + 1; // closing "
        finalBuf[fi] = 0 as char8;
        lexSetLastTokenText(lex, finalBuf);
    }

    func lexScanString(lex: *Lexer) {
        lexMarkStart(lex);
        lexScanStringFrom(lex);
    }

    func lexScanChar(lex: *Lexer) {
        lexMarkStart(lex);
        // Collect the prefix for the token text
        var prefix: String = "";
        var prefixLen: int = 0;
        if lex.pos > lex.startPos {
            let plen: int = lex.pos - lex.startPos;
            let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8;
            var pi: int = 0;
            while pi < plen {
                pbuf[pi] = lex.source[lex.startPos + pi] as char8;
                pi = pi + 1;
            }
            pbuf[plen] = 0 as char8;
            prefix = pbuf;
            prefixLen = plen;
        }
        var resolved: char8 = 0 as char8;
        if lexPeek(lex, 0) == 39 { discard lexAdvance(lex); } // opening '
        if lexIsAtEnd(lex) {
            lexEmitDiag(lex, "unterminated char literal");
            lexEmitToken(lex, tkCharLiteral);
            return;
        }
        if lexPeek(lex, 0) == 10 { // \n
            lexEmitDiag(lex, "newline in char literal");
        } else if lexPeek(lex, 0) == 92 { // backslash
            discard lexAdvance(lex);
            if !lexIsAtEnd(lex) {
                let ec: uint32 = lexAdvance(lex);
                if ec == 110 { resolved = 10 as char8; }      // \n
                else if ec == 114 { resolved = 13 as char8; } // \r
                else if ec == 116 { resolved = 9 as char8; }  // \t
                else if ec == 48 { resolved = 0 as char8; }   // \0
                else if ec == 92 { resolved = 92 as char8; }  // \\
                else if ec == 34 { resolved = 34 as char8; }  // \"
                else if ec == 39 { resolved = 39 as char8; }  // \'
                else { resolved = ec as char8; }
            }
        } else {
            resolved = lexAdvance(lex) as char8;
        }
        if lexIsAtEnd(lex) || lexPeek(lex, 0) != 39 {
            lexEmitDiag(lex, "expected closing ' for char literal");
        } else {
            discard lexAdvance(lex); // closing '
        }
        lexEmitToken(lex, tkCharLiteral);
        // Replace token text with resolved version
        let totalLen: int = prefixLen + 1 + 1 + 1;
        let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8;
        var fi: int = 0;
        var pi2: int = 0;
        while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; }
        finalBuf[fi] = 39 as char8; fi = fi + 1; // opening '
        finalBuf[fi] = resolved; fi = fi + 1;     // resolved char
        finalBuf[fi] = 39 as char8; fi = fi + 1;  // closing '
        finalBuf[fi] = 0 as char8;
        lexSetLastTokenText(lex, finalBuf);
    }

    // ---------------------------------------------------------------------------
    // Symbols / operators
    // ---------------------------------------------------------------------------

    func lexScanSymbol(lex: *Lexer) {
        lexMarkStart(lex);
        let c: uint32 = lexAdvance(lex);

        // Single-char tokens
        if c == 40 { lexEmitToken(lex, tkLParen); return; }
        if c == 41 { lexEmitToken(lex, tkRParen); return; }
        if c == 123 { lexEmitToken(lex, tkLBrace); return; }
        if c == 125 { lexEmitToken(lex, tkRBrace); return; }
        if c == 91 { lexEmitToken(lex, tkLBracket); return; }
        if c == 93 { lexEmitToken(lex, tkRBracket); return; }
        if c == 44 { lexEmitToken(lex, tkComma); return; }
        if c == 59 { lexEmitToken(lex, tkSemicolon); return; }
        if c == 64 { lexEmitToken(lex, tkAt); return; }
        if c == 63 { lexEmitToken(lex, tkQuestion); return; }
        if c == 126 { lexEmitToken(lex, tkTilde); return; }

        // : :: 
        if c == 58 {
            if lexMatch(lex, 58) { lexEmitToken(lex, tkColonColon); }
            else { lexEmitToken(lex, tkColon); }
            return;
        }

        // . .. ... ..=
        if c == 46 {
            if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 46 {
                discard lexAdvance(lex); discard lexAdvance(lex);
                lexEmitToken(lex, tkDotDotDot); return;
            }
            if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 61 {
                discard lexAdvance(lex); discard lexAdvance(lex);
                lexEmitToken(lex, tkDotDotEqual); return;
            }
            if lexMatch(lex, 46) { lexEmitToken(lex, tkDotDot); return; }
            lexEmitToken(lex, tkDot); return;
        }

        // - -> -- -=
        if c == 45 {
            if lexMatch(lex, 62) { lexEmitToken(lex, tkArrow); return; }
            if lexMatch(lex, 45) { lexEmitToken(lex, tkMinusMinus); return; }
            if lexMatch(lex, 61) { lexEmitToken(lex, tkMinusAssign); return; }
            lexEmitToken(lex, tkMinus); return;
        }

        // + ++ +=
        if c == 43 {
            if lexMatch(lex, 43) { lexEmitToken(lex, tkPlusPlus); return; }
            if lexMatch(lex, 61) { lexEmitToken(lex, tkPlusAssign); return; }
            lexEmitToken(lex, tkPlus); return;
        }

        // * ** *=
        if c == 42 {
            if lexMatch(lex, 42) { lexEmitToken(lex, tkStarStar); return; }
            if lexMatch(lex, 61) { lexEmitToken(lex, tkStarAssign); return; }
            lexEmitToken(lex, tkStar); return;
        }

        // / /=
        if c == 47 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkSlashAssign); return; }
            lexEmitToken(lex, tkSlash); return;
        }

        // % %=
        if c == 37 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkPercentAssign); return; }
            lexEmitToken(lex, tkPercent); return;
        }

        // = == =>
        if c == 61 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkEq); return; }
            if lexMatch(lex, 62) { lexEmitToken(lex, tkFatArrow); return; }
            lexEmitToken(lex, tkAssign); return;
        }

        // ! !=
        if c == 33 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkNe); return; }
            lexEmitToken(lex, tkBang); return;
        }

        // < <= << <<=
        if c == 60 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkLe); return; }
            if lexMatch(lex, 60) {
                if lexMatch(lex, 61) { lexEmitToken(lex, tkShlAssign); return; }
                lexEmitToken(lex, tkShl); return;
            }
            lexEmitToken(lex, tkLt); return;
        }

        // > >= >> >>=
        if c == 62 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkGe); return; }
            if lexMatch(lex, 62) {
                if lexMatch(lex, 61) { lexEmitToken(lex, tkShrAssign); return; }
                lexEmitToken(lex, tkShr); return;
            }
            lexEmitToken(lex, tkGt); return;
        }

        // & && &=
        if c == 38 {
            if lexMatch(lex, 38) { lexEmitToken(lex, tkAmpAmp); return; }
            if lexMatch(lex, 61) { lexEmitToken(lex, tkAmpAssign); return; }
            lexEmitToken(lex, tkAmp); return;
        }

        // | || |=
        if c == 124 {
            if lexMatch(lex, 124) { lexEmitToken(lex, tkPipePipe); return; }
            if lexMatch(lex, 61) { lexEmitToken(lex, tkPipeAssign); return; }
            lexEmitToken(lex, tkPipe); return;
        }

        // ^ ^=
        if c == 94 {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkCaretAssign); return; }
            lexEmitToken(lex, tkCaret); return;
        }

        // # intrinsics
        if c == 35 {
            if lexMatchStr(lex, "line") { lexEmitToken(lex, tkHashLine); return; }
            if lexMatchStr(lex, "column") { lexEmitToken(lex, tkHashColumn); return; }
            if lexMatchStr(lex, "file") { lexEmitToken(lex, tkHashFile); return; }
            if lexMatchStr(lex, "function") { lexEmitToken(lex, tkHashFunction); return; }
            if lexMatchStr(lex, "date") { lexEmitToken(lex, tkHashDate); return; }
            if lexMatchStr(lex, "time") { lexEmitToken(lex, tkHashTime); return; }
            if lexMatchStr(lex, "module") { lexEmitToken(lex, tkHashModule); return; }
            lexEmitToken(lex, tkHash); return;
        }

        lexEmitDiag(lex, "unexpected character");
        lexEmitToken(lex, tkUnknown);
    }

    // ---------------------------------------------------------------------------
    // Next token
    // ---------------------------------------------------------------------------

    func lexNextToken(lex: *Lexer) {
        lexSkipWhitespace(lex);

        if lexIsAtEnd(lex) {
            lexMarkStart(lex);
            lexEmitToken(lex, tkEndOfFile);
            return;
        }
        let c: uint32 = lexPeek(lex, 0);

        if c == 10 { // \n
            lexMarkStart(lex);
            discard lexAdvance(lex);
            lexEmitToken(lex, tkNewLine);
            return;
        }

        // String prefixes: f" c8" c16" c32"
        // Keep `f` in token text so the parser can detect interpolating strings.
        if c == 102 && lexPeek(lex, 1) == 34 { // f"
            lexMarkStart(lex);          // start at 'f'
            discard lexAdvance(lex);    // consume f; startPos still at f
            lexScanStringFrom(lex);     // does not re-mark — prefix = "f"
            return;
        }
        if c == 99 { // 'c'
            let d: uint32 = lexPeek(lex, 1);
            if d == 56 && lexPeek(lex, 2) == 34 { // c8"
                discard lexAdvance(lex); discard lexAdvance(lex); // c 8
                lexScanString(lex); return;
            }
            if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 34 { // c16"
                discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
                lexScanString(lex); return;
            }
            if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 34 { // c32"
                discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
                lexScanString(lex); return;
            }
        }

        if c == 34 { // "
            lexScanString(lex); return;
        }

        // Backtick raw string
        if c == 96 { // backtick
            lexScanBacktickString(lex); return;
        }

        // Char prefixes: c8' c16' c32'
        if c == 99 { // 'c'
            let d: uint32 = lexPeek(lex, 1);
            if d == 56 && lexPeek(lex, 2) == 39 { // c8'
                discard lexAdvance(lex); discard lexAdvance(lex);
                lexScanChar(lex); return;
            }
            if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 39 { // c16'
                discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
                lexScanChar(lex); return;
            }
            if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 39 { // c32'
                discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
                lexScanChar(lex); return;
            }
        }

        if c == 39 { // '
            // Lifetime 'a vs char literal 'x' / '\n'
            // Lifetime: ' + ident-start, and the char after that is NOT closing '
            let n1: uint32 = lexPeek(lex, 1);
            let n2: uint32 = lexPeek(lex, 2);
            if Lex_IsIdentStart(n1) && n2 != 39 && n2 != 0 {
                lexMarkStart(lex);
                discard lexAdvance(lex); // '
                while !lexIsAtEnd(lex) && Lex_IsIdentChar(lexPeek(lex, 0)) {
                    discard lexAdvance(lex);
                }
                lexEmitToken(lex, tkLifetime);
                return;
            }
            lexScanChar(lex); return;
        }

        if Lex_IsIdentStart(c) {
            lexScanIdent(lex); return;
        }

        if Lex_IsDigit(c) {
            lexScanNumber(lex); return;
        }

        lexScanSymbol(lex);
    }

    // ---------------------------------------------------------------------------
    // Tokenize — main entry point
    // ---------------------------------------------------------------------------

    func Lexer_Tokenize(source: String) -> *Lexer {
        let lex: *Lexer = bux_alloc(sizeof(Lexer)) as *Lexer;
        lex.source = source;
        lex.sourceLen = bux_strlen(source) as int;
        lex.pos = 0;
        lex.line = 1;
        lex.column = 1;
        lex.startLine = 0;
        lex.startColumn = 0;
        lex.startPos = 0;
        let tokBuf: *LexToken = bux_alloc(maxTokens as uint * sizeof(LexToken)) as *LexToken;
        let diagBuf: *LexerDiag = bux_alloc(maxDiags as uint * sizeof(LexerDiag)) as *LexerDiag;
        lex.tokens = tokBuf;
        lex.diags = diagBuf;
        lex.tokenCount = 0;
        lex.diagCount = 0;

        while true {
            lexNextToken(lex);
            if lex.tokenCount >= maxTokens {
                lexEmitDiag(lex, "too many tokens");
                break;
            }
            if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile {
                break;
            }
        }
        return lex;
    }

    func Lexer_DiagCount(lex: *Lexer) -> int {
        return lex.diagCount;
    }

    func Lexer_Free(lex: *Lexer) {
        bux_free(lex.tokens as *void);
        bux_free(lex.diags as *void);
        bux_free(lex as *void);
    }
}
