// lexer.bux — Lexer state machine (ported from lexer.nim)
// Tokenizes Bux source into a stream of tokens.
module Lexer {

extern func bux_strlen(s: String) -> uint;

// ---------------------------------------------------------------------------
// Character helpers (wrap C ctype)
// ---------------------------------------------------------------------------

func Lex_IsDigit(c: uint32) -> bool {
    return c >= 48 && c <= 57; // '0'..'9'
}

func Lex_IsHexDigit(c: uint32) -> bool {
    if c >= 48 && c <= 57 { return true; }  // 0-9
    if c >= 65 && c <= 70 { return true; }  // A-F
    if c >= 97 && c <= 102 { return true; } // a-f
    return false;
}

func Lex_IsBinDigit(c: uint32) -> bool {
    return c == 48 || c == 49; // '0' or '1'
}

func Lex_IsOctDigit(c: uint32) -> bool {
    return c >= 48 && c <= 55; // '0'..'7'
}

func Lex_IsIdentStart(c: uint32) -> bool {
    if c >= 97 && c <= 122 { return true; }  // a-z
    if c >= 65 && c <= 90 { return true; }   // A-Z
    return c == 95; // '_'
}

func Lex_IsIdentChar(c: uint32) -> bool {
    if Lex_IsIdentStart(c) { return true; }
    return Lex_IsDigit(c);
}

// ---------------------------------------------------------------------------
// Lexer state
// ---------------------------------------------------------------------------

const maxTokens: int = 4096;
const maxDiags: int = 128;

struct LexerDiag {
    line: uint32,
    column: uint32,
    message: String,
}

struct Lexer {
    source: String,
    sourceLen: int,
    pos: int,
    line: uint32,
    column: uint32,
    startLine: uint32,
    startColumn: uint32,
    startPos: int,
    tokenCount: int,
    tokens: *LexToken,
    diagCount: int,
    diags: *LexerDiag,
}

struct LexToken {
    kind: int,
    text: String,
    line: uint32,
    column: uint32,
}

// ---------------------------------------------------------------------------
// Init
// ---------------------------------------------------------------------------

func Lexer_Init(source: String) -> Lexer {
    let len: uint = bux_strlen(source);
    let tokBuf: *LexToken = bux_alloc(maxTokens as uint * sizeof(LexToken)) as *LexToken;
    let diagBuf: *LexerDiag = bux_alloc(maxDiags as uint * sizeof(LexerDiag)) as *LexerDiag;
    return Lexer {
        source: source, sourceLen: len as int,
        pos: 0, line: 1, column: 1,
        startLine: 0, startColumn: 0, startPos: 0,
        tokenCount: 0, tokens: tokBuf,
        diagCount: 0, diags: diagBuf
    };
}

// ---------------------------------------------------------------------------
// Core primitives
// ---------------------------------------------------------------------------

func lexIsAtEnd(lex: *Lexer) -> bool {
    return lex.pos >= lex.sourceLen;
}

func lexPeek(lex: *Lexer, ahead: int) -> uint32 {
    let i: int = lex.pos + ahead;
    if i < lex.sourceLen {
        return lex.source[i] as uint32;
    }
    return 0;
}

func lexAdvance(lex: *Lexer) -> uint32 {
    let c: uint32 = lexPeek(lex, 0);
    if !lexIsAtEnd(lex) {
        lex.pos = lex.pos + 1;
        if c == 10 { // '\n'
            lex.line = lex.line + 1;
            lex.column = 1;
        } else {
            lex.column = lex.column + 1;
        }
    }
    return c;
}

func lexMatch(lex: *Lexer, expected: uint32) -> bool {
    if lexIsAtEnd(lex) { return false; }
    if lexPeek(lex, 0) != expected { return false; }
    discard lexAdvance(lex);
    return true;
}

func lexMatchStr(lex: *Lexer, s: String) -> bool {
    let len: uint = bux_strlen(s);
    var i: int = 0;
    while i < (len as int) {
        if lexPeek(lex, i) != (s[i] as uint32) {
            return false;
        }
        i = i + 1;
    }
    i = 0;
    while i < (len as int) {
        discard lexAdvance(lex);
        i = i + 1;
    }
    return true;
}

// ---------------------------------------------------------------------------
// Token emission
// ---------------------------------------------------------------------------

func lexMarkStart(lex: *Lexer) {
    lex.startLine = lex.line;
    lex.startColumn = lex.column;
    lex.startPos = lex.pos;
}

func lexMakeToken(lex: *Lexer, kind: int) -> LexToken {
    var text: String = "";
    let endPos: int = lex.pos;
    let startPos: int = lex.startPos;
    if endPos > startPos {
        let len: int = endPos - startPos;
        let buf: *char8 = bux_alloc((len + 1) as uint) as *char8;
        var i: int = 0;
        while i < len {
            buf[i] = lex.source[startPos + i] as char8;
            i = i + 1;
        }
        buf[len] = 0 as char8;
        text = buf;
    } else {
        text = "";
    }
    return LexToken {
        kind: kind, text: text,
        line: lex.startLine, column: lex.startColumn
    };
}

func lexEmitToken(lex: *Lexer, kind: int) {
    let tok: LexToken = lexMakeToken(lex, kind);
    if lex.tokenCount < maxTokens {
        lex.tokens[lex.tokenCount] = tok;
        lex.tokenCount = lex.tokenCount + 1;
    }
}

func lexEmitDiag(lex: *Lexer, msg: String) {
    if lex.diagCount < maxDiags {
        lex.diags[lex.diagCount] = LexerDiag {
            line: lex.line, column: lex.column, message: msg
        };
        lex.diagCount = lex.diagCount + 1;
    }
}

// ---------------------------------------------------------------------------
// Whitespace / comments
// ---------------------------------------------------------------------------

func lexSkipLineComment(lex: *Lexer) {
    while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 10 { // '\n'
        discard lexAdvance(lex);
    }
}

func lexSkipBlockComment(lex: *Lexer) {
    var depth: int = 1;
    while !lexIsAtEnd(lex) && depth > 0 {
        if lexPeek(lex, 0) == 47 && lexPeek(lex, 1) == 42 { // /*
            discard lexAdvance(lex);
            discard lexAdvance(lex);
            depth = depth + 1;
        } else if lexPeek(lex, 0) == 42 && lexPeek(lex, 1) == 47 { // */
            discard lexAdvance(lex);
            discard lexAdvance(lex);
            depth = depth - 1;
        } else {
            discard lexAdvance(lex);
        }
    }
    if depth > 0 {
        lexEmitDiag(lex, "unterminated block comment");
    }
}

func lexSkipWhitespace(lex: *Lexer) {
    while !lexIsAtEnd(lex) {
        let c: uint32 = lexPeek(lex, 0);
        if c == 32 || c == 9 || c == 13 { // space, tab, CR
            discard lexAdvance(lex);
        } else if c == 47 && lexPeek(lex, 1) == 47 { // //
            lexSkipLineComment(lex);
        } else if c == 47 && lexPeek(lex, 1) == 42 { // /*
            discard lexAdvance(lex);
            discard lexAdvance(lex);
            lexSkipBlockComment(lex);
        } else {
            break;
        }
    }
}

// ---------------------------------------------------------------------------
// Identifiers
// ---------------------------------------------------------------------------

func lexIsIdentKind(tok: int) -> bool {
    return tok == tkFunc || tok == tkLet || tok == tkVar || tok == tkConst
        || tok == tkType || tok == tkStruct || tok == tkEnum || tok == tkUnion
        || tok == tkInterface || tok == tkExtend || tok == tkModule || tok == tkImport
        || tok == tkPub || tok == tkExtern || tok == tkIf || tok == tkElse
        || tok == tkWhile || tok == tkDo || tok == tkLoop || tok == tkFor
        || tok == tkIn || tok == tkBreak || tok == tkContinue || tok == tkReturn
        || tok == tkMatch || tok == tkAs || tok == tkIs || tok == tkNull
        || tok == tkSelf || tok == tkSuper;
}

func lexKeywordKind(text: String) -> int {
    if String_Eq(text, "true") { return tkBoolLiteral; }
    if String_Eq(text, "false") { return tkBoolLiteral; }
    if String_Eq(text, "func") { return tkFunc; }
    if String_Eq(text, "let") { return tkLet; }
    if String_Eq(text, "var") { return tkVar; }
    if String_Eq(text, "const") { return tkConst; }
    if String_Eq(text, "type") { return tkType; }
    if String_Eq(text, "struct") { return tkStruct; }
    if String_Eq(text, "enum") { return tkEnum; }
    if String_Eq(text, "union") { return tkUnion; }
    if String_Eq(text, "interface") { return tkInterface; }
    if String_Eq(text, "extend") { return tkExtend; }
    if String_Eq(text, "module") { return tkModule; }
    if String_Eq(text, "import") { return tkImport; }
    if String_Eq(text, "pub") { return tkPub; }
    if String_Eq(text, "extern") { return tkExtern; }
    if String_Eq(text, "if") { return tkIf; }
    if String_Eq(text, "else") { return tkElse; }
    if String_Eq(text, "while") { return tkWhile; }
    if String_Eq(text, "do") { return tkDo; }
    if String_Eq(text, "loop") { return tkLoop; }
    if String_Eq(text, "for") { return tkFor; }
    if String_Eq(text, "in") { return tkIn; }
    if String_Eq(text, "break") { return tkBreak; }
    if String_Eq(text, "continue") { return tkContinue; }
    if String_Eq(text, "return") { return tkReturn; }
    if String_Eq(text, "match") { return tkMatch; }
    if String_Eq(text, "as") { return tkAs; }
    if String_Eq(text, "is") { return tkIs; }
    if String_Eq(text, "null") { return tkNull; }
    if String_Eq(text, "self") { return tkSelf; }
    if String_Eq(text, "super") { return tkSuper; }
    if String_Eq(text, "sizeof") { return tkSizeOf; }
    return tkIdent;
}

func lexScanIdent(lex: *Lexer) {
    lexMarkStart(lex);
    while !lexIsAtEnd(lex) && Lex_IsIdentChar(lexPeek(lex, 0)) {
        discard lexAdvance(lex);
    }
    let tok: LexToken = lexMakeToken(lex, tkIdent);
    let kind: int = lexKeywordKind(tok.text);
    tok.kind = kind;
    if lex.tokenCount < maxTokens {
        lex.tokens[lex.tokenCount] = tok;
        lex.tokenCount = lex.tokenCount + 1;
    }
}

// ---------------------------------------------------------------------------
// Numbers
// ---------------------------------------------------------------------------

func lexScanDigits(lex: *Lexer) {
    while !lexIsAtEnd(lex) && Lex_IsDigit(lexPeek(lex, 0)) {
        discard lexAdvance(lex);
    }
}

func lexScanHexDigits(lex: *Lexer) {
    while !lexIsAtEnd(lex) && Lex_IsHexDigit(lexPeek(lex, 0)) {
        discard lexAdvance(lex);
    }
}

func lexScanNumber(lex: *Lexer) {
    lexMarkStart(lex);
    var isFloat: bool = false;

    if lexPeek(lex, 0) == 48 { // '0'
        let p1: uint32 = lexPeek(lex, 1);
        if p1 == 120 || p1 == 88 || p1 == 98 || p1 == 66 || p1 == 111 || p1 == 79 { // x X b B o O
            discard lexAdvance(lex); // 0
            discard lexAdvance(lex); // prefix
            if p1 == 120 || p1 == 88 { lexScanHexDigits(lex); }
            else if p1 == 98 || p1 == 66 { 
                while !lexIsAtEnd(lex) && Lex_IsBinDigit(lexPeek(lex, 0)) {
                    discard lexAdvance(lex);
                }
            }
            else {
                while !lexIsAtEnd(lex) && Lex_IsOctDigit(lexPeek(lex, 0)) {
                    discard lexAdvance(lex);
                }
            }
            lexEmitToken(lex, tkIntLiteral);
            return;
        }
    }

    lexScanDigits(lex);

    if lexPeek(lex, 0) == 46 && Lex_IsDigit(lexPeek(lex, 1)) { // . digit
        isFloat = true;
        discard lexAdvance(lex); // .
        lexScanDigits(lex);
    }

    let e: uint32 = lexPeek(lex, 0);
    if e == 101 || e == 69 { // e E
        isFloat = true;
        discard lexAdvance(lex);
        let sign: uint32 = lexPeek(lex, 0);
        if sign == 43 || sign == 45 { discard lexAdvance(lex); } // + -
        lexScanDigits(lex);
    }

    if isFloat {
        lexEmitToken(lex, tkFloatLiteral);
    } else {
        lexEmitToken(lex, tkIntLiteral);
    }
}

// ---------------------------------------------------------------------------
// Strings and chars
// ---------------------------------------------------------------------------

func lexScanString(lex: *Lexer) {
    lexMarkStart(lex);
    if lexPeek(lex, 0) == 34 { discard lexAdvance(lex); } // opening "
    while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 34 { // closing "
        if lexPeek(lex, 0) == 10 { // \n
            lexEmitDiag(lex, "unterminated string literal");
            break;
        }
        if lexPeek(lex, 0) == 92 { // backslash
            discard lexAdvance(lex);
            if !lexIsAtEnd(lex) { discard lexAdvance(lex); } // skip escape char
        } else {
            discard lexAdvance(lex);
        }
    }
    if lexIsAtEnd(lex) {
        lexEmitDiag(lex, "unterminated string literal");
    } else {
        discard lexAdvance(lex); // closing "
    }
    lexEmitToken(lex, tkStringLiteral);
}

func lexScanChar(lex: *Lexer) {
    lexMarkStart(lex);
    if lexPeek(lex, 0) == 39 { discard lexAdvance(lex); } // opening '
    if lexIsAtEnd(lex) {
        lexEmitDiag(lex, "unterminated char literal");
        lexEmitToken(lex, tkCharLiteral);
        return;
    }
    if lexPeek(lex, 0) == 10 { // \n
        lexEmitDiag(lex, "newline in char literal");
    } else if lexPeek(lex, 0) == 92 { // backslash
        discard lexAdvance(lex);
        if !lexIsAtEnd(lex) { discard lexAdvance(lex); }
    } else {
        discard lexAdvance(lex);
    }
    if lexIsAtEnd(lex) || lexPeek(lex, 0) != 39 {
        lexEmitDiag(lex, "expected closing ' for char literal");
    } else {
        discard lexAdvance(lex); // closing '
    }
    lexEmitToken(lex, tkCharLiteral);
}

// ---------------------------------------------------------------------------
// Symbols / operators
// ---------------------------------------------------------------------------

func lexScanSymbol(lex: *Lexer) {
    lexMarkStart(lex);
    let c: uint32 = lexAdvance(lex);

    // Single-char tokens
    if c == 40 { lexEmitToken(lex, tkLParen); return; }
    if c == 41 { lexEmitToken(lex, tkRParen); return; }
    if c == 123 { lexEmitToken(lex, tkLBrace); return; }
    if c == 125 { lexEmitToken(lex, tkRBrace); return; }
    if c == 91 { lexEmitToken(lex, tkLBracket); return; }
    if c == 93 { lexEmitToken(lex, tkRBracket); return; }
    if c == 44 { lexEmitToken(lex, tkComma); return; }
    if c == 59 { lexEmitToken(lex, tkSemicolon); return; }
    if c == 64 { lexEmitToken(lex, tkAt); return; }
    if c == 63 { lexEmitToken(lex, tkQuestion); return; }
    if c == 126 { lexEmitToken(lex, tkTilde); return; }

    // : :: 
    if c == 58 {
        if lexMatch(lex, 58) { lexEmitToken(lex, tkColonColon); }
        else { lexEmitToken(lex, tkColon); }
        return;
    }

    // . .. ... ..=
    if c == 46 {
        if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 46 {
            discard lexAdvance(lex); discard lexAdvance(lex);
            lexEmitToken(lex, tkDotDotDot); return;
        }
        if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 61 {
            discard lexAdvance(lex); discard lexAdvance(lex);
            lexEmitToken(lex, tkDotDotEqual); return;
        }
        if lexMatch(lex, 46) { lexEmitToken(lex, tkDotDot); return; }
        lexEmitToken(lex, tkDot); return;
    }

    // - -> -- -=
    if c == 45 {
        if lexMatch(lex, 62) { lexEmitToken(lex, tkArrow); return; }
        if lexMatch(lex, 45) { lexEmitToken(lex, tkMinusMinus); return; }
        if lexMatch(lex, 61) { lexEmitToken(lex, tkMinusAssign); return; }
        lexEmitToken(lex, tkMinus); return;
    }

    // + ++ +=
    if c == 43 {
        if lexMatch(lex, 43) { lexEmitToken(lex, tkPlusPlus); return; }
        if lexMatch(lex, 61) { lexEmitToken(lex, tkPlusAssign); return; }
        lexEmitToken(lex, tkPlus); return;
    }

    // * ** *=
    if c == 42 {
        if lexMatch(lex, 42) { lexEmitToken(lex, tkStarStar); return; }
        if lexMatch(lex, 61) { lexEmitToken(lex, tkStarAssign); return; }
        lexEmitToken(lex, tkStar); return;
    }

    // / /=
    if c == 47 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkSlashAssign); return; }
        lexEmitToken(lex, tkSlash); return;
    }

    // % %=
    if c == 37 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkPercentAssign); return; }
        lexEmitToken(lex, tkPercent); return;
    }

    // = == =>
    if c == 61 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkEq); return; }
        if lexMatch(lex, 62) { lexEmitToken(lex, tkFatArrow); return; }
        lexEmitToken(lex, tkAssign); return;
    }

    // ! !=
    if c == 33 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkNe); return; }
        lexEmitToken(lex, tkBang); return;
    }

    // < <= << <<=
    if c == 60 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkLe); return; }
        if lexMatch(lex, 60) {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkShlAssign); return; }
            lexEmitToken(lex, tkShl); return;
        }
        lexEmitToken(lex, tkLt); return;
    }

    // > >= >> >>=
    if c == 62 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkGe); return; }
        if lexMatch(lex, 62) {
            if lexMatch(lex, 61) { lexEmitToken(lex, tkShrAssign); return; }
            lexEmitToken(lex, tkShr); return;
        }
        lexEmitToken(lex, tkGt); return;
    }

    // & && &=
    if c == 38 {
        if lexMatch(lex, 38) { lexEmitToken(lex, tkAmpAmp); return; }
        if lexMatch(lex, 61) { lexEmitToken(lex, tkAmpAssign); return; }
        lexEmitToken(lex, tkAmp); return;
    }

    // | || |=
    if c == 124 {
        if lexMatch(lex, 124) { lexEmitToken(lex, tkPipePipe); return; }
        if lexMatch(lex, 61) { lexEmitToken(lex, tkPipeAssign); return; }
        lexEmitToken(lex, tkPipe); return;
    }

    // ^ ^=
    if c == 94 {
        if lexMatch(lex, 61) { lexEmitToken(lex, tkCaretAssign); return; }
        lexEmitToken(lex, tkCaret); return;
    }

    // # intrinsics
    if c == 35 {
        if lexMatchStr(lex, "line") { lexEmitToken(lex, tkHashLine); return; }
        if lexMatchStr(lex, "column") { lexEmitToken(lex, tkHashColumn); return; }
        if lexMatchStr(lex, "file") { lexEmitToken(lex, tkHashFile); return; }
        if lexMatchStr(lex, "function") { lexEmitToken(lex, tkHashFunction); return; }
        if lexMatchStr(lex, "date") { lexEmitToken(lex, tkHashDate); return; }
        if lexMatchStr(lex, "time") { lexEmitToken(lex, tkHashTime); return; }
        if lexMatchStr(lex, "module") { lexEmitToken(lex, tkHashModule); return; }
        lexEmitToken(lex, tkHash); return;
    }

    lexEmitDiag(lex, "unexpected character");
    lexEmitToken(lex, tkUnknown);
}

// ---------------------------------------------------------------------------
// Next token
// ---------------------------------------------------------------------------

func lexNextToken(lex: *Lexer) {
    lexSkipWhitespace(lex);

    if lexIsAtEnd(lex) {
        lexMarkStart(lex);
        lexEmitToken(lex, tkEndOfFile);
        return;
    }

    let c: uint32 = lexPeek(lex, 0);

    if c == 10 { // \n
        lexMarkStart(lex);
        discard lexAdvance(lex);
        lexEmitToken(lex, tkNewLine);
        return;
    }

    // String prefixes: c8" c16" c32"
    if c == 99 { // 'c'
        let d: uint32 = lexPeek(lex, 1);
        if d == 56 && lexPeek(lex, 2) == 34 { // c8"
            discard lexAdvance(lex); discard lexAdvance(lex); // c 8
            lexScanString(lex); return;
        }
        if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 34 { // c16"
            discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
            lexScanString(lex); return;
        }
        if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 34 { // c32"
            discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
            lexScanString(lex); return;
        }
    }

    if c == 34 { // "
        lexScanString(lex); return;
    }

    // Char prefixes: c8' c16' c32'
    if c == 99 { // 'c'
        let d: uint32 = lexPeek(lex, 1);
        if d == 56 && lexPeek(lex, 2) == 39 { // c8'
            discard lexAdvance(lex); discard lexAdvance(lex);
            lexScanChar(lex); return;
        }
        if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 39 { // c16'
            discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
            lexScanChar(lex); return;
        }
        if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 39 { // c32'
            discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
            lexScanChar(lex); return;
        }
    }

    if c == 39 { // '
        lexScanChar(lex); return;
    }

    if Lex_IsIdentStart(c) {
        lexScanIdent(lex); return;
    }

    if Lex_IsDigit(c) {
        lexScanNumber(lex); return;
    }

    lexScanSymbol(lex);
}

// ---------------------------------------------------------------------------
// Tokenize — main entry point
// ---------------------------------------------------------------------------

func Lexer_Tokenize(source: String) -> *Lexer {
    let lex: *Lexer = bux_alloc(sizeof(Lexer)) as *Lexer;
    lex.source = source;
    lex.sourceLen = bux_strlen(source) as int;
    lex.pos = 0;
    lex.line = 1;
    lex.column = 1;
    lex.startLine = 0;
    lex.startColumn = 0;
    lex.startPos = 0;
    let tokBuf: *LexToken = bux_alloc(maxTokens as uint * sizeof(LexToken)) as *LexToken;
    let diagBuf: *LexerDiag = bux_alloc(maxDiags as uint * sizeof(LexerDiag)) as *LexerDiag;
    lex.tokens = tokBuf;
    lex.diags = diagBuf;
    lex.tokenCount = 0;
    lex.diagCount = 0;

    while true {
        lexNextToken(lex);
        if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile {
            break;
        }
    }
    return lex;
}

func Lexer_TokenCount(lex: *Lexer) -> int {
    return lex.tokenCount;
}

func Lexer_GetToken(lex: *Lexer, index: int) -> LexToken {
    if index >= 0 && index < lex.tokenCount {
        return lex.tokens[index];
    }
    var empty: LexToken;
    return empty;
}

func Lexer_DiagCount(lex: *Lexer) -> int {
    return lex.diagCount;
}

func Lexer_Free(lex: *Lexer) {
    bux_free(lex.tokens as *void);
    bux_free(lex.diags as *void);
    bux_free(lex as *void);
}