697 lines
21 KiB
Plaintext
697 lines
21 KiB
Plaintext
// lexer.bux — Lexer state machine (ported from lexer.nim)
|
|
// Tokenizes Bux source into a stream of tokens.
|
|
module Lexer {
|
|
|
|
extern func bux_strlen(s: String) -> uint;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Character helpers (wrap C ctype)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func Lex_IsDigit(c: uint32) -> bool {
|
|
return c >= 48 && c <= 57; // '0'..'9'
|
|
}
|
|
|
|
func Lex_IsHexDigit(c: uint32) -> bool {
|
|
if c >= 48 && c <= 57 { return true; } // 0-9
|
|
if c >= 65 && c <= 70 { return true; } // A-F
|
|
if c >= 97 && c <= 102 { return true; } // a-f
|
|
return false;
|
|
}
|
|
|
|
func Lex_IsBinDigit(c: uint32) -> bool {
|
|
return c == 48 || c == 49; // '0' or '1'
|
|
}
|
|
|
|
func Lex_IsOctDigit(c: uint32) -> bool {
|
|
return c >= 48 && c <= 55; // '0'..'7'
|
|
}
|
|
|
|
func Lex_IsIdentStart(c: uint32) -> bool {
|
|
if c >= 97 && c <= 122 { return true; } // a-z
|
|
if c >= 65 && c <= 90 { return true; } // A-Z
|
|
return c == 95; // '_'
|
|
}
|
|
|
|
func Lex_IsIdentChar(c: uint32) -> bool {
|
|
if Lex_IsIdentStart(c) { return true; }
|
|
return Lex_IsDigit(c);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Lexer state
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const maxTokens: int = 4096;
|
|
const maxDiags: int = 128;
|
|
|
|
struct LexerDiag {
|
|
line: uint32,
|
|
column: uint32,
|
|
message: String,
|
|
}
|
|
|
|
struct Lexer {
|
|
source: String,
|
|
sourceLen: int,
|
|
pos: int,
|
|
line: uint32,
|
|
column: uint32,
|
|
startLine: uint32,
|
|
startColumn: uint32,
|
|
startPos: int,
|
|
tokenCount: int,
|
|
tokens: *LexToken,
|
|
diagCount: int,
|
|
diags: *LexerDiag,
|
|
}
|
|
|
|
struct LexToken {
|
|
kind: int,
|
|
text: String,
|
|
line: uint32,
|
|
column: uint32,
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Init
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func Lexer_Init(source: String) -> Lexer {
|
|
let len: uint = bux_strlen(source);
|
|
let tokBuf: *LexToken = bux_alloc(maxTokens as uint * sizeof(LexToken)) as *LexToken;
|
|
let diagBuf: *LexerDiag = bux_alloc(maxDiags as uint * sizeof(LexerDiag)) as *LexerDiag;
|
|
return Lexer {
|
|
source: source, sourceLen: len as int,
|
|
pos: 0, line: 1, column: 1,
|
|
startLine: 0, startColumn: 0, startPos: 0,
|
|
tokenCount: 0, tokens: tokBuf,
|
|
diagCount: 0, diags: diagBuf
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Core primitives
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexIsAtEnd(lex: *Lexer) -> bool {
|
|
return lex.pos >= lex.sourceLen;
|
|
}
|
|
|
|
func lexPeek(lex: *Lexer, ahead: int) -> uint32 {
|
|
let i: int = lex.pos + ahead;
|
|
if i < lex.sourceLen {
|
|
return lex.source[i] as uint32;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
func lexAdvance(lex: *Lexer) -> uint32 {
|
|
let c: uint32 = lexPeek(lex, 0);
|
|
if !lexIsAtEnd(lex) {
|
|
lex.pos = lex.pos + 1;
|
|
if c == 10 { // '\n'
|
|
lex.line = lex.line + 1;
|
|
lex.column = 1;
|
|
} else {
|
|
lex.column = lex.column + 1;
|
|
}
|
|
}
|
|
return c;
|
|
}
|
|
|
|
func lexMatch(lex: *Lexer, expected: uint32) -> bool {
|
|
if lexIsAtEnd(lex) { return false; }
|
|
if lexPeek(lex, 0) != expected { return false; }
|
|
discard lexAdvance(lex);
|
|
return true;
|
|
}
|
|
|
|
func lexMatchStr(lex: *Lexer, s: String) -> bool {
|
|
let len: uint = bux_strlen(s);
|
|
var i: int = 0;
|
|
while i < (len as int) {
|
|
if lexPeek(lex, i) != (s[i] as uint32) {
|
|
return false;
|
|
}
|
|
i = i + 1;
|
|
}
|
|
i = 0;
|
|
while i < (len as int) {
|
|
discard lexAdvance(lex);
|
|
i = i + 1;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Token emission
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexMarkStart(lex: *Lexer) {
|
|
lex.startLine = lex.line;
|
|
lex.startColumn = lex.column;
|
|
lex.startPos = lex.pos;
|
|
}
|
|
|
|
func lexMakeToken(lex: *Lexer, kind: int) -> LexToken {
|
|
var text: String = "";
|
|
let endPos: int = lex.pos;
|
|
let startPos: int = lex.startPos;
|
|
if endPos > startPos {
|
|
let len: int = endPos - startPos;
|
|
let buf: *char8 = bux_alloc((len + 1) as uint) as *char8;
|
|
var i: int = 0;
|
|
while i < len {
|
|
buf[i] = lex.source[startPos + i] as char8;
|
|
i = i + 1;
|
|
}
|
|
buf[len] = 0 as char8;
|
|
text = buf;
|
|
} else {
|
|
text = "";
|
|
}
|
|
return LexToken {
|
|
kind: kind, text: text,
|
|
line: lex.startLine, column: lex.startColumn
|
|
};
|
|
}
|
|
|
|
func lexEmitToken(lex: *Lexer, kind: int) {
|
|
let tok: LexToken = lexMakeToken(lex, kind);
|
|
if lex.tokenCount < maxTokens {
|
|
lex.tokens[lex.tokenCount] = tok;
|
|
lex.tokenCount = lex.tokenCount + 1;
|
|
}
|
|
}
|
|
|
|
func lexEmitDiag(lex: *Lexer, msg: String) {
|
|
if lex.diagCount < maxDiags {
|
|
lex.diags[lex.diagCount] = LexerDiag {
|
|
line: lex.line, column: lex.column, message: msg
|
|
};
|
|
lex.diagCount = lex.diagCount + 1;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Whitespace / comments
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexSkipLineComment(lex: *Lexer) {
|
|
while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 10 { // '\n'
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
|
|
func lexSkipBlockComment(lex: *Lexer) {
|
|
var depth: int = 1;
|
|
while !lexIsAtEnd(lex) && depth > 0 {
|
|
if lexPeek(lex, 0) == 47 && lexPeek(lex, 1) == 42 { // /*
|
|
discard lexAdvance(lex);
|
|
discard lexAdvance(lex);
|
|
depth = depth + 1;
|
|
} else if lexPeek(lex, 0) == 42 && lexPeek(lex, 1) == 47 { // */
|
|
discard lexAdvance(lex);
|
|
discard lexAdvance(lex);
|
|
depth = depth - 1;
|
|
} else {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
if depth > 0 {
|
|
lexEmitDiag(lex, "unterminated block comment");
|
|
}
|
|
}
|
|
|
|
func lexSkipWhitespace(lex: *Lexer) {
|
|
while !lexIsAtEnd(lex) {
|
|
let c: uint32 = lexPeek(lex, 0);
|
|
if c == 32 || c == 9 || c == 13 { // space, tab, CR
|
|
discard lexAdvance(lex);
|
|
} else if c == 47 && lexPeek(lex, 1) == 47 { // //
|
|
lexSkipLineComment(lex);
|
|
} else if c == 47 && lexPeek(lex, 1) == 42 { // /*
|
|
discard lexAdvance(lex);
|
|
discard lexAdvance(lex);
|
|
lexSkipBlockComment(lex);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Identifiers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexIsIdentKind(tok: int) -> bool {
|
|
return tok == tkFunc || tok == tkLet || tok == tkVar || tok == tkConst
|
|
|| tok == tkType || tok == tkStruct || tok == tkEnum || tok == tkUnion
|
|
|| tok == tkInterface || tok == tkExtend || tok == tkModule || tok == tkImport
|
|
|| tok == tkPub || tok == tkExtern || tok == tkIf || tok == tkElse
|
|
|| tok == tkWhile || tok == tkDo || tok == tkLoop || tok == tkFor
|
|
|| tok == tkIn || tok == tkBreak || tok == tkContinue || tok == tkReturn
|
|
|| tok == tkMatch || tok == tkAs || tok == tkIs || tok == tkNull
|
|
|| tok == tkSelf || tok == tkSuper;
|
|
}
|
|
|
|
func lexKeywordKind(text: String) -> int {
|
|
if String_Eq(text, "true") { return tkBoolLiteral; }
|
|
if String_Eq(text, "false") { return tkBoolLiteral; }
|
|
if String_Eq(text, "func") { return tkFunc; }
|
|
if String_Eq(text, "let") { return tkLet; }
|
|
if String_Eq(text, "var") { return tkVar; }
|
|
if String_Eq(text, "const") { return tkConst; }
|
|
if String_Eq(text, "type") { return tkType; }
|
|
if String_Eq(text, "struct") { return tkStruct; }
|
|
if String_Eq(text, "enum") { return tkEnum; }
|
|
if String_Eq(text, "union") { return tkUnion; }
|
|
if String_Eq(text, "interface") { return tkInterface; }
|
|
if String_Eq(text, "extend") { return tkExtend; }
|
|
if String_Eq(text, "module") { return tkModule; }
|
|
if String_Eq(text, "import") { return tkImport; }
|
|
if String_Eq(text, "pub") { return tkPub; }
|
|
if String_Eq(text, "extern") { return tkExtern; }
|
|
if String_Eq(text, "if") { return tkIf; }
|
|
if String_Eq(text, "else") { return tkElse; }
|
|
if String_Eq(text, "while") { return tkWhile; }
|
|
if String_Eq(text, "do") { return tkDo; }
|
|
if String_Eq(text, "loop") { return tkLoop; }
|
|
if String_Eq(text, "for") { return tkFor; }
|
|
if String_Eq(text, "in") { return tkIn; }
|
|
if String_Eq(text, "break") { return tkBreak; }
|
|
if String_Eq(text, "continue") { return tkContinue; }
|
|
if String_Eq(text, "return") { return tkReturn; }
|
|
if String_Eq(text, "match") { return tkMatch; }
|
|
if String_Eq(text, "as") { return tkAs; }
|
|
if String_Eq(text, "is") { return tkIs; }
|
|
if String_Eq(text, "null") { return tkNull; }
|
|
if String_Eq(text, "self") { return tkSelf; }
|
|
if String_Eq(text, "super") { return tkSuper; }
|
|
if String_Eq(text, "sizeof") { return tkSizeOf; }
|
|
return tkIdent;
|
|
}
|
|
|
|
func lexScanIdent(lex: *Lexer) {
|
|
lexMarkStart(lex);
|
|
while !lexIsAtEnd(lex) && Lex_IsIdentChar(lexPeek(lex, 0)) {
|
|
discard lexAdvance(lex);
|
|
}
|
|
let tok: LexToken = lexMakeToken(lex, tkIdent);
|
|
let kind: int = lexKeywordKind(tok.text);
|
|
tok.kind = kind;
|
|
if lex.tokenCount < maxTokens {
|
|
lex.tokens[lex.tokenCount] = tok;
|
|
lex.tokenCount = lex.tokenCount + 1;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Numbers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexScanDigits(lex: *Lexer) {
|
|
while !lexIsAtEnd(lex) && Lex_IsDigit(lexPeek(lex, 0)) {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
|
|
func lexScanHexDigits(lex: *Lexer) {
|
|
while !lexIsAtEnd(lex) && Lex_IsHexDigit(lexPeek(lex, 0)) {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
|
|
func lexScanNumber(lex: *Lexer) {
|
|
lexMarkStart(lex);
|
|
var isFloat: bool = false;
|
|
|
|
if lexPeek(lex, 0) == 48 { // '0'
|
|
let p1: uint32 = lexPeek(lex, 1);
|
|
if p1 == 120 || p1 == 88 || p1 == 98 || p1 == 66 || p1 == 111 || p1 == 79 { // x X b B o O
|
|
discard lexAdvance(lex); // 0
|
|
discard lexAdvance(lex); // prefix
|
|
if p1 == 120 || p1 == 88 { lexScanHexDigits(lex); }
|
|
else if p1 == 98 || p1 == 66 {
|
|
while !lexIsAtEnd(lex) && Lex_IsBinDigit(lexPeek(lex, 0)) {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
else {
|
|
while !lexIsAtEnd(lex) && Lex_IsOctDigit(lexPeek(lex, 0)) {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
lexEmitToken(lex, tkIntLiteral);
|
|
return;
|
|
}
|
|
}
|
|
|
|
lexScanDigits(lex);
|
|
|
|
if lexPeek(lex, 0) == 46 && Lex_IsDigit(lexPeek(lex, 1)) { // . digit
|
|
isFloat = true;
|
|
discard lexAdvance(lex); // .
|
|
lexScanDigits(lex);
|
|
}
|
|
|
|
let e: uint32 = lexPeek(lex, 0);
|
|
if e == 101 || e == 69 { // e E
|
|
isFloat = true;
|
|
discard lexAdvance(lex);
|
|
let sign: uint32 = lexPeek(lex, 0);
|
|
if sign == 43 || sign == 45 { discard lexAdvance(lex); } // + -
|
|
lexScanDigits(lex);
|
|
}
|
|
|
|
if isFloat {
|
|
lexEmitToken(lex, tkFloatLiteral);
|
|
} else {
|
|
lexEmitToken(lex, tkIntLiteral);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Strings and chars
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexScanString(lex: *Lexer) {
|
|
lexMarkStart(lex);
|
|
if lexPeek(lex, 0) == 34 { discard lexAdvance(lex); } // opening "
|
|
while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 34 { // closing "
|
|
if lexPeek(lex, 0) == 10 { // \n
|
|
lexEmitDiag(lex, "unterminated string literal");
|
|
break;
|
|
}
|
|
if lexPeek(lex, 0) == 92 { // backslash
|
|
discard lexAdvance(lex);
|
|
if !lexIsAtEnd(lex) { discard lexAdvance(lex); } // skip escape char
|
|
} else {
|
|
discard lexAdvance(lex);
|
|
}
|
|
}
|
|
if lexIsAtEnd(lex) {
|
|
lexEmitDiag(lex, "unterminated string literal");
|
|
} else {
|
|
discard lexAdvance(lex); // closing "
|
|
}
|
|
lexEmitToken(lex, tkStringLiteral);
|
|
}
|
|
|
|
func lexScanChar(lex: *Lexer) {
|
|
lexMarkStart(lex);
|
|
if lexPeek(lex, 0) == 39 { discard lexAdvance(lex); } // opening '
|
|
if lexIsAtEnd(lex) {
|
|
lexEmitDiag(lex, "unterminated char literal");
|
|
lexEmitToken(lex, tkCharLiteral);
|
|
return;
|
|
}
|
|
if lexPeek(lex, 0) == 10 { // \n
|
|
lexEmitDiag(lex, "newline in char literal");
|
|
} else if lexPeek(lex, 0) == 92 { // backslash
|
|
discard lexAdvance(lex);
|
|
if !lexIsAtEnd(lex) { discard lexAdvance(lex); }
|
|
} else {
|
|
discard lexAdvance(lex);
|
|
}
|
|
if lexIsAtEnd(lex) || lexPeek(lex, 0) != 39 {
|
|
lexEmitDiag(lex, "expected closing ' for char literal");
|
|
} else {
|
|
discard lexAdvance(lex); // closing '
|
|
}
|
|
lexEmitToken(lex, tkCharLiteral);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Symbols / operators
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexScanSymbol(lex: *Lexer) {
|
|
lexMarkStart(lex);
|
|
let c: uint32 = lexAdvance(lex);
|
|
|
|
// Single-char tokens
|
|
if c == 40 { lexEmitToken(lex, tkLParen); return; }
|
|
if c == 41 { lexEmitToken(lex, tkRParen); return; }
|
|
if c == 123 { lexEmitToken(lex, tkLBrace); return; }
|
|
if c == 125 { lexEmitToken(lex, tkRBrace); return; }
|
|
if c == 91 { lexEmitToken(lex, tkLBracket); return; }
|
|
if c == 93 { lexEmitToken(lex, tkRBracket); return; }
|
|
if c == 44 { lexEmitToken(lex, tkComma); return; }
|
|
if c == 59 { lexEmitToken(lex, tkSemicolon); return; }
|
|
if c == 64 { lexEmitToken(lex, tkAt); return; }
|
|
if c == 63 { lexEmitToken(lex, tkQuestion); return; }
|
|
if c == 126 { lexEmitToken(lex, tkTilde); return; }
|
|
|
|
// : ::
|
|
if c == 58 {
|
|
if lexMatch(lex, 58) { lexEmitToken(lex, tkColonColon); }
|
|
else { lexEmitToken(lex, tkColon); }
|
|
return;
|
|
}
|
|
|
|
// . .. ... ..=
|
|
if c == 46 {
|
|
if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 46 {
|
|
discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexEmitToken(lex, tkDotDotDot); return;
|
|
}
|
|
if lexPeek(lex, 0) == 46 && lexPeek(lex, 1) == 61 {
|
|
discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexEmitToken(lex, tkDotDotEqual); return;
|
|
}
|
|
if lexMatch(lex, 46) { lexEmitToken(lex, tkDotDot); return; }
|
|
lexEmitToken(lex, tkDot); return;
|
|
}
|
|
|
|
// - -> -- -=
|
|
if c == 45 {
|
|
if lexMatch(lex, 62) { lexEmitToken(lex, tkArrow); return; }
|
|
if lexMatch(lex, 45) { lexEmitToken(lex, tkMinusMinus); return; }
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkMinusAssign); return; }
|
|
lexEmitToken(lex, tkMinus); return;
|
|
}
|
|
|
|
// + ++ +=
|
|
if c == 43 {
|
|
if lexMatch(lex, 43) { lexEmitToken(lex, tkPlusPlus); return; }
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkPlusAssign); return; }
|
|
lexEmitToken(lex, tkPlus); return;
|
|
}
|
|
|
|
// * ** *=
|
|
if c == 42 {
|
|
if lexMatch(lex, 42) { lexEmitToken(lex, tkStarStar); return; }
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkStarAssign); return; }
|
|
lexEmitToken(lex, tkStar); return;
|
|
}
|
|
|
|
// / /=
|
|
if c == 47 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkSlashAssign); return; }
|
|
lexEmitToken(lex, tkSlash); return;
|
|
}
|
|
|
|
// % %=
|
|
if c == 37 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkPercentAssign); return; }
|
|
lexEmitToken(lex, tkPercent); return;
|
|
}
|
|
|
|
// = == =>
|
|
if c == 61 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkEq); return; }
|
|
if lexMatch(lex, 62) { lexEmitToken(lex, tkFatArrow); return; }
|
|
lexEmitToken(lex, tkAssign); return;
|
|
}
|
|
|
|
// ! !=
|
|
if c == 33 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkNe); return; }
|
|
lexEmitToken(lex, tkBang); return;
|
|
}
|
|
|
|
// < <= << <<=
|
|
if c == 60 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkLe); return; }
|
|
if lexMatch(lex, 60) {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkShlAssign); return; }
|
|
lexEmitToken(lex, tkShl); return;
|
|
}
|
|
lexEmitToken(lex, tkLt); return;
|
|
}
|
|
|
|
// > >= >> >>=
|
|
if c == 62 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkGe); return; }
|
|
if lexMatch(lex, 62) {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkShrAssign); return; }
|
|
lexEmitToken(lex, tkShr); return;
|
|
}
|
|
lexEmitToken(lex, tkGt); return;
|
|
}
|
|
|
|
// & && &=
|
|
if c == 38 {
|
|
if lexMatch(lex, 38) { lexEmitToken(lex, tkAmpAmp); return; }
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkAmpAssign); return; }
|
|
lexEmitToken(lex, tkAmp); return;
|
|
}
|
|
|
|
// | || |=
|
|
if c == 124 {
|
|
if lexMatch(lex, 124) { lexEmitToken(lex, tkPipePipe); return; }
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkPipeAssign); return; }
|
|
lexEmitToken(lex, tkPipe); return;
|
|
}
|
|
|
|
// ^ ^=
|
|
if c == 94 {
|
|
if lexMatch(lex, 61) { lexEmitToken(lex, tkCaretAssign); return; }
|
|
lexEmitToken(lex, tkCaret); return;
|
|
}
|
|
|
|
// # intrinsics
|
|
if c == 35 {
|
|
if lexMatchStr(lex, "line") { lexEmitToken(lex, tkHashLine); return; }
|
|
if lexMatchStr(lex, "column") { lexEmitToken(lex, tkHashColumn); return; }
|
|
if lexMatchStr(lex, "file") { lexEmitToken(lex, tkHashFile); return; }
|
|
if lexMatchStr(lex, "function") { lexEmitToken(lex, tkHashFunction); return; }
|
|
if lexMatchStr(lex, "date") { lexEmitToken(lex, tkHashDate); return; }
|
|
if lexMatchStr(lex, "time") { lexEmitToken(lex, tkHashTime); return; }
|
|
if lexMatchStr(lex, "module") { lexEmitToken(lex, tkHashModule); return; }
|
|
lexEmitToken(lex, tkHash); return;
|
|
}
|
|
|
|
lexEmitDiag(lex, "unexpected character");
|
|
lexEmitToken(lex, tkUnknown);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Next token
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func lexNextToken(lex: *Lexer) {
|
|
lexSkipWhitespace(lex);
|
|
|
|
if lexIsAtEnd(lex) {
|
|
lexMarkStart(lex);
|
|
lexEmitToken(lex, tkEndOfFile);
|
|
return;
|
|
}
|
|
|
|
let c: uint32 = lexPeek(lex, 0);
|
|
|
|
if c == 10 { // \n
|
|
lexMarkStart(lex);
|
|
discard lexAdvance(lex);
|
|
lexEmitToken(lex, tkNewLine);
|
|
return;
|
|
}
|
|
|
|
// String prefixes: c8" c16" c32"
|
|
if c == 99 { // 'c'
|
|
let d: uint32 = lexPeek(lex, 1);
|
|
if d == 56 && lexPeek(lex, 2) == 34 { // c8"
|
|
discard lexAdvance(lex); discard lexAdvance(lex); // c 8
|
|
lexScanString(lex); return;
|
|
}
|
|
if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 34 { // c16"
|
|
discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexScanString(lex); return;
|
|
}
|
|
if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 34 { // c32"
|
|
discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexScanString(lex); return;
|
|
}
|
|
}
|
|
|
|
if c == 34 { // "
|
|
lexScanString(lex); return;
|
|
}
|
|
|
|
// Char prefixes: c8' c16' c32'
|
|
if c == 99 { // 'c'
|
|
let d: uint32 = lexPeek(lex, 1);
|
|
if d == 56 && lexPeek(lex, 2) == 39 { // c8'
|
|
discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexScanChar(lex); return;
|
|
}
|
|
if d == 49 && lexPeek(lex, 2) == 54 && lexPeek(lex, 3) == 39 { // c16'
|
|
discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexScanChar(lex); return;
|
|
}
|
|
if d == 51 && lexPeek(lex, 2) == 50 && lexPeek(lex, 3) == 39 { // c32'
|
|
discard lexAdvance(lex); discard lexAdvance(lex); discard lexAdvance(lex);
|
|
lexScanChar(lex); return;
|
|
}
|
|
}
|
|
|
|
if c == 39 { // '
|
|
lexScanChar(lex); return;
|
|
}
|
|
|
|
if Lex_IsIdentStart(c) {
|
|
lexScanIdent(lex); return;
|
|
}
|
|
|
|
if Lex_IsDigit(c) {
|
|
lexScanNumber(lex); return;
|
|
}
|
|
|
|
lexScanSymbol(lex);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tokenize — main entry point
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func Lexer_Tokenize(source: String) -> *Lexer {
|
|
let lex: *Lexer = bux_alloc(sizeof(Lexer)) as *Lexer;
|
|
lex.source = source;
|
|
lex.sourceLen = bux_strlen(source) as int;
|
|
lex.pos = 0;
|
|
lex.line = 1;
|
|
lex.column = 1;
|
|
lex.startLine = 0;
|
|
lex.startColumn = 0;
|
|
lex.startPos = 0;
|
|
let tokBuf: *LexToken = bux_alloc(maxTokens as uint * sizeof(LexToken)) as *LexToken;
|
|
let diagBuf: *LexerDiag = bux_alloc(maxDiags as uint * sizeof(LexerDiag)) as *LexerDiag;
|
|
lex.tokens = tokBuf;
|
|
lex.diags = diagBuf;
|
|
lex.tokenCount = 0;
|
|
lex.diagCount = 0;
|
|
|
|
while true {
|
|
lexNextToken(lex);
|
|
if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile {
|
|
break;
|
|
}
|
|
}
|
|
return lex;
|
|
}
|
|
|
|
func Lexer_TokenCount(lex: *Lexer) -> int {
|
|
return lex.tokenCount;
|
|
}
|
|
|
|
func Lexer_GetToken(lex: *Lexer, index: int) -> LexToken {
|
|
if index >= 0 && index < lex.tokenCount {
|
|
return lex.tokens[index];
|
|
}
|
|
var empty: LexToken;
|
|
return empty;
|
|
}
|
|
|
|
func Lexer_DiagCount(lex: *Lexer) -> int {
|
|
return lex.diagCount;
|
|
}
|
|
|
|
func Lexer_Free(lex: *Lexer) {
|
|
bux_free(lex.tokens as *void);
|
|
bux_free(lex.diags as *void);
|
|
bux_free(lex as *void);
|
|
}
|