selfhost: fix parser infinite loop + lexer token limit

- Parser_Parse: add beforePos safeguard to skip unrecognized tokens
  instead of looping forever when parserParseDecl returns null
- Lexer: increase maxTokens 4096 -> 32768; add explicit break on limit
- CLI: null-check ReadFile result before String_Eq
- Nim CLI: add -O2 to cc invocation for faster self-hosted builds
This commit is contained in:
2026-06-01 14:03:54 +03:00
parent 55beb23220
commit e8d8b62c9d
7 changed files with 25 additions and 9 deletions
+5 -2
View File
@@ -42,7 +42,7 @@ func Lex_IsIdentChar(c: uint32) -> bool {
// Lexer state
// ---------------------------------------------------------------------------
const maxTokens: int = 4096;
const maxTokens: int = 32768;
const maxDiags: int = 128;
struct LexerDiag {
@@ -587,7 +587,6 @@ func lexNextToken(lex: *Lexer) {
lexEmitToken(lex, tkEndOfFile);
return;
}
let c: uint32 = lexPeek(lex, 0);
if c == 10 { // \n
@@ -673,6 +672,10 @@ func Lexer_Tokenize(source: String) -> *Lexer {
while true {
lexNextToken(lex);
if lex.tokenCount >= maxTokens {
lexEmitDiag(lex, "too many tokens");
break;
}
if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile {
break;
}