selfhost: fix parser infinite loop + lexer token limit
- Parser_Parse: add beforePos safeguard to skip unrecognized tokens instead of looping forever when parserParseDecl returns null - Lexer: increase maxTokens 4096 -> 32768; add explicit break on limit - CLI: null-check ReadFile result before String_Eq - Nim CLI: add -O2 to cc invocation for faster self-hosted builds
This commit is contained in:
@@ -106,7 +106,7 @@ func Cli_Compile(source: String, sourceName: String) -> String {
|
||||
|
||||
func Cli_Build(srcPath: String, outPath: String) -> int {
|
||||
let source: String = ReadFile(srcPath);
|
||||
if String_Eq(source, "") || !FileExists(srcPath) {
|
||||
if source == null as String || String_Eq(source, "") || !FileExists(srcPath) {
|
||||
Print("Error: cannot read source file: ");
|
||||
PrintLine(srcPath);
|
||||
return 1;
|
||||
@@ -142,7 +142,7 @@ func Cli_Check(srcPath: String) -> int {
|
||||
Print("Check: ");
|
||||
PrintLine(srcPath);
|
||||
let source: String = ReadFile(srcPath);
|
||||
if String_Eq(source, "") {
|
||||
if source == null as String || String_Eq(source, "") {
|
||||
PrintLine("Error: cannot read source file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func Lex_IsIdentChar(c: uint32) -> bool {
|
||||
// Lexer state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const maxTokens: int = 4096;
|
||||
const maxTokens: int = 32768;
|
||||
const maxDiags: int = 128;
|
||||
|
||||
struct LexerDiag {
|
||||
@@ -587,7 +587,6 @@ func lexNextToken(lex: *Lexer) {
|
||||
lexEmitToken(lex, tkEndOfFile);
|
||||
return;
|
||||
}
|
||||
|
||||
let c: uint32 = lexPeek(lex, 0);
|
||||
|
||||
if c == 10 { // \n
|
||||
@@ -673,6 +672,10 @@ func Lexer_Tokenize(source: String) -> *Lexer {
|
||||
|
||||
while true {
|
||||
lexNextToken(lex);
|
||||
if lex.tokenCount >= maxTokens {
|
||||
lexEmitDiag(lex, "too many tokens");
|
||||
break;
|
||||
}
|
||||
if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1195,12 +1195,17 @@ func Parser_Parse(tokens: *LexToken, tokenCount: int) -> *Module {
|
||||
discard parserAdvance(p);
|
||||
continue;
|
||||
}
|
||||
let beforePos: int = p.pos;
|
||||
let decl: *Decl = parserParseDecl(p);
|
||||
if decl != null as *Decl {
|
||||
decl.childDecl2 = mod.firstItem; // push front
|
||||
mod.firstItem = decl;
|
||||
mod.itemCount = mod.itemCount + 1;
|
||||
}
|
||||
// Infinite-loop safeguard: if no progress, skip token
|
||||
if p.pos == beforePos {
|
||||
discard parserAdvance(p);
|
||||
}
|
||||
}
|
||||
|
||||
return mod;
|
||||
|
||||
Reference in New Issue
Block a user