From e8d8b62c9d4fca7bf60bb760d769df1f7ad1e06a Mon Sep 17 00:00:00 2001 From: dimgigov Date: Mon, 1 Jun 2026 14:03:54 +0300 Subject: [PATCH] selfhost: fix parser infinite loop + lexer token limit - Parser_Parse: add beforePos safeguard to skip unrecognized tokens instead of looping forever when parserParseDecl returns null - Lexer: increase maxTokens 4096 -> 32768; add explicit break on limit - CLI: null-check ReadFile result before String_Eq - Nim CLI: add -O2 to cc invocation for faster self-hosted builds --- _selfhost/src/cli.bux | 4 ++-- _selfhost/src/lexer.bux | 7 +++++-- _selfhost/src/parser.bux | 5 +++++ src/cli.nim | 2 +- src_bux/cli.bux | 4 ++-- src_bux/lexer.bux | 7 +++++-- src_bux/parser.bux | 5 +++++ 7 files changed, 25 insertions(+), 9 deletions(-) diff --git a/_selfhost/src/cli.bux b/_selfhost/src/cli.bux index 86b9415..9a91efa 100644 --- a/_selfhost/src/cli.bux +++ b/_selfhost/src/cli.bux @@ -106,7 +106,7 @@ func Cli_Compile(source: String, sourceName: String) -> String { func Cli_Build(srcPath: String, outPath: String) -> int { let source: String = ReadFile(srcPath); - if String_Eq(source, "") || !FileExists(srcPath) { + if source == null as String || String_Eq(source, "") || !FileExists(srcPath) { Print("Error: cannot read source file: "); PrintLine(srcPath); return 1; @@ -142,7 +142,7 @@ func Cli_Check(srcPath: String) -> int { Print("Check: "); PrintLine(srcPath); let source: String = ReadFile(srcPath); - if String_Eq(source, "") { + if source == null as String || String_Eq(source, "") { PrintLine("Error: cannot read source file"); return 1; } diff --git a/_selfhost/src/lexer.bux b/_selfhost/src/lexer.bux index 9ae37eb..637e7b7 100644 --- a/_selfhost/src/lexer.bux +++ b/_selfhost/src/lexer.bux @@ -42,7 +42,7 @@ func Lex_IsIdentChar(c: uint32) -> bool { // Lexer state // --------------------------------------------------------------------------- -const maxTokens: int = 4096; +const maxTokens: int = 32768; const maxDiags: int = 128; struct LexerDiag { @@ -587,7 +587,6 @@ func lexNextToken(lex: *Lexer) { lexEmitToken(lex, tkEndOfFile); return; } - let c: uint32 = lexPeek(lex, 0); if c == 10 { // \n @@ -673,6 +672,10 @@ func Lexer_Tokenize(source: String) -> *Lexer { while true { lexNextToken(lex); + if lex.tokenCount >= maxTokens { + lexEmitDiag(lex, "too many tokens"); + break; + } if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile { break; } diff --git a/_selfhost/src/parser.bux b/_selfhost/src/parser.bux index c606e5d..a663f43 100644 --- a/_selfhost/src/parser.bux +++ b/_selfhost/src/parser.bux @@ -1195,12 +1195,17 @@ func Parser_Parse(tokens: *LexToken, tokenCount: int) -> *Module { discard parserAdvance(p); continue; } + let beforePos: int = p.pos; let decl: *Decl = parserParseDecl(p); if decl != null as *Decl { decl.childDecl2 = mod.firstItem; // push front mod.firstItem = decl; mod.itemCount = mod.itemCount + 1; } + // Infinite-loop safeguard: if no progress, skip token + if p.pos == beforePos { + discard parserAdvance(p); + } } return mod; diff --git a/src/cli.nim b/src/cli.nim index 2021d25..fba848d 100644 --- a/src/cli.nim +++ b/src/cli.nim @@ -536,7 +536,7 @@ proc cmdBuild*(args: seq[string], opts: GlobalOptions): int = # Compile with cc let outputName = if man.name != "": man.name else: "bux_out" let outputFile = buildDir / outputName - let ccCmd = &"cc -pthread -o {outputFile} {cFile} {runtimeDst} {ioDst} -lm 2>&1" + let ccCmd = &"cc -O2 -pthread -o {outputFile} {cFile} {runtimeDst} {ioDst} -lm 2>&1" if opts.verbose: printInfo(&"running: {ccCmd}", useColor) let (output, exitCode) = execCmdEx(ccCmd) diff --git a/src_bux/cli.bux b/src_bux/cli.bux index 86b9415..9a91efa 100644 --- a/src_bux/cli.bux +++ b/src_bux/cli.bux @@ -106,7 +106,7 @@ func Cli_Compile(source: String, sourceName: String) -> String { func Cli_Build(srcPath: String, outPath: String) -> int { let source: String = ReadFile(srcPath); - if String_Eq(source, "") || !FileExists(srcPath) { + if source == null as String || String_Eq(source, "") || !FileExists(srcPath) { Print("Error: cannot read source file: "); PrintLine(srcPath); return 1; @@ -142,7 +142,7 @@ func Cli_Check(srcPath: String) -> int { Print("Check: "); PrintLine(srcPath); let source: String = ReadFile(srcPath); - if String_Eq(source, "") { + if source == null as String || String_Eq(source, "") { PrintLine("Error: cannot read source file"); return 1; } diff --git a/src_bux/lexer.bux b/src_bux/lexer.bux index 9ae37eb..637e7b7 100644 --- a/src_bux/lexer.bux +++ b/src_bux/lexer.bux @@ -42,7 +42,7 @@ func Lex_IsIdentChar(c: uint32) -> bool { // Lexer state // --------------------------------------------------------------------------- -const maxTokens: int = 4096; +const maxTokens: int = 32768; const maxDiags: int = 128; struct LexerDiag { @@ -587,7 +587,6 @@ func lexNextToken(lex: *Lexer) { lexEmitToken(lex, tkEndOfFile); return; } - let c: uint32 = lexPeek(lex, 0); if c == 10 { // \n @@ -673,6 +672,10 @@ func Lexer_Tokenize(source: String) -> *Lexer { while true { lexNextToken(lex); + if lex.tokenCount >= maxTokens { + lexEmitDiag(lex, "too many tokens"); + break; + } if lex.tokens[lex.tokenCount - 1].kind == tkEndOfFile { break; } diff --git a/src_bux/parser.bux b/src_bux/parser.bux index c606e5d..a663f43 100644 --- a/src_bux/parser.bux +++ b/src_bux/parser.bux @@ -1195,12 +1195,17 @@ func Parser_Parse(tokens: *LexToken, tokenCount: int) -> *Module { discard parserAdvance(p); continue; } + let beforePos: int = p.pos; let decl: *Decl = parserParseDecl(p); if decl != null as *Decl { decl.childDecl2 = mod.firstItem; // push front mod.firstItem = decl; mod.itemCount = mod.itemCount + 1; } + // Infinite-loop safeguard: if no progress, skip token + if p.pos == beforePos { + discard parserAdvance(p); + } } return mod;