From a45a2ecd49b3e10a72d616add1c25e7990fbacaa Mon Sep 17 00:00:00 2001 From: dimgigov Date: Fri, 5 Jun 2026 18:24:59 +0300 Subject: [PATCH] fix: escape sequences in strings, --version/--help, manifest parser, debug cleanup - Lexer (Nim + Bux): resolve escape sequences (\n, \t, \r, etc) instead of keeping raw source text in token. This was the root cause of \n being double-escaped to \\n in generated C code. - Manifest parser (src_bux/manifest.bux): use bux_strlen instead of broken String_SplitCount(s, "") hack for string length; fix quote stripping. - CLI (src_bux/cli.bux): --version/--help now recognized alongside version/help. - Remove ~30+ debug Print statements from cli, parser, lexer, sema, main. - Clean up unneeded Print/PrintInt extern declarations. - _selfhost/src/: synced from src_bux/. --- _selfhost/src/Main.bux | 4 +- _selfhost/src/cli.bux | 78 ++++++------------------------ _selfhost/src/lexer.bux | 97 ++++++++++++++++++++++++++++++++++++-- _selfhost/src/manifest.bux | 9 ++-- _selfhost/src/parser.bux | 4 +- src/lexer.nim | 29 ++++++++---- src_bux/cli.bux | 78 ++++++------------------------ src_bux/lexer.bux | 97 ++++++++++++++++++++++++++++++++++++-- src_bux/main.bux | 4 +- src_bux/manifest.bux | 9 ++-- src_bux/parser.bux | 4 +- 11 files changed, 255 insertions(+), 158 deletions(-) diff --git a/_selfhost/src/Main.bux b/_selfhost/src/Main.bux index cabd012..88940f3 100644 --- a/_selfhost/src/Main.bux +++ b/_selfhost/src/Main.bux @@ -4,15 +4,13 @@ module Main { // C runtime for command-line args extern func bux_argc() -> int; extern func bux_argv(index: int) -> String; +extern func bux_alloc(size: uint) -> *void; // Forward declaration from Cli module func Cli_Run(args: *String, argCount: int) -> int; func Main() -> int { let count: int = bux_argc(); - Print("Main count="); - PrintInt(count); - PrintLine(""); // Allocate array of String pointers let args: *String = bux_alloc(count as uint * 8) as *String; var i: int = 0; diff --git a/_selfhost/src/cli.bux b/_selfhost/src/cli.bux index d06db7c..b653a7f 100644 --- a/_selfhost/src/cli.bux +++ b/_selfhost/src/cli.bux @@ -398,8 +398,6 @@ func Cli_CollectStdlibImports(mod: *Module, outPaths: *String, maxCount: int, st } func Cli_MergeFileInto(target: *Module, path: String, skipNames: *String, skipCount: int) -> int { - Print("DEBUG: MergeFileInto "); - PrintLine(path); if !FileExists(path) { Print("Error: stdlib file not found: "); PrintLine(path); @@ -445,8 +443,6 @@ func Cli_MergeFileInto(target: *Module, path: String, skipNames: *String, skipCo } decl = next; } - Print("DEBUG: MergeFileInto done, added="); - PrintInt(added); return added; } @@ -467,15 +463,12 @@ func Cli_CopyModuleDecls(target: *Module, source: *Module) { } func Cli_BuildProject(projectDir: String) -> int { - PrintLine("DEBUG: Cli_BuildProject start"); let man: Manifest = Manifest_Load(bux_path_join(projectDir, "bux.toml")); - PrintLine("DEBUG: manifest loaded"); var outName: String = man.name; if String_Eq(outName, "") { outName = "bux_out"; } let srcDir: String = bux_path_join(projectDir, "src"); - PrintLine("DEBUG: srcDir built"); if !DirExists(srcDir) { Print("Error: no src/ directory in "); PrintLine(projectDir); @@ -553,9 +546,6 @@ func Cli_BuildProject(projectDir: String) -> int { i = i + 1; } - Print("DEBUG: userMerged.itemCount="); - PrintInt(userMerged.itemCount); - // Collect user declaration names for shadow detection let maxNames: int = 2048; let userNames: *String = bux_alloc(maxNames * 8) as *String; @@ -596,46 +586,13 @@ func Cli_BuildProject(projectDir: String) -> int { } // Copy user declarations into merged (user shadows stdlib) - PrintLine("DEBUG: before copy"); - var countTest: int = 0; - var dtest: *Decl = userMerged.firstItem; - while dtest != null as *Decl && countTest < 10000 { - dtest = dtest.childDecl2; - countTest = countTest + 1; - } - Print("userMerged actual count="); - PrintInt(countTest); - Print("userMerged.itemCount="); - PrintInt(userMerged.itemCount); - var du: *Decl = userMerged.firstItem; - while du != null as *Decl { - if du.kind == dkFunc { - Print("user func "); - PrintLine(du.strValue); - } - if du.kind == dkStruct { - Print("user struct "); - Print(du.strValue); - PrintInt(du.fieldCount); - } - if du.kind == dkImpl { - Print("user impl "); - PrintLine(du.strValue); - } - du = du.childDecl2; - } Cli_CopyModuleDecls(merged, userMerged); - PrintLine("DEBUG: copy done"); - Print("merged.itemCount="); - PrintInt(merged.itemCount); - PrintLine("DEBUG: copy done"); Print("Merged "); PrintInt(merged.itemCount); PrintLine(" declarations"); // Semantic analysis - PrintLine("DEBUG: about to run sema"); PrintLine("Running sema..."); let sema: *Sema = Sema_Analyze(merged); if Sema_HasError(sema) { @@ -653,14 +610,6 @@ func Cli_BuildProject(projectDir: String) -> int { // HIR lowering PrintLine("Lowering to HIR..."); - var di: *Decl = merged.firstItem; - while di != null as *Decl { - if di.kind == dkFunc { - Print("decl func "); - PrintLine(di.strValue); - } - di = di.childDecl2; - } let hirMod: *HirModule = HirLower_LowerModule(merged, sema); // C code generation @@ -730,7 +679,6 @@ func Cli_BuildProject(projectDir: String) -> int { // --------------------------------------------------------------------------- func Cli_RunProject(projectDir: String) -> int { - PrintLine("DEBUG: Cli_RunProject start"); let rc: int = Cli_BuildProject(projectDir); if rc != 0 { return rc; @@ -751,12 +699,23 @@ func Cli_RunProject(projectDir: String) -> int { // --------------------------------------------------------------------------- func Cli_Run(args: *String, argCount: int) -> int { - Print("Cli_Run argCount="); - PrintInt(argCount); if argCount < 2 { PrintLine("Bux Self-Hosting Compiler v0.2.0"); PrintLine("Usage: buxc [args]"); - PrintLine("Commands: build, check, run, version"); + PrintLine("Commands: build, check, run, project, help, version"); + return 0; + } + + let cmd: String = args[1]; + if String_Eq(cmd, "version") || String_Eq(cmd, "--version") || String_Eq(cmd, "-v") { + PrintLine("Bux 0.2.0 (self-hosting bootstrap)"); + return 0; + } + + if String_Eq(cmd, "help") || String_Eq(cmd, "--help") || String_Eq(cmd, "-h") { + PrintLine("Bux Self-Hosting Compiler v0.2.0"); + PrintLine("Usage: buxc [args]"); + PrintLine("Commands: build, check, run, project, help, version"); PrintLine("Pipeline modules:"); PrintLine(" Lexer ✅ 695 lines"); PrintLine(" Parser ✅ 1004 lines"); @@ -767,14 +726,6 @@ func Cli_Run(args: *String, argCount: int) -> int { return 0; } - let cmd: String = args[1]; - Print("cmd="); - PrintLine(cmd); - if String_Eq(cmd, "version") { - PrintLine("Bux 0.2.0 (self-hosting bootstrap)"); - return 0; - } - if String_Eq(cmd, "check") { if argCount < 3 { PrintLine("Usage: buxc check "); @@ -798,7 +749,6 @@ func Cli_Run(args: *String, argCount: int) -> int { } if String_Eq(cmd, "run") { - PrintLine("DEBUG: run command"); let dir: String = "."; if argCount >= 3 { dir = args[2]; } return Cli_RunProject(dir); diff --git a/_selfhost/src/lexer.bux b/_selfhost/src/lexer.bux index 0c5f8e6..91e02f1 100644 --- a/_selfhost/src/lexer.bux +++ b/_selfhost/src/lexer.bux @@ -185,6 +185,12 @@ func lexEmitToken(lex: *Lexer, kind: int) { } } +func lexSetLastTokenText(lex: *Lexer, text: String) { + if lex.tokenCount > 0 { + lex.tokens[lex.tokenCount - 1].text = text; + } +} + func lexEmitDiag(lex: *Lexer, msg: String) { if lex.diagCount < maxDiags { lex.diags[lex.diagCount] = LexerDiag { @@ -400,6 +406,25 @@ func lexScanBacktickString(lex: *Lexer) { func lexScanString(lex: *Lexer) { lexMarkStart(lex); + // Collect the prefix (before opening quote) for the token text + var prefix: String = ""; + var prefixLen: int = 0; + if lex.pos > lex.startPos { + let plen: int = lex.pos - lex.startPos; + let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8; + var pi: int = 0; + while pi < plen { + pbuf[pi] = lex.source[lex.startPos + pi] as char8; + pi = pi + 1; + } + pbuf[plen] = 0 as char8; + prefix = pbuf; + prefixLen = plen; + } + // Allocate buffer for resolved content (max = remaining source length) + let maxLen: int = 4096; + let resolved: *char8 = bux_alloc(maxLen as uint) as *char8; + var rpos: int = 0; if lexPeek(lex, 0) == 34 { discard lexAdvance(lex); } // opening " while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 34 { // closing " if lexPeek(lex, 0) == 10 { // \n @@ -408,21 +433,64 @@ func lexScanString(lex: *Lexer) { } if lexPeek(lex, 0) == 92 { // backslash discard lexAdvance(lex); - if !lexIsAtEnd(lex) { discard lexAdvance(lex); } // skip escape char + if !lexIsAtEnd(lex) { + let ec: uint32 = lexAdvance(lex); + var rc: char8 = ec as char8; + if ec == 110 { rc = 10 as char8; } // \n + else if ec == 114 { rc = 13 as char8; } // \r + else if ec == 116 { rc = 9 as char8; } // \t + else if ec == 48 { rc = 0 as char8; } // \0 + else if ec == 92 { rc = 92 as char8; } // \\ + else if ec == 34 { rc = 34 as char8; } // \" + else if ec == 39 { rc = 39 as char8; } // \' + resolved[rpos] = rc; + rpos = rpos + 1; + } } else { - discard lexAdvance(lex); + let c: uint32 = lexAdvance(lex); + resolved[rpos] = c as char8; + rpos = rpos + 1; } } + resolved[rpos] = 0 as char8; if lexIsAtEnd(lex) { lexEmitDiag(lex, "unterminated string literal"); } else { discard lexAdvance(lex); // closing " } lexEmitToken(lex, tkStringLiteral); + // Replace token text with resolved version: prefix + " + resolved + " + let totalLen: int = prefixLen + 1 + rpos + 1; + let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8; + var fi: int = 0; + var pi2: int = 0; + while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; } + finalBuf[fi] = 34 as char8; fi = fi + 1; // opening " + var ri: int = 0; + while ri < rpos { finalBuf[fi] = resolved[ri]; fi = fi + 1; ri = ri + 1; } + finalBuf[fi] = 34 as char8; fi = fi + 1; // closing " + finalBuf[fi] = 0 as char8; + lexSetLastTokenText(lex, finalBuf); } func lexScanChar(lex: *Lexer) { lexMarkStart(lex); + // Collect the prefix for the token text + var prefix: String = ""; + var prefixLen: int = 0; + if lex.pos > lex.startPos { + let plen: int = lex.pos - lex.startPos; + let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8; + var pi: int = 0; + while pi < plen { + pbuf[pi] = lex.source[lex.startPos + pi] as char8; + pi = pi + 1; + } + pbuf[plen] = 0 as char8; + prefix = pbuf; + prefixLen = plen; + } + var resolved: char8 = 0 as char8; if lexPeek(lex, 0) == 39 { discard lexAdvance(lex); } // opening ' if lexIsAtEnd(lex) { lexEmitDiag(lex, "unterminated char literal"); @@ -433,9 +501,19 @@ func lexScanChar(lex: *Lexer) { lexEmitDiag(lex, "newline in char literal"); } else if lexPeek(lex, 0) == 92 { // backslash discard lexAdvance(lex); - if !lexIsAtEnd(lex) { discard lexAdvance(lex); } + if !lexIsAtEnd(lex) { + let ec: uint32 = lexAdvance(lex); + if ec == 110 { resolved = 10 as char8; } // \n + else if ec == 114 { resolved = 13 as char8; } // \r + else if ec == 116 { resolved = 9 as char8; } // \t + else if ec == 48 { resolved = 0 as char8; } // \0 + else if ec == 92 { resolved = 92 as char8; } // \\ + else if ec == 34 { resolved = 34 as char8; } // \" + else if ec == 39 { resolved = 39 as char8; } // \' + else { resolved = ec as char8; } + } } else { - discard lexAdvance(lex); + resolved = lexAdvance(lex) as char8; } if lexIsAtEnd(lex) || lexPeek(lex, 0) != 39 { lexEmitDiag(lex, "expected closing ' for char literal"); @@ -443,6 +521,17 @@ func lexScanChar(lex: *Lexer) { discard lexAdvance(lex); // closing ' } lexEmitToken(lex, tkCharLiteral); + // Replace token text with resolved version + let totalLen: int = prefixLen + 1 + 1 + 1; + let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8; + var fi: int = 0; + var pi2: int = 0; + while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; } + finalBuf[fi] = 39 as char8; fi = fi + 1; // opening ' + finalBuf[fi] = resolved; fi = fi + 1; // resolved char + finalBuf[fi] = 39 as char8; fi = fi + 1; // closing ' + finalBuf[fi] = 0 as char8; + lexSetLastTokenText(lex, finalBuf); } // --------------------------------------------------------------------------- diff --git a/_selfhost/src/manifest.bux b/_selfhost/src/manifest.bux index e8cb3a0..47df46d 100644 --- a/_selfhost/src/manifest.bux +++ b/_selfhost/src/manifest.bux @@ -2,6 +2,8 @@ // Parses package metadata: name, version, type, build output. module Manifest { +extern func bux_strlen(s: String) -> uint; + // --------------------------------------------------------------------------- // Manifest struct // --------------------------------------------------------------------------- @@ -56,9 +58,10 @@ func Manifest_Parse(content: String) -> Manifest { // Strip quotes from value var val: String = rawVal; if String_StartsWith(val, "\"") && String_EndsWith(val, "\"") { - let len: uint = String_SplitCount(val, ""); // dummy to get length indirectly - // Simple strip: just use the string between quotes - val = String_Slice(rawVal, 1, String_SplitCount(rawVal, "") as uint - 2); + let vlen: uint = bux_strlen(val); + if vlen >= 2 { + val = String_Slice(val, 1, vlen - 2); + } } if String_Eq(currentSection, "Package") { diff --git a/_selfhost/src/parser.bux b/_selfhost/src/parser.bux index cedcb4e..ce699a8 100644 --- a/_selfhost/src/parser.bux +++ b/_selfhost/src/parser.bux @@ -1326,7 +1326,9 @@ func parserParseDecl(p: *Parser) -> *Decl { discard parserAdvance(p); // async return parserParseFuncDecl(p, isPublic, false, true); } - if kind == tkFunc { return parserParseFuncDecl(p, isPublic, false, false); } + if kind == tkFunc { + return parserParseFuncDecl(p, isPublic, false, false); + } if kind == tkStruct { return parserParseStructDecl(p, isPublic); } if kind == tkEnum { return parserParseEnumDecl(p, isPublic); } if kind == tkImport { return parserParseImportDecl(p, isPublic); } diff --git a/src/lexer.nim b/src/lexer.nim index 5ba66a7..9fc41f3 100644 --- a/src/lexer.nim +++ b/src/lexer.nim @@ -260,6 +260,8 @@ proc scanString(lex: var Lexer, startLoc: SourceLocation, prefixLen: int): Token let startPos = lex.pos - prefixLen # prefixLen characters before the opening quote were already consumed by caller # but we need to handle the quote itself + # Collect resolved string content to properly handle escape sequences + var resolved = "" if lex.peek() == '"': discard lex.advance() while not lex.isAtEnd() and lex.peek() != '"': @@ -268,14 +270,19 @@ proc scanString(lex: var Lexer, startLoc: SourceLocation, prefixLen: int): Token break if lex.peek() == '\\': discard lex.advance() - discard lex.scanEscapeSequence() + resolved.add(lex.scanEscapeSequence()) else: - discard lex.advance() + resolved.add(lex.advance()) if lex.isAtEnd(): lex.emitError(startLoc, "unterminated string literal") else: discard lex.advance() # closing " - result = lex.makeToken(tkStringLiteral, startLoc, startPos) + # Rebuild text with resolved escapes: prefix + " + resolved + " + var text = lex.source[startPos ..< startPos + prefixLen] + text.add('"') + text.add(resolved) + text.add('"') + result = Token(kind: tkStringLiteral, text: text, loc: startLoc) proc scanBacktickString(lex: var Lexer, startLoc: SourceLocation): Token = ## Scan a backtick-delimited raw string literal: content is literal, @@ -291,23 +298,29 @@ proc scanBacktickString(lex: var Lexer, startLoc: SourceLocation): Token = proc scanChar(lex: var Lexer, startLoc: SourceLocation, prefixLen: int): Token = let startPos = lex.pos - prefixLen + # Collect resolved char content to properly handle escape sequences + var resolved = "" if lex.peek() == '\'': discard lex.advance() if lex.isAtEnd(): lex.emitError(startLoc, "unterminated char literal") - return lex.makeToken(tkCharLiteral, startLoc, startPos) - if lex.peek() == '\n': + elif lex.peek() == '\n': lex.emitError(lex.currentLocation(), "newline in char literal") elif lex.peek() == '\\': discard lex.advance() - discard lex.scanEscapeSequence() + resolved.add(lex.scanEscapeSequence()) else: - discard lex.advance() + resolved.add(lex.advance()) if lex.isAtEnd() or lex.peek() != '\'': lex.emitError(lex.currentLocation(), "expected closing ' for char literal") else: discard lex.advance() - result = lex.makeToken(tkCharLiteral, startLoc, startPos) + # Rebuild text with resolved escape: prefix + ' + resolved + ' + var text = lex.source[startPos ..< startPos + prefixLen] + text.add('\'') + text.add(resolved) + text.add('\'') + result = Token(kind: tkCharLiteral, text: text, loc: startLoc) # --------------------------------------------------------------------------- # Symbols / operators diff --git a/src_bux/cli.bux b/src_bux/cli.bux index d06db7c..b653a7f 100644 --- a/src_bux/cli.bux +++ b/src_bux/cli.bux @@ -398,8 +398,6 @@ func Cli_CollectStdlibImports(mod: *Module, outPaths: *String, maxCount: int, st } func Cli_MergeFileInto(target: *Module, path: String, skipNames: *String, skipCount: int) -> int { - Print("DEBUG: MergeFileInto "); - PrintLine(path); if !FileExists(path) { Print("Error: stdlib file not found: "); PrintLine(path); @@ -445,8 +443,6 @@ func Cli_MergeFileInto(target: *Module, path: String, skipNames: *String, skipCo } decl = next; } - Print("DEBUG: MergeFileInto done, added="); - PrintInt(added); return added; } @@ -467,15 +463,12 @@ func Cli_CopyModuleDecls(target: *Module, source: *Module) { } func Cli_BuildProject(projectDir: String) -> int { - PrintLine("DEBUG: Cli_BuildProject start"); let man: Manifest = Manifest_Load(bux_path_join(projectDir, "bux.toml")); - PrintLine("DEBUG: manifest loaded"); var outName: String = man.name; if String_Eq(outName, "") { outName = "bux_out"; } let srcDir: String = bux_path_join(projectDir, "src"); - PrintLine("DEBUG: srcDir built"); if !DirExists(srcDir) { Print("Error: no src/ directory in "); PrintLine(projectDir); @@ -553,9 +546,6 @@ func Cli_BuildProject(projectDir: String) -> int { i = i + 1; } - Print("DEBUG: userMerged.itemCount="); - PrintInt(userMerged.itemCount); - // Collect user declaration names for shadow detection let maxNames: int = 2048; let userNames: *String = bux_alloc(maxNames * 8) as *String; @@ -596,46 +586,13 @@ func Cli_BuildProject(projectDir: String) -> int { } // Copy user declarations into merged (user shadows stdlib) - PrintLine("DEBUG: before copy"); - var countTest: int = 0; - var dtest: *Decl = userMerged.firstItem; - while dtest != null as *Decl && countTest < 10000 { - dtest = dtest.childDecl2; - countTest = countTest + 1; - } - Print("userMerged actual count="); - PrintInt(countTest); - Print("userMerged.itemCount="); - PrintInt(userMerged.itemCount); - var du: *Decl = userMerged.firstItem; - while du != null as *Decl { - if du.kind == dkFunc { - Print("user func "); - PrintLine(du.strValue); - } - if du.kind == dkStruct { - Print("user struct "); - Print(du.strValue); - PrintInt(du.fieldCount); - } - if du.kind == dkImpl { - Print("user impl "); - PrintLine(du.strValue); - } - du = du.childDecl2; - } Cli_CopyModuleDecls(merged, userMerged); - PrintLine("DEBUG: copy done"); - Print("merged.itemCount="); - PrintInt(merged.itemCount); - PrintLine("DEBUG: copy done"); Print("Merged "); PrintInt(merged.itemCount); PrintLine(" declarations"); // Semantic analysis - PrintLine("DEBUG: about to run sema"); PrintLine("Running sema..."); let sema: *Sema = Sema_Analyze(merged); if Sema_HasError(sema) { @@ -653,14 +610,6 @@ func Cli_BuildProject(projectDir: String) -> int { // HIR lowering PrintLine("Lowering to HIR..."); - var di: *Decl = merged.firstItem; - while di != null as *Decl { - if di.kind == dkFunc { - Print("decl func "); - PrintLine(di.strValue); - } - di = di.childDecl2; - } let hirMod: *HirModule = HirLower_LowerModule(merged, sema); // C code generation @@ -730,7 +679,6 @@ func Cli_BuildProject(projectDir: String) -> int { // --------------------------------------------------------------------------- func Cli_RunProject(projectDir: String) -> int { - PrintLine("DEBUG: Cli_RunProject start"); let rc: int = Cli_BuildProject(projectDir); if rc != 0 { return rc; @@ -751,12 +699,23 @@ func Cli_RunProject(projectDir: String) -> int { // --------------------------------------------------------------------------- func Cli_Run(args: *String, argCount: int) -> int { - Print("Cli_Run argCount="); - PrintInt(argCount); if argCount < 2 { PrintLine("Bux Self-Hosting Compiler v0.2.0"); PrintLine("Usage: buxc [args]"); - PrintLine("Commands: build, check, run, version"); + PrintLine("Commands: build, check, run, project, help, version"); + return 0; + } + + let cmd: String = args[1]; + if String_Eq(cmd, "version") || String_Eq(cmd, "--version") || String_Eq(cmd, "-v") { + PrintLine("Bux 0.2.0 (self-hosting bootstrap)"); + return 0; + } + + if String_Eq(cmd, "help") || String_Eq(cmd, "--help") || String_Eq(cmd, "-h") { + PrintLine("Bux Self-Hosting Compiler v0.2.0"); + PrintLine("Usage: buxc [args]"); + PrintLine("Commands: build, check, run, project, help, version"); PrintLine("Pipeline modules:"); PrintLine(" Lexer ✅ 695 lines"); PrintLine(" Parser ✅ 1004 lines"); @@ -767,14 +726,6 @@ func Cli_Run(args: *String, argCount: int) -> int { return 0; } - let cmd: String = args[1]; - Print("cmd="); - PrintLine(cmd); - if String_Eq(cmd, "version") { - PrintLine("Bux 0.2.0 (self-hosting bootstrap)"); - return 0; - } - if String_Eq(cmd, "check") { if argCount < 3 { PrintLine("Usage: buxc check "); @@ -798,7 +749,6 @@ func Cli_Run(args: *String, argCount: int) -> int { } if String_Eq(cmd, "run") { - PrintLine("DEBUG: run command"); let dir: String = "."; if argCount >= 3 { dir = args[2]; } return Cli_RunProject(dir); diff --git a/src_bux/lexer.bux b/src_bux/lexer.bux index 0c5f8e6..91e02f1 100644 --- a/src_bux/lexer.bux +++ b/src_bux/lexer.bux @@ -185,6 +185,12 @@ func lexEmitToken(lex: *Lexer, kind: int) { } } +func lexSetLastTokenText(lex: *Lexer, text: String) { + if lex.tokenCount > 0 { + lex.tokens[lex.tokenCount - 1].text = text; + } +} + func lexEmitDiag(lex: *Lexer, msg: String) { if lex.diagCount < maxDiags { lex.diags[lex.diagCount] = LexerDiag { @@ -400,6 +406,25 @@ func lexScanBacktickString(lex: *Lexer) { func lexScanString(lex: *Lexer) { lexMarkStart(lex); + // Collect the prefix (before opening quote) for the token text + var prefix: String = ""; + var prefixLen: int = 0; + if lex.pos > lex.startPos { + let plen: int = lex.pos - lex.startPos; + let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8; + var pi: int = 0; + while pi < plen { + pbuf[pi] = lex.source[lex.startPos + pi] as char8; + pi = pi + 1; + } + pbuf[plen] = 0 as char8; + prefix = pbuf; + prefixLen = plen; + } + // Allocate buffer for resolved content (max = remaining source length) + let maxLen: int = 4096; + let resolved: *char8 = bux_alloc(maxLen as uint) as *char8; + var rpos: int = 0; if lexPeek(lex, 0) == 34 { discard lexAdvance(lex); } // opening " while !lexIsAtEnd(lex) && lexPeek(lex, 0) != 34 { // closing " if lexPeek(lex, 0) == 10 { // \n @@ -408,21 +433,64 @@ func lexScanString(lex: *Lexer) { } if lexPeek(lex, 0) == 92 { // backslash discard lexAdvance(lex); - if !lexIsAtEnd(lex) { discard lexAdvance(lex); } // skip escape char + if !lexIsAtEnd(lex) { + let ec: uint32 = lexAdvance(lex); + var rc: char8 = ec as char8; + if ec == 110 { rc = 10 as char8; } // \n + else if ec == 114 { rc = 13 as char8; } // \r + else if ec == 116 { rc = 9 as char8; } // \t + else if ec == 48 { rc = 0 as char8; } // \0 + else if ec == 92 { rc = 92 as char8; } // \\ + else if ec == 34 { rc = 34 as char8; } // \" + else if ec == 39 { rc = 39 as char8; } // \' + resolved[rpos] = rc; + rpos = rpos + 1; + } } else { - discard lexAdvance(lex); + let c: uint32 = lexAdvance(lex); + resolved[rpos] = c as char8; + rpos = rpos + 1; } } + resolved[rpos] = 0 as char8; if lexIsAtEnd(lex) { lexEmitDiag(lex, "unterminated string literal"); } else { discard lexAdvance(lex); // closing " } lexEmitToken(lex, tkStringLiteral); + // Replace token text with resolved version: prefix + " + resolved + " + let totalLen: int = prefixLen + 1 + rpos + 1; + let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8; + var fi: int = 0; + var pi2: int = 0; + while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; } + finalBuf[fi] = 34 as char8; fi = fi + 1; // opening " + var ri: int = 0; + while ri < rpos { finalBuf[fi] = resolved[ri]; fi = fi + 1; ri = ri + 1; } + finalBuf[fi] = 34 as char8; fi = fi + 1; // closing " + finalBuf[fi] = 0 as char8; + lexSetLastTokenText(lex, finalBuf); } func lexScanChar(lex: *Lexer) { lexMarkStart(lex); + // Collect the prefix for the token text + var prefix: String = ""; + var prefixLen: int = 0; + if lex.pos > lex.startPos { + let plen: int = lex.pos - lex.startPos; + let pbuf: *char8 = bux_alloc((plen + 1) as uint) as *char8; + var pi: int = 0; + while pi < plen { + pbuf[pi] = lex.source[lex.startPos + pi] as char8; + pi = pi + 1; + } + pbuf[plen] = 0 as char8; + prefix = pbuf; + prefixLen = plen; + } + var resolved: char8 = 0 as char8; if lexPeek(lex, 0) == 39 { discard lexAdvance(lex); } // opening ' if lexIsAtEnd(lex) { lexEmitDiag(lex, "unterminated char literal"); @@ -433,9 +501,19 @@ func lexScanChar(lex: *Lexer) { lexEmitDiag(lex, "newline in char literal"); } else if lexPeek(lex, 0) == 92 { // backslash discard lexAdvance(lex); - if !lexIsAtEnd(lex) { discard lexAdvance(lex); } + if !lexIsAtEnd(lex) { + let ec: uint32 = lexAdvance(lex); + if ec == 110 { resolved = 10 as char8; } // \n + else if ec == 114 { resolved = 13 as char8; } // \r + else if ec == 116 { resolved = 9 as char8; } // \t + else if ec == 48 { resolved = 0 as char8; } // \0 + else if ec == 92 { resolved = 92 as char8; } // \\ + else if ec == 34 { resolved = 34 as char8; } // \" + else if ec == 39 { resolved = 39 as char8; } // \' + else { resolved = ec as char8; } + } } else { - discard lexAdvance(lex); + resolved = lexAdvance(lex) as char8; } if lexIsAtEnd(lex) || lexPeek(lex, 0) != 39 { lexEmitDiag(lex, "expected closing ' for char literal"); @@ -443,6 +521,17 @@ func lexScanChar(lex: *Lexer) { discard lexAdvance(lex); // closing ' } lexEmitToken(lex, tkCharLiteral); + // Replace token text with resolved version + let totalLen: int = prefixLen + 1 + 1 + 1; + let finalBuf: *char8 = bux_alloc((totalLen + 1) as uint) as *char8; + var fi: int = 0; + var pi2: int = 0; + while pi2 < prefixLen { finalBuf[fi] = prefix[pi2] as char8; fi = fi + 1; pi2 = pi2 + 1; } + finalBuf[fi] = 39 as char8; fi = fi + 1; // opening ' + finalBuf[fi] = resolved; fi = fi + 1; // resolved char + finalBuf[fi] = 39 as char8; fi = fi + 1; // closing ' + finalBuf[fi] = 0 as char8; + lexSetLastTokenText(lex, finalBuf); } // --------------------------------------------------------------------------- diff --git a/src_bux/main.bux b/src_bux/main.bux index cabd012..88940f3 100644 --- a/src_bux/main.bux +++ b/src_bux/main.bux @@ -4,15 +4,13 @@ module Main { // C runtime for command-line args extern func bux_argc() -> int; extern func bux_argv(index: int) -> String; +extern func bux_alloc(size: uint) -> *void; // Forward declaration from Cli module func Cli_Run(args: *String, argCount: int) -> int; func Main() -> int { let count: int = bux_argc(); - Print("Main count="); - PrintInt(count); - PrintLine(""); // Allocate array of String pointers let args: *String = bux_alloc(count as uint * 8) as *String; var i: int = 0; diff --git a/src_bux/manifest.bux b/src_bux/manifest.bux index e8cb3a0..47df46d 100644 --- a/src_bux/manifest.bux +++ b/src_bux/manifest.bux @@ -2,6 +2,8 @@ // Parses package metadata: name, version, type, build output. module Manifest { +extern func bux_strlen(s: String) -> uint; + // --------------------------------------------------------------------------- // Manifest struct // --------------------------------------------------------------------------- @@ -56,9 +58,10 @@ func Manifest_Parse(content: String) -> Manifest { // Strip quotes from value var val: String = rawVal; if String_StartsWith(val, "\"") && String_EndsWith(val, "\"") { - let len: uint = String_SplitCount(val, ""); // dummy to get length indirectly - // Simple strip: just use the string between quotes - val = String_Slice(rawVal, 1, String_SplitCount(rawVal, "") as uint - 2); + let vlen: uint = bux_strlen(val); + if vlen >= 2 { + val = String_Slice(val, 1, vlen - 2); + } } if String_Eq(currentSection, "Package") { diff --git a/src_bux/parser.bux b/src_bux/parser.bux index cedcb4e..ce699a8 100644 --- a/src_bux/parser.bux +++ b/src_bux/parser.bux @@ -1326,7 +1326,9 @@ func parserParseDecl(p: *Parser) -> *Decl { discard parserAdvance(p); // async return parserParseFuncDecl(p, isPublic, false, true); } - if kind == tkFunc { return parserParseFuncDecl(p, isPublic, false, false); } + if kind == tkFunc { + return parserParseFuncDecl(p, isPublic, false, false); + } if kind == tkStruct { return parserParseStructDecl(p, isPublic); } if kind == tkEnum { return parserParseEnumDecl(p, isPublic); } if kind == tkImport { return parserParseImportDecl(p, isPublic); }