From 0d70b0d02f1e2e1e016c2fafa120834d79ad0767 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 3 Nov 2026 14:18:37 -0600 Subject: [PATCH 1/5] Handle eval correctly --- run-tests.py | 1 - simplecpp.cpp | 187 ++++++++++++++++++++++++++++++++++++++++++++------ simplecpp.h | 14 +++- test.cpp | 118 +++++++++++++++++++++++++++++++ 4 files changed, 298 insertions(+), 22 deletions(-) diff --git a/run-tests.py b/run-tests.py index 71d59b15..a8f26e32 100644 --- a/run-tests.py +++ b/run-tests.py @@ -77,7 +77,6 @@ def cleanup(out: str) -> str: 'macro_fn_disable_expand.c', 'macro_paste_commaext.c', 'macro_paste_hard.c', - 'macro_rescan_varargs.c', # todo, high priority 'c99-6_10_3_4_p6.c', diff --git a/simplecpp.cpp b/simplecpp.cpp index 438288e5..4cb0e32c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1702,7 +1702,7 @@ namespace simplecpp { } else if (rawtok->op == '(') { macro2tok = output2.back(); } - if (!macro2tok || !macro2tok->name) + if (!macro2tok || !macro2tok->name || macro2tok->isSkippedByRescan()) break; if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; @@ -1732,8 +1732,11 @@ namespace simplecpp { } if (!rawtok2 || par != 1U) break; + const Token * const outputBack = output2.cback(); if (macro->second.expand(output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != nullptr) break; + if (output2.cback() == outputBack) + skipTrailingName(output2); rawtok = rawtok2->next; } output.takeTokens(output2); @@ -2164,10 +2167,10 @@ namespace simplecpp { return functionLike() ? parametertokens2.back()->next : nameTokInst->next; } - /** Returns the macro to expand when the last token of @p temp is the name of a - * function-like macro and the tokens after @p tok supply its arguments; nullptr otherwise */ - static const Macro *rescanMacro(const TokenList &temp, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros) { - if (!temp.cback() || !temp.cback()->name || !sameline(tok, tok->next) || tok->next->op != '(') + /** Returns the macro when the last token of @p temp is the name of a + * function-like macro that the rescan can still expand; nullptr otherwise */ + static const Macro *trailingMacro(const TokenList &temp, const MacroMap ¯os, const std::set &expandedmacros) { + if (!temp.cback() || !temp.cback()->name || temp.cback()->isSkippedByRescan()) return nullptr; const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) @@ -2177,6 +2180,22 @@ namespace simplecpp { return &it->second; } + /** Returns the macro to expand when the last token of @p temp is the name of a + * function-like macro and the tokens after @p tok supply its arguments; nullptr otherwise */ + static const Macro *rescanMacro(const TokenList &temp, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros) { + if (!sameline(tok, tok->next) || tok->next->op != '(') + return nullptr; + return trailingMacro(temp, macros, expandedmacros); + } + + /** A macro call that expanded to nothing came after the last token of + * @p output: the rescan has passed that token, so when it is the name of + * a function-like macro, a '(' that comes later does not start a call. */ + static void skipTrailingName(TokenList &output) { + if (output.back() && output.back()->name) + output.back()->setSkippedByRescan(true); + } + /** Expand while the expansion result in @p temp ends with the name of a * function-like macro whose arguments are supplied by the tokens after @p tok. * Each round consumes that macro call from the token stream. The @p gatherCall @@ -2196,18 +2215,144 @@ namespace simplecpp { output.takeTokens(temp); output.deleteToken(output.back()); calledMacro->expand(temp, loc, temp2.cfront(), macros, expandedmacros); + if (temp.empty()) + skipTrailingName(output); tok = closingPar; } output.takeTokens(temp); return tok->next; } + /** Expand the call of the function-like macro @p calledMacro whose name is the + * only token in @p tokens and whose arguments follow the replacement-list + * token @p tok. Parameters are substituted before the rescan, so parameters + * that expand to nothing are skipped and the arguments may come from a + * parameter whose expansion starts with '(' (that expansion may hold more + * than the arguments, so the macro name is rescanned together with it). + * The expansion result is rescanned against the tokens that follow. + * Returns false when there is no call; otherwise @p next is set to the + * first token that was not consumed. */ + bool expandCallAfter(TokenList &output, TokenList &tokens, const Macro &calledMacro, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, const Token *&next) const { + const Token *tok2 = nullptr; + bool argumentsFromParameter = false; + for (const Token *after = tok->next; sameline(tok, after); after = after->next) { + if (after->op == '(') { + tok2 = appendTokens(tokens, loc, after, macros, expandedmacros, parametertokens); + break; + } + if (!expandArg(tokens, after, loc, macros, expandedmacros, parametertokens)) + break; + if (tokens.cfront()->next) { + if (tokens.cfront()->next->op == '(') { + tok2 = after; + argumentsFromParameter = true; + } + break; + } + } + if (!tok2) + return false; + const Token * const outputBack = output.cback(); + TokenList temp(files); + if (argumentsFromParameter) + rescanArg(temp, tokens, loc, macros, expandedmacros); + else + calledMacro.expand(temp, loc, tokens.cfront(), macros, expandedmacros); + next = recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens); + if (output.cback() == outputBack) + skipTrailingName(output); + return true; + } + const Token *recursiveExpandToken(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + if (sameline(tok, tok->next) && tok->next->op != '(') { + // the arguments of a trailing macro name can come from the parameters that follow + if (const Macro * const calledMacro = trailingMacro(temp, macros, expandedmacros)) { + TokenList tokens(files); + tokens.push_back(new Token(*temp.cback())); + TokenList output2(files); + const Token *next = nullptr; + if (expandCallAfter(output2, tokens, *calledMacro, loc, tok, macros, expandedmacros, parametertokens, next)) { + output.takeTokens(temp); + output.deleteToken(output.back()); + if (output2.empty()) + skipTrailingName(output); + output.takeTokens(output2); + return next; + } + } + } return expandTrailingCalls(output, temp, loc, tok, macros, expandedmacros, [&](TokenList &temp2, const Token *lpar) { return appendTokens(temp2, loc, lpar, macros, expandedmacros, parametertokens); }); } + /** Rescan a macro argument after it has been substituted into the replacement + * list: move @p tokens to @p output, expanding each call of a function-like + * macro that is complete within @p tokens. The argument pre-expansion leaves + * such a call behind when the macro name and its '(' were separated by tokens + * that expanded to nothing, as in "f EMPTY() (1)". The result of each call is + * rescanned against the tokens that follow it. Names of macros that are being + * expanded are marked so that they are never expanded later. */ + void rescanArg(TokenList &output, TokenList &tokens, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros) const { + bool hasMacroName = false; + for (Token *tok = tokens.front(); tok; tok = tok->next) { + tok->setSkippedByRescan(false); + if (tok->name && macros.find(tok->str()) != macros.end()) + hasMacroName = true; + } + if (!hasMacroName) { + output.takeTokens(tokens); + return; + } + + const auto gatherCall = [](TokenList &temp2, const Token *lpar) -> const Token * { + unsigned int par = 0; + for (const Token *tok2 = lpar; tok2; tok2 = tok2->next) { + temp2.push_back(new Token(*tok2)); + if (tok2->op == '(') { + ++par; + } else if (tok2->op == ')') { + if (--par == 0U) + return tok2; + } + } + return nullptr; + }; + + for (const Token *tok = tokens.cfront(); tok;) { + const MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); + if (it == macros.end()) { + output.push_back(new Token(*tok)); + tok = tok->next; + continue; + } + const Macro &calledMacro = it->second; + if (expandedmacros.find(tok->str()) != expandedmacros.end()) { + output.push_back(new Token(*tok)); + output.back()->markExpandedFrom(&calledMacro); + tok = tok->next; + continue; + } + TokenList call(files); + call.push_back(new Token(tok->str(), loc)); + const Token *closingPar = nullptr; + if (calledMacro.functionLike() && !tok->isExpandedFrom(&calledMacro) && tok->next && tok->next->op == '(') + closingPar = gatherCall(call, tok->next); + if (!closingPar) { + output.push_back(new Token(*tok)); + tok = tok->next; + continue; + } + TokenList temp(files); + calledMacro.expand(temp, loc, call.cfront(), macros, expandedmacros); + if (temp.empty()) + skipTrailingName(output); + tok = expandTrailingCalls(output, temp, loc, closingPar, macros, expandedmacros, gatherCall); + } + tokens.clear(); + } + const Token *expandToken(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { @@ -2222,7 +2367,14 @@ namespace simplecpp { if (tok->str() == "__VA_ARGS__" && temp.empty() && output.cback() && output.cback()->str() == "," && tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") output.deleteToken(output.back()); - return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); + const bool substituted = !temp.empty(); + TokenList rescanned(files); + rescanArg(rescanned, temp, loc, macros, expandedmacros); + // the argument held only macro calls that expanded to nothing: + // the rescan has passed the token before it + if (substituted && rescanned.empty()) + skipTrailingName(output); + return recursiveExpandToken(output, rescanned, loc, tok, macros, expandedmacros, parametertokens); } } @@ -2236,6 +2388,8 @@ namespace simplecpp { if (!calledMacro.functionLike()) { TokenList temp(files); calledMacro.expand(temp, loc, tok, macros, expandedmacros); + if (temp.empty()) + skipTrailingName(output); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); } if (!sameline(tok, tok->next)) { @@ -2244,22 +2398,12 @@ namespace simplecpp { } TokenList tokens(files); tokens.push_back(new Token(*tok)); - const Token * tok2 = nullptr; - if (tok->next->op == '(') { - tok2 = appendTokens(tokens, loc, tok->next, macros, expandedmacros, parametertokens); - } - else if (expandArg(tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { - tokens.front()->location = loc; - if (tokens.cfront()->next && tokens.cfront()->next->op == '(') - tok2 = tok->next; - } - if (!tok2) { + const Token *next = nullptr; + if (!expandCallAfter(output, tokens, calledMacro, loc, tok, macros, expandedmacros, parametertokens, next)) { output.push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } - TokenList temp(files); - calledMacro.expand(temp, loc, tokens.cfront(), macros, expandedmacros); - return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens); + return next; } if (tok->str() == DEFINED) { @@ -2310,8 +2454,11 @@ namespace simplecpp { if (variadic && argnr + 1U >= parametertokens.size()) return true; - for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U]; partok = partok->next) + for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U]; partok = partok->next) { output.push_back(new Token(*partok)); + // the substituted tokens are rescanned again + output.back()->setSkippedByRescan(false); + } return true; } diff --git a/simplecpp.h b/simplecpp.h index f4dca183..90f284a2 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -165,7 +165,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom), mSkippedByRescan(tok.mSkippedByRescan) {} Token &operator=(const Token &tok) = delete; @@ -223,6 +223,17 @@ namespace simplecpp { mExpandedFrom.insert(m); } + /** The macro rescan has examined this token and passed it: a function-like + * macro name that was not followed by '(' at that point. A '(' that comes + * later does not make it a macro call, until the token is substituted into + * a replacement list and rescanned again. */ + bool isSkippedByRescan() const { + return mSkippedByRescan; + } + void setSkippedByRescan(bool skipped) { + mSkippedByRescan = skipped; + } + void printAll() const; void printOut() const; private: @@ -237,6 +248,7 @@ namespace simplecpp { TokenString string; std::set mExpandedFrom; + bool mSkippedByRescan{}; }; /** Output from preprocessor */ diff --git a/test.cpp b/test.cpp index 4ef9bdd2..ec5dfabf 100644 --- a/test.cpp +++ b/test.cpp @@ -1038,6 +1038,123 @@ static void define27() ASSERT_EQUALS("", preprocess(code4)); } +static void define28() +{ + // a function-like macro name and its '(' are separated by a call that + // expands to nothing: the call is found when the substituted argument is + // rescanned, but not when the tokens are rescanned only once + const char code1[] = "#define EMPTY()\n" + "#define EVAL(...) __VA_ARGS__\n" + "#define HELLO() \"Hello World\"\n" + "EVAL ( HELLO EMPTY() () )\n" + "HELLO EMPTY() ()\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\"Hello World\"\n" + "HELLO ( )", preprocess(code1)); + + // the macro that is being expanded is not expanded by that rescan, and the + // name stays unexpandable afterwards + const char code2[] = "#define EMPTY()\n" + "#define EVAL(...) __VA_ARGS__\n" + "#define ID(x) x\n" + "EVAL(EVAL EMPTY() (1)) ID(EVAL(EVAL EMPTY() (2)))\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "EVAL ( 1 ) EVAL ( 2 )", preprocess(code2)); + + // calls found by the rescan are nested and their results are rescanned + // against the tokens that follow + const char code3[] = "#define EMPTY()\n" + "#define EVAL(...) __VA_ARGS__\n" + "#define ID(x) x\n" + "#define HELLO() \"Hello World\"\n" + "#define G1() G2\n" + "#define G2() \"g2\"\n" + "EVAL(ID EMPTY() (HELLO EMPTY() ())) EVAL(G1 EMPTY() () ())\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\"Hello World\" \"g2\"", preprocess(code3)); + + // parameters are substituted before the rescan: an empty parameter does + // not separate a macro name from its '(', and a parameter may hold more + // than the arguments of the call + const char code4[] = "#define EMPTY()\n" + "#define HELLO() \"Hello World\"\n" + "#define F(x) HELLO x ()\n" + "#define G(x, y) x y\n" + "#define H(x) EMPTY x\n" + "F() F(EMPTY()) F(1) G(HELLO, ()) H(() 1)\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\"Hello World\" \"Hello World\" HELLO 1 ( ) \"Hello World\" 1", preprocess(code4)); + + // a macro name that the rescan has passed is not a call: DEFER and OBSTRUCT + // delay the expansion by one and two rescans + const char code5[] = "#define EMPTY()\n" + "#define DEFER(id) id EMPTY()\n" + "#define OBSTRUCT(...) __VA_ARGS__ DEFER(EMPTY)()\n" + "#define EXPAND(...) __VA_ARGS__\n" + "#define A() 123\n" + "DEFER(A)() EXPAND(DEFER(A)())\n" + "OBSTRUCT(A)() EXPAND(OBSTRUCT(A)()) EXPAND(EXPAND(OBSTRUCT(A)()))\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "A ( ) 123\n" + "A EMPTY ( ) ( ) A ( ) 123", preprocess(code5)); + + // recursion through deferred expansion + const char code6[] = "#define EMPTY()\n" + "#define DEFER(id) id EMPTY()\n" + "#define OBSTRUCT(...) __VA_ARGS__ DEFER(EMPTY)()\n" + "#define EXPAND(...) __VA_ARGS__\n" + "#define EVAL(...) EVAL1(EVAL1(EVAL1(__VA_ARGS__)))\n" + "#define EVAL1(...) EVAL2(EVAL2(EVAL2(__VA_ARGS__)))\n" + "#define EVAL2(...) EVAL3(EVAL3(EVAL3(__VA_ARGS__)))\n" + "#define EVAL3(...) __VA_ARGS__\n" + "#define PRIMITIVE_CAT(a, ...) a ## __VA_ARGS__\n" + "#define DEC(x) PRIMITIVE_CAT(DEC_, x)\n" + "#define DEC_0 0\n" + "#define DEC_1 0\n" + "#define DEC_2 1\n" + "#define DEC_3 2\n" + "#define CHECK_N(x, n, ...) n\n" + "#define CHECK(...) CHECK_N(__VA_ARGS__, 0,)\n" + "#define PROBE(x) x, 1,\n" + "#define NOT(x) CHECK(PRIMITIVE_CAT(NOT_, x))\n" + "#define NOT_0 PROBE(~)\n" + "#define COMPL(b) PRIMITIVE_CAT(COMPL_, b)\n" + "#define COMPL_0 1\n" + "#define COMPL_1 0\n" + "#define BOOL(x) COMPL(NOT(x))\n" + "#define IIF(c) PRIMITIVE_CAT(IIF_, c)\n" + "#define IIF_0(t, ...) __VA_ARGS__\n" + "#define IIF_1(t, ...) t\n" + "#define IF(c) IIF(BOOL(c))\n" + "#define EAT(...)\n" + "#define WHEN(c) IF(c)(EXPAND, EAT)\n" + "#define REPEAT(count, macro, ...) WHEN(count)(OBSTRUCT(REPEAT_INDIRECT)()(DEC(count), macro, __VA_ARGS__) OBSTRUCT(macro)(DEC(count), __VA_ARGS__))\n" + "#define REPEAT_INDIRECT() REPEAT\n" + "#define M(i, _) i\n" + "REPEAT(2, M, ~)\n" + "EVAL(REPEAT(3, M, ~))\n"; + ASSERT_EQUALS(std::string(32, '\n') + + "REPEAT_INDIRECT ( ) ( 1 , M , ~ ) M ( 1 , ~ )\n" + "0 1 2", preprocess(code6)); +} + static void define_invalid_1() { @@ -4638,6 +4755,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(define25); TEST_CASE(define26); TEST_CASE(define27); + TEST_CASE(define28); TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_invalid_3); From 27ef941e455a82d06a3a0c05783ccc6f0c1acfe8 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 4 Nov 2026 09:05:50 -0600 Subject: [PATCH 2/5] Cleanup --- simplecpp.cpp | 329 ++++++++++++++++++++++---------------------------- simplecpp.h | 19 +-- test.cpp | 8 +- 3 files changed, 155 insertions(+), 201 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4cb0e32c..73dc1a3a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1702,7 +1702,7 @@ namespace simplecpp { } else if (rawtok->op == '(') { macro2tok = output2.back(); } - if (!macro2tok || !macro2tok->name || macro2tok->isSkippedByRescan()) + if (!macro2tok || !macro2tok->name || macro2tok->skippedByRescan) break; if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; @@ -1732,11 +1732,8 @@ namespace simplecpp { } if (!rawtok2 || par != 1U) break; - const Token * const outputBack = output2.cback(); if (macro->second.expand(output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != nullptr) break; - if (output2.cback() == outputBack) - skipTrailingName(output2); rawtok = rawtok2->next; } output.takeTokens(output2); @@ -1923,6 +1920,10 @@ namespace simplecpp { return ~0U; } + bool isParameter(const Token *tok) const { + return tok->name && getArgNum(tok->str()) < args.size(); + } + std::vector getMacroParameters(const Token *nameTokInst, bool calledInDefine) const { if (!nameTokInst->next || nameTokInst->next->op != '(' || !functionLike()) return {}; @@ -2164,13 +2165,17 @@ namespace simplecpp { if (!parametertokens1.empty()) parametertokens1.swap(parametertokens2); + // the call expanded to nothing: the rescan has passed the token before it + if (output.back() == output_end_1) + skipTrailingName(output); + return functionLike() ? parametertokens2.back()->next : nameTokInst->next; } /** Returns the macro when the last token of @p temp is the name of a * function-like macro that the rescan can still expand; nullptr otherwise */ static const Macro *trailingMacro(const TokenList &temp, const MacroMap ¯os, const std::set &expandedmacros) { - if (!temp.cback() || !temp.cback()->name || temp.cback()->isSkippedByRescan()) + if (!temp.cback() || !temp.cback()->name || temp.cback()->skippedByRescan) return nullptr; const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) @@ -2180,177 +2185,132 @@ namespace simplecpp { return &it->second; } - /** Returns the macro to expand when the last token of @p temp is the name of a - * function-like macro and the tokens after @p tok supply its arguments; nullptr otherwise */ - static const Macro *rescanMacro(const TokenList &temp, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros) { - if (!sameline(tok, tok->next) || tok->next->op != '(') - return nullptr; - return trailingMacro(temp, macros, expandedmacros); - } - /** A macro call that expanded to nothing came after the last token of * @p output: the rescan has passed that token, so when it is the name of * a function-like macro, a '(' that comes later does not start a call. */ static void skipTrailingName(TokenList &output) { if (output.back() && output.back()->name) - output.back()->setSkippedByRescan(true); + output.back()->skippedByRescan = true; + } + + /** Copy the call of the function-like macro named by @p nameTok whose + * "( ... )" follows @p tok into @p call, stopping before @p end. Returns + * the matching ')', or nullptr when there is no complete call. */ + static const Token *gatherCall(TokenList &call, const Token *nameTok, const Token *tok, const Token *end = nullptr) { + if (!sameline(tok, tok->next) || tok->next->op != '(') + return nullptr; + call.push_back(new Token(nameTok->str(), tok->next->location)); + unsigned int par = 0; + for (const Token *tok2 = tok->next; tok2 && tok2 != end; tok2 = tok2->next) { + call.push_back(new Token(*tok2)); + if (tok2->op == '(') { + ++par; + } else if (tok2->op == ')') { + if (--par == 0U) + return tok2; + } + } + return nullptr; + } + + /** Gather the call of the function-like macro named by @p nameTok whose + * arguments follow the replacement-list token @p tok into @p call. + * Parameters are substituted before the rescan, so parameters that expand + * to nothing are skipped and the arguments can come from a parameter whose + * expansion starts with '('; that expansion is copied whole since it may + * hold more than the arguments. Returns the ')' from the replacement list, + * the parameter that supplied the arguments, or nullptr when there is no + * call. */ + const Token *gatherCallAfter(TokenList &call, const Token *nameTok, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + TokenList args(files); + const Token *last = nullptr; + for (const Token *after = tok->next; sameline(tok, after); after = after->next) { + if (after->op == '(') { + last = appendTokens(args, loc, after, macros, expandedmacros, parametertokens); + break; + } + if (!expandArg(args, after, loc, macros, expandedmacros, parametertokens)) + break; + // a parameter that expands to nothing is substituted before the rescan, + // so it does not separate the macro name from its '(' + if (args.cfront()) { + if (args.cfront()->op == '(') + last = after; + break; + } + } + if (!last) + return nullptr; + call.push_back(new Token(nameTok->str(), args.cfront()->location)); + call.takeTokens(args); + return last; } /** Expand while the expansion result in @p temp ends with the name of a * function-like macro whose arguments are supplied by the tokens after @p tok. * Each round consumes that macro call from the token stream. The @p gatherCall - * callback copies the "( ... )" tokens that follow @p lpar into its output list - * and returns the matching ')', or nullptr when no complete call is available. - * Returns the first token that was not consumed (tok->next when no call was - * expanded). */ + * callback copies the macro name and its call into its list and returns the + * last token of the call, or nullptr when no complete call is available (see + * gatherCall and gatherCallAfter). Returns the first token that was not + * consumed (tok->next when no call was expanded). */ template const Token *expandTrailingCalls(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, GatherCall gatherCall) const { - while (const Macro * const calledMacro = rescanMacro(temp, tok, macros, expandedmacros)) { - TokenList temp2(files); - temp2.push_back(new Token(temp.cback()->str(), tok->next->location)); - - const Token * const closingPar = gatherCall(temp2, tok->next); - if (!closingPar) + while (const Macro * const calledMacro = trailingMacro(temp, macros, expandedmacros)) { + TokenList call(files); + const Token * const last = gatherCall(call, temp.cback(), tok); + if (!last) break; output.takeTokens(temp); output.deleteToken(output.back()); - calledMacro->expand(temp, loc, temp2.cfront(), macros, expandedmacros); + if (last->name) { + // the arguments came from a parameter that may hold more than the call + temp.takeTokens(call); + rescanArg(temp, loc, macros, expandedmacros); + } else { + calledMacro->expand(temp, loc, call.cfront(), macros, expandedmacros); + } if (temp.empty()) skipTrailingName(output); - tok = closingPar; + tok = last; } output.takeTokens(temp); return tok->next; } - /** Expand the call of the function-like macro @p calledMacro whose name is the - * only token in @p tokens and whose arguments follow the replacement-list - * token @p tok. Parameters are substituted before the rescan, so parameters - * that expand to nothing are skipped and the arguments may come from a - * parameter whose expansion starts with '(' (that expansion may hold more - * than the arguments, so the macro name is rescanned together with it). - * The expansion result is rescanned against the tokens that follow. - * Returns false when there is no call; otherwise @p next is set to the - * first token that was not consumed. */ - bool expandCallAfter(TokenList &output, TokenList &tokens, const Macro &calledMacro, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, const Token *&next) const { - const Token *tok2 = nullptr; - bool argumentsFromParameter = false; - for (const Token *after = tok->next; sameline(tok, after); after = after->next) { - if (after->op == '(') { - tok2 = appendTokens(tokens, loc, after, macros, expandedmacros, parametertokens); - break; - } - if (!expandArg(tokens, after, loc, macros, expandedmacros, parametertokens)) - break; - if (tokens.cfront()->next) { - if (tokens.cfront()->next->op == '(') { - tok2 = after; - argumentsFromParameter = true; - } - break; - } - } - if (!tok2) - return false; - const Token * const outputBack = output.cback(); - TokenList temp(files); - if (argumentsFromParameter) - rescanArg(temp, tokens, loc, macros, expandedmacros); - else - calledMacro.expand(temp, loc, tokens.cfront(), macros, expandedmacros); - next = recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens); - if (output.cback() == outputBack) - skipTrailingName(output); - return true; - } - const Token *recursiveExpandToken(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { - if (sameline(tok, tok->next) && tok->next->op != '(') { - // the arguments of a trailing macro name can come from the parameters that follow - if (const Macro * const calledMacro = trailingMacro(temp, macros, expandedmacros)) { - TokenList tokens(files); - tokens.push_back(new Token(*temp.cback())); - TokenList output2(files); - const Token *next = nullptr; - if (expandCallAfter(output2, tokens, *calledMacro, loc, tok, macros, expandedmacros, parametertokens, next)) { - output.takeTokens(temp); - output.deleteToken(output.back()); - if (output2.empty()) - skipTrailingName(output); - output.takeTokens(output2); - return next; - } - } + // the arguments of a trailing macro name can only follow as '(' or as + // parameters, which are substituted before the rescan + if (!sameline(tok, tok->next) || (tok->next->op != '(' && !isParameter(tok->next))) { + output.takeTokens(temp); + return tok->next; } - return expandTrailingCalls(output, temp, loc, tok, macros, expandedmacros, [&](TokenList &temp2, const Token *lpar) { - return appendTokens(temp2, loc, lpar, macros, expandedmacros, parametertokens); + return expandTrailingCalls(output, temp, loc, tok, macros, expandedmacros, [&](TokenList &call, const Token *nameTok, const Token *tok2) { + return gatherCallAfter(call, nameTok, loc, tok2, macros, expandedmacros, parametertokens); }); } - /** Rescan a macro argument after it has been substituted into the replacement - * list: move @p tokens to @p output, expanding each call of a function-like - * macro that is complete within @p tokens. The argument pre-expansion leaves - * such a call behind when the macro name and its '(' were separated by tokens - * that expanded to nothing, as in "f EMPTY() (1)". The result of each call is - * rescanned against the tokens that follow it. Names of macros that are being - * expanded are marked so that they are never expanded later. */ - void rescanArg(TokenList &output, TokenList &tokens, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros) const { - bool hasMacroName = false; - for (Token *tok = tokens.front(); tok; tok = tok->next) { - tok->setSkippedByRescan(false); - if (tok->name && macros.find(tok->str()) != macros.end()) - hasMacroName = true; - } - if (!hasMacroName) { - output.takeTokens(tokens); - return; - } - - const auto gatherCall = [](TokenList &temp2, const Token *lpar) -> const Token * { - unsigned int par = 0; - for (const Token *tok2 = lpar; tok2; tok2 = tok2->next) { - temp2.push_back(new Token(*tok2)); - if (tok2->op == '(') { - ++par; - } else if (tok2->op == ')') { - if (--par == 0U) - return tok2; - } - } - return nullptr; - }; - - for (const Token *tok = tokens.cfront(); tok;) { - const MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); - if (it == macros.end()) { - output.push_back(new Token(*tok)); - tok = tok->next; - continue; - } - const Macro &calledMacro = it->second; - if (expandedmacros.find(tok->str()) != expandedmacros.end()) { - output.push_back(new Token(*tok)); - output.back()->markExpandedFrom(&calledMacro); - tok = tok->next; - continue; - } - TokenList call(files); - call.push_back(new Token(tok->str(), loc)); - const Token *closingPar = nullptr; - if (calledMacro.functionLike() && !tok->isExpandedFrom(&calledMacro) && tok->next && tok->next->op == '(') - closingPar = gatherCall(call, tok->next); - if (!closingPar) { - output.push_back(new Token(*tok)); - tok = tok->next; - continue; - } + /** Rescan the macro argument in @p tokens after it has been substituted into + * the replacement list, expanding each call of a function-like macro that is + * complete within it. The argument pre-expansion leaves such a call behind + * when the macro name and its '(' were separated by tokens that expanded to + * nothing, as in "f EMPTY() (1)". The result of each call is rescanned + * against the tokens that follow it. Names of macros that are being expanded + * are marked so that they are never expanded later. */ + void rescanArg(TokenList &tokens, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros) const { + TokenList input(files); + input.takeTokens(tokens); + for (const Token *tok = input.cfront(); tok;) { TokenList temp(files); - calledMacro.expand(temp, loc, call.cfront(), macros, expandedmacros); - if (temp.empty()) - skipTrailingName(output); - tok = expandTrailingCalls(output, temp, loc, closingPar, macros, expandedmacros, gatherCall); + temp.push_back(new Token(*tok)); + if (tok->name && expandedmacros.find(tok->str()) != expandedmacros.end()) { + const MacroMap::const_iterator it = macros.find(tok->str()); + if (it != macros.end()) + temp.back()->markExpandedFrom(&it->second); + } + tok = expandTrailingCalls(tokens, temp, loc, tok, macros, expandedmacros, [](TokenList &call, const Token *nameTok, const Token *tok2) { + return gatherCall(call, nameTok, tok2); + }); } - tokens.clear(); } const Token *expandToken(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { @@ -2363,18 +2323,19 @@ namespace simplecpp { // Macro parameter.. { TokenList temp(files); - if (expandArg(temp, tok, loc, macros, expandedmacros, parametertokens)) { + bool hasMacroName = false; + if (expandArg(temp, tok, loc, macros, expandedmacros, parametertokens, &hasMacroName)) { if (tok->str() == "__VA_ARGS__" && temp.empty() && output.cback() && output.cback()->str() == "," && tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") output.deleteToken(output.back()); - const bool substituted = !temp.empty(); - TokenList rescanned(files); - rescanArg(rescanned, temp, loc, macros, expandedmacros); - // the argument held only macro calls that expanded to nothing: - // the rescan has passed the token before it - if (substituted && rescanned.empty()) - skipTrailingName(output); - return recursiveExpandToken(output, rescanned, loc, tok, macros, expandedmacros, parametertokens); + if (hasMacroName && !temp.empty()) { + rescanArg(temp, loc, macros, expandedmacros); + // the argument held only macro calls that expanded to nothing: + // the rescan has passed the token before it + if (temp.empty()) + skipTrailingName(output); + } + return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); } } @@ -2392,18 +2353,10 @@ namespace simplecpp { skipTrailingName(output); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); } - if (!sameline(tok, tok->next)) { - output.push_back(newMacroToken(tok->str(), loc, true, tok)); - return tok->next; - } - TokenList tokens(files); - tokens.push_back(new Token(*tok)); - const Token *next = nullptr; - if (!expandCallAfter(output, tokens, calledMacro, loc, tok, macros, expandedmacros, parametertokens, next)) { - output.push_back(newMacroToken(tok->str(), loc, true, tok)); - return tok->next; - } - return next; + // the call's arguments follow in the replacement list + TokenList temp(files); + temp.push_back(newMacroToken(tok->str(), loc, true, tok)); + return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); } if (tok->str() == DEFINED) { @@ -2454,26 +2407,34 @@ namespace simplecpp { if (variadic && argnr + 1U >= parametertokens.size()) return true; - for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U]; partok = partok->next) { + for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U]; partok = partok->next) output.push_back(new Token(*partok)); - // the substituted tokens are rescanned again - output.back()->setSkippedByRescan(false); - } return true; } - bool expandArg(TokenList &output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + /** Append the fully macro-expanded argument for parameter @p tok to @p output. + * @p hasMacroName, when given, is set when the argument held a macro name + * (otherwise the output cannot hold one). Returns false when @p tok is not + * a parameter. */ + bool expandArg(TokenList &output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, bool *hasMacroName = nullptr) const { if (!tok->name) return false; const unsigned int argnr = getArgNum(tok->str()); if (argnr >= args.size()) return false; + if (hasMacroName) + *hasMacroName = false; if (variadic && argnr + 1U >= parametertokens.size()) // empty variadic parameter return true; const Token * const argEnd = parametertokens[argnr + 1U]; + // NOLINTNEXTLINE(misc-const-correctness) - technically correct but used to access non-const data + Token * const outputEnd = output.back(); + bool macroName = false; for (const Token *partok = parametertokens[argnr]->next; partok != argEnd;) { const MacroMap::const_iterator it = macros.find(partok->str()); + if (it != macros.end()) + macroName = true; if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation expandedmacros2.erase(name()); @@ -2484,18 +2445,8 @@ namespace simplecpp { // macro whose arguments are supplied by the remaining argument tokens std::set expandedmacros3(expandedmacros); expandedmacros3.erase(name()); - partok = expandTrailingCalls(output, temp, loc, partok->previous, macros, expandedmacros3, [&](TokenList &temp2, const Token *lpar) -> const Token * { - unsigned int par = 0; - for (const Token *tok2 = lpar; tok2 != argEnd; tok2 = tok2->next) { - temp2.push_back(new Token(*tok2)); - if (tok2->op == '(') { - ++par; - } else if (tok2->op == ')') { - if (--par == 0U) - return tok2; - } - } - return nullptr; + partok = expandTrailingCalls(output, temp, loc, partok->previous, macros, expandedmacros3, [&](TokenList &call, const Token *nameTok, const Token *tok2) { + return gatherCall(call, nameTok, tok2, argEnd); }); } else { output.takeTokens(temp); @@ -2508,6 +2459,14 @@ namespace simplecpp { } if (tok->whitespaceahead && output.back()) output.back()->whitespaceahead = true; + if (macroName) { + // the substituted tokens are rescanned again, so the marks that the + // expansions above left on them do not apply + for (Token *tok2 = outputEnd ? outputEnd->next : output.front(); tok2; tok2 = tok2->next) + tok2->skippedByRescan = false; + if (hasMacroName) + *hasMacroName = true; + } return true; } diff --git a/simplecpp.h b/simplecpp.h index 90f284a2..09350e20 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -165,7 +165,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom), mSkippedByRescan(tok.mSkippedByRescan) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), skippedByRescan(tok.skippedByRescan), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} Token &operator=(const Token &tok) = delete; @@ -191,6 +191,11 @@ namespace simplecpp { bool name; bool number; bool whitespaceahead; + /** The macro rescan has examined this token and passed it: a function-like + * macro name that was not followed by '(' at that point. A '(' that comes + * later does not make it a macro call, until the token is substituted into + * a replacement list and rescanned again. */ + bool skippedByRescan{}; Location location; Token *previous{}; Token *next{}; @@ -223,17 +228,6 @@ namespace simplecpp { mExpandedFrom.insert(m); } - /** The macro rescan has examined this token and passed it: a function-like - * macro name that was not followed by '(' at that point. A '(' that comes - * later does not make it a macro call, until the token is substituted into - * a replacement list and rescanned again. */ - bool isSkippedByRescan() const { - return mSkippedByRescan; - } - void setSkippedByRescan(bool skipped) { - mSkippedByRescan = skipped; - } - void printAll() const; void printOut() const; private: @@ -248,7 +242,6 @@ namespace simplecpp { TokenString string; std::set mExpandedFrom; - bool mSkippedByRescan{}; }; /** Output from preprocessor */ diff --git a/test.cpp b/test.cpp index ec5dfabf..caba223a 100644 --- a/test.cpp +++ b/test.cpp @@ -1066,21 +1066,23 @@ static void define28() "EVAL ( 1 ) EVAL ( 2 )", preprocess(code2)); // calls found by the rescan are nested and their results are rescanned - // against the tokens that follow + // against the tokens that follow, which can be a parameter const char code3[] = "#define EMPTY()\n" "#define EVAL(...) __VA_ARGS__\n" "#define ID(x) x\n" "#define HELLO() \"Hello World\"\n" "#define G1() G2\n" "#define G2() \"g2\"\n" - "EVAL(ID EMPTY() (HELLO EMPTY() ())) EVAL(G1 EMPTY() () ())\n"; + "#define F(x) G1 () x\n" + "EVAL(ID EMPTY() (HELLO EMPTY() ())) EVAL(G1 EMPTY() () ()) F(())\n"; ASSERT_EQUALS("\n" "\n" "\n" "\n" "\n" "\n" - "\"Hello World\" \"g2\"", preprocess(code3)); + "\n" + "\"Hello World\" \"g2\" \"g2\"", preprocess(code3)); // parameters are substituted before the rescan: an empty parameter does // not separate a macro name from its '(', and a parameter may hold more From 15f01b82d460f714de0c7aeb44aab31872534f3f Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 5 Nov 2026 10:40:00 -0600 Subject: [PATCH 3/5] Fix shadow warnings --- simplecpp.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 73dc1a3a..0c1669a8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2222,27 +2222,27 @@ namespace simplecpp { * the parameter that supplied the arguments, or nullptr when there is no * call. */ const Token *gatherCallAfter(TokenList &call, const Token *nameTok, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { - TokenList args(files); + TokenList callArgs(files); const Token *last = nullptr; for (const Token *after = tok->next; sameline(tok, after); after = after->next) { if (after->op == '(') { - last = appendTokens(args, loc, after, macros, expandedmacros, parametertokens); + last = appendTokens(callArgs, loc, after, macros, expandedmacros, parametertokens); break; } - if (!expandArg(args, after, loc, macros, expandedmacros, parametertokens)) + if (!expandArg(callArgs, after, loc, macros, expandedmacros, parametertokens)) break; // a parameter that expands to nothing is substituted before the rescan, // so it does not separate the macro name from its '(' - if (args.cfront()) { - if (args.cfront()->op == '(') + if (callArgs.cfront()) { + if (callArgs.cfront()->op == '(') last = after; break; } } if (!last) return nullptr; - call.push_back(new Token(nameTok->str(), args.cfront()->location)); - call.takeTokens(args); + call.push_back(new Token(nameTok->str(), callArgs.cfront()->location)); + call.takeTokens(callArgs); return last; } From 05d017cc280288e7d244dfd099f7b28838458489 Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 7 Nov 2026 21:32:56 -0600 Subject: [PATCH 4/5] Fix another test case --- simplecpp.cpp | 68 ++++++++++++++++++++++++++++++++------------------- test.cpp | 21 ++++++++++++++++ 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1f715bf0..4661e869 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2252,35 +2252,53 @@ namespace simplecpp { /** Gather the call of the function-like macro named by @p nameTok whose * arguments follow the replacement-list token @p tok into @p call. - * Parameters are substituted before the rescan, so parameters that expand - * to nothing are skipped and the arguments can come from a parameter whose - * expansion starts with '('; that expansion is copied whole since it may - * hold more than the arguments. Returns the ')' from the replacement list, - * the parameter that supplied the arguments, or nullptr when there is no - * call. */ + * Parameters are substituted before the rescan, so the parentheses are + * counted in the substituted tokens: parameters that expand to nothing are + * skipped, and the '(' and the ')' of the call can each come from the + * replacement list or from a parameter. Returns the replacement-list token + * that supplied the ')', or nullptr when there is no complete call. When + * that token is a parameter, its expansion is copied whole since it may + * hold more than the end of the call. */ const Token *gatherCallAfter(TokenList &call, const Token *nameTok, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, OutputList *outputList) const { TokenList callArgs(files); - const Token *last = nullptr; - for (const Token *after = tok->next; sameline(tok, after); after = after->next) { - if (after->op == '(') { - last = appendTokens(callArgs, loc, after, macros, expandedmacros, parametertokens, outputList); - break; + unsigned int par = 0; + for (const Token *after = tok->next; sameline(tok, after);) { + if (par > 0U && after->op == '#' && sameline(after, after->next) && (after->next->op != '#' || sameline(after, after->next->next))) { + // # and ## keep the parentheses balanced + if (after->next->op == '#') + after = expandHashHash(callArgs, loc, after, macros, expandedmacros, parametertokens, outputList, false); + else + after = expandHash(callArgs, loc, after, expandedmacros, parametertokens, outputList); + continue; } - if (!expandArg(callArgs, after, loc, macros, expandedmacros, parametertokens, outputList)) - break; - // a parameter that expands to nothing is substituted before the rescan, - // so it does not separate the macro name from its '(' - if (callArgs.cfront()) { - if (callArgs.cfront()->op == '(') - last = after; - break; + const Token * const end = callArgs.cback(); + if (!expandArg(callArgs, after, loc, macros, expandedmacros, parametertokens, outputList)) { + // only parameters, which can expand to nothing, can come between + // the macro name and its '(' + if (par == 0U && after->op != '(') + return nullptr; + callArgs.push_back(new Token(*after)); + if (after->macro.empty() && par > 0U) + callArgs.back()->macro = name(); } + for (const Token *tok2 = end ? end->next : callArgs.cfront(); tok2; tok2 = tok2->next) { + if (tok2->op == '(') { + ++par; + } else if (par == 0U) { + // the parameter does not start with '(' + return nullptr; + } else if (tok2->op == ')' && --par == 0U) { + // the tokens of the call must be on the same line + for (Token *tok3 = callArgs.front(); tok3 != tok2->next; tok3 = tok3->next) + tok3->location = callArgs.cfront()->location; + call.push_back(new Token(nameTok->str(), callArgs.cfront()->location)); + call.takeTokens(callArgs); + return after; + } + } + after = after->next; } - if (!last) - return nullptr; - call.push_back(new Token(nameTok->str(), callArgs.cfront()->location)); - call.takeTokens(callArgs); - return last; + return nullptr; } /** Expand while the expansion result in @p temp ends with the name of a @@ -2300,7 +2318,7 @@ namespace simplecpp { output.takeTokens(temp); output.deleteToken(output.back()); if (last->name) { - // the arguments came from a parameter that may hold more than the call + // the call ended in a parameter that may hold more than the call temp.takeTokens(call); rescanArg(temp, loc, macros, expandedmacros, outputList); } else { diff --git a/test.cpp b/test.cpp index be2017bd..2f26d77c 100644 --- a/test.cpp +++ b/test.cpp @@ -1158,6 +1158,27 @@ static void define28() ASSERT_EQUALS(std::string(32, '\n') + "REPEAT_INDIRECT ( ) ( 1 , M , ~ ) M ( 1 , ~ )\n" "0 1 2", preprocess(code6)); + + // the parentheses of a call are counted after the parameters are + // substituted, so its '(' and its ')' can each come from a parameter. The + // tokens that a parameter holds after the ')' are rescanned with the + // tokens that follow. + const char code7[] = "#define LP (\n" + "#define RP )\n" + "#define F(x) [x]\n" + "#define G(x) \n" + "#define A(f, x) f x 1)\n" + "#define B(x) F(1 x 2)\n" + "#define C(x) F(1 x (2))\n" + "A(F, (LP)) B(RP) B(RP LP) C(RP G)\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "[ ( ) 1 ] [ 1 ] 2 ) [ 1 ] ( 2 ) [ 1 ] < 2 > )", preprocess(code7)); } From 08c7c439b4afc09f43a82593614de35a9a7676ae Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 7 Nov 2026 21:46:37 -0600 Subject: [PATCH 5/5] Fix nesting --- simplecpp.cpp | 65 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4661e869..1e4f2d95 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2209,19 +2209,29 @@ namespace simplecpp { return callEnd->next; } - /** Returns the macro when the last token of @p temp is the name of a - * function-like macro that the rescan can still expand; nullptr otherwise */ - static const Macro *trailingMacro(const TokenList &temp, const MacroMap ¯os, const std::set &expandedmacros) { - if (!temp.cback() || !temp.cback()->name || temp.cback()->skippedByRescan) + /** Returns the macro when @p tok is the name of a function-like macro + * that the rescan can still expand; nullptr otherwise */ + static const Macro *callableMacro(const Token *tok, const MacroMap ¯os, const std::set &expandedmacros) { + if (!tok || !tok->name || tok->skippedByRescan) return nullptr; - const MacroMap::const_iterator it = macros.find(temp.cback()->str()); - if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) + const MacroMap::const_iterator it = macros.find(tok->str()); + if (it == macros.end() || expandedmacros.find(tok->str()) != expandedmacros.end()) return nullptr; - if (!it->second.functionLike() || temp.cback()->isExpandedFrom(&it->second)) + if (!it->second.functionLike() || tok->isExpandedFrom(&it->second)) return nullptr; return &it->second; } + /** Mark @p tok when it is the name of a macro that is being expanded, so + * that it is never expanded later */ + static void markExpanding(Token *tok, const MacroMap ¯os, const std::set &expandedmacros) { + if (!tok->name || expandedmacros.find(tok->str()) == expandedmacros.end()) + return; + const MacroMap::const_iterator it = macros.find(tok->str()); + if (it != macros.end()) + tok->markExpandedFrom(&it->second); + } + /** A macro call that expanded to nothing came after the last token of * @p output: the rescan has passed that token, so when it is the name of * a function-like macro, a '(' that comes later does not start a call. */ @@ -2310,7 +2320,7 @@ namespace simplecpp { * consumed (tok->next when no call was expanded). */ template const Token *expandTrailingCalls(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, OutputList *outputList, GatherCall gatherCall) const { - while (const Macro * const calledMacro = trailingMacro(temp, macros, expandedmacros)) { + while (const Macro * const calledMacro = callableMacro(temp.cback(), macros, expandedmacros)) { TokenList call(files); const Token * const last = gatherCall(call, temp.cback(), tok); if (!last) @@ -2350,18 +2360,28 @@ namespace simplecpp { * when the macro name and its '(' were separated by tokens that expanded to * nothing, as in "f EMPTY() (1)". The result of each call is rescanned * against the tokens that follow it. Names of macros that are being expanded - * are marked so that they are never expanded later. */ + * are marked so that they are never expanded later. The marks that the + * pre-expansion left on the tokens do not apply to this rescan. */ void rescanArg(TokenList &tokens, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, OutputList *outputList) const { + // Usually the pre-expansion has left no call behind. Then the tokens are + // only changed in place, which avoids copying them at each nesting level. + bool hasCall = false; + for (Token *tok = tokens.front(); tok; tok = tok->next) { + tok->skippedByRescan = false; + if (hasCall) + continue; + hasCall = sameline(tok, tok->next) && tok->next->op == '(' && callableMacro(tok, macros, expandedmacros) != nullptr; + if (!hasCall) + markExpanding(tok, macros, expandedmacros); + } + if (!hasCall) + return; TokenList input(files); input.takeTokens(tokens); for (const Token *tok = input.cfront(); tok;) { TokenList temp(files); temp.push_back(new Token(*tok)); - if (tok->name && expandedmacros.find(tok->str()) != expandedmacros.end()) { - const MacroMap::const_iterator it = macros.find(tok->str()); - if (it != macros.end()) - temp.back()->markExpandedFrom(&it->second); - } + markExpanding(temp.back(), macros, expandedmacros); tok = expandTrailingCalls(tokens, temp, loc, tok, macros, expandedmacros, outputList, [](TokenList &call, const Token *nameTok, const Token *tok2) { return gatherCall(call, nameTok, tok2); }); @@ -2470,8 +2490,9 @@ namespace simplecpp { /** Append the fully macro-expanded argument for parameter @p tok to @p output. * @p hasMacroName, when given, is set when the argument held a macro name - * (otherwise the output cannot hold one). Returns false when @p tok is not - * a parameter. */ + * (otherwise the output cannot hold one); the caller then passes the output + * to rescanArg, which clears the marks that the pre-expansion left on it. + * Returns false when @p tok is not a parameter. */ bool expandArg(TokenList &output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, OutputList *outputList, bool *hasMacroName = nullptr) const { if (!tok->name) return false; @@ -2515,12 +2536,14 @@ namespace simplecpp { if (tok->whitespaceahead && output.back()) output.back()->whitespaceahead = true; if (macroName) { - // the substituted tokens are rescanned again, so the marks that the - // expansions above left on them do not apply - for (Token *tok2 = outputEnd ? outputEnd->next : output.front(); tok2; tok2 = tok2->next) - tok2->skippedByRescan = false; - if (hasMacroName) + if (hasMacroName) { *hasMacroName = true; + } else { + // the substituted tokens are rescanned again, so the marks that the + // expansions above left on them do not apply + for (Token *tok2 = outputEnd ? outputEnd->next : output.front(); tok2; tok2 = tok2->next) + tok2->skippedByRescan = false; + } } return true; }