From eca9122eb746025943e7beb2cf927140a0391175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 16:26:10 +0100 Subject: [PATCH 01/28] Add regression tests for macro expansion C11 6.10.3.5 examples 3, 4, 5 and 7, the C23 __VA_OPT__ examples and invalid ## pastes. Known bugs are tested with the new TODO_ASSERT_EQUALS. Tests for Token::macro, the locations of expanded tokens and the macro usage, which are used by cppcheck. A new helper expandedTokens() writes each output token with its macro name and location; the tokenMacro tests use it. Co-Authored-By: Claude Opus 5.5 --- test.cpp | 460 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 398 insertions(+), 62 deletions(-) diff --git a/test.cpp b/test.cpp index a7db4c3c..2a9ebd8b 100644 --- a/test.cpp +++ b/test.cpp @@ -45,6 +45,7 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) #define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while (false) +#define TODO_ASSERT_EQUALS(wanted, current, actual) (todoAssertEquals((wanted), (current), (actual), __LINE__)) static std::string pprint(const std::string &in) { @@ -84,6 +85,23 @@ static int assertEquals(const long long &expected, const long long &actual, int return assertEquals(std::to_string(expected), std::to_string(actual), line); } +/** + * For a known bug: actual must be the current (wrong) value. When the bug is + * fixed and actual is the wanted value, the assertion fails so that it is + * changed to ASSERT_EQUALS. + */ +static void todoAssertEquals(const std::string &wanted, const std::string ¤t, const std::string &actual, int line) +{ + if (wanted == actual) { + numberOfFailedAssertions++; + std::cerr << "------ TODO assertion fixed (" << inputString(USE_INPUT) << ")---------" << std::endl; + std::cerr << "line test.cpp:" << line << std::endl; + std::cerr << "change TODO_ASSERT_EQUALS to ASSERT_EQUALS" << std::endl; + return; + } + assertEquals(current, actual, line); +} + static void assertThrowFailed(int line) { numberOfFailedAssertions++; @@ -267,6 +285,43 @@ static std::string toString(const simplecpp::OutputList &outputList) return ostr.str(); } +/** + * Preprocess the code. Each output token is written as str(line:col), or + * str@macro(line:col) when it comes from a macro expansion. + */ +static std::string expandedTokens(const char code[], const simplecpp::DUI &dui = simplecpp::DUI(), std::list *macroUsage = nullptr) +{ + std::vector files; + simplecpp::FileDataCache cache; + simplecpp::TokenList tokens(files); + const simplecpp::TokenList rawtokens = makeTokenList(code, files); + simplecpp::preprocess(tokens, rawtokens, files, cache, dui, nullptr, macroUsage); + std::string ret; + for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) { + if (!ret.empty()) + ret += ' '; + ret += tok->str(); + if (!tok->macro.empty()) + ret += '@' + tok->macro; + ret += '(' + std::to_string(tok->location.line) + ':' + std::to_string(tok->location.col) + ')'; + } + return ret; +} + +/** Each macro usage as "name line:col use line:col", with " unknown" if the value is not known */ +static std::string toString(const std::list ¯oUsage) +{ + std::string ret; + for (const simplecpp::MacroUsage &mu : macroUsage) { + ret += mu.macroName + ' ' + std::to_string(mu.macroLocation.line) + ':' + std::to_string(mu.macroLocation.col) + + " use " + std::to_string(mu.useLocation.line) + ':' + std::to_string(mu.useLocation.col); + if (!mu.macroValueKnown) + ret += " unknown"; + ret += '\n'; + } + return ret; +} + static void backslash() { // preprocessed differently @@ -2243,6 +2298,231 @@ static void hashhash_punctuator_invalid() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '+' and 'x' yields an invalid token.\n", toString(outputList)); } +static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 +{ + const char code[] = + "#define x 3\n" + "#define f(a) f(x * (a))\n" + "#undef x\n" + "#define x 2\n" + "#define g f\n" + "#define z z[0]\n" + "#define h g(~\n" + "#define m(a) a(w)\n" + "#define w 0,1\n" + "#define t(a) a\n" + "#define p() int\n" + "#define q(x) x\n" + "#define r(x,y) x ## y\n" + "#define str(x) # x\n" + "f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);\n" + "g(x+(3,4)-w) | h 5) & m\n" + "(f)^m(m);\n" + "p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };\n" + "char c[2][6] = { str(hello), str() };\n"; + // TODO: r(,) leaves an empty token + TODO_ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + preprocess(code)); +} + +static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 +{ + const char code[] = + "#define str(s) # s\n" + "#define xstr(s) str(s)\n" + "#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n" + " x ## s, x ## t)\n" + "#define INCFILE(n) vers ## n\n" + "#define glue(a, b) a ## b\n" + "#define xglue(a, b) glue(a, b)\n" + "#define HIGHLOW \"hello\"\n" + "#define LOW LOW \", world\"\n" + "debug(1, 2);\n" + "fputs(str(strncmp(\"abc\\0d\", \"abc\", '\\4') // this goes away\n" + " == 0) str(: @\\n), s);\n" + "xstr(INCFILE(2).h)\n" + "glue(HIGH, LOW);\n" + "xglue(HIGH, LOW)\n"; + // the comments are removed before preprocessing, like cppcheck and the simplecpp program do + simplecpp::DUI dui; + dui.removeComments = true; + // TODO: the space before == is lost after the comment and newline in the # argument + TODO_ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" + "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0\"\n" + "\": @\\n\" , s ) ;\n" + "\"vers2.h\"\n" + "\"hello\" ;\n" + "\"hello\" \", world\"", + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" + "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') ==0\"\n" + "\": @\\n\" , s ) ;\n" + "\"vers2.h\"\n" + "\"hello\" ;\n" + "\"hello\" \", world\"", + preprocess(code, dui)); +} + +static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 +{ + const char code[] = + "#define t(x,y,z) x ## y ## z\n" + "int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),\n" + " t(10,,), t(,11,), t(,,12), t(,,) };\n"; + // TODO: t(,,) leaves an empty token + TODO_ASSERT_EQUALS("\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + "\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + preprocess(code)); +} + +static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 +{ + const char code[] = + "#define debug(...) fprintf(stderr, __VA_ARGS__)\n" + "#define showlist(...) puts(#__VA_ARGS__)\n" + "#define report(test, ...) ((test)?puts(#test):\\\n" + " printf(__VA_ARGS__))\n" + "debug(\"Flag\");\n" + "debug(\"X = %d\\n\", x);\n" + "showlist(The first, second, and third items.);\n" + "report(x>y, \"x is %d but y is %d\", x, y);\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "fprintf ( stderr , \"Flag\" ) ;\n" + "fprintf ( stderr , \"X = %d\\n\" , x ) ;\n" + "puts ( \"The first, second, and third items.\" ) ;\n" + "( ( x > y ) ? puts ( \"x>y\" ) : printf ( \"x is %d but y is %d\" , x , y ) ) ;", + preprocess(code)); +} + +static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE +{ + const char code[] = + "#define F(...) f(0 __VA_OPT__(,) __VA_ARGS__)\n" + "#define G(X, ...) f(0, X __VA_OPT__(,) __VA_ARGS__)\n" + "#define SDEF(sname, ...) S sname __VA_OPT__(= { __VA_ARGS__ })\n" + "#define EMP\n" + "F(a,b,c)\n" + "F()\n" + "F(EMP)\n" + "G(a,b,c)\n" + "G(a,)\n" + "G(a)\n" + "SDEF(foo);\n" + "SDEF(bar, 1, 2);\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "f ( 0 , a , b , c )\n" + "f ( 0 )\n" + "f ( 0 )\n" + "f ( 0 , a , b , c )\n" + "f ( 0 , a )\n" + "f ( 0 , a )\n" + "S foo ;\n" + "S bar = { 1 , 2 } ;", + preprocess(code)); +} + +static void hashhash_empty_placemarker() +{ + const char code[] = + "#define r(x,y) x ## y\n" + "{ r(,) }\n"; + std::vector files; + simplecpp::FileDataCache cache; + simplecpp::TokenList tokens(files); + const simplecpp::TokenList rawtokens = makeTokenList(code, files); + simplecpp::preprocess(tokens, rawtokens, files, cache, simplecpp::DUI()); + std::string str; + for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) + str += "[" + tok->str() + "]"; + // TODO: pasting two empty arguments gives an empty token + TODO_ASSERT_EQUALS("[{][}]", "[{][][}]", str); +} + +static void hashhash_invalid_dotdot() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(.,.)\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '.' and '.' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_invalid_name_string() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(a,\"s\")\n"; + simplecpp::OutputList outputList; + // TODO: a"s" is not a valid token, gcc reports an error + TODO_ASSERT_EQUALS("", "\na\"s\"", preprocess(code, &outputList)); + TODO_ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining 'a' and '\"s\"' yields an invalid token.\n", "", toString(outputList)); +} + static void hashhash_va_args_unexpected() { const char code[] = @@ -4284,36 +4564,19 @@ static void stringify1() ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); } +// Token::macro, the locations of expanded tokens and the macro usage are used by cppcheck static void tokenMacro1() { const char code[] = "#define A 123\n" "A"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - ASSERT_EQUALS("A", tokenList.cback()->macro); + ASSERT_EQUALS("123@A(2:1)", expandedTokens(code)); } static void tokenMacro2() { const char code[] = "#define ADD(X,Y) X+Y\n" "ADD(1,2)"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("", tok->macro); - tok = tok->next; - ASSERT_EQUALS("+", tok->str()); - ASSERT_EQUALS("ADD", tok->macro); - tok = tok->next; - ASSERT_EQUALS("2", tok->str()); - ASSERT_EQUALS("", tok->macro); + ASSERT_EQUALS("1(2:1) +@ADD(2:1) 2(2:1)", expandedTokens(code)); } static void tokenMacro3() @@ -4321,20 +4584,9 @@ static void tokenMacro3() const char code[] = "#define ADD(X,Y) X+Y\n" "#define FRED 1\n" "ADD(FRED,2)"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("FRED", tok->macro); - tok = tok->next; - ASSERT_EQUALS("+", tok->str()); - ASSERT_EQUALS("ADD", tok->macro); - tok = tok->next; - ASSERT_EQUALS("2", tok->str()); - ASSERT_EQUALS("", tok->macro); + std::list macroUsage; + ASSERT_EQUALS("1@FRED(3:1) +@ADD(3:1) 2(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("FRED 2:9 use 3:1\nADD 1:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro4() @@ -4342,14 +4594,9 @@ static void tokenMacro4() const char code[] = "#define A B\n" "#define B 1\n" "A"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token * const tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("A", tok->macro); + std::list macroUsage; + ASSERT_EQUALS("1@A(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 3:1\nB 2:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro5() @@ -4357,14 +4604,7 @@ static void tokenMacro5() const char code[] = "#define SET_BPF(code) (code)\n" "#define SET_BPF_JUMP(code) SET_BPF(D | code)\n" "SET_BPF_JUMP(A | B | C);"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token * const tok = tokenList.cfront()->next; - ASSERT_EQUALS("D", tok->str()); - ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); + ASSERT_EQUALS("(@SET_BPF(3:1) D@SET_BPF_JUMP(3:1) |@SET_BPF_JUMP(3:1) A(3:1) |(3:1) B(3:1) |(3:1) C(3:1) )@SET_BPF(3:1) ;(3:24)", expandedTokens(code)); } static void tokenMacro6() @@ -4375,21 +4615,100 @@ static void tokenMacro6() "FILE()\n" "LINE()\n" "COUNTER()\n"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok; + ASSERT_EQUALS("\"\"@__FILE__(4:1) 5@__LINE__(5:1) 0@__COUNTER__(6:1)", expandedTokens(code)); +} - tok = tokenList.cfront(); - ASSERT_EQUALS("__FILE__", tok->macro); +static void tokenMacro7() +{ + const char code[] = "#define A 1+2\n" + "x A y\n"; + std::list macroUsage; + ASSERT_EQUALS("x(2:1) 1@A(2:3) +@A(2:3) 2@A(2:3) y(2:5)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 2:3\n", toString(macroUsage)); +} - tok = tok->next; - ASSERT_EQUALS("__LINE__", tok->macro); +static void tokenMacro8() +{ + // the call g(2) is completed with tokens after the expansion of f. + // The tokens get the location of the ( and the macro g + const char code[] = "#define f g\n" + "#define g(x) x+1\n" + "f(2)\n"; + std::list macroUsage; + ASSERT_EQUALS("2(3:2) +@g(3:2) 1@g(3:2)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("g 2:9 use 3:2\nf 1:9 use 3:1\n", toString(macroUsage)); +} - tok = tok->next; - ASSERT_EQUALS("__COUNTER__", tok->macro); +static void tokenMacro9() +{ + // the arguments of g( come after the expansion of f + const char code[] = "#define g(x) x+1\n" + "#define f g(\n" + "f 2)\n"; + std::list macroUsage; + ASSERT_EQUALS("2(3:3) +@g(3:3) 1@g(3:3)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("g 1:9 use 3:3\nf 2:9 use 3:1\n", toString(macroUsage)); +} + +static void tokenMacro10() +{ + // the results of # and ## have no macro name + const char code[] = "#define str(x) #x\n" + "#define cat(a,b) a##b\n" + "str(a b) cat(x,y)\n"; + std::list macroUsage; + ASSERT_EQUALS("\"a b\"(3:1) xy(3:10)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("str 1:9 use 3:1\ncat 2:9 use 3:10\n", toString(macroUsage)); +} + +static void tokenMacro11() +{ + const char code[] = "#define F(...) f(__VA_ARGS__)\n" + "F(1,2)\n"; + std::list macroUsage; + ASSERT_EQUALS("f@F(2:1) (@F(2:1) 1(2:1) ,(2:1) 2(2:1) )@F(2:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("F 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro12() +{ + const char code[] = "#define f f\n" + "f\n"; + std::list macroUsage; + ASSERT_EQUALS("f@f(2:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("f 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro13() +{ + // all tokens of the expansion get the location of the macro name + const char code[] = "#define ADD(X,Y) X+Y\n" + "ADD(1,\n" + " 2)\n" + "z\n"; + std::list macroUsage; + ASSERT_EQUALS("1(2:1) +@ADD(2:1) 2(2:1) z(4:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("ADD 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro14() +{ + // A is used in the argument and in the body, so it is listed twice + const char code[] = "#define A 1\n" + "#define B(x) x A\n" + "B(A)\n"; + std::list macroUsage; + ASSERT_EQUALS("1@A(3:1) 1@A(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 3:1\nA 1:9 use 3:1\nB 2:9 use 3:1\n", toString(macroUsage)); +} + +static void tokenMacro15() +{ + simplecpp::DUI dui; + dui.defines.emplace_back("D=1+2"); + std::list macroUsage; + ASSERT_EQUALS("x(1:1) 1@D(1:3) +@D(1:3) 2@D(1:3)", expandedTokens("x D\n", dui, ¯oUsage)); + ASSERT_EQUALS("D 1:1 use 1:3 unknown\n", toString(macroUsage)); } static void undef() @@ -5191,6 +5510,11 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(define6); TEST_CASE(define7); TEST_CASE(define8); + TEST_CASE(macro_c11_example3); + TEST_CASE(macro_c11_example4); + TEST_CASE(macro_c11_example5); + TEST_CASE(macro_c11_example7); + TEST_CASE(macro_c23_va_opt_examples); TEST_CASE(define9); TEST_CASE(define10); TEST_CASE(define11); @@ -5314,6 +5638,9 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash_empty_va_args); TEST_CASE(hashhash_punctuator); TEST_CASE(hashhash_punctuator_invalid); + TEST_CASE(hashhash_empty_placemarker); + TEST_CASE(hashhash_invalid_dotdot); + TEST_CASE(hashhash_invalid_name_string); TEST_CASE(hashhash_va_args_unexpected); // C standard, 5.1.1.2, paragraph 4: @@ -5445,6 +5772,15 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(tokenMacro4); TEST_CASE(tokenMacro5); TEST_CASE(tokenMacro6); + TEST_CASE(tokenMacro7); + TEST_CASE(tokenMacro8); + TEST_CASE(tokenMacro9); + TEST_CASE(tokenMacro10); + TEST_CASE(tokenMacro11); + TEST_CASE(tokenMacro12); + TEST_CASE(tokenMacro13); + TEST_CASE(tokenMacro14); + TEST_CASE(tokenMacro15); TEST_CASE(undef); From 131c265c2b9257818f6d977e4261d59e04acb396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:07:52 +0100 Subject: [PATCH 02/28] Move the ## checks out of Macro::expandHashHash() checkHashHashOperands(), checkPunctuatorPaste() and checkUniversalCharacterPaste() only depend on the tokens, not on the expansion state, so a new expander can use them too. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 81 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a6adddad..3ab37279 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2679,20 +2679,9 @@ namespace simplecpp { throw invalidHashHash::unexpectedNewline(tok->location, name()); const bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); - // A is a punctuator; the result is checked when B has been expanded - const bool punctuatorA = (!A->name && !A->number && !A->str().empty() && !canBeConcatenatedStringOrChar); const Token * const B = tok->next->next; - if (!punctuatorA) { - if (!B->name && !B->number && B->op && !B->isOneOf("#=")) - throw invalidHashHash::unexpectedToken(tok->location, name(), B); - if (B->op == '=') - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); - } - - // Superficial check; more in-depth would in theory be possible _after_ expandArg - if (canBeConcatenatedStringOrChar && (B->number || !B->name)) - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); + checkHashHashOperands(tok->location, name(), A, B); TokenList tokensB(files); const Token *nextTok = B->next; @@ -2721,24 +2710,16 @@ namespace simplecpp { strAB = ","; } else { - if (punctuatorA && !isValidPunctuatorPaste(A->str() + tokensB.cfront()->str())) - throw invalidHashHash::cannotCombine(tok->location, name(), A, tokensB.cfront()); + checkPunctuatorPaste(tok->location, name(), A, tokensB.cfront()); strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); } } else { - if (punctuatorA && !isValidPunctuatorPaste(A->str() + B->str())) - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); + checkPunctuatorPaste(tok->location, name(), A, B); strAB = A->str() + B->str(); } - // producing universal character is undefined behavior - if (A->previous && A->previous->str() == "\\") { - if (strAB[0] == 'u' && strAB.size() == 5) - throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); - if (strAB[0] == 'U' && strAB.size() == 9) - throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); - } + checkUniversalCharacterPaste(tok->location, name(), A, strAB); if (varargs && tokensB.empty() && tok->previous->str() == ",") { output.deleteToken(A); @@ -2780,6 +2761,60 @@ namespace simplecpp { return nextTok; } + /** A is a punctuator, not a name, number or literal. Pasting it is checked after B is expanded. */ + static bool isPunctuatorOperand(const Token *A) { + return !A->name && !A->number && !A->str().empty() && !isStringLiteral(A->str()) && !isCharLiteral(A->str()); + } + + /** + * Check the operands of A ## B before B is expanded. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ##, already expanded + * @param B the token after ## in the macro definition + * @throws invalidHashHash if A and B can not be pasted + */ + static void checkHashHashOperands(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { + if (!isPunctuatorOperand(A)) { + if (!B->name && !B->number && B->op && !B->isOneOf("#=")) + throw invalidHashHash::unexpectedToken(loc, macroName, B); + if (B->op == '=') + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + // Superficial check; more in-depth would in theory be possible _after_ expandArg + if ((isStringLiteral(A->str()) || isCharLiteral(A->str())) && (B->number || !B->name)) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + /** + * When A is a punctuator, check that pasting A and B gives a valid token. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ## + * @param B the first token after ##, after expansion + * @throws invalidHashHash if the result is not a valid token + */ + static void checkPunctuatorPaste(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { + if (isPunctuatorOperand(A) && !isValidPunctuatorPaste(A->str() + B->str())) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + /** + * Producing a universal character name with ## is undefined behavior. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ## + * @param strAB the pasted text + * @throws invalidHashHash if A follows a backslash and strAB is a universal character name + */ + static void checkUniversalCharacterPaste(const Location &loc, const std::string ¯oName, const Token *A, const std::string &strAB) { + if (!A->previous || A->previous->str() != "\\") + return; + if ((strAB[0] == 'u' && strAB.size() == 5) || (strAB[0] == 'U' && strAB.size() == 9)) + throw invalidHashHash::universalCharacterUB(loc, macroName, A, strAB); + } + static bool isReplaced(const std::set &expandedmacros) { // return true if size > 1 auto it = expandedmacros.cbegin(); From 3dbca5f2f460424b5eac1c48b2b1e6998cbd07bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:14:36 +0100 Subject: [PATCH 03/28] ##: pasting a name or number with a string or character literal is an error Only an encoding prefix such as L, u8 or R gives a valid token, as in L ## "s". Before, cat(a,"s") gave the invalid token a"s". Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 13 +++++++++---- test.cpp | 28 +++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3ab37279..5a25d283 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2710,12 +2710,12 @@ namespace simplecpp { strAB = ","; } else { - checkPunctuatorPaste(tok->location, name(), A, tokensB.cfront()); + checkPaste(tok->location, name(), A, tokensB.cfront()); strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); } } else { - checkPunctuatorPaste(tok->location, name(), A, B); + checkPaste(tok->location, name(), A, B); strAB = A->str() + B->str(); } @@ -2788,16 +2788,21 @@ namespace simplecpp { } /** - * When A is a punctuator, check that pasting A and B gives a valid token. + * Check that pasting A and B gives a valid token, when A is a punctuator + * or B is a string or character literal. * @param loc the location of the ## * @param macroName the macro that is expanded * @param A the token before ## * @param B the first token after ##, after expansion * @throws invalidHashHash if the result is not a valid token */ - static void checkPunctuatorPaste(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { + static void checkPaste(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { if (isPunctuatorOperand(A) && !isValidPunctuatorPaste(A->str() + B->str())) throw invalidHashHash::cannotCombine(loc, macroName, A, B); + // a name or number followed by a literal: only an encoding prefix such as L or u8 is valid + if ((A->name || A->number) && (isStringLiteral(B->str()) || isCharLiteral(B->str())) && + !(A->name && isStringLiteralPrefix(A->str()))) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); } /** diff --git a/test.cpp b/test.cpp index 2a9ebd8b..1e1484a8 100644 --- a/test.cpp +++ b/test.cpp @@ -2518,9 +2518,29 @@ static void hashhash_invalid_name_string() "#define cat(a,b) a##b\n" "cat(a,\"s\")\n"; simplecpp::OutputList outputList; - // TODO: a"s" is not a valid token, gcc reports an error - TODO_ASSERT_EQUALS("", "\na\"s\"", preprocess(code, &outputList)); - TODO_ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining 'a' and '\"s\"' yields an invalid token.\n", "", toString(outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining 'a' and '\"s\"' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_invalid_number_string() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(1,\"s\")\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '1' and '\"s\"' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_encoding_prefix() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "#define WIDE(s) L ## s\n" + "cat(L,\"s\") cat(u8,\"s\") cat(R,\"(x)\") cat(L,'c') WIDE(\"s\")\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n\nL\"s\" u8\"s\" R\"(x)\" L'c' L\"s\"", preprocess(code, &outputList)); + ASSERT_EQUALS("", toString(outputList)); } static void hashhash_va_args_unexpected() @@ -5641,6 +5661,8 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash_empty_placemarker); TEST_CASE(hashhash_invalid_dotdot); TEST_CASE(hashhash_invalid_name_string); + TEST_CASE(hashhash_invalid_number_string); + TEST_CASE(hashhash_encoding_prefix); TEST_CASE(hashhash_va_args_unexpected); // C standard, 5.1.1.2, paragraph 4: From 6a576ded5323bca794601f6ba7b07a3f370f5d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:23:48 +0100 Subject: [PATCH 04/28] Move the # string building out of Macro::expandHash() stringify() only depends on the tokens, so a new expander can use it too. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 58 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5a25d283..c8cc893b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -733,6 +733,41 @@ static std::string stringifyToken(const std::string &str) return ret; } +/** + * The result of the # operator: the tokens as a string literal (C 6.10.3.2). + * The tokens are separated by a space where there was whitespace between them. + * @param tokens the argument of #, not macro expanded + * @param loc location for the warning + * @param macroName the macro that is expanded, for the warning + * @param outputList if not null, a warning is added when a final backslash is dropped + * @return the string literal, including the quotes + */ +static std::string stringify(const simplecpp::TokenList &tokens, const simplecpp::Location &loc, const std::string ¯oName, simplecpp::OutputList *outputList) +{ + std::string str; + for (const simplecpp::Token *tok = tokens.cfront(), *next; tok; tok = next) { + next = tok->next; + str += stringifyToken(tok->str()); + if (next && tok->whitespaceahead) + str += ' '; + } + // An odd number of trailing backslashes would escape the closing quote and + // yield an unterminated string literal (undefined behavior, C 6.10.3.2p2). + // Drop the final backslash, as gcc and clang do. + std::size_t trailing = 0; + while (trailing < str.size() && str[str.size() - 1U - trailing] == '\\') + ++trailing; + if (trailing % 2U == 1U) { + str.pop_back(); + if (outputList) { + outputList->emplace_back(simplecpp::Output::PORTABILITY_STRINGIFY, loc, + "Invalid string literal when expanding \'" + macroName + "\', ignoring final \'\\\'. " + "Undefined behavior according to C standard chapter 6.10.3.2, paragraph 2."); + } + } + return '\"' + str + '\"'; +} + void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location) { if (fileIndex_ != location.fileIndex || line >= location.line) { @@ -2634,28 +2669,7 @@ namespace simplecpp { TokenList tokenListHash(files); const MacroMap macros2; // temporarily bypass macro expansion tok = expandToken(tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens, outputList); - std::string str; - for (const Token *hashtok = tokenListHash.cfront(), *next; hashtok; hashtok = next) { - next = hashtok->next; - str += stringifyToken(hashtok->str()); - if (next && hashtok->whitespaceahead) - str += ' '; - } - // An odd number of trailing backslashes would escape the closing quote and - // yield an unterminated string literal (undefined behavior, C 6.10.3.2p2). - // Drop the final backslash, as gcc and clang do. - std::size_t trailing = 0; - while (trailing < str.size() && str[str.size() - 1U - trailing] == '\\') - ++trailing; - if (trailing % 2U == 1U) { - str.pop_back(); - if (outputList) { - outputList->emplace_back(Output::PORTABILITY_STRINGIFY, loc, - "Invalid string literal when expanding \'" + name() + "\', ignoring final \'\\\'. " - "Undefined behavior according to C standard chapter 6.10.3.2, paragraph 2."); - } - } - output.push_back(newMacroToken('\"' + str + '\"', loc, isReplaced(expandedmacros))); + output.push_back(newMacroToken(stringify(tokenListHash, loc, name(), outputList), loc, isReplaced(expandedmacros))); return tok; } From 191359daaf067efd3cfd98b1ea3b3b838d63dcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:30:36 +0100 Subject: [PATCH 05/28] Add combineWithNext() for the token joins in TokenList::combineOperators() No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c8cc893b..aedff0b3 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1352,6 +1352,13 @@ static bool isAnonymousReferenceParameter(const simplecpp::Token *tok) return false; } +/** Append the next token to tok, and delete the next token */ +static void combineWithNext(simplecpp::TokenList &tokens, simplecpp::Token *tok) +{ + tok->setstr(tok->str() + tok->next->str()); + tokens.deleteToken(tok->next); +} + void simplecpp::TokenList::combineOperators() { std::stack> executableScope{{false}}; @@ -1363,24 +1370,19 @@ void simplecpp::TokenList::combineOperators() // ellipsis ... if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) && tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) { - tok->setstr("..."); - deleteToken(tok->next); - deleteToken(tok->next); + combineWithNext(*this, tok); + combineWithNext(*this, tok); continue; } // float literals.. if (tok->previous && tok->previous->number && sameline(tok->previous, tok) && tok->previous->str().find_first_of("._") == std::string::npos) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); - if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp") && !isAlternativeAndBitandBitor(tok->next)))) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); - } - } - if (tok->next && tok->next->number) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp") && !isAlternativeAndBitandBitor(tok->next)))) + combineWithNext(*this, tok); } + if (tok->next && tok->next->number) + combineWithNext(*this, tok); } // match: [0-9.]+E [+-] [0-9]+ const char lastChar = tok->str()[tok->str().size() - 1]; @@ -1388,9 +1390,8 @@ void simplecpp::TokenList::combineOperators() ((!isHex(tok->str()) && (lastChar == 'E' || lastChar == 'e')) || (isHex(tok->str()) && (lastChar == 'P' || lastChar == 'p'))) && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { - tok->setstr(tok->str() + tok->next->op + tok->next->next->str()); - deleteToken(tok->next); - deleteToken(tok->next); + combineWithNext(*this, tok); + combineWithNext(*this, tok); } if (tok->op == '\0' || !tok->next || tok->next->op == '\0') @@ -1405,23 +1406,17 @@ void simplecpp::TokenList::combineOperators() // void f(x&=2) if (tok->op == '&' && !executableScope.top() && isAnonymousReferenceParameter(tok)) continue; - tok->setstr(tok->str() + "="); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if ((tok->op == '|' || tok->op == '&') && tok->op == tok->next->op) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if (tok->op == ':' && tok->next->op == ':') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if (tok->op == '-' && tok->next->op == '>') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if ((tok->op == '<' || tok->op == '>') && tok->op == tok->next->op) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); if (tok->next && tok->next->op == '=' && tok->next->next && tok->next->next->op != '=') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } } else if ((tok->op == '+' || tok->op == '-') && tok->op == tok->next->op) { if (tok->location.col + 1U != tok->next->location.col) @@ -1430,8 +1425,7 @@ void simplecpp::TokenList::combineOperators() continue; if (tok->next->next && tok->next->next->number) continue; - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } } } From 6a9acd6d56ae44a0ef17b12640813601bfbde08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:37:06 +0100 Subject: [PATCH 06/28] #: keep the space after an operator of two characters combineOperators() did not copy whitespaceahead from the second character, so str(a == 0) gave "a ==0" (C11 6.10.3.5 EXAMPLE 4). Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 3 ++- test.cpp | 57 ++++++++++++++++++++++----------------------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index aedff0b3..e9e0078b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1352,10 +1352,11 @@ static bool isAnonymousReferenceParameter(const simplecpp::Token *tok) return false; } -/** Append the next token to tok, and delete the next token */ +/** Append the next token to tok, and delete the next token. tok gets the whitespace after the next token. */ static void combineWithNext(simplecpp::TokenList &tokens, simplecpp::Token *tok) { tok->setstr(tok->str() + tok->next->str()); + tok->whitespaceahead = tok->next->whitespaceahead; tokens.deleteToken(tok->next); } diff --git a/test.cpp b/test.cpp index 1e1484a8..455f8619 100644 --- a/test.cpp +++ b/test.cpp @@ -1867,6 +1867,14 @@ static void hashMacroTrailingWhitespace() "\"a b\"", preprocess(code, dui)); } +static void hashCombinedOperator() +{ + // the whitespace after an operator of two characters is kept + const char code[] = "#define str(x) #x\n" + "str(a == 0) str(a -> b) str(x... y)\n"; + ASSERT_EQUALS("\n\"a == 0\" \"a -> b\" \"x... y\"", preprocess(code)); +} + static void hashEscapeTrailingBackslash() { // an odd number of trailing backslashes would escape the closing quote, @@ -2383,38 +2391,22 @@ static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 // the comments are removed before preprocessing, like cppcheck and the simplecpp program do simplecpp::DUI dui; dui.removeComments = true; - // TODO: the space before == is lost after the comment and newline in the # argument - TODO_ASSERT_EQUALS("\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" - "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0\"\n" - "\": @\\n\" , s ) ;\n" - "\"vers2.h\"\n" - "\"hello\" ;\n" - "\"hello\" \", world\"", - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" - "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') ==0\"\n" - "\": @\\n\" , s ) ;\n" - "\"vers2.h\"\n" - "\"hello\" ;\n" - "\"hello\" \", world\"", - preprocess(code, dui)); + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" + "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0\"\n" + "\": @\\n\" , s ) ;\n" + "\"vers2.h\"\n" + "\"hello\" ;\n" + "\"hello\" \", world\"", + preprocess(code, dui)); } static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 @@ -5625,6 +5617,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashEscape); TEST_CASE(hashComment); TEST_CASE(hashMacroTrailingWhitespace); + TEST_CASE(hashCombinedOperator); TEST_CASE(hashEscapeTrailingBackslash); TEST_CASE(hashhash1); TEST_CASE(hashhash2); From 92b451a1083d6e9d66fae5cc56f5d61269c09d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 09:55:20 +0100 Subject: [PATCH 07/28] Move the argument count check into Macro::getMacroParameters() It now returns the argument delimiters of a valid call, or throws wrongNumberOfParameters, so a new expander can use it. Tests are added for the argument count. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 49 ++++++++++++++++++++++++++++++------------------- test.cpp | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e9e0078b..dcd7abd6 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2199,10 +2199,33 @@ namespace simplecpp { return ~0U; } + /** + * Find the arguments of a call of this macro. + * @param nameTokInst the macro name in the call + * @param calledInDefine the call is in a #define, so the arguments must be on the same line + * @return the tokens that delimit the arguments: the '(', each ',' between two arguments and the ')'. + * Empty when the macro is not function-like or the name is not followed by '('. + * @throws wrongNumberOfParameters if a function-like macro has the wrong number of arguments + */ std::vector getMacroParameters(const Token *nameTokInst, bool calledInDefine) const { - if (!nameTokInst->next || nameTokInst->next->op != '(' || !functionLike()) + if (!functionLike()) return {}; + std::vector parametertokens; + if (nameTokInst->next && nameTokInst->next->op == '(') + parametertokens = findArgumentDelimiters(nameTokInst, calledInDefine); + + if (variadic) { + if (parametertokens.size() < args.size()) + throw wrongNumberOfParameters(nameTokInst->location, name()); + } else { + if (parametertokens.size() != args.size() + (args.empty() ? 2U : 1U)) + throw wrongNumberOfParameters(nameTokInst->location, name()); + } + return parametertokens; + } + /** The '(', each ',' between two arguments and the ')' of the call. nameTokInst is followed by '('. */ + std::vector findArgumentDelimiters(const Token *nameTokInst, bool calledInDefine) const { std::vector parametertokens; parametertokens.emplace_back(nameTokInst->next); unsigned int par = 0U; @@ -2291,26 +2314,14 @@ namespace simplecpp { const bool calledInDefine = (loc.fileIndex != nameTokInst->location.fileIndex || loc.line < nameTokInst->location.line); - std::vector parametertokens1(getMacroParameters(nameTokInst, calledInDefine)); - - if (functionLike()) { - // No arguments => not macro expansion - if (nameTokInst->next && nameTokInst->next->op != '(') { - output.push_back(new Token(nameTokInst->str(), loc)); - return nameTokInst->next; - } - - // Parse macro-call - if (variadic) { - if (parametertokens1.size() < args.size()) { - throw wrongNumberOfParameters(nameTokInst->location, name()); - } - } else { - if (parametertokens1.size() != args.size() + (args.empty() ? 2U : 1U)) - throw wrongNumberOfParameters(nameTokInst->location, name()); - } + // No arguments => not macro expansion + if (functionLike() && nameTokInst->next && nameTokInst->next->op != '(') { + output.push_back(new Token(nameTokInst->str(), loc)); + return nameTokInst->next; } + std::vector parametertokens1(getMacroParameters(nameTokInst, calledInDefine)); + // If macro call uses __COUNTER__ then expand that first TokenList tokensparams(files); std::vector parametertokens2; diff --git a/test.cpp b/test.cpp index 455f8619..6bf34b22 100644 --- a/test.cpp +++ b/test.cpp @@ -1154,6 +1154,42 @@ static void define_invalid_3() ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); } +static void define_wrong_number_of_arguments() +{ + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b) a+b\nf(1)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b) a+b\nf(1,2,3)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b,...) a+b\nf(1)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n1", preprocess("#define f(a,...) a __VA_ARGS__\nf(1)\n", &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // a function-like macro name that is not followed by ( is not expanded + simplecpp::OutputList outputList; + ASSERT_EQUALS("\nf ;", preprocess("#define f(a) a\nf;\n", &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // TODO: f takes no arguments, gcc reports an error + simplecpp::OutputList outputList; + TODO_ASSERT_EQUALS("", "\n1", preprocess("#define f() 1\nf(1)\n", &outputList)); + TODO_ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", "", toString(outputList)); + } +} + static void define_invalid_4() { const char code[] = "#define X(...)\n" @@ -5550,6 +5586,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(define_invalid_2); TEST_CASE(define_invalid_3); TEST_CASE(define_invalid_4); + TEST_CASE(define_wrong_number_of_arguments); TEST_CASE(define_define_1); TEST_CASE(define_define_2); TEST_CASE(define_define_3); From df3b4c0ceda38c9b5eccdda789ca346f97467119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:04:16 +0100 Subject: [PATCH 08/28] Add Macro::expandBuiltin() for __FILE__, __LINE__ and __COUNTER__ No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index dcd7abd6..c693d0f0 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2286,6 +2286,29 @@ namespace simplecpp { return sameline(lpar,tok) ? tok : nullptr; } + /** + * Expand the built-in macros __FILE__, __LINE__ and __COUNTER__. + * The usage must already be added to usageList, __COUNTER__ is the number of earlier usages. + * @param output destination tokenlist + * @param loc location of the usage + * @param nameTokInst the macro name + * @return true if the macro is built-in and its value is added to output + */ + bool expandBuiltin(TokenList &output, const Location &loc, const Token *nameTokInst) const { + std::string value; + if (nameTokInst->str() == "__FILE__") + value = '\"' + output.file(loc) + '\"'; + else if (nameTokInst->str() == "__LINE__") + value = toString(loc.line); + else if (nameTokInst->str() == "__COUNTER__") + value = toString(usageList.size() - 1U); + else + return false; + output.push_back(new Token(value, loc)); + output.back()->macro = nameTokInst->str(); + return true; + } + const Token * expand(TokenList & output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros, OutputList *outputList) const { expandedmacros.insert(nameTokInst->str()); @@ -2295,21 +2318,8 @@ namespace simplecpp { usageList.emplace_back(loc); - if (nameTokInst->str() == "__FILE__") { - output.push_back(new Token('\"'+output.file(loc)+'\"', loc)); - output.back()->macro = "__FILE__"; - return nameTokInst->next; - } - if (nameTokInst->str() == "__LINE__") { - output.push_back(new Token(toString(loc.line), loc)); - output.back()->macro = "__LINE__"; + if (expandBuiltin(output, loc, nameTokInst)) return nameTokInst->next; - } - if (nameTokInst->str() == "__COUNTER__") { - output.push_back(new Token(toString(usageList.size()-1U), loc)); - output.back()->macro = "__COUNTER__"; - return nameTokInst->next; - } const bool calledInDefine = (loc.fileIndex != nameTokInst->location.fileIndex || loc.line < nameTokInst->location.line); From 52fb0925ac422c037612356a606413d401fddc8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:04:31 +0100 Subject: [PATCH 09/28] Tests: a function-like macro name at the end of the file should not be an error gcc outputs the name, simplecpp reports wrong number of parameters. Co-Authored-By: Claude Opus 5.5 --- test.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test.cpp b/test.cpp index 6bf34b22..ad0354d5 100644 --- a/test.cpp +++ b/test.cpp @@ -1150,8 +1150,9 @@ static void define_invalid_3() const char code[] = "#define R()\n" "R\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); + // TODO: a function-like macro name at the end of the file is not a macro call, gcc outputs R + TODO_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); + TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); } static void define_wrong_number_of_arguments() @@ -1195,8 +1196,9 @@ static void define_invalid_4() const char code[] = "#define X(...)\n" "X\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); + // TODO: a function-like macro name at the end of the file is not a macro call, gcc outputs X + TODO_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); + TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); } static void define_define_1() From 953f255375f34e343859fec13f6b8279bd1d3bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:11:20 +0100 Subject: [PATCH 10/28] Add Macro::expandDefined() for defined in a replacement list No behaviour change. A test is added for defined at the end of the file. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 75 +++++++++++++++++++++++++++++++-------------------- test.cpp | 11 ++++++++ 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c693d0f0..ef357cde 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2570,35 +2570,9 @@ namespace simplecpp { return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens, outputList); } - if (tok->str() == DEFINED) { - const Token * const tok2 = tok->next; - const Token * const tok3 = tok2 ? tok2->next : nullptr; - const Token * const tok4 = tok3 ? tok3->next : nullptr; - const Token *defToken = nullptr; - const Token *lastToken = nullptr; - if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { - defToken = tok3; - lastToken = tok4; - } else if (sameline(tok,tok2) && tok2->name) { - defToken = lastToken = tok2; - } - if (defToken) { - std::string macroName = defToken->str(); - if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { - TokenList temp(files); - if (expandArg(temp, defToken, parametertokens)) - macroName = temp.cback()->str(); - if (expandArg(temp, defToken->next->next->next, parametertokens)) - macroName += temp.cback() ? temp.cback()->str() : ""; - else - macroName += defToken->next->next->next->str(); - lastToken = defToken->next->next->next; - } - const bool def = (macros.find(macroName) != macros.end()); - output.push_back(newMacroToken(def ? "1" : "0", loc, true)); - return lastToken->next; - } - } + const Token *next = nullptr; + if (tok->str() == DEFINED && expandDefined(output, loc, tok, macros, parametertokens, next)) + return next; output.push_back(newMacroToken(tok->str(), loc, true, tok)); if (it != macros.end()) @@ -2606,6 +2580,49 @@ namespace simplecpp { return tok->next; } + /** + * Expand "defined X" or "defined(X)" in the replacement list to 1 or 0. + * The operand can be a parameter, or two names pasted with ##. + * @param output destination tokenlist + * @param loc location for the result + * @param tok the "defined" token + * @param macros all macros + * @param parametertokens arguments of this macro call + * @param next output: the token after the operand + * @return false if there is no operand, then nothing is added to output + */ + bool expandDefined(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::vector ¶metertokens, const Token *&next) const { + const Token * const tok2 = tok->next; + const Token * const tok3 = tok2 ? tok2->next : nullptr; + const Token * const tok4 = tok3 ? tok3->next : nullptr; + const Token *defToken = nullptr; + const Token *lastToken = nullptr; + if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { + defToken = tok3; + lastToken = tok4; + } else if (sameline(tok,tok2) && tok2->name) { + defToken = lastToken = tok2; + } + if (defToken) { + std::string macroName = defToken->str(); + if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { + TokenList temp(files); + if (expandArg(temp, defToken, parametertokens)) + macroName = temp.cback()->str(); + if (expandArg(temp, defToken->next->next->next, parametertokens)) + macroName += temp.cback() ? temp.cback()->str() : ""; + else + macroName += defToken->next->next->next->str(); + lastToken = defToken->next->next->next; + } + const bool def = (macros.find(macroName) != macros.end()); + output.push_back(newMacroToken(def ? "1" : "0", loc, true)); + next = lastToken->next; + return true; + } + return false; + } + bool expandArg(TokenList &output, const Token *tok, const std::vector ¶metertokens) const { if (!tok->name) return false; diff --git a/test.cpp b/test.cpp index ad0354d5..64ca3f7c 100644 --- a/test.cpp +++ b/test.cpp @@ -3005,6 +3005,16 @@ static void ifDefinedHashHash2() ASSERT_EQUALS("\n0", preprocess(code, &outputList)); } +static void definedAtEndOfFile() +{ + // "defined" from a macro expansion is undefined behavior; gcc outputs "defined" here. + // The operand is the last token of the file. + const char code[] = "#define X\n" + "#define D defined X\n" + "D"; + ASSERT_EQUALS("\n\n1", preprocess(code)); +} + static void ifLogical() { const char code[] = "#if defined(A) || defined(B)\n" @@ -5736,6 +5746,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(ifDefinedInvalid2); TEST_CASE(ifDefinedHashHash); TEST_CASE(ifDefinedHashHash2); + TEST_CASE(definedAtEndOfFile); TEST_CASE(ifLogical); TEST_CASE(ifLogicalPrecedence); TEST_CASE(ifSizeof); From d3c9a695d15d4d6e038e266e30f54a32f7ddcee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:17:26 +0100 Subject: [PATCH 11/28] Add Macro::copyMacroCall() for copying a macro call to one line No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ef357cde..39d8c35a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1913,6 +1913,34 @@ namespace simplecpp { return valueDefinedInCode_; } + /** + * Copy a macro call "NAME( ... )" to a tokenlist with no line breaks: + * all tokens get the location of the name. + * @param dest destination tokenlist + * @param nameTok the macro name, followed by '(' + * @return the token after the ')', or nullptr if the call is not terminated + * @throws invalidDirectiveAsMacroParameter if there is a preprocessor directive in the arguments + */ + static const Token *copyMacroCall(TokenList &dest, const Token *nameTok) { + const Location &loc = nameTok->location; + dest.push_back(new Token(nameTok->str(), loc, nameTok->whitespaceahead)); + const Token *tok = nameTok->next; + dest.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + tok = tok->next; + int par = 1; + while (tok && par > 0) { + if (tok->op == '(') + ++par; + else if (tok->op == ')') + --par; + else if (tok->op == '#' && !sameline(tok->previous, tok)) + throw invalidDirectiveAsMacroParameter(tok->location); + dest.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + tok = tok->next; + } + return tok; + } + /** * Expand macro. This will recursively expand inner macros. * @param output destination tokenlist @@ -1939,24 +1967,9 @@ namespace simplecpp { TokenList output2(inputFiles); if (functionLike() && rawtok->next && rawtok->next->op == '(') { - // Copy macro call to a new tokenlist with no linebreaks const Token * const rawtok1 = rawtok; TokenList rawtokens2(inputFiles); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - int par = 1; - while (rawtok && par > 0) { - if (rawtok->op == '(') - ++par; - else if (rawtok->op == ')') - --par; - else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) - throw invalidDirectiveAsMacroParameter(rawtok->location); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - } + rawtok = copyMacroCall(rawtokens2, rawtok); if (expand(output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros, outputList)) rawtok = rawtok1->next; } else { From 3bd22f4ea620dd08413a6e8f921351822e920426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:24:07 +0100 Subject: [PATCH 12/28] Add setLocation() for setting the location of all tokens in a list No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 39d8c35a..e45f28cc 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1845,6 +1845,13 @@ namespace simplecpp { class Macro; using MacroMap = std::unordered_map; + /** Set the location of all tokens */ + static void setLocation(TokenList &tokens, const Location &loc) + { + for (Token *tok = tokens.front(); tok; tok = tok->next) + tok->location = loc; + } + class Macro { public: explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), valueDefinedInCode_(false) {} @@ -2294,8 +2301,7 @@ namespace simplecpp { tok = tok->next; } } - for (Token *tok2 = tokens.front(); tok2; tok2 = tok2->next) - tok2->location = lpar->location; + setLocation(tokens, lpar->location); return sameline(lpar,tok) ? tok : nullptr; } @@ -2750,8 +2756,7 @@ namespace simplecpp { // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. // TODO The question is whether the ## or varargs may still apply, and how to provoke? if (expandArg(tokensB, B, parametertokens)) { - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); } else { tokensB.push_back(new Token(*B)); tokensB.back()->location = loc; @@ -2786,8 +2791,7 @@ namespace simplecpp { } else if (strAB != "," && macros.find(strAB) == macros.end()) { A->setstr(strAB); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); output.takeTokens(tokensB); } else if (sameline(B, nextTok) && sameline(B, nextTok->next) && nextTok->op == '#' && nextTok->next->op == '#') { TokenList output2(files); @@ -2812,8 +2816,7 @@ namespace simplecpp { expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens, outputList); else output.takeTokens(tokens); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); output.takeTokens(tokensB); } } From 306faa9ab5736e39bea76c8d389f2f0e9e55ac66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 10:48:13 +0100 Subject: [PATCH 13/28] Token::setExpandedFrom() only updates the expanded-from set The whitespace is copied in Macro::newMacroToken() instead. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 13 +++++++++++-- simplecpp.h | 3 +-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e45f28cc..728450be 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2105,13 +2105,22 @@ namespace simplecpp { } }; private: - /** Create new token where Token::macro is set for replaced tokens */ + /** + * Create a new token in the expansion of this macro. + * @param str the token text + * @param loc the location + * @param replaced Token::macro is set to the name of this macro + * @param expandedFromToken if not null, the new token copies its "expanded from" set and + * its whitespace, and this macro is added to the set + */ Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=nullptr) const { auto *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); - if (expandedFromToken) + if (expandedFromToken) { tok->setExpandedFrom(expandedFromToken, this); + tok->whitespaceahead = expandedFromToken->whitespaceahead; + } return tok; } diff --git a/simplecpp.h b/simplecpp.h index 2b560d27..d0bc1a1b 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -210,11 +210,10 @@ namespace simplecpp { return tok; } + /** Copy the "expanded from" set of tok, and add m */ void setExpandedFrom(const Token *tok, const Macro* m) { mExpandedFrom = tok->mExpandedFrom; mExpandedFrom.insert(m); - if (tok->whitespaceahead) - whitespaceahead = true; } bool isExpandedFrom(const Macro* m) const { return mExpandedFrom.find(m) != mExpandedFrom.end(); From 2677821546b95396a96f7d1022440ffbbc194443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 11:36:28 +0100 Subject: [PATCH 14/28] Rename Token::mExpandedFrom to mHideSet setExpandedFrom(), markExpandedFrom() and isExpandedFrom() are renamed to setHideSet(), addToHideSet() and isInHideSet(). No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 14 +++++++------- simplecpp.h | 23 +++++++++++++---------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 728450be..bc42b3e4 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2006,7 +2006,7 @@ namespace simplecpp { if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; const MacroMap::const_iterator macro = macros.find(macro2tok->str()); - if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isExpandedFrom(¯o->second)) + if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isInHideSet(¯o->second)) break; TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); @@ -2110,15 +2110,15 @@ namespace simplecpp { * @param str the token text * @param loc the location * @param replaced Token::macro is set to the name of this macro - * @param expandedFromToken if not null, the new token copies its "expanded from" set and - * its whitespace, and this macro is added to the set + * @param expandedFromToken if not null, the new token copies its hide set and + * its whitespace, and this macro is added to the hide set */ Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=nullptr) const { auto *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); if (expandedFromToken) { - tok->setExpandedFrom(expandedFromToken, this); + tok->setHideSet(expandedFromToken, this); tok->whitespaceahead = expandedFromToken->whitespaceahead; } return tok; @@ -2508,7 +2508,7 @@ namespace simplecpp { const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) return nullptr; - if (!it->second.functionLike() || temp.cback()->isExpandedFrom(&it->second)) + if (!it->second.functionLike() || temp.cback()->isInHideSet(&it->second)) return nullptr; return &it->second; } @@ -2604,7 +2604,7 @@ namespace simplecpp { output.push_back(newMacroToken(tok->str(), loc, true, tok)); if (it != macros.end()) - output.back()->markExpandedFrom(&it->second); + output.back()->addToHideSet(&it->second); return tok->next; } @@ -2680,7 +2680,7 @@ namespace simplecpp { const Token * const argEnd = parametertokens[argnr + 1U]; for (const Token *partok = parametertokens[argnr]->next; partok != argEnd;) { const MacroMap::const_iterator it = macros.find(partok->str()); - if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { + if (it != macros.end() && !partok->isInHideSet(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation expandedmacros2.erase(name()); TokenList temp(files); diff --git a/simplecpp.h b/simplecpp.h index d0bc1a1b..ca48b1e7 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -165,7 +165,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mHideSet(tok.mHideSet) {} Token &operator=(const Token &tok) = delete; @@ -210,16 +210,19 @@ namespace simplecpp { return tok; } - /** Copy the "expanded from" set of tok, and add m */ - void setExpandedFrom(const Token *tok, const Macro* m) { - mExpandedFrom = tok->mExpandedFrom; - mExpandedFrom.insert(m); + /** + * The hide set holds the macros that this token must not be expanded into. + * Set it to the hide set of tok, with m added. + */ + void setHideSet(const Token *tok, const Macro* m) { + mHideSet = tok->mHideSet; + mHideSet.insert(m); } - bool isExpandedFrom(const Macro* m) const { - return mExpandedFrom.find(m) != mExpandedFrom.end(); + bool isInHideSet(const Macro* m) const { + return mHideSet.find(m) != mHideSet.end(); } - void markExpandedFrom(const Macro* m) { - mExpandedFrom.insert(m); + void addToHideSet(const Macro* m) { + mHideSet.insert(m); } void printAll() const; @@ -235,7 +238,7 @@ namespace simplecpp { TokenString string; - std::set mExpandedFrom; + std::set mHideSet; }; /** Output from preprocessor */ From f20321968bba43cbeead0b610c1121ab8effaf5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 11:43:15 +0100 Subject: [PATCH 15/28] The hide set of a token holds macro names instead of Macro pointers The names are what Prosser's algorithm uses, and they can not dangle after #undef. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 10 +++++----- simplecpp.h | 18 ++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index bc42b3e4..cc7708a2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2006,7 +2006,7 @@ namespace simplecpp { if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; const MacroMap::const_iterator macro = macros.find(macro2tok->str()); - if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isInHideSet(¯o->second)) + if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isInHideSet(macro->first)) break; TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); @@ -2118,7 +2118,7 @@ namespace simplecpp { if (replaced) tok->macro = nameTokDef->str(); if (expandedFromToken) { - tok->setHideSet(expandedFromToken, this); + tok->setHideSet(expandedFromToken, name()); tok->whitespaceahead = expandedFromToken->whitespaceahead; } return tok; @@ -2508,7 +2508,7 @@ namespace simplecpp { const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) return nullptr; - if (!it->second.functionLike() || temp.cback()->isInHideSet(&it->second)) + if (!it->second.functionLike() || temp.cback()->isInHideSet(it->first)) return nullptr; return &it->second; } @@ -2604,7 +2604,7 @@ namespace simplecpp { output.push_back(newMacroToken(tok->str(), loc, true, tok)); if (it != macros.end()) - output.back()->addToHideSet(&it->second); + output.back()->addToHideSet(it->first); return tok->next; } @@ -2680,7 +2680,7 @@ namespace simplecpp { const Token * const argEnd = parametertokens[argnr + 1U]; for (const Token *partok = parametertokens[argnr]->next; partok != argEnd;) { const MacroMap::const_iterator it = macros.find(partok->str()); - if (it != macros.end() && !partok->isInHideSet(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { + if (it != macros.end() && !partok->isInHideSet(it->first) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation expandedmacros2.erase(name()); TokenList temp(files); diff --git a/simplecpp.h b/simplecpp.h index ca48b1e7..c7ce660c 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -117,8 +117,6 @@ namespace simplecpp { }; #endif // defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) - class Macro; - /** * Location in source code */ @@ -212,17 +210,17 @@ namespace simplecpp { /** * The hide set holds the macros that this token must not be expanded into. - * Set it to the hide set of tok, with m added. + * Set it to the hide set of tok, with the macro macroName added. */ - void setHideSet(const Token *tok, const Macro* m) { + void setHideSet(const Token *tok, const TokenString ¯oName) { mHideSet = tok->mHideSet; - mHideSet.insert(m); + mHideSet.insert(macroName); } - bool isInHideSet(const Macro* m) const { - return mHideSet.find(m) != mHideSet.end(); + bool isInHideSet(const TokenString ¯oName) const { + return mHideSet.find(macroName) != mHideSet.end(); } - void addToHideSet(const Macro* m) { - mHideSet.insert(m); + void addToHideSet(const TokenString ¯oName) { + mHideSet.insert(macroName); } void printAll() const; @@ -238,7 +236,7 @@ namespace simplecpp { TokenString string; - std::set mHideSet; + std::set mHideSet; }; /** Output from preprocessor */ From 6604e52a46f91a47bd719aec667d898120c0baf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 16:26:10 +0100 Subject: [PATCH 16/28] Add HideSetExpander, macro expansion with hide sets DUI::hideSetExpansion (simplecpp option -hideset-expansion) selects it. It implements Prosser's algorithm for object-like and function-like macros: arguments are expanded once, and the result of an expansion is pushed back onto the input, so a call whose arguments follow in the source is found by normal rescanning. testrunner runs all tests with both macro expansions; HIDESET_EXPANSION_TODO() skips a test that the new one can not handle yet. The macro usage in the tests is sorted, its order depends on the standard library. Co-Authored-By: Claude Opus 5.5 --- main.cpp | 7 + simplecpp.cpp | 206 ++++++++++++++++++++++++- simplecpp.h | 7 + test.cpp | 413 ++++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 584 insertions(+), 49 deletions(-) diff --git a/main.cpp b/main.cpp index ec0fc5c7..1aef98db 100644 --- a/main.cpp +++ b/main.cpp @@ -153,6 +153,12 @@ int main(int argc, char **argv) found = true; } break; + case 'h': + if (std::strcmp(arg, "-hideset-expansion")==0) { + dui.hideSetExpansion = true; + found = true; + } + break; } if (!found) { std::cout << "error: option '" << arg << "' is unknown." << std::endl; @@ -187,6 +193,7 @@ int main(int argc, char **argv) std::cout << " -e Output errors only." << std::endl; std::cout << " -f Fail when errors were encountered (exitcode 1)." << std::endl; std::cout << " -l Print lines numbers." << std::endl; + std::cout << " -hideset-expansion Use the new macro expansion with hide sets (work in progress)." << std::endl; return 0; } diff --git a/simplecpp.cpp b/simplecpp.cpp index cc7708a2..b7629bce 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -27,12 +27,15 @@ #include #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -1852,7 +1855,10 @@ namespace simplecpp { tok->location = loc; } + class HideSetExpander; + class Macro { + friend class HideSetExpander; public: explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), valueDefinedInCode_(false) {} @@ -2937,6 +2943,199 @@ namespace simplecpp { }; } +namespace simplecpp { + /** + * Macro expansion with hide sets (Prosser's algorithm, used by DUI::hideSetExpansion). + * Each token has a hide set: the macros that it must not be expanded into. + * The result of an expansion is pushed back onto the input and scanned again, so a + * function-like macro whose arguments follow the expansion in the source is found by + * normal rescanning. + * Work in progress: #, ##, __VA_OPT__ and "defined" in a replacement list are not handled yet. + */ + class HideSetExpander { + public: + HideSetExpander(const MacroMap ¯os, std::vector &files) + : macros(macros), files(files) {} + + HideSetExpander(const HideSetExpander&) = delete; + HideSetExpander &operator=(const HideSetExpander&) = delete; + + /** + * Expand the macro at rawtok. + * @param output destination tokenlist + * @param rawtok the macro name in the source + * @return the first source token after the expansion + * @throws Macro::Error if the macro can not be expanded + */ + const Token *expand(TokenList &output, const Token *rawtok) { + loc = rawtok->location; + Input input(rawtok->next); + input.pending.push_back(new Token(*rawtok)); + expandInput(output, input); + return input.raw; + } + + private: + /** The input of an expansion: tokens to scan again, then the source */ + struct Input { + explicit Input(const Token *raw) : raw(raw) {} + Input(const Input&) = delete; + Input &operator=(const Input&) = delete; + ~Input() { + for (const Token * const tok : pending) + delete tok; + } + + /** The next token, read from the source if there are no pending tokens; nullptr at the end */ + Token *read() { + if (!pending.empty()) { + std::unique_ptr tok(pending.front()); + pending.pop_front(); + return tok.release(); + } + if (!raw) + return nullptr; + if (raw->op == '#' && !sameline(raw->previous, raw)) + throw Macro::invalidDirectiveAsMacroParameter(raw->location); + const Token * const tok = raw; + raw = raw->next; + return new Token(*tok); + } + + /** The next token, without reading it */ + const Token *peek() const { + return pending.empty() ? raw : pending.front(); + } + + /** Put tokens in front of the pending tokens */ + void pushFront(TokenList &tokens) { + std::deque front; + while (tokens.front()) { + front.push_back(new Token(*tokens.front())); + tokens.deleteToken(tokens.front()); + } + pending.insert(pending.begin(), front.begin(), front.end()); + } + + /** tokens to scan again, the next one is at the front */ + std::deque pending; + /** next token in the source, it is only read for the arguments of a macro call */ + const Token *raw; + }; + + static Macro::Error unsupported(const Location &loc, const std::string &what) { + return {loc, what + " is not supported by the hide-set expansion yet"}; + } + + /** Expand the pending tokens of the input. Source tokens are only read for macro arguments. */ + void expandInput(TokenList &output, Input &input) { + while (!input.pending.empty()) { + Token * const tok = input.read(); + const MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); + if (it == macros.end() || tok->isInHideSet(it->first)) { + output.push_back(tok); + continue; + } + const Macro ¯o = it->second; + + if (macro.functionLike()) { + const Token * const next = input.peek(); + if (!next || next->op != '(') { + output.push_back(tok); + continue; + } + } + + macro.usageList.emplace_back(loc); + if (macro.expandBuiltin(output, loc, tok)) { + delete tok; + continue; + } + + std::set hideSet; + std::vector arguments; + if (macro.functionLike()) { + TokenList call(files); + call.push_back(tok); + readCall(call, input); + const std::vector delimiters = macro.getMacroParameters(call.cfront(), false); + // the hide set of the name and the ) have the macro calls that this call + // was produced by, the arguments are not affected by them + const std::set &hs1 = call.cfront()->hideSet(); + const std::set &hs2 = delimiters.back()->hideSet(); + std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(hideSet, hideSet.begin())); + arguments = expandArguments(macro, delimiters); + } else { + hideSet = tok->hideSet(); + delete tok; + } + hideSet.insert(macro.name()); + + TokenList result(files); + substitute(result, macro, arguments, hideSet); + input.pushFront(result); + } + } + + /** Read "( ... )" after the macro name in call */ + static void readCall(TokenList &call, Input &input) { + int par = 0; + while (Token * const tok = input.read()) { + call.push_back(tok); + if (tok->op == '(') + ++par; + else if (tok->op == ')' && --par == 0) + return; + } + } + + /** Each argument of the call, fully macro expanded */ + std::vector expandArguments(const Macro ¯o, const std::vector &delimiters) { + std::vector arguments; + for (std::size_t i = 0; i < macro.args.size(); ++i) { + arguments.emplace_back(files); + if (i + 1U >= delimiters.size()) + continue; // empty variadic argument + Input input(nullptr); + for (const Token *tok = delimiters[i]->next; tok != delimiters[i + 1U]; tok = tok->next) + input.pending.push_back(new Token(*tok)); + expandInput(arguments.back(), input); + } + return arguments; + } + + /** The replacement list of the macro, with the parameters replaced by the arguments */ + void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::set &hideSet) const { + if (macro.variadicOpt) + throw unsupported(loc, "__VA_OPT__"); + for (const Token *tok = macro.valueToken; tok != macro.endToken; tok = tok->next) { + if (tok->op == '#') + throw unsupported(loc, "# and ##"); + if (tok->str() == DEFINED) + throw unsupported(loc, "defined"); + const unsigned int argnr = tok->name ? macro.getArgNum(tok->str()) : ~0U; + if (argnr < arguments.size()) { + for (const Token *argtok = arguments[argnr].cfront(); argtok; argtok = argtok->next) + result.push_back(new Token(*argtok)); + } else { + result.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + result.back()->macro = macro.name(); + } + } + for (Token *tok = result.front(); tok; tok = tok->next) { + tok->location = loc; + for (const TokenString &name : hideSet) + tok->addToHideSet(name); + } + } + + const MacroMap ¯os; + std::vector &files; + /** location of the macro name in the source, all tokens of the expansion get it */ + Location loc; + }; +} + namespace simplecpp { #ifdef __CYGWIN__ @@ -3865,7 +4064,12 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token if (it != macros.end()) { simplecpp::TokenList value(files); try { - tok1 = it->second.expand(value, tok, macros, files, outputList); + if (dui.hideSetExpansion) { + simplecpp::HideSetExpander expander(macros, files); + tok1 = expander.expand(value, tok); + } else { + tok1 = it->second.expand(value, tok, macros, files, outputList); + } } catch (const simplecpp::Macro::invalidDirectiveAsMacroParameter &err) { if (outputList) { simplecpp::Output out{ diff --git a/simplecpp.h b/simplecpp.h index c7ce660c..d4f83d2f 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -222,6 +222,12 @@ namespace simplecpp { void addToHideSet(const TokenString ¯oName) { mHideSet.insert(macroName); } + const std::set &hideSet() const { + return mHideSet; + } + void setHideSet(std::set hideSet) { + mHideSet = std::move(hideSet); + } void printAll() const; void printOut() const; @@ -277,6 +283,7 @@ namespace simplecpp { std::string std; bool clearIncludeCache{}; bool removeComments{}; /** remove comment tokens from included files */ + bool hideSetExpansion{}; /** use the new macro expansion with hide sets (work in progress) */ }; /** List of tokens. */ diff --git a/test.cpp b/test.cpp index 64ca3f7c..ce37fd01 100644 --- a/test.cpp +++ b/test.cpp @@ -5,6 +5,8 @@ #include "simplecpp.h" +#include + #include #include #include @@ -41,7 +43,13 @@ namespace { } static Input USE_INPUT = Input::Stringstream; +/** the tests are run with the old macro expansion and with the new one with hide sets */ +static bool USE_HIDESET_EXPANSION = false; static int numberOfFailedAssertions = 0; +static int numberOfHideSetTodos = 0; + +/** Skip the rest of the test when it is run with the new macro expansion, which can not handle it yet */ +#define HIDESET_EXPANSION_TODO() do { if (USE_HIDESET_EXPANSION) { ++numberOfHideSetTodos; return; } } while (false) #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) #define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while (false) @@ -58,12 +66,13 @@ static std::string pprint(const std::string &in) return ret; } -static const char* inputString(Input input) { +static std::string inputString(Input input) { + const std::string expansion = USE_HIDESET_EXPANSION ? ", hide-set expansion" : ""; switch (input) { case Input::Stringstream: - return "Stringstream"; + return "Stringstream" + expansion; case Input::CharBuffer: - return "CharBuffer"; + return "CharBuffer" + expansion; } return ""; // unreachable - needed for GCC and Visual Studio } @@ -125,6 +134,14 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) +/** simplecpp::preprocess() with the macro expansion selected by USE_HIDESET_EXPANSION */ +static void runPreprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, simplecpp::FileDataCache &cache, const simplecpp::DUI &dui, simplecpp::OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr) +{ + simplecpp::DUI dui2(dui); + dui2.hideSetExpansion = USE_HIDESET_EXPANSION; + simplecpp::preprocess(output, rawtokens, files, cache, dui2, outputList, macroUsage, ifCond); +} + static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), const simplecpp::DUI &dui = {}, simplecpp::OutputList *outputList = nullptr) { switch (USE_INPUT) { @@ -184,7 +201,7 @@ static std::string preprocess(const char code[], std::size_t size, const simplec if (dui.removeComments) tokens.removeComments(); simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList, macroUsage, ifCond); + runPreprocess(tokens2, tokens, files, cache, dui, outputList, macroUsage, ifCond); simplecpp::cleanup(cache); return tokens2.stringify(); } @@ -285,17 +302,15 @@ static std::string toString(const simplecpp::OutputList &outputList) return ostr.str(); } -/** - * Preprocess the code. Each output token is written as str(line:col), or - * str@macro(line:col) when it comes from a macro expansion. - */ +// Preprocess the code. Each output token is written as str(line:col), or +// str@macro(line:col) when it comes from a macro expansion. static std::string expandedTokens(const char code[], const simplecpp::DUI &dui = simplecpp::DUI(), std::list *macroUsage = nullptr) { std::vector files; simplecpp::FileDataCache cache; simplecpp::TokenList tokens(files); const simplecpp::TokenList rawtokens = makeTokenList(code, files); - simplecpp::preprocess(tokens, rawtokens, files, cache, dui, nullptr, macroUsage); + runPreprocess(tokens, rawtokens, files, cache, dui, nullptr, macroUsage); std::string ret; for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) { if (!ret.empty()) @@ -308,17 +323,24 @@ static std::string expandedTokens(const char code[], const simplecpp::DUI &dui = return ret; } -/** Each macro usage as "name line:col use line:col", with " unknown" if the value is not known */ +/** + * Each macro usage as "name line:col use line:col", with " unknown" if the value is not known. + * The lines are sorted, the order of the macro usage depends on the standard library. + */ static std::string toString(const std::list ¯oUsage) { - std::string ret; + std::vector lines; for (const simplecpp::MacroUsage &mu : macroUsage) { - ret += mu.macroName + ' ' + std::to_string(mu.macroLocation.line) + ':' + std::to_string(mu.macroLocation.col) + - " use " + std::to_string(mu.useLocation.line) + ':' + std::to_string(mu.useLocation.col); + std::string line = mu.macroName + ' ' + std::to_string(mu.macroLocation.line) + ':' + std::to_string(mu.macroLocation.col) + + " use " + std::to_string(mu.useLocation.line) + ':' + std::to_string(mu.useLocation.col); if (!mu.macroValueKnown) - ret += " unknown"; - ret += '\n'; + line += " unknown"; + lines.push_back(std::move(line)); } + std::sort(lines.begin(), lines.end()); + std::string ret; + for (const std::string &line : lines) + ret += line + '\n'; return ret; } @@ -865,6 +887,7 @@ static void define13() static void define14() // #296 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define bar(x) x % 2\n" "#define foo(x) printf(#x \"\\n\")\n" "\n" @@ -877,6 +900,7 @@ static void define14() // #296 static void define15() // #231 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CAT(a, b) CAT2(a, b)\n" "#define CAT2(a, b) a ## b\n" "#define FOO x\n" @@ -893,6 +917,7 @@ static void define15() // #231 static void define16() // #201 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define ALL_COLORS(warm_colors) \\\n" " X(Blue) \\\n" " X(Green) \\\n" @@ -929,6 +954,7 @@ static void define16() // #201 static void define17() // #185 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define at(x, y) x##y\n" "#define b(...) \\\n" "aa(__VA_ARGS__, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , \\\n" @@ -957,6 +983,7 @@ static void define17() // #185 static void define18() // #130 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define MAC2STR(x) x[0],x[1],x[2],x[3],x[4],x[5]\n" "#define FT_DEBUG(fmt, args...) if(pGlobalCtx && pGlobalCtx->debug_level>=2) printf(\"FT-dbg: \"fmt, ##args)\n" "\n" @@ -969,6 +996,7 @@ static void define18() // #130 static void define19() // #124 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(tok) tok##suffix\n" "\n" "CONCAT(Test);\n" @@ -993,6 +1021,7 @@ static void define20() // #113 static void define21() // #66 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "FIGHT_FOO(1, GETMYID(a));\n"; @@ -1003,6 +1032,7 @@ static void define21() // #66 static void define22() // #40 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define COUNTER_NAME(NAME, ...) NAME##Count\n" "#define COMMA ,\n" "\n" @@ -1150,9 +1180,15 @@ static void define_invalid_3() const char code[] = "#define R()\n" "R\n"; simplecpp::OutputList outputList; - // TODO: a function-like macro name at the end of the file is not a macro call, gcc outputs R - TODO_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); - TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); + // a function-like macro name at the end of the file is not a macro call, gcc outputs R + if (USE_HIDESET_EXPANSION) { + ASSERT_EQUALS("\nR", preprocess(code, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } else { + // TODO: the old macro expansion reports an error + TODO_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); + TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); + } } static void define_wrong_number_of_arguments() @@ -1196,9 +1232,15 @@ static void define_invalid_4() const char code[] = "#define X(...)\n" "X\n"; simplecpp::OutputList outputList; - // TODO: a function-like macro name at the end of the file is not a macro call, gcc outputs X - TODO_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); - TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); + // a function-like macro name at the end of the file is not a macro call, gcc outputs X + if (USE_HIDESET_EXPANSION) { + ASSERT_EQUALS("\nX", preprocess(code, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } else { + // TODO: the old macro expansion reports an error + TODO_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); + TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); + } } static void define_define_1() @@ -1219,6 +1261,7 @@ static void define_define_2() static void define_define_3() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define ABC 123\n" "#define A(B) A##B\n" "A(BC)"; @@ -1282,6 +1325,7 @@ static void define_define_9() // line break in nested macro call static void define_define_10() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define glue(a, b) a ## b\n" "#define xglue(a, b) glue(a, b)\n" "#define AB 1\n" @@ -1292,6 +1336,7 @@ static void define_define_10() static void define_define_11() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define XY(x, y) x ## y\n" "#define XY2(x, y) XY(x, y)\n" "#define PORT XY2(P, 2)\n" @@ -1302,6 +1347,7 @@ static void define_define_11() static void define_define_11a() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A_B_C 0x1\n" "#define A_ADDRESS 0x00001000U\n" "#define A ((uint32_t ) A_ADDRESS)\n" @@ -1320,6 +1366,7 @@ static void define_define_11a() static void define_define_12() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define XY(Z) Z\n" "#define X(ID) X##ID(0)\n" "X(Y)\n"; @@ -1336,6 +1383,7 @@ static void define_define_13() // issue #49 - empty macro static void define_define_14() // issue #58 - endless recursion { + HIDESET_EXPANSION_TODO(); // unterminated macro call const char code[] = "#define z f(w\n" "#define f()\n" "#define w f(z\n" @@ -1354,6 +1402,7 @@ static void define_define_15() // issue #72 without __VA_ARGS__ static void define_define_16() // issue #72 with __VA_ARGS__ { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define ab(a, b) a##b\n" "#define foo(...) ab(f, 2) (__VA_ARGS__)\n" "#define f2(x, y) x y\n" @@ -1389,6 +1438,7 @@ static void define_define_19() // #292 static void define_define_20() // #384 arg contains comma { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define Z_IS_ENABLED1(config_macro) Z_IS_ENABLED2(_XXXX##config_macro)\n" "#define _XXXX1 _YYYY,\n" "#define Z_IS_ENABLED2(one_or_two_args) Z_IS_ENABLED3(one_or_two_args 1, 0)\n" @@ -1421,6 +1471,7 @@ static void define_define_21() // #397 DEBRACKET macro static void define_define_22() // #400 inner macro not expanded after hash hash { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define FOO(a) CAT(DO, STUFF)(1,2)\n" "#define DOSTUFF(a, b) CAT(3, 4)\n" "#define CAT(a, b) a##b\n" @@ -1430,6 +1481,7 @@ static void define_define_22() // #400 inner macro not expanded after hash hash static void define_define_23() // #403 crash (infinite recursion) { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define C_(x, y) x ## y\n" "#define C(x, y) C_(x, y)\n" "#define X(func) C(Y, C(func, Z))\n" @@ -1457,6 +1509,7 @@ static void define_va_args_1() static void define_va_args_2() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(X,...) X(#__VA_ARGS__)\n" "A(f,123);"; ASSERT_EQUALS("\nf ( \"123\" ) ;", preprocess(code)); @@ -1471,6 +1524,7 @@ static void define_va_args_3() // min number of arguments static void define_va_args_4() // cppcheck trac #9754 { + HIDESET_EXPANSION_TODO(); // comma before an empty __VA_ARGS__ const char code[] = "#define A(x, y, ...) printf(x, y, __VA_ARGS__)\n" "A(1, 2)\n"; ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); @@ -1478,6 +1532,7 @@ static void define_va_args_4() // cppcheck trac #9754 static void define_va_opt_1() { + HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define p1(fmt, args...) printf(fmt __VA_OPT__(,) args)\n" "p1(\"hello\");\n" "p1(\"%s\", \"hello\");\n"; @@ -1489,6 +1544,7 @@ static void define_va_opt_1() static void define_va_opt_2() { + HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define err(...)\\\n" "__VA_OPT__(\\\n" "printf(__VA_ARGS__);\\\n" @@ -1599,6 +1655,7 @@ static void define_va_opt_7() static void define_va_opt_8() { + HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define f(...) #__VA_OPT__(x)\n" "const char* v1 = f();\n"; @@ -1719,7 +1776,7 @@ static void error4() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtoken = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtoken, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtoken, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1732,7 +1789,7 @@ static void error5() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1745,7 +1802,7 @@ static void error6() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error \n", toString(outputList)); } @@ -1757,7 +1814,7 @@ static void error7() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error blabla\n", toString(outputList)); } @@ -1769,12 +1826,13 @@ static void error8() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error blabla\n", toString(outputList)); } static void garbage() { + HIDESET_EXPANSION_TODO(); // # and ## simplecpp::OutputList outputList; outputList.clear(); @@ -1809,6 +1867,7 @@ static void garbage_endif() static void hash() { + HIDESET_EXPANSION_TODO(); // # and ## ASSERT_EQUALS("x = \"1\"", preprocess("x=#__LINE__")); const char code[] = "#define a(x) #x\n" @@ -1839,6 +1898,7 @@ static void hash() static void hashEscape() { + HIDESET_EXPANSION_TODO(); // # and ## // a \ is only inserted before " and \ inside string literals and character constants const char code[] = "#define str(x) #x\n" "str(\\n)\n" @@ -1858,6 +1918,7 @@ static void hashEscape() static void hashComment() { + HIDESET_EXPANSION_TODO(); // # and ## // a comment is replaced by one space character simplecpp::DUI dui; dui.removeComments = true; @@ -1883,6 +1944,7 @@ static void hashComment() static void hashMacroTrailingWhitespace() { + HIDESET_EXPANSION_TODO(); // # and ## // whitespace at the end of a replacement list does not count simplecpp::DUI dui; dui.removeComments = true; @@ -1907,6 +1969,7 @@ static void hashMacroTrailingWhitespace() static void hashCombinedOperator() { + HIDESET_EXPANSION_TODO(); // # and ## // the whitespace after an operator of two characters is kept const char code[] = "#define str(x) #x\n" "str(a == 0) str(a -> b) str(x... y)\n"; @@ -1915,6 +1978,7 @@ static void hashCombinedOperator() static void hashEscapeTrailingBackslash() { + HIDESET_EXPANSION_TODO(); // # and ## // an odd number of trailing backslashes would escape the closing quote, // the final backslash is dropped (like gcc and clang) and a warning is reported const char msg[] = "Invalid string literal when expanding 'str', ignoring final '\\'. " @@ -1974,6 +2038,7 @@ static void hashEscapeTrailingBackslash() static void hashhash1() // #4703 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define MACRO( A, B, C ) class A##B##C##Creator {};\n" "MACRO( B\t, U , G )"; ASSERT_EQUALS("\nclass BUGCreator { } ;", preprocess(code)); @@ -1981,6 +2046,7 @@ static void hashhash1() // #4703 static void hashhash2() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(x) a##x\n" "#define B 0\n" "A(B)"; @@ -1989,6 +2055,7 @@ static void hashhash2() static void hashhash3() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(B) A##B\n" "#define a(B) A(B)\n" "a(A(B))"; @@ -1997,6 +2064,7 @@ static void hashhash3() static void hashhash4() // nonstandard gcc/clang extension for empty varargs { + HIDESET_EXPANSION_TODO(); // # and ## const char *code; code = "#define A(x,y...) a(x,##y)\n" @@ -2011,6 +2079,7 @@ static void hashhash4() // nonstandard gcc/clang extension for empty varargs static void hashhash4a() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "#define FIGHT_BAR(c, args...) bar(c, ##args)\n" @@ -2026,6 +2095,7 @@ static void hashhash5() static void hashhash6() { + HIDESET_EXPANSION_TODO(); // # and ## const char *code; code = "#define A(X, ...) LOG(X, ##__VA_ARGS__)\n" @@ -2041,6 +2111,7 @@ static void hashhash6() static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) { + HIDESET_EXPANSION_TODO(); // # and ## const char *code; code = "#define hash_hash # ## #\n" @@ -2050,6 +2121,7 @@ static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) static void hashhash8() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define a(xy) x##y = xy\n" "a(123);"; ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); @@ -2057,6 +2129,7 @@ static void hashhash8() static void hashhash9() { + HIDESET_EXPANSION_TODO(); // # and ## const char * code = "#define ADD_OPERATOR(OP) void operator OP ## = (void) { x = x OP 1; }\n" "ADD_OPERATOR(+);\n" "ADD_OPERATOR(-);\n" @@ -2104,6 +2177,7 @@ static void hashhash9() static void hashhash10() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define x # #\n" "x"; ASSERT_EQUALS("# #", preprocess(code)); @@ -2111,6 +2185,7 @@ static void hashhash10() static void hashhash11() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define x # # #\n" "x"; ASSERT_EQUALS("# # #", preprocess(code)); @@ -2118,6 +2193,7 @@ static void hashhash11() static void hashhash12() { + HIDESET_EXPANSION_TODO(); // # and ## { const char code[] = "#define MAX_FOO 1\n" "#define MAX_FOO_AA 2\n" @@ -2155,6 +2231,7 @@ static void hashhash12() static void hashhash13() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define X(x) x##U\n" "X((1<<1)-1)"; ASSERT_EQUALS("\n( 1 << 1 ) - 1U", preprocess(code)); @@ -2166,6 +2243,7 @@ static void hashhash13() static void hashhash_string_literal() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define UL(x) x##_ul\n" "\"ABC\"_ul;\n" @@ -2176,6 +2254,7 @@ static void hashhash_string_literal() static void hashhash_string_wrapped() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define STR(x) CONCAT(x,s)\n" @@ -2186,6 +2265,7 @@ static void hashhash_string_wrapped() static void hashhash_char_literal() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CH(x) x##_ch\n" "CH('a');"; @@ -2195,6 +2275,7 @@ static void hashhash_char_literal() static void hashhash_multichar_literal() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CH(x) x##_ch\n" "CH('abcd');"; @@ -2204,6 +2285,7 @@ static void hashhash_multichar_literal() static void hashhash_char_escaped() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CH(x) x##_ch\n" "CH('\\'');"; @@ -2213,6 +2295,7 @@ static void hashhash_char_escaped() static void hashhash_string_nothing() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\",);"; @@ -2222,6 +2305,7 @@ static void hashhash_string_nothing() static void hashhash_string_char() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\", 'c');"; @@ -2232,6 +2316,7 @@ static void hashhash_string_char() static void hashhash_string_name() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define LIT _literal\n" @@ -2245,6 +2330,7 @@ static void hashhash_string_name() static void hashhashhash_int_literal() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define CONCAT(a,b,c) a##b##c\n" "#define PASTER(a,b,c) CONCAT(a,b,c)\n" @@ -2255,6 +2341,7 @@ static void hashhashhash_int_literal() static void hashhash_int_literal() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define PASTE(a,b) a##b\n" "PASTE(123,_i);\n" @@ -2265,6 +2352,7 @@ static void hashhash_int_literal() static void hashhash_invalid_1() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define f(a) (##x)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2273,6 +2361,7 @@ static void hashhash_invalid_1() static void hashhash_invalid_2() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define f(a) (x##)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2281,6 +2370,7 @@ static void hashhash_invalid_2() static void hashhash_invalid_string_number() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define BAD(x) x##12345\nBAD(\"ABC\")\n"; @@ -2291,6 +2381,7 @@ static void hashhash_invalid_string_number() static void hashhash_invalid_missing_args() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define BAD(x) ##x\nBAD()\n"; @@ -2301,6 +2392,7 @@ static void hashhash_invalid_missing_args() static void hashhash_null_stmt() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "# define B(x) C ## x\n" "#\n" @@ -2313,6 +2405,7 @@ static void hashhash_null_stmt() static void hashhash_empty_va_args() { + HIDESET_EXPANSION_TODO(); // # and ## // #395 hash hash with an empty __VA_ARGS__ in a macro const char code[] = "#define CAT(a, ...) a##__VA_ARGS__\n" @@ -2324,6 +2417,7 @@ static void hashhash_empty_va_args() static void hashhash_punctuator() // #724 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "#define MINUS_GT -##>\n" @@ -2336,6 +2430,7 @@ static void hashhash_punctuator() // #724 static void hashhash_punctuator_invalid() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "cat(+,x)\n"; @@ -2346,6 +2441,7 @@ static void hashhash_punctuator_invalid() static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define x 3\n" "#define f(a) f(x * (a))\n" @@ -2410,6 +2506,7 @@ static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define str(s) # s\n" "#define xstr(s) str(s)\n" @@ -2449,6 +2546,7 @@ static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define t(x,y,z) x ## y ## z\n" "int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),\n" @@ -2465,6 +2563,7 @@ static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define debug(...) fprintf(stderr, __VA_ARGS__)\n" "#define showlist(...) puts(#__VA_ARGS__)\n" @@ -2487,6 +2586,7 @@ static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE { + HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define F(...) f(0 __VA_OPT__(,) __VA_ARGS__)\n" "#define G(X, ...) f(0, X __VA_OPT__(,) __VA_ARGS__)\n" @@ -2517,6 +2617,7 @@ static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE static void hashhash_empty_placemarker() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define r(x,y) x ## y\n" "{ r(,) }\n"; @@ -2524,7 +2625,7 @@ static void hashhash_empty_placemarker() simplecpp::FileDataCache cache; simplecpp::TokenList tokens(files); const simplecpp::TokenList rawtokens = makeTokenList(code, files); - simplecpp::preprocess(tokens, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens, rawtokens, files, cache, simplecpp::DUI()); std::string str; for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) str += "[" + tok->str() + "]"; @@ -2534,6 +2635,7 @@ static void hashhash_empty_placemarker() static void hashhash_invalid_dotdot() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "cat(.,.)\n"; @@ -2544,6 +2646,7 @@ static void hashhash_invalid_dotdot() static void hashhash_invalid_name_string() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "cat(a,\"s\")\n"; @@ -2554,6 +2657,7 @@ static void hashhash_invalid_name_string() static void hashhash_invalid_number_string() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "cat(1,\"s\")\n"; @@ -2564,6 +2668,7 @@ static void hashhash_invalid_number_string() static void hashhash_encoding_prefix() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define cat(a,b) a##b\n" "#define WIDE(s) L ## s\n" @@ -2575,6 +2680,7 @@ static void hashhash_encoding_prefix() static void hashhash_va_args_unexpected() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define C(...)!##__VA_ARGS__\n" "C(1)\n"; @@ -2585,6 +2691,7 @@ static void hashhash_va_args_unexpected() static void hashhash_universal_character() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(x,y) x##y\nint A(\\u01,04);\n"; simplecpp::OutputList outputList; @@ -2594,6 +2701,7 @@ static void hashhash_universal_character() static void hashhash_universal_character_2() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(x,y) x##y\nint A(\\U0104, 0104);\n"; simplecpp::OutputList outputList; @@ -2942,6 +3050,7 @@ static void ifDefinedNoPar() static void ifDefinedNested() { + HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define FOODEF defined(FOO)\n" "#if FOODEF\n" "X\n" @@ -2954,6 +3063,7 @@ static void ifDefinedNested() static void ifDefinedNestedNoPar() { + HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define FOODEF defined FOO\n" "#if FOODEF\n" "X\n" @@ -2982,6 +3092,7 @@ static void ifDefinedInvalid2() static void ifDefinedHashHash() { + HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define ENABLE(FEATURE) defined ENABLE_##FEATURE\n" "#define ENABLE_FOO 1\n" "#if ENABLE(FOO)\n" @@ -2996,6 +3107,7 @@ static void ifDefinedHashHash() static void ifDefinedHashHash2() { + HIDESET_EXPANSION_TODO(); // # and ## // #409 // do not crash when expanding P() (as ## rhs is "null") // note: gcc outputs "defined E" @@ -3007,6 +3119,7 @@ static void ifDefinedHashHash2() static void definedAtEndOfFile() { + HIDESET_EXPANSION_TODO(); // defined in a replacement list // "defined" from a macro expansion is undefined behavior; gcc outputs "defined" here. // The operand is the last token of the file. const char code[] = "#define X\n" @@ -3604,7 +3717,7 @@ static void missingHeader2() const simplecpp::TokenList rawtokens = makeTokenList(code,files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -3638,7 +3751,7 @@ static void nestedInclude() simplecpp::TokenList tokens2(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } @@ -3656,7 +3769,7 @@ static void systemInclude() simplecpp::TokenList tokens2(files); simplecpp::DUI dui; dui.includePaths.emplace_back("include"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -3698,7 +3811,7 @@ static void circularInclude() const simplecpp::TokenList rawtokens = makeTokenList(code, files, "test.cpp"); cache = simplecpp::load(rawtokens, filenames, dui, &outputList, std::move(cache)); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); } ASSERT_EQUALS("", toString(outputList)); @@ -3723,7 +3836,7 @@ static void multiline2() rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -3738,7 +3851,7 @@ static void multiline3() // #28 - macro with multiline comment rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -3754,7 +3867,7 @@ static void multiline4() // #28 - macro with multiline comment rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n\n1", tokens2.stringify()); } @@ -4077,6 +4190,7 @@ static void include2() static void include3() // #16 - crash when expanding macro from header { + HIDESET_EXPANSION_TODO(); // # and ## const char code_c[] = "#include \"A.h\"\n" "glue(1,2,3,4)\n"; const char code_h[] = "#define glue(a,b,c,d) a##b##c##d\n"; @@ -4097,7 +4211,7 @@ static void include3() // #16 - crash when expanding macro from header simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n1234", out.stringify()); } @@ -4125,7 +4239,7 @@ static void include4() // #27 - -include simplecpp::DUI dui; dui.includePaths.emplace_back("."); dui.includes.emplace_back("27.h"); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("123", out.stringify()); } @@ -4151,7 +4265,7 @@ static void include5() // #3 - handle #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -4171,7 +4285,7 @@ static void include6() // #57 - incomplete macro #include MACRO(,) cache.insert({"57.c", rawtokens}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(out, rawtokens, files, cache, simplecpp::DUI()); } @@ -4197,7 +4311,7 @@ static void include7() // #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -4236,7 +4350,7 @@ static void include9() simplecpp::DUI dui; dui.includePaths.emplace_back("."); dui.removeComments = true; - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify()); } @@ -4292,7 +4406,7 @@ static void include11() // #669 - -include with preprocess() simplecpp::DUI dui; dui.includes.emplace_back("inc.h"); dui.includes.emplace_back("missing.h"); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui, &outputList); + runPreprocess(out, rawtokens_c, files, cache, dui, &outputList); ASSERT_EQUALS("123", out.stringify()); ASSERT_EQUALS(1, outputList.size()); @@ -4619,7 +4733,7 @@ static void stringify1() simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); } @@ -4646,11 +4760,12 @@ static void tokenMacro3() "ADD(FRED,2)"; std::list macroUsage; ASSERT_EQUALS("1@FRED(3:1) +@ADD(3:1) 2(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); - ASSERT_EQUALS("FRED 2:9 use 3:1\nADD 1:9 use 3:1\n", toString(macroUsage)); + ASSERT_EQUALS("ADD 1:9 use 3:1\nFRED 2:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro4() { + HIDESET_EXPANSION_TODO(); // Token::macro and location rules const char code[] = "#define A B\n" "#define B 1\n" "A"; @@ -4689,6 +4804,7 @@ static void tokenMacro7() static void tokenMacro8() { + HIDESET_EXPANSION_TODO(); // Token::macro and location rules // the call g(2) is completed with tokens after the expansion of f. // The tokens get the location of the ( and the macro g const char code[] = "#define f g\n" @@ -4696,29 +4812,31 @@ static void tokenMacro8() "f(2)\n"; std::list macroUsage; ASSERT_EQUALS("2(3:2) +@g(3:2) 1@g(3:2)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); - ASSERT_EQUALS("g 2:9 use 3:2\nf 1:9 use 3:1\n", toString(macroUsage)); + ASSERT_EQUALS("f 1:9 use 3:1\ng 2:9 use 3:2\n", toString(macroUsage)); } static void tokenMacro9() { + HIDESET_EXPANSION_TODO(); // Token::macro and location rules // the arguments of g( come after the expansion of f const char code[] = "#define g(x) x+1\n" "#define f g(\n" "f 2)\n"; std::list macroUsage; ASSERT_EQUALS("2(3:3) +@g(3:3) 1@g(3:3)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); - ASSERT_EQUALS("g 1:9 use 3:3\nf 2:9 use 3:1\n", toString(macroUsage)); + ASSERT_EQUALS("f 2:9 use 3:1\ng 1:9 use 3:3\n", toString(macroUsage)); } static void tokenMacro10() { + HIDESET_EXPANSION_TODO(); // # and ## // the results of # and ## have no macro name const char code[] = "#define str(x) #x\n" "#define cat(a,b) a##b\n" "str(a b) cat(x,y)\n"; std::list macroUsage; ASSERT_EQUALS("\"a b\"(3:1) xy(3:10)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); - ASSERT_EQUALS("str 1:9 use 3:1\ncat 2:9 use 3:10\n", toString(macroUsage)); + ASSERT_EQUALS("cat 2:9 use 3:10\nstr 1:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro11() @@ -4771,6 +4889,183 @@ static void tokenMacro15() ASSERT_EQUALS("D 1:1 use 1:3 unknown\n", toString(macroUsage)); } +// Rescanning and hide sets. The results are the same as gcc gives. +static void hideSetExpansion_recursive() +{ + const char code[] = "#define f f\n" + "f\n"; + ASSERT_EQUALS("\n" + "f", preprocess(code)); +} + +static void hideSetExpansion_mutual() +{ + const char code[] = "#define a b\n" + "#define b a\n" + "a b\n"; + ASSERT_EQUALS("\n" + "\n" + "a b", preprocess(code)); +} + +static void hideSetExpansion_std_example() +{ + const char code[] = "#define f(a) a*g\n" + "#define g(a) f(a)\n" + "f(2)(9)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 * 9 * g", preprocess(code)); +} + +static void hideSetExpansion_trailing() +{ + const char code[] = "#define f g\n" + "#define g(x) x+1\n" + "f(2)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 + 1", preprocess(code)); +} + +static void hideSetExpansion_trailing_args() +{ + const char code[] = "#define g(x) x+1\n" + "#define f g(\n" + "f 2)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 + 1", preprocess(code)); +} + +static void hideSetExpansion_args_lines() +{ + const char code[] = "#define ADD(X,Y) X+Y\n" + "ADD(1,\n" + " 2)\n" + "z\n"; + ASSERT_EQUALS("\n" + "1 + 2\n" + "\n" + "z", preprocess(code)); +} + +static void hideSetExpansion_nested() +{ + const char code[] = "#define A(x) [x]\n" + "#define B(x) A(x) A(A(x))\n" + "B(1)\n"; + ASSERT_EQUALS("\n" + "\n" + "[ 1 ] [ [ 1 ] ]", preprocess(code)); +} + +static void hideSetExpansion_arg_self() +{ + const char code[] = "#define f(x) x\n" + "f(f)(1)\n"; + if (USE_HIDESET_EXPANSION) + ASSERT_EQUALS("\n" + "f ( 1 )", preprocess(code)); + else // TODO: the old macro expansion is wrong + TODO_ASSERT_EQUALS("\n" + "f ( 1 )", "\n" + "1", preprocess(code)); +} + +static void hideSetExpansion_blue_in_arg() +{ + const char code[] = "#define f(x) g(x)\n" + "#define g(x) f(x)\n" + "f(f(1))\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( f ( 1 ) )", preprocess(code)); +} + +static void hideSetExpansion_paren_in_arg() +{ + const char code[] = "#define f(x,y) (x)(y)\n" + "f((1,2),3)\n"; + ASSERT_EQUALS("\n" + "( ( 1 , 2 ) ) ( 3 )", preprocess(code)); +} + +static void hideSetExpansion_not_call() +{ + const char code[] = "#define f(x) x\n" + "f ; f\n" + "(1)\n"; + ASSERT_EQUALS("\n" + "f ; 1", preprocess(code)); +} + +static void hideSetExpansion_obj_to_fn() +{ + const char code[] = "#define LPAREN (\n" + "#define f(x) [x]\n" + "f LPAREN 1)\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( 1 )", preprocess(code)); +} + +static void hideSetExpansion_va_args() +{ + const char code[] = "#define F(...) f(__VA_ARGS__)\n" + "F(1,2) F()\n"; + ASSERT_EQUALS("\n" + "f ( 1 , 2 ) f ( )", preprocess(code)); +} + +static void hideSetExpansion_counter() +{ + const char code[] = "#define C __COUNTER__\n" + "C C __COUNTER__\n"; + ASSERT_EQUALS("\n" + "0 1 2", preprocess(code)); +} + +static void hideSetExpansion_line() +{ + const char code[] = "#define L __LINE__\n" + "L\n"; + ASSERT_EQUALS("\n" + "2", preprocess(code)); +} + +static void hideSetExpansion_deep() +{ + const char code[] = "#define a(x) b(x)\n" + "#define b(x) c(x)\n" + "#define c(x) x x\n" + "a(a(1))\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "1 1 1 1", preprocess(code)); +} + +static void hideSetExpansion_q_example() +{ + const char code[] = "#define q(x) x\n" + "#define r(x,y) x y\n" + "q(r)(q,1)\n"; + ASSERT_EQUALS("\n" + "\n" + "q 1", preprocess(code)); +} + +static void hideSetExpansion_empty() +{ + const char code[] = "#define E\n" + "#define f(x) \n" + "f(E) f()\n"; + ASSERT_EQUALS("\n" + "\n" + "< > < >", preprocess(code)); +} + static void undef() { const char code[] = "#define A\n" @@ -5214,7 +5509,7 @@ static void preprocess_files() ASSERT_EQUALS("", *files.cbegin()); simplecpp::FileDataCache cache; - simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); + runPreprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("", *files.cbegin()); } @@ -5231,7 +5526,7 @@ static void preprocess_files() ASSERT_EQUALS("test.cpp", *files.cbegin()); simplecpp::FileDataCache cache; - simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); + runPreprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("test.cpp", *files.cbegin()); } @@ -5846,6 +6141,24 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(tokenMacro13); TEST_CASE(tokenMacro14); TEST_CASE(tokenMacro15); + TEST_CASE(hideSetExpansion_recursive); + TEST_CASE(hideSetExpansion_mutual); + TEST_CASE(hideSetExpansion_std_example); + TEST_CASE(hideSetExpansion_trailing); + TEST_CASE(hideSetExpansion_trailing_args); + TEST_CASE(hideSetExpansion_args_lines); + TEST_CASE(hideSetExpansion_nested); + TEST_CASE(hideSetExpansion_arg_self); + TEST_CASE(hideSetExpansion_blue_in_arg); + TEST_CASE(hideSetExpansion_paren_in_arg); + TEST_CASE(hideSetExpansion_not_call); + TEST_CASE(hideSetExpansion_obj_to_fn); + TEST_CASE(hideSetExpansion_va_args); + TEST_CASE(hideSetExpansion_counter); + TEST_CASE(hideSetExpansion_line); + TEST_CASE(hideSetExpansion_deep); + TEST_CASE(hideSetExpansion_q_example); + TEST_CASE(hideSetExpansion_empty); TEST_CASE(undef); @@ -5897,5 +6210,9 @@ int main(int argc, char **argv) { runTests(argc, argv, Input::Stringstream); runTests(argc, argv, Input::CharBuffer); + USE_HIDESET_EXPANSION = true; + runTests(argc, argv, Input::Stringstream); + if (numberOfHideSetTodos > 0) + std::cout << "Tests skipped with the hide-set expansion: " << numberOfHideSetTodos << std::endl; return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From ec5d9345b1838c18636a5b3c74cdf3c924aaba68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 13:57:33 +0100 Subject: [PATCH 17/28] HideSetExpander: get the arguments as written before they are expanded # needs the arguments that are not expanded. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index b7629bce..0bfc74a8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3064,7 +3064,7 @@ namespace simplecpp { const std::set &hs1 = call.cfront()->hideSet(); const std::set &hs2 = delimiters.back()->hideSet(); std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(hideSet, hideSet.begin())); - arguments = expandArguments(macro, delimiters); + arguments = expandArguments(getArguments(macro, delimiters)); } else { hideSet = tok->hideSet(); delete tok; @@ -3089,21 +3089,32 @@ namespace simplecpp { } } - /** Each argument of the call, fully macro expanded */ - std::vector expandArguments(const Macro ¯o, const std::vector &delimiters) { + /** Each argument of the call, as written in the call */ + std::vector getArguments(const Macro ¯o, const std::vector &delimiters) const { std::vector arguments; for (std::size_t i = 0; i < macro.args.size(); ++i) { arguments.emplace_back(files); if (i + 1U >= delimiters.size()) continue; // empty variadic argument - Input input(nullptr); for (const Token *tok = delimiters[i]->next; tok != delimiters[i + 1U]; tok = tok->next) - input.pending.push_back(new Token(*tok)); - expandInput(arguments.back(), input); + arguments.back().push_back(new Token(*tok)); } return arguments; } + /** Each argument fully macro expanded, on its own */ + std::vector expandArguments(const std::vector &arguments) { + std::vector expanded; + for (const TokenList &argument : arguments) { + expanded.emplace_back(files); + Input input(nullptr); + for (const Token *tok = argument.cfront(); tok; tok = tok->next) + input.pending.push_back(new Token(*tok)); + expandInput(expanded.back(), input); + } + return expanded; + } + /** The replacement list of the macro, with the parameters replaced by the arguments */ void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::set &hideSet) const { if (macro.variadicOpt) From 06d3eec5fa052c17f891880effff4a05d55dfec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 16:26:10 +0100 Subject: [PATCH 18/28] HideSetExpander: handle # # is an operator in a function-like macro when it is followed by a parameter; the argument as written in the call is turned into a string with stringify(). The whitespace after a parameter comes after its argument, and the expansion is followed by the whitespace after the macro call. Output: a # from a macro expansion at the start of a line gets a space before it, so it is not read as a directive, as gcc and clang do (clang tests hash_space.c and macro_not_define.c). Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 64 ++++++++++++++++----- test.cpp | 151 +++++++++++++++++++++++++------------------------- 2 files changed, 127 insertions(+), 88 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0bfc74a8..df4ede39 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -671,7 +671,9 @@ std::string simplecpp::TokenList::stringify(bool linenrs) const ret << loc.line << ": "; } - if (sameline(tok->previous, tok)) + // a # from a macro expansion at the start of a line gets a space + // before it, so the output is not read as a directive (as gcc and clang do) + if (sameline(tok->previous, tok) || (tok->op == '#' && !tok->macro.empty())) ret << ' '; ret << tok->str(); @@ -2950,12 +2952,12 @@ namespace simplecpp { * The result of an expansion is pushed back onto the input and scanned again, so a * function-like macro whose arguments follow the expansion in the source is found by * normal rescanning. - * Work in progress: #, ##, __VA_OPT__ and "defined" in a replacement list are not handled yet. + * Work in progress: ##, __VA_OPT__ and "defined" in a replacement list are not handled yet. */ class HideSetExpander { public: - HideSetExpander(const MacroMap ¯os, std::vector &files) - : macros(macros), files(files) {} + HideSetExpander(const MacroMap ¯os, std::vector &files, OutputList *outputList) + : macros(macros), files(files), outputList(outputList) {} HideSetExpander(const HideSetExpander&) = delete; HideSetExpander &operator=(const HideSetExpander&) = delete; @@ -3054,6 +3056,8 @@ namespace simplecpp { std::set hideSet; std::vector arguments; + std::vector expandedArguments; + bool whitespaceAfter = tok->whitespaceahead; if (macro.functionLike()) { TokenList call(files); call.push_back(tok); @@ -3064,7 +3068,9 @@ namespace simplecpp { const std::set &hs1 = call.cfront()->hideSet(); const std::set &hs2 = delimiters.back()->hideSet(); std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(hideSet, hideSet.begin())); - arguments = expandArguments(getArguments(macro, delimiters)); + arguments = getArguments(macro, delimiters); + expandedArguments = expandArguments(arguments); + whitespaceAfter = delimiters.back()->whitespaceahead; } else { hideSet = tok->hideSet(); delete tok; @@ -3072,7 +3078,7 @@ namespace simplecpp { hideSet.insert(macro.name()); TokenList result(files); - substitute(result, macro, arguments, hideSet); + substitute(result, macro, arguments, expandedArguments, hideSet, whitespaceAfter); input.pushFront(result); } } @@ -3115,19 +3121,44 @@ namespace simplecpp { return expanded; } - /** The replacement list of the macro, with the parameters replaced by the arguments */ - void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::set &hideSet) const { + /** The parameter number of tok in the replacement list, or ~0U if it is not a parameter */ + static unsigned int getParameter(const Macro ¯o, const Token *tok) { + return (tok && tok != macro.endToken && tok->name) ? macro.getArgNum(tok->str()) : ~0U; + } + + /** + * The replacement list of the macro, with the parameters replaced by the arguments. + * @param result destination tokenlist + * @param macro the macro + * @param arguments the arguments as written in the call, for # + * @param expandedArguments the arguments, macro expanded + * @param hideSet the hide set of the result + * @param whitespaceAfter there is whitespace after the macro call + */ + void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::vector &expandedArguments, const std::set &hideSet, bool whitespaceAfter) const { if (macro.variadicOpt) throw unsupported(loc, "__VA_OPT__"); for (const Token *tok = macro.valueToken; tok != macro.endToken; tok = tok->next) { - if (tok->op == '#') - throw unsupported(loc, "# and ##"); + const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; + if (tok->op == '#' && next && next->op == '#') + throw unsupported(loc, "##"); if (tok->str() == DEFINED) throw unsupported(loc, "defined"); - const unsigned int argnr = tok->name ? macro.getArgNum(tok->str()) : ~0U; - if (argnr < arguments.size()) { - for (const Token *argtok = arguments[argnr].cfront(); argtok; argtok = argtok->next) + // # is an operator in a function-like macro when it is followed by a parameter + const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; + if (hashArg < arguments.size()) { + result.push_back(new Token(stringify(arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); + result.back()->macro = macro.name(); + tok = next; + continue; + } + const unsigned int argnr = getParameter(macro, tok); + if (argnr < expandedArguments.size()) { + for (const Token *argtok = expandedArguments[argnr].cfront(); argtok; argtok = argtok->next) result.push_back(new Token(*argtok)); + // the whitespace after the parameter comes after the argument + if (tok->whitespaceahead && result.back()) + result.back()->whitespaceahead = true; } else { result.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); result.back()->macro = macro.name(); @@ -3138,10 +3169,15 @@ namespace simplecpp { for (const TokenString &name : hideSet) tok->addToHideSet(name); } + // whitespace at the end of the replacement list does not count, the + // expansion is followed by the whitespace after the macro call + if (result.back()) + result.back()->whitespaceahead = whitespaceAfter; } const MacroMap ¯os; std::vector &files; + OutputList *outputList; /** location of the macro name in the source, all tokens of the expansion get it */ Location loc; }; @@ -4076,7 +4112,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token simplecpp::TokenList value(files); try { if (dui.hideSetExpansion) { - simplecpp::HideSetExpander expander(macros, files); + simplecpp::HideSetExpander expander(macros, files, outputList); tok1 = expander.expand(value, tok); } else { tok1 = it->second.expand(value, tok, macros, files, outputList); diff --git a/test.cpp b/test.cpp index ce37fd01..9500d797 100644 --- a/test.cpp +++ b/test.cpp @@ -887,7 +887,6 @@ static void define13() static void define14() // #296 { - HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define bar(x) x % 2\n" "#define foo(x) printf(#x \"\\n\")\n" "\n" @@ -900,7 +899,7 @@ static void define14() // #296 static void define15() // #231 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CAT(a, b) CAT2(a, b)\n" "#define CAT2(a, b) a ## b\n" "#define FOO x\n" @@ -917,7 +916,6 @@ static void define15() // #231 static void define16() // #201 { - HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define ALL_COLORS(warm_colors) \\\n" " X(Blue) \\\n" " X(Green) \\\n" @@ -954,7 +952,7 @@ static void define16() // #201 static void define17() // #185 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define at(x, y) x##y\n" "#define b(...) \\\n" "aa(__VA_ARGS__, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , \\\n" @@ -983,7 +981,7 @@ static void define17() // #185 static void define18() // #130 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define MAC2STR(x) x[0],x[1],x[2],x[3],x[4],x[5]\n" "#define FT_DEBUG(fmt, args...) if(pGlobalCtx && pGlobalCtx->debug_level>=2) printf(\"FT-dbg: \"fmt, ##args)\n" "\n" @@ -996,7 +994,7 @@ static void define18() // #130 static void define19() // #124 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(tok) tok##suffix\n" "\n" "CONCAT(Test);\n" @@ -1021,7 +1019,7 @@ static void define20() // #113 static void define21() // #66 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "FIGHT_FOO(1, GETMYID(a));\n"; @@ -1032,7 +1030,7 @@ static void define21() // #66 static void define22() // #40 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define COUNTER_NAME(NAME, ...) NAME##Count\n" "#define COMMA ,\n" "\n" @@ -1261,7 +1259,7 @@ static void define_define_2() static void define_define_3() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define ABC 123\n" "#define A(B) A##B\n" "A(BC)"; @@ -1325,7 +1323,7 @@ static void define_define_9() // line break in nested macro call static void define_define_10() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define glue(a, b) a ## b\n" "#define xglue(a, b) glue(a, b)\n" "#define AB 1\n" @@ -1336,7 +1334,7 @@ static void define_define_10() static void define_define_11() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define XY(x, y) x ## y\n" "#define XY2(x, y) XY(x, y)\n" "#define PORT XY2(P, 2)\n" @@ -1347,7 +1345,7 @@ static void define_define_11() static void define_define_11a() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A_B_C 0x1\n" "#define A_ADDRESS 0x00001000U\n" "#define A ((uint32_t ) A_ADDRESS)\n" @@ -1366,7 +1364,7 @@ static void define_define_11a() static void define_define_12() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define XY(Z) Z\n" "#define X(ID) X##ID(0)\n" "X(Y)\n"; @@ -1402,7 +1400,7 @@ static void define_define_15() // issue #72 without __VA_ARGS__ static void define_define_16() // issue #72 with __VA_ARGS__ { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define ab(a, b) a##b\n" "#define foo(...) ab(f, 2) (__VA_ARGS__)\n" "#define f2(x, y) x y\n" @@ -1438,7 +1436,7 @@ static void define_define_19() // #292 static void define_define_20() // #384 arg contains comma { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define Z_IS_ENABLED1(config_macro) Z_IS_ENABLED2(_XXXX##config_macro)\n" "#define _XXXX1 _YYYY,\n" "#define Z_IS_ENABLED2(one_or_two_args) Z_IS_ENABLED3(one_or_two_args 1, 0)\n" @@ -1471,7 +1469,7 @@ static void define_define_21() // #397 DEBRACKET macro static void define_define_22() // #400 inner macro not expanded after hash hash { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define FOO(a) CAT(DO, STUFF)(1,2)\n" "#define DOSTUFF(a, b) CAT(3, 4)\n" "#define CAT(a, b) a##b\n" @@ -1481,7 +1479,7 @@ static void define_define_22() // #400 inner macro not expanded after hash hash static void define_define_23() // #403 crash (infinite recursion) { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define C_(x, y) x ## y\n" "#define C(x, y) C_(x, y)\n" "#define X(func) C(Y, C(func, Z))\n" @@ -1509,7 +1507,6 @@ static void define_va_args_1() static void define_va_args_2() { - HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define A(X,...) X(#__VA_ARGS__)\n" "A(f,123);"; ASSERT_EQUALS("\nf ( \"123\" ) ;", preprocess(code)); @@ -1832,7 +1829,7 @@ static void error8() static void garbage() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## simplecpp::OutputList outputList; outputList.clear(); @@ -1867,7 +1864,6 @@ static void garbage_endif() static void hash() { - HIDESET_EXPANSION_TODO(); // # and ## ASSERT_EQUALS("x = \"1\"", preprocess("x=#__LINE__")); const char code[] = "#define a(x) #x\n" @@ -1898,7 +1894,6 @@ static void hash() static void hashEscape() { - HIDESET_EXPANSION_TODO(); // # and ## // a \ is only inserted before " and \ inside string literals and character constants const char code[] = "#define str(x) #x\n" "str(\\n)\n" @@ -1918,7 +1913,6 @@ static void hashEscape() static void hashComment() { - HIDESET_EXPANSION_TODO(); // # and ## // a comment is replaced by one space character simplecpp::DUI dui; dui.removeComments = true; @@ -1944,7 +1938,6 @@ static void hashComment() static void hashMacroTrailingWhitespace() { - HIDESET_EXPANSION_TODO(); // # and ## // whitespace at the end of a replacement list does not count simplecpp::DUI dui; dui.removeComments = true; @@ -1969,7 +1962,6 @@ static void hashMacroTrailingWhitespace() static void hashCombinedOperator() { - HIDESET_EXPANSION_TODO(); // # and ## // the whitespace after an operator of two characters is kept const char code[] = "#define str(x) #x\n" "str(a == 0) str(a -> b) str(x... y)\n"; @@ -1978,7 +1970,6 @@ static void hashCombinedOperator() static void hashEscapeTrailingBackslash() { - HIDESET_EXPANSION_TODO(); // # and ## // an odd number of trailing backslashes would escape the closing quote, // the final backslash is dropped (like gcc and clang) and a warning is reported const char msg[] = "Invalid string literal when expanding 'str', ignoring final '\\'. " @@ -2038,7 +2029,7 @@ static void hashEscapeTrailingBackslash() static void hashhash1() // #4703 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define MACRO( A, B, C ) class A##B##C##Creator {};\n" "MACRO( B\t, U , G )"; ASSERT_EQUALS("\nclass BUGCreator { } ;", preprocess(code)); @@ -2046,7 +2037,7 @@ static void hashhash1() // #4703 static void hashhash2() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x) a##x\n" "#define B 0\n" "A(B)"; @@ -2055,7 +2046,7 @@ static void hashhash2() static void hashhash3() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(B) A##B\n" "#define a(B) A(B)\n" "a(A(B))"; @@ -2064,7 +2055,7 @@ static void hashhash3() static void hashhash4() // nonstandard gcc/clang extension for empty varargs { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define A(x,y...) a(x,##y)\n" @@ -2079,7 +2070,7 @@ static void hashhash4() // nonstandard gcc/clang extension for empty varargs static void hashhash4a() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "#define FIGHT_BAR(c, args...) bar(c, ##args)\n" @@ -2095,7 +2086,7 @@ static void hashhash5() static void hashhash6() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define A(X, ...) LOG(X, ##__VA_ARGS__)\n" @@ -2111,7 +2102,7 @@ static void hashhash6() static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define hash_hash # ## #\n" @@ -2121,7 +2112,7 @@ static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) static void hashhash8() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define a(xy) x##y = xy\n" "a(123);"; ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); @@ -2129,7 +2120,7 @@ static void hashhash8() static void hashhash9() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char * code = "#define ADD_OPERATOR(OP) void operator OP ## = (void) { x = x OP 1; }\n" "ADD_OPERATOR(+);\n" "ADD_OPERATOR(-);\n" @@ -2177,23 +2168,35 @@ static void hashhash9() static void hashhash10() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x # #\n" "x"; - ASSERT_EQUALS("# #", preprocess(code)); + // a space is added before the #, as gcc and clang do, so the output is not read as a directive + ASSERT_EQUALS(" # #", preprocess(code)); +} + +static void hashFromMacroAtLineStart() +{ + // clang testsuite hash_space.c: a space before the # so that the output is not read as a directive + const char code[] = "#define HASH #\n" + "HASH define foo bar\n"; + if (USE_HIDESET_EXPANSION) + ASSERT_EQUALS("\n # define foo bar", preprocess(code)); + else // TODO: the old macro expansion gives the # the location of the #define + TODO_ASSERT_EQUALS("\n # define foo bar", " #\ndefine foo bar", preprocess(code)); } static void hashhash11() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x # # #\n" "x"; - ASSERT_EQUALS("# # #", preprocess(code)); + ASSERT_EQUALS(" # # #", preprocess(code)); } static void hashhash12() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## { const char code[] = "#define MAX_FOO 1\n" "#define MAX_FOO_AA 2\n" @@ -2231,7 +2234,7 @@ static void hashhash12() static void hashhash13() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define X(x) x##U\n" "X((1<<1)-1)"; ASSERT_EQUALS("\n( 1 << 1 ) - 1U", preprocess(code)); @@ -2243,7 +2246,7 @@ static void hashhash13() static void hashhash_string_literal() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define UL(x) x##_ul\n" "\"ABC\"_ul;\n" @@ -2254,7 +2257,7 @@ static void hashhash_string_literal() static void hashhash_string_wrapped() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define STR(x) CONCAT(x,s)\n" @@ -2265,7 +2268,7 @@ static void hashhash_string_wrapped() static void hashhash_char_literal() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('a');"; @@ -2275,7 +2278,7 @@ static void hashhash_char_literal() static void hashhash_multichar_literal() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('abcd');"; @@ -2285,7 +2288,7 @@ static void hashhash_multichar_literal() static void hashhash_char_escaped() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('\\'');"; @@ -2295,7 +2298,7 @@ static void hashhash_char_escaped() static void hashhash_string_nothing() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\",);"; @@ -2305,7 +2308,7 @@ static void hashhash_string_nothing() static void hashhash_string_char() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\", 'c');"; @@ -2316,7 +2319,7 @@ static void hashhash_string_char() static void hashhash_string_name() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define LIT _literal\n" @@ -2330,7 +2333,7 @@ static void hashhash_string_name() static void hashhashhash_int_literal() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b,c) a##b##c\n" "#define PASTER(a,b,c) CONCAT(a,b,c)\n" @@ -2341,7 +2344,7 @@ static void hashhashhash_int_literal() static void hashhash_int_literal() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define PASTE(a,b) a##b\n" "PASTE(123,_i);\n" @@ -2352,7 +2355,7 @@ static void hashhash_int_literal() static void hashhash_invalid_1() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define f(a) (##x)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2361,7 +2364,7 @@ static void hashhash_invalid_1() static void hashhash_invalid_2() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define f(a) (x##)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2370,7 +2373,7 @@ static void hashhash_invalid_2() static void hashhash_invalid_string_number() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define BAD(x) x##12345\nBAD(\"ABC\")\n"; @@ -2381,7 +2384,7 @@ static void hashhash_invalid_string_number() static void hashhash_invalid_missing_args() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define BAD(x) ##x\nBAD()\n"; @@ -2392,7 +2395,7 @@ static void hashhash_invalid_missing_args() static void hashhash_null_stmt() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "# define B(x) C ## x\n" "#\n" @@ -2405,7 +2408,7 @@ static void hashhash_null_stmt() static void hashhash_empty_va_args() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## // #395 hash hash with an empty __VA_ARGS__ in a macro const char code[] = "#define CAT(a, ...) a##__VA_ARGS__\n" @@ -2417,7 +2420,7 @@ static void hashhash_empty_va_args() static void hashhash_punctuator() // #724 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "#define MINUS_GT -##>\n" @@ -2430,7 +2433,7 @@ static void hashhash_punctuator() // #724 static void hashhash_punctuator_invalid() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(+,x)\n"; @@ -2441,7 +2444,7 @@ static void hashhash_punctuator_invalid() static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x 3\n" "#define f(a) f(x * (a))\n" @@ -2506,7 +2509,7 @@ static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define str(s) # s\n" "#define xstr(s) str(s)\n" @@ -2546,7 +2549,7 @@ static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define t(x,y,z) x ## y ## z\n" "int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),\n" @@ -2563,7 +2566,6 @@ static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 { - HIDESET_EXPANSION_TODO(); // # and ## const char code[] = "#define debug(...) fprintf(stderr, __VA_ARGS__)\n" "#define showlist(...) puts(#__VA_ARGS__)\n" @@ -2617,7 +2619,7 @@ static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE static void hashhash_empty_placemarker() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define r(x,y) x ## y\n" "{ r(,) }\n"; @@ -2635,7 +2637,7 @@ static void hashhash_empty_placemarker() static void hashhash_invalid_dotdot() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(.,.)\n"; @@ -2646,7 +2648,7 @@ static void hashhash_invalid_dotdot() static void hashhash_invalid_name_string() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(a,\"s\")\n"; @@ -2657,7 +2659,7 @@ static void hashhash_invalid_name_string() static void hashhash_invalid_number_string() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(1,\"s\")\n"; @@ -2668,7 +2670,7 @@ static void hashhash_invalid_number_string() static void hashhash_encoding_prefix() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "#define WIDE(s) L ## s\n" @@ -2680,7 +2682,7 @@ static void hashhash_encoding_prefix() static void hashhash_va_args_unexpected() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define C(...)!##__VA_ARGS__\n" "C(1)\n"; @@ -2691,7 +2693,7 @@ static void hashhash_va_args_unexpected() static void hashhash_universal_character() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x,y) x##y\nint A(\\u01,04);\n"; simplecpp::OutputList outputList; @@ -2701,7 +2703,7 @@ static void hashhash_universal_character() static void hashhash_universal_character_2() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x,y) x##y\nint A(\\U0104, 0104);\n"; simplecpp::OutputList outputList; @@ -3092,7 +3094,7 @@ static void ifDefinedInvalid2() static void ifDefinedHashHash() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define ENABLE(FEATURE) defined ENABLE_##FEATURE\n" "#define ENABLE_FOO 1\n" "#if ENABLE(FOO)\n" @@ -3107,7 +3109,7 @@ static void ifDefinedHashHash() static void ifDefinedHashHash2() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## // #409 // do not crash when expanding P() (as ## rhs is "null") // note: gcc outputs "defined E" @@ -4190,7 +4192,7 @@ static void include2() static void include3() // #16 - crash when expanding macro from header { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## const char code_c[] = "#include \"A.h\"\n" "glue(1,2,3,4)\n"; const char code_h[] = "#define glue(a,b,c,d) a##b##c##d\n"; @@ -4829,7 +4831,7 @@ static void tokenMacro9() static void tokenMacro10() { - HIDESET_EXPANSION_TODO(); // # and ## + HIDESET_EXPANSION_TODO(); // ## // the results of # and ## have no macro name const char code[] = "#define str(x) #x\n" "#define cat(a,b) a##b\n" @@ -5975,6 +5977,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash9); TEST_CASE(hashhash10); // #108 : #define x # # TEST_CASE(hashhash11); // #60: #define x # # # + TEST_CASE(hashFromMacroAtLineStart); TEST_CASE(hashhash12); TEST_CASE(hashhash13); TEST_CASE(hashhash_string_literal); From b59b1051115ba53ed49098ad7ef6453d58a4d145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 14:53:07 +0100 Subject: [PATCH 19/28] HideSetExpander: add getOperand() for what a replacement list token is replaced by ## needs the operands before they are joined. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 57 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index df4ede39..f48d812a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3126,6 +3126,37 @@ namespace simplecpp { return (tok && tok != macro.endToken && tok->name) ? macro.getArgNum(tok->str()) : ~0U; } + /** + * The tokens that a token in the replacement list is replaced by: a parameter by its + * argument, # and a parameter by a string literal, and other tokens by a copy. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; for # it is moved to the parameter + * @param arguments the arguments as written in the call + * @param expandedArguments the arguments, macro expanded + * @param expand use the expanded argument for a parameter + */ + void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const std::vector &arguments, const std::vector &expandedArguments, bool expand) const { + const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; + // # is an operator in a function-like macro when it is followed by a parameter + const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; + if (hashArg < arguments.size()) { + operand.push_back(new Token(stringify(arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); + operand.back()->macro = macro.name(); + tok = next; + return; + } + const unsigned int argnr = getParameter(macro, tok); + if (argnr < arguments.size()) { + const TokenList &argument = expand ? expandedArguments[argnr] : arguments[argnr]; + for (const Token *argtok = argument.cfront(); argtok; argtok = argtok->next) + operand.push_back(new Token(*argtok)); + return; + } + operand.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + operand.back()->macro = macro.name(); + } + /** * The replacement list of the macro, with the parameters replaced by the arguments. * @param result destination tokenlist @@ -3144,25 +3175,13 @@ namespace simplecpp { throw unsupported(loc, "##"); if (tok->str() == DEFINED) throw unsupported(loc, "defined"); - // # is an operator in a function-like macro when it is followed by a parameter - const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; - if (hashArg < arguments.size()) { - result.push_back(new Token(stringify(arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); - result.back()->macro = macro.name(); - tok = next; - continue; - } - const unsigned int argnr = getParameter(macro, tok); - if (argnr < expandedArguments.size()) { - for (const Token *argtok = expandedArguments[argnr].cfront(); argtok; argtok = argtok->next) - result.push_back(new Token(*argtok)); - // the whitespace after the parameter comes after the argument - if (tok->whitespaceahead && result.back()) - result.back()->whitespaceahead = true; - } else { - result.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); - result.back()->macro = macro.name(); - } + const bool parameter = getParameter(macro, tok) < arguments.size(); + TokenList operand(files); + getOperand(operand, macro, tok, arguments, expandedArguments, true); + result.takeTokens(operand); + // the whitespace after a parameter comes after its argument + if (parameter && tok->whitespaceahead && result.back()) + result.back()->whitespaceahead = true; } for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; From a62865ef03102a4b0216b4bf107b90cd961438a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 14:58:58 +0100 Subject: [PATCH 20/28] HideSetExpander: handle ## As in C11 6.10.3.3: an argument that is an operand of ## is not macro expanded, an empty argument is a placemarker, and the pasted token is rescanned. The ## checks of the old expansion are used. Like the old expansion, ", ## __VA_ARGS__" removes the comma when there are no variable arguments, and a string literal followed by a name is not pasted. With the hide-set expansion, 141 of the 155 clang/gcc testsuite tests pass (140 with the old expansion). The new test macro TODO_OLD_EXPANSION_ASSERT_EQUALS() is for known bugs of the old expansion. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 80 +++++++++++++++-- test.cpp | 232 +++++++++++++++++++------------------------------- 2 files changed, 161 insertions(+), 151 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f48d812a..d20de326 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2952,7 +2952,7 @@ namespace simplecpp { * The result of an expansion is pushed back onto the input and scanned again, so a * function-like macro whose arguments follow the expansion in the source is found by * normal rescanning. - * Work in progress: ##, __VA_OPT__ and "defined" in a replacement list are not handled yet. + * Work in progress: __VA_OPT__ and "defined" in a replacement list are not handled yet. */ class HideSetExpander { public: @@ -3126,6 +3126,57 @@ namespace simplecpp { return (tok && tok != macro.endToken && tok->name) ? macro.getArgNum(tok->str()) : ~0U; } + /** tok and the token after it are ## in the replacement list of macro */ + static bool isHashHash(const Macro ¯o, const Token *tok) { + if (!tok || tok == macro.endToken || tok->op != '#') + return false; + const Token * const next = tok->next; + return next && next != macro.endToken && next->op == '#' && + sameline(tok, next) && next->location.col == tok->location.col + 1U; + } + + /** + * A ## B: paste the last token of result with the first token of operand. + * An empty operand is a placemarker: pasting it gives the other operand. + * @param result the replacement so far, it ends with A + * @param operand B, its tokens are moved to result + * @param previousEmpty A is a placemarker + * @param macro the macro + * @param hashHash the ## in the replacement list + * @param operandTok the token after ## in the replacement list + * @throws invalidHashHash if the result is not a valid token + */ + static void paste(TokenList &result, TokenList &operand, bool previousEmpty, const Macro ¯o, const Token *hashHash, const Token *operandTok) { + Token * const A = result.back(); + const bool varargs = macro.variadic && !macro.args.empty() && operandTok->str() == macro.args.back(); + if (operand.empty()) { + // gcc extension: in ", ## __VA_ARGS__" the comma is removed when there are no variable arguments + if (varargs && A && !previousEmpty && A->op == ',') + result.deleteToken(A); + return; + } + if (previousEmpty || !A) { + result.takeTokens(operand); + return; + } + Macro::checkHashHashOperands(hashHash->location, macro.name(), A, operandTok); + // ", ## __VA_ARGS__" with variable arguments: nothing is pasted + // a string or character literal followed by a name (user-defined literal) is not pasted either + if ((varargs && A->op == ',') || isStringLiteral(A->str()) || isCharLiteral(A->str())) { + result.takeTokens(operand); + return; + } + const Token * const B = operand.cfront(); + Macro::checkPaste(hashHash->location, macro.name(), A, B); + const std::string strAB = A->str() + B->str(); + Macro::checkUniversalCharacterPaste(hashHash->location, macro.name(), A, strAB); + A->setstr(strAB); + A->whitespaceahead = B->whitespaceahead; + A->macro = macro.name(); + operand.deleteToken(operand.front()); + result.takeTokens(operand); + } + /** * The tokens that a token in the replacement list is replaced by: a parameter by its * argument, # and a parameter by a string literal, and other tokens by a copy. @@ -3169,20 +3220,37 @@ namespace simplecpp { void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::vector &expandedArguments, const std::set &hideSet, bool whitespaceAfter) const { if (macro.variadicOpt) throw unsupported(loc, "__VA_OPT__"); + const Token *hashHash = nullptr; // the ## before the current operand + bool previousEmpty = false; // the previous operand was empty, it is a placemarker for (const Token *tok = macro.valueToken; tok != macro.endToken; tok = tok->next) { - const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; - if (tok->op == '#' && next && next->op == '#') - throw unsupported(loc, "##"); + if (isHashHash(macro, tok)) { + if (!result.back() && !previousEmpty) + throw Macro::invalidHashHash(tok->location, macro.name(), "Missing first argument"); + hashHash = tok; + tok = tok->next; + continue; + } if (tok->str() == DEFINED) throw unsupported(loc, "defined"); + const Token * const operandTok = tok; const bool parameter = getParameter(macro, tok) < arguments.size(); + // an argument is not expanded when it is an operand of ## + const bool expand = !hashHash && !isHashHash(macro, tok->next); TokenList operand(files); - getOperand(operand, macro, tok, arguments, expandedArguments, true); - result.takeTokens(operand); + getOperand(operand, macro, tok, arguments, expandedArguments, expand); + const bool empty = operand.empty(); + if (hashHash) + paste(result, operand, previousEmpty, macro, hashHash, operandTok); + else + result.takeTokens(operand); // the whitespace after a parameter comes after its argument if (parameter && tok->whitespaceahead && result.back()) result.back()->whitespaceahead = true; + previousEmpty = empty && (hashHash ? previousEmpty : true); + hashHash = nullptr; } + if (hashHash) + throw Macro::invalidHashHash::unexpectedNewline(hashHash->location, macro.name()); for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; for (const TokenString &name : hideSet) diff --git a/test.cpp b/test.cpp index 9500d797..fcff596e 100644 --- a/test.cpp +++ b/test.cpp @@ -54,6 +54,8 @@ static int numberOfHideSetTodos = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) #define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while (false) #define TODO_ASSERT_EQUALS(wanted, current, actual) (todoAssertEquals((wanted), (current), (actual), __LINE__)) +/** A known bug of the old macro expansion that the hide-set expansion does not have */ +#define TODO_OLD_EXPANSION_ASSERT_EQUALS(wanted, current, actual) (todoOldExpansionAssertEquals((wanted), (current), (actual), __LINE__)) static std::string pprint(const std::string &in) { @@ -111,6 +113,14 @@ static void todoAssertEquals(const std::string &wanted, const std::string &curre assertEquals(current, actual, line); } +static void todoOldExpansionAssertEquals(const std::string &wanted, const std::string ¤t, const std::string &actual, int line) +{ + if (USE_HIDESET_EXPANSION) + assertEquals(wanted, actual, line); + else + todoAssertEquals(wanted, current, actual, line); +} + static void assertThrowFailed(int line) { numberOfFailedAssertions++; @@ -899,7 +909,6 @@ static void define14() // #296 static void define15() // #231 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CAT(a, b) CAT2(a, b)\n" "#define CAT2(a, b) a ## b\n" "#define FOO x\n" @@ -952,7 +961,6 @@ static void define16() // #201 static void define17() // #185 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define at(x, y) x##y\n" "#define b(...) \\\n" "aa(__VA_ARGS__, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , \\\n" @@ -981,7 +989,6 @@ static void define17() // #185 static void define18() // #130 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define MAC2STR(x) x[0],x[1],x[2],x[3],x[4],x[5]\n" "#define FT_DEBUG(fmt, args...) if(pGlobalCtx && pGlobalCtx->debug_level>=2) printf(\"FT-dbg: \"fmt, ##args)\n" "\n" @@ -994,7 +1001,6 @@ static void define18() // #130 static void define19() // #124 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(tok) tok##suffix\n" "\n" "CONCAT(Test);\n" @@ -1019,7 +1025,6 @@ static void define20() // #113 static void define21() // #66 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "FIGHT_FOO(1, GETMYID(a));\n"; @@ -1030,7 +1035,6 @@ static void define21() // #66 static void define22() // #40 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define COUNTER_NAME(NAME, ...) NAME##Count\n" "#define COMMA ,\n" "\n" @@ -1178,15 +1182,10 @@ static void define_invalid_3() const char code[] = "#define R()\n" "R\n"; simplecpp::OutputList outputList; - // a function-like macro name at the end of the file is not a macro call, gcc outputs R - if (USE_HIDESET_EXPANSION) { - ASSERT_EQUALS("\nR", preprocess(code, &outputList)); - ASSERT_EQUALS("", toString(outputList)); - } else { - // TODO: the old macro expansion reports an error - TODO_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); - TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); - } + // a function-like macro name at the end of the file is not a macro call, gcc outputs R. + // TODO: the old macro expansion reports an error + TODO_OLD_EXPANSION_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); + TODO_OLD_EXPANSION_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); } static void define_wrong_number_of_arguments() @@ -1230,15 +1229,10 @@ static void define_invalid_4() const char code[] = "#define X(...)\n" "X\n"; simplecpp::OutputList outputList; - // a function-like macro name at the end of the file is not a macro call, gcc outputs X - if (USE_HIDESET_EXPANSION) { - ASSERT_EQUALS("\nX", preprocess(code, &outputList)); - ASSERT_EQUALS("", toString(outputList)); - } else { - // TODO: the old macro expansion reports an error - TODO_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); - TODO_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); - } + // a function-like macro name at the end of the file is not a macro call, gcc outputs X. + // TODO: the old macro expansion reports an error + TODO_OLD_EXPANSION_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); + TODO_OLD_EXPANSION_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); } static void define_define_1() @@ -1259,7 +1253,6 @@ static void define_define_2() static void define_define_3() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define ABC 123\n" "#define A(B) A##B\n" "A(BC)"; @@ -1323,7 +1316,6 @@ static void define_define_9() // line break in nested macro call static void define_define_10() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define glue(a, b) a ## b\n" "#define xglue(a, b) glue(a, b)\n" "#define AB 1\n" @@ -1334,7 +1326,6 @@ static void define_define_10() static void define_define_11() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define XY(x, y) x ## y\n" "#define XY2(x, y) XY(x, y)\n" "#define PORT XY2(P, 2)\n" @@ -1345,7 +1336,6 @@ static void define_define_11() static void define_define_11a() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A_B_C 0x1\n" "#define A_ADDRESS 0x00001000U\n" "#define A ((uint32_t ) A_ADDRESS)\n" @@ -1364,7 +1354,6 @@ static void define_define_11a() static void define_define_12() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define XY(Z) Z\n" "#define X(ID) X##ID(0)\n" "X(Y)\n"; @@ -1400,7 +1389,6 @@ static void define_define_15() // issue #72 without __VA_ARGS__ static void define_define_16() // issue #72 with __VA_ARGS__ { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define ab(a, b) a##b\n" "#define foo(...) ab(f, 2) (__VA_ARGS__)\n" "#define f2(x, y) x y\n" @@ -1436,7 +1424,6 @@ static void define_define_19() // #292 static void define_define_20() // #384 arg contains comma { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define Z_IS_ENABLED1(config_macro) Z_IS_ENABLED2(_XXXX##config_macro)\n" "#define _XXXX1 _YYYY,\n" "#define Z_IS_ENABLED2(one_or_two_args) Z_IS_ENABLED3(one_or_two_args 1, 0)\n" @@ -1469,7 +1456,6 @@ static void define_define_21() // #397 DEBRACKET macro static void define_define_22() // #400 inner macro not expanded after hash hash { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define FOO(a) CAT(DO, STUFF)(1,2)\n" "#define DOSTUFF(a, b) CAT(3, 4)\n" "#define CAT(a, b) a##b\n" @@ -1479,7 +1465,6 @@ static void define_define_22() // #400 inner macro not expanded after hash hash static void define_define_23() // #403 crash (infinite recursion) { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define C_(x, y) x ## y\n" "#define C(x, y) C_(x, y)\n" "#define X(func) C(Y, C(func, Z))\n" @@ -1829,7 +1814,6 @@ static void error8() static void garbage() { - HIDESET_EXPANSION_TODO(); // ## simplecpp::OutputList outputList; outputList.clear(); @@ -2029,7 +2013,6 @@ static void hashEscapeTrailingBackslash() static void hashhash1() // #4703 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define MACRO( A, B, C ) class A##B##C##Creator {};\n" "MACRO( B\t, U , G )"; ASSERT_EQUALS("\nclass BUGCreator { } ;", preprocess(code)); @@ -2037,7 +2020,6 @@ static void hashhash1() // #4703 static void hashhash2() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x) a##x\n" "#define B 0\n" "A(B)"; @@ -2046,7 +2028,6 @@ static void hashhash2() static void hashhash3() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(B) A##B\n" "#define a(B) A(B)\n" "a(A(B))"; @@ -2055,7 +2036,6 @@ static void hashhash3() static void hashhash4() // nonstandard gcc/clang extension for empty varargs { - HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define A(x,y...) a(x,##y)\n" @@ -2070,7 +2050,6 @@ static void hashhash4() // nonstandard gcc/clang extension for empty varargs static void hashhash4a() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define GETMYID(a) ((a))+1\n" "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" "#define FIGHT_BAR(c, args...) bar(c, ##args)\n" @@ -2086,7 +2065,6 @@ static void hashhash5() static void hashhash6() { - HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define A(X, ...) LOG(X, ##__VA_ARGS__)\n" @@ -2102,7 +2080,6 @@ static void hashhash6() static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) { - HIDESET_EXPANSION_TODO(); // ## const char *code; code = "#define hash_hash # ## #\n" @@ -2112,7 +2089,6 @@ static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) static void hashhash8() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define a(xy) x##y = xy\n" "a(123);"; ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); @@ -2120,7 +2096,6 @@ static void hashhash8() static void hashhash9() { - HIDESET_EXPANSION_TODO(); // ## const char * code = "#define ADD_OPERATOR(OP) void operator OP ## = (void) { x = x OP 1; }\n" "ADD_OPERATOR(+);\n" "ADD_OPERATOR(-);\n" @@ -2168,11 +2143,11 @@ static void hashhash9() static void hashhash10() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x # #\n" "x"; - // a space is added before the #, as gcc and clang do, so the output is not read as a directive - ASSERT_EQUALS(" # #", preprocess(code)); + // a space is added before the #, as gcc and clang do, so the output is not read as a directive. + // TODO: the old macro expansion gives the tokens the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # #", " # #", preprocess(code)); } static void hashFromMacroAtLineStart() @@ -2180,23 +2155,20 @@ static void hashFromMacroAtLineStart() // clang testsuite hash_space.c: a space before the # so that the output is not read as a directive const char code[] = "#define HASH #\n" "HASH define foo bar\n"; - if (USE_HIDESET_EXPANSION) - ASSERT_EQUALS("\n # define foo bar", preprocess(code)); - else // TODO: the old macro expansion gives the # the location of the #define - TODO_ASSERT_EQUALS("\n # define foo bar", " #\ndefine foo bar", preprocess(code)); + // TODO: the old macro expansion gives the # the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # define foo bar", " #\ndefine foo bar", preprocess(code)); } static void hashhash11() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x # # #\n" "x"; - ASSERT_EQUALS(" # # #", preprocess(code)); + // TODO: the old macro expansion gives the tokens the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # # #", " # # #", preprocess(code)); } static void hashhash12() { - HIDESET_EXPANSION_TODO(); // ## { const char code[] = "#define MAX_FOO 1\n" "#define MAX_FOO_AA 2\n" @@ -2234,7 +2206,6 @@ static void hashhash12() static void hashhash13() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define X(x) x##U\n" "X((1<<1)-1)"; ASSERT_EQUALS("\n( 1 << 1 ) - 1U", preprocess(code)); @@ -2246,7 +2217,6 @@ static void hashhash13() static void hashhash_string_literal() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define UL(x) x##_ul\n" "\"ABC\"_ul;\n" @@ -2257,7 +2227,6 @@ static void hashhash_string_literal() static void hashhash_string_wrapped() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define STR(x) CONCAT(x,s)\n" @@ -2268,7 +2237,6 @@ static void hashhash_string_wrapped() static void hashhash_char_literal() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('a');"; @@ -2278,7 +2246,6 @@ static void hashhash_char_literal() static void hashhash_multichar_literal() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('abcd');"; @@ -2288,7 +2255,6 @@ static void hashhash_multichar_literal() static void hashhash_char_escaped() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CH(x) x##_ch\n" "CH('\\'');"; @@ -2298,7 +2264,6 @@ static void hashhash_char_escaped() static void hashhash_string_nothing() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\",);"; @@ -2308,7 +2273,6 @@ static void hashhash_string_nothing() static void hashhash_string_char() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "CONCAT(\"ABC\", 'c');"; @@ -2319,7 +2283,6 @@ static void hashhash_string_char() static void hashhash_string_name() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b) a##b\n" "#define LIT _literal\n" @@ -2328,12 +2291,13 @@ static void hashhash_string_name() // TODO is this correct? clang fails because that's not really a valid thing but gcc seems to accept it // see https://gist.github.com/patrickdowling/877a25294f069bf059f3b07f9b5b7039 - ASSERT_EQUALS("\n\n\"string\" LIT ;", preprocess(code)); + // "string" and LIT are not pasted, so the result is rescanned and LIT is expanded. + // TODO: the old macro expansion does not rescan LIT + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n\n\"string\" _literal ;", "\n\n\"string\" LIT ;", preprocess(code)); } static void hashhashhash_int_literal() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define CONCAT(a,b,c) a##b##c\n" "#define PASTER(a,b,c) CONCAT(a,b,c)\n" @@ -2344,7 +2308,6 @@ static void hashhashhash_int_literal() static void hashhash_int_literal() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define PASTE(a,b) a##b\n" "PASTE(123,_i);\n" @@ -2355,7 +2318,6 @@ static void hashhash_int_literal() static void hashhash_invalid_1() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define f(a) (##x)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2364,7 +2326,6 @@ static void hashhash_invalid_1() static void hashhash_invalid_2() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define f(a) (x##)\nf(1)\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); @@ -2373,7 +2334,6 @@ static void hashhash_invalid_2() static void hashhash_invalid_string_number() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define BAD(x) x##12345\nBAD(\"ABC\")\n"; @@ -2384,7 +2344,6 @@ static void hashhash_invalid_string_number() static void hashhash_invalid_missing_args() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define BAD(x) ##x\nBAD()\n"; @@ -2395,7 +2354,6 @@ static void hashhash_invalid_missing_args() static void hashhash_null_stmt() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "# define B(x) C ## x\n" "#\n" @@ -2408,7 +2366,6 @@ static void hashhash_null_stmt() static void hashhash_empty_va_args() { - HIDESET_EXPANSION_TODO(); // ## // #395 hash hash with an empty __VA_ARGS__ in a macro const char code[] = "#define CAT(a, ...) a##__VA_ARGS__\n" @@ -2420,7 +2377,6 @@ static void hashhash_empty_va_args() static void hashhash_punctuator() // #724 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "#define MINUS_GT -##>\n" @@ -2433,7 +2389,6 @@ static void hashhash_punctuator() // #724 static void hashhash_punctuator_invalid() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(+,x)\n"; @@ -2444,7 +2399,6 @@ static void hashhash_punctuator_invalid() static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define x 3\n" "#define f(a) f(x * (a))\n" @@ -2465,51 +2419,50 @@ static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 "(f)^m(m);\n" "p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };\n" "char c[2][6] = { str(hello), str() };\n"; - // TODO: r(,) leaves an empty token - TODO_ASSERT_EQUALS("\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" - "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" - "^ m ( 0 , 1 ) ;\n" - "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" - "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" - "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" - "^ m ( 0 , 1 ) ;\n" - "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" - "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", - preprocess(code)); + // TODO: the old macro expansion leaves an empty token for r(,) + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + preprocess(code)); } static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define str(s) # s\n" "#define xstr(s) str(s)\n" @@ -2549,19 +2502,18 @@ static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define t(x,y,z) x ## y ## z\n" "int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),\n" " t(10,,), t(,11,), t(,,12), t(,,) };\n"; - // TODO: t(,,) leaves an empty token - TODO_ASSERT_EQUALS("\n" - "int j [ ] = { 123 , 45 , 67 , 89 ,\n" - "10 , 11 , 12 , } ;", - "\n" - "int j [ ] = { 123 , 45 , 67 , 89 ,\n" - "10 , 11 , 12 , } ;", - preprocess(code)); + // TODO: the old macro expansion leaves an empty token for t(,,) + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + "\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + preprocess(code)); } static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 @@ -2619,7 +2571,6 @@ static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE static void hashhash_empty_placemarker() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define r(x,y) x ## y\n" "{ r(,) }\n"; @@ -2631,13 +2582,12 @@ static void hashhash_empty_placemarker() std::string str; for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) str += "[" + tok->str() + "]"; - // TODO: pasting two empty arguments gives an empty token - TODO_ASSERT_EQUALS("[{][}]", "[{][][}]", str); + // TODO: with the old macro expansion, pasting two empty arguments gives an empty token + TODO_OLD_EXPANSION_ASSERT_EQUALS("[{][}]", "[{][][}]", str); } static void hashhash_invalid_dotdot() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(.,.)\n"; @@ -2648,7 +2598,6 @@ static void hashhash_invalid_dotdot() static void hashhash_invalid_name_string() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(a,\"s\")\n"; @@ -2659,7 +2608,6 @@ static void hashhash_invalid_name_string() static void hashhash_invalid_number_string() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "cat(1,\"s\")\n"; @@ -2670,7 +2618,6 @@ static void hashhash_invalid_number_string() static void hashhash_encoding_prefix() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define cat(a,b) a##b\n" "#define WIDE(s) L ## s\n" @@ -2682,7 +2629,6 @@ static void hashhash_encoding_prefix() static void hashhash_va_args_unexpected() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define C(...)!##__VA_ARGS__\n" "C(1)\n"; @@ -2693,7 +2639,6 @@ static void hashhash_va_args_unexpected() static void hashhash_universal_character() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x,y) x##y\nint A(\\u01,04);\n"; simplecpp::OutputList outputList; @@ -2703,7 +2648,6 @@ static void hashhash_universal_character() static void hashhash_universal_character_2() { - HIDESET_EXPANSION_TODO(); // ## const char code[] = "#define A(x,y) x##y\nint A(\\U0104, 0104);\n"; simplecpp::OutputList outputList; @@ -3094,7 +3038,7 @@ static void ifDefinedInvalid2() static void ifDefinedHashHash() { - HIDESET_EXPANSION_TODO(); // ## + HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define ENABLE(FEATURE) defined ENABLE_##FEATURE\n" "#define ENABLE_FOO 1\n" "#if ENABLE(FOO)\n" @@ -3109,7 +3053,7 @@ static void ifDefinedHashHash() static void ifDefinedHashHash2() { - HIDESET_EXPANSION_TODO(); // ## + HIDESET_EXPANSION_TODO(); // defined in a replacement list // #409 // do not crash when expanding P() (as ## rhs is "null") // note: gcc outputs "defined E" @@ -4192,7 +4136,6 @@ static void include2() static void include3() // #16 - crash when expanding macro from header { - HIDESET_EXPANSION_TODO(); // ## const char code_c[] = "#include \"A.h\"\n" "glue(1,2,3,4)\n"; const char code_h[] = "#define glue(a,b,c,d) a##b##c##d\n"; @@ -4831,7 +4774,7 @@ static void tokenMacro9() static void tokenMacro10() { - HIDESET_EXPANSION_TODO(); // ## + HIDESET_EXPANSION_TODO(); // Token::macro and location rules // the results of # and ## have no macro name const char code[] = "#define str(x) #x\n" "#define cat(a,b) a##b\n" @@ -4966,13 +4909,12 @@ static void hideSetExpansion_arg_self() { const char code[] = "#define f(x) x\n" "f(f)(1)\n"; - if (USE_HIDESET_EXPANSION) - ASSERT_EQUALS("\n" - "f ( 1 )", preprocess(code)); - else // TODO: the old macro expansion is wrong - TODO_ASSERT_EQUALS("\n" - "f ( 1 )", "\n" - "1", preprocess(code)); + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "f ( 1 )", + "\n" + "1", + preprocess(code)); } static void hideSetExpansion_blue_in_arg() From 3d44fcc11eab0b32afe59ec6363c97e0d99112dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:13:57 +0100 Subject: [PATCH 21/28] HideSetExpander: add MacroCall and readMacroCall() MacroCall has what the replacement of a macro call depends on: the hide set, the arguments and the whitespace after the call. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 101 +++++++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d20de326..68bd762f 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3025,6 +3025,18 @@ namespace simplecpp { const Token *raw; }; + /** What a macro call is replaced by depends on */ + struct MacroCall { + /** the hide set of the result */ + std::set hideSet; + /** the arguments as written in the call */ + std::vector arguments; + /** the arguments, macro expanded */ + std::vector expandedArguments; + /** there is whitespace after the call */ + bool whitespaceAfter{}; + }; + static Macro::Error unsupported(const Location &loc, const std::string &what) { return {loc, what + " is not supported by the hide-set expansion yet"}; } @@ -3054,35 +3066,44 @@ namespace simplecpp { continue; } - std::set hideSet; - std::vector arguments; - std::vector expandedArguments; - bool whitespaceAfter = tok->whitespaceahead; - if (macro.functionLike()) { - TokenList call(files); - call.push_back(tok); - readCall(call, input); - const std::vector delimiters = macro.getMacroParameters(call.cfront(), false); - // the hide set of the name and the ) have the macro calls that this call - // was produced by, the arguments are not affected by them - const std::set &hs1 = call.cfront()->hideSet(); - const std::set &hs2 = delimiters.back()->hideSet(); - std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(hideSet, hideSet.begin())); - arguments = getArguments(macro, delimiters); - expandedArguments = expandArguments(arguments); - whitespaceAfter = delimiters.back()->whitespaceahead; - } else { - hideSet = tok->hideSet(); - delete tok; - } - hideSet.insert(macro.name()); - + const MacroCall call = readMacroCall(macro, tok, input); TokenList result(files); - substitute(result, macro, arguments, expandedArguments, hideSet, whitespaceAfter); + substitute(result, macro, call); input.pushFront(result); } } + /** + * Read a macro call: the name, and "( ... )" for a function-like macro. + * @param macro the macro + * @param nameTok the macro name, it is deleted + * @param input the input after the name + * @throws wrongNumberOfParameters if a function-like macro has the wrong number of arguments + */ + MacroCall readMacroCall(const Macro ¯o, Token *nameTok, Input &input) { + MacroCall call; + if (!macro.functionLike()) { + call.hideSet = nameTok->hideSet(); + call.whitespaceAfter = nameTok->whitespaceahead; + delete nameTok; + } else { + TokenList tokens(files); + tokens.push_back(nameTok); + readCall(tokens, input); + const std::vector delimiters = macro.getMacroParameters(tokens.cfront(), false); + // the name and the ) have the hide sets of the macro calls that this call + // was produced by, the arguments are not affected by them + const std::set &hs1 = tokens.cfront()->hideSet(); + const std::set &hs2 = delimiters.back()->hideSet(); + std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(call.hideSet, call.hideSet.begin())); + call.arguments = getArguments(macro, delimiters); + call.expandedArguments = expandArguments(call.arguments); + call.whitespaceAfter = delimiters.back()->whitespaceahead; + } + call.hideSet.insert(macro.name()); + return call; + } + /** Read "( ... )" after the macro name in call */ static void readCall(TokenList &call, Input &input) { int par = 0; @@ -3183,23 +3204,22 @@ namespace simplecpp { * @param operand destination tokenlist * @param macro the macro * @param tok the token in the replacement list; for # it is moved to the parameter - * @param arguments the arguments as written in the call - * @param expandedArguments the arguments, macro expanded + * @param call the macro call * @param expand use the expanded argument for a parameter */ - void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const std::vector &arguments, const std::vector &expandedArguments, bool expand) const { + void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call, bool expand) const { const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; // # is an operator in a function-like macro when it is followed by a parameter const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; - if (hashArg < arguments.size()) { - operand.push_back(new Token(stringify(arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); + if (hashArg < call.arguments.size()) { + operand.push_back(new Token(stringify(call.arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); operand.back()->macro = macro.name(); tok = next; return; } const unsigned int argnr = getParameter(macro, tok); - if (argnr < arguments.size()) { - const TokenList &argument = expand ? expandedArguments[argnr] : arguments[argnr]; + if (argnr < call.arguments.size()) { + const TokenList &argument = expand ? call.expandedArguments[argnr] : call.arguments[argnr]; for (const Token *argtok = argument.cfront(); argtok; argtok = argtok->next) operand.push_back(new Token(*argtok)); return; @@ -3210,14 +3230,11 @@ namespace simplecpp { /** * The replacement list of the macro, with the parameters replaced by the arguments. - * @param result destination tokenlist - * @param macro the macro - * @param arguments the arguments as written in the call, for # - * @param expandedArguments the arguments, macro expanded - * @param hideSet the hide set of the result - * @param whitespaceAfter there is whitespace after the macro call + * @param result destination tokenlist + * @param macro the macro + * @param call the macro call */ - void substitute(TokenList &result, const Macro ¯o, const std::vector &arguments, const std::vector &expandedArguments, const std::set &hideSet, bool whitespaceAfter) const { + void substitute(TokenList &result, const Macro ¯o, const MacroCall &call) const { if (macro.variadicOpt) throw unsupported(loc, "__VA_OPT__"); const Token *hashHash = nullptr; // the ## before the current operand @@ -3233,11 +3250,11 @@ namespace simplecpp { if (tok->str() == DEFINED) throw unsupported(loc, "defined"); const Token * const operandTok = tok; - const bool parameter = getParameter(macro, tok) < arguments.size(); + const bool parameter = getParameter(macro, tok) < call.arguments.size(); // an argument is not expanded when it is an operand of ## const bool expand = !hashHash && !isHashHash(macro, tok->next); TokenList operand(files); - getOperand(operand, macro, tok, arguments, expandedArguments, expand); + getOperand(operand, macro, tok, call, expand); const bool empty = operand.empty(); if (hashHash) paste(result, operand, previousEmpty, macro, hashHash, operandTok); @@ -3253,13 +3270,13 @@ namespace simplecpp { throw Macro::invalidHashHash::unexpectedNewline(hashHash->location, macro.name()); for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; - for (const TokenString &name : hideSet) + for (const TokenString &name : call.hideSet) tok->addToHideSet(name); } // whitespace at the end of the replacement list does not count, the // expansion is followed by the whitespace after the macro call if (result.back()) - result.back()->whitespaceahead = whitespaceAfter; + result.back()->whitespaceahead = call.whitespaceAfter; } const MacroMap ¯os; From 6431981b56a719921c4a3ad2aae8e2dc283d5a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:16:06 +0100 Subject: [PATCH 22/28] HideSetExpander: add newToken() for tokens from the replacement list Token::macro is set in one place. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 68bd762f..f1c52b65 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3198,6 +3198,13 @@ namespace simplecpp { result.takeTokens(operand); } + /** A new token from the replacement list of macro, Token::macro is the macro name */ + Token *newToken(const TokenString &str, const Macro ¯o, bool whitespaceahead) const { + auto *tok = new Token(str, loc, whitespaceahead); + tok->macro = macro.name(); + return tok; + } + /** * The tokens that a token in the replacement list is replaced by: a parameter by its * argument, # and a parameter by a string literal, and other tokens by a copy. @@ -3212,8 +3219,7 @@ namespace simplecpp { // # is an operator in a function-like macro when it is followed by a parameter const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; if (hashArg < call.arguments.size()) { - operand.push_back(new Token(stringify(call.arguments[hashArg], loc, macro.name(), outputList), loc, next->whitespaceahead)); - operand.back()->macro = macro.name(); + operand.push_back(newToken(stringify(call.arguments[hashArg], loc, macro.name(), outputList), macro, next->whitespaceahead)); tok = next; return; } @@ -3224,8 +3230,7 @@ namespace simplecpp { operand.push_back(new Token(*argtok)); return; } - operand.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); - operand.back()->macro = macro.name(); + operand.push_back(newToken(tok->str(), macro, tok->whitespaceahead)); } /** From f2da8f7362cee63534dd1e0b7d27ef520abdc943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:18:11 +0100 Subject: [PATCH 23/28] Add Token::addToHideSet() for a set of macro names No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 3 +-- simplecpp.h | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f1c52b65..5d1a86f4 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3275,8 +3275,7 @@ namespace simplecpp { throw Macro::invalidHashHash::unexpectedNewline(hashHash->location, macro.name()); for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; - for (const TokenString &name : call.hideSet) - tok->addToHideSet(name); + tok->addToHideSet(call.hideSet); } // whitespace at the end of the replacement list does not count, the // expansion is followed by the whitespace after the macro call diff --git a/simplecpp.h b/simplecpp.h index d4f83d2f..4f119efd 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -222,6 +222,9 @@ namespace simplecpp { void addToHideSet(const TokenString ¯oName) { mHideSet.insert(macroName); } + void addToHideSet(const std::set &hideSet) { + mHideSet.insert(hideSet.cbegin(), hideSet.cend()); + } const std::set &hideSet() const { return mHideSet; } From 9d2bbe1b2bfccf25745a1dea41ca17d5ac5f3115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:20:48 +0100 Subject: [PATCH 24/28] Split Macro::expandDefined() into findDefinedOperand() and definedMacroName() They do not depend on how the old expansion stores the arguments, so the hide-set expansion can use them. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 92 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5d1a86f4..d133c0b1 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2616,6 +2616,55 @@ namespace simplecpp { return tok->next; } + /** + * Find the operand of "defined" in a replacement list: "defined X", "defined(X)" or "defined X ## Y". + * @param tok the "defined" token + * @param first output: X + * @param second output: Y for "X ## Y", otherwise nullptr + * @param last output: the last token of the operand + * @return false if there is no operand + */ + static bool findDefinedOperand(const Token *tok, const Token *&first, const Token *&second, const Token *&last) { + const Token * const tok2 = tok->next; + const Token * const tok3 = tok2 ? tok2->next : nullptr; + const Token * const tok4 = tok3 ? tok3->next : nullptr; + first = last = second = nullptr; + if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { + first = tok3; + last = tok4; + } else if (sameline(tok,tok2) && tok2->name) { + first = last = tok2; + } + if (!first) + return false; + if (first->next && first->next->op == '#' && first->next->next && first->next->next->op == '#' && first->next->next->next && first->next->next->next->name && sameline(first,first->next->next->next)) { + second = first->next->next->next; + last = second; + } + return true; + } + + /** + * The name of the macro that "defined" checks. + * @param first X in "defined X" + * @param second Y in "defined X ## Y", or nullptr + * @param argText for "X ## Y": gives the text of a parameter, and returns false if the token is not a parameter + */ + template + static std::string definedMacroName(const Token *first, const Token *second, ArgText argText) { + std::string macroName = first->str(); + if (second) { + std::string text; + if (argText(first, text)) + macroName = text; + if (argText(second, text)) + macroName += text; + else + macroName += second->str(); + } + return macroName; + } + /** * Expand "defined X" or "defined(X)" in the replacement list to 1 or 0. * The operand can be a parameter, or two names pasted with ##. @@ -2628,35 +2677,22 @@ namespace simplecpp { * @return false if there is no operand, then nothing is added to output */ bool expandDefined(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::vector ¶metertokens, const Token *&next) const { - const Token * const tok2 = tok->next; - const Token * const tok3 = tok2 ? tok2->next : nullptr; - const Token * const tok4 = tok3 ? tok3->next : nullptr; - const Token *defToken = nullptr; - const Token *lastToken = nullptr; - if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { - defToken = tok3; - lastToken = tok4; - } else if (sameline(tok,tok2) && tok2->name) { - defToken = lastToken = tok2; - } - if (defToken) { - std::string macroName = defToken->str(); - if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { - TokenList temp(files); - if (expandArg(temp, defToken, parametertokens)) - macroName = temp.cback()->str(); - if (expandArg(temp, defToken->next->next->next, parametertokens)) - macroName += temp.cback() ? temp.cback()->str() : ""; - else - macroName += defToken->next->next->next->str(); - lastToken = defToken->next->next->next; - } - const bool def = (macros.find(macroName) != macros.end()); - output.push_back(newMacroToken(def ? "1" : "0", loc, true)); - next = lastToken->next; + const Token *first; + const Token *second; + const Token *last; + if (!findDefinedOperand(tok, first, second, last)) + return false; + TokenList temp(files); + const std::string macroName = definedMacroName(first, second, [&](const Token *argtok, std::string &text) { + if (!expandArg(temp, argtok, parametertokens)) + return false; + text = temp.cback() ? temp.cback()->str() : ""; return true; - } - return false; + }); + const bool def = (macros.find(macroName) != macros.end()); + output.push_back(newMacroToken(def ? "1" : "0", loc, true)); + next = last->next; + return true; } bool expandArg(TokenList &output, const Token *tok, const std::vector ¶metertokens) const { From 7f11b76d3034ae45e50780b2dbf0426795a3c69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:23:18 +0100 Subject: [PATCH 25/28] HideSetExpander: handle defined in a replacement list findDefinedOperand() and definedMacroName() are used, as in the old expansion. 5 more tests pass with the hide-set expansion. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 22 ++++++++++++++++++---- test.cpp | 5 ----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d133c0b1..eaaea433 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2988,7 +2988,7 @@ namespace simplecpp { * The result of an expansion is pushed back onto the input and scanned again, so a * function-like macro whose arguments follow the expansion in the source is found by * normal rescanning. - * Work in progress: __VA_OPT__ and "defined" in a replacement list are not handled yet. + * Work in progress: __VA_OPT__ is not handled yet. */ class HideSetExpander { public: @@ -3243,7 +3243,7 @@ namespace simplecpp { /** * The tokens that a token in the replacement list is replaced by: a parameter by its - * argument, # and a parameter by a string literal, and other tokens by a copy. + * argument, # and a parameter by a string literal, "defined X" by 1 or 0, and other tokens by a copy. * @param operand destination tokenlist * @param macro the macro * @param tok the token in the replacement list; for # it is moved to the parameter @@ -3251,6 +3251,22 @@ namespace simplecpp { * @param expand use the expanded argument for a parameter */ void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call, bool expand) const { + // "defined X" is replaced by 1 or 0 + const Token *first; + const Token *second; + const Token *last; + if (tok->str() == DEFINED && Macro::findDefinedOperand(tok, first, second, last)) { + const std::string macroName = Macro::definedMacroName(first, second, [&](const Token *argtok, std::string &text) { + const unsigned int argnr = getParameter(macro, argtok); + if (argnr >= call.arguments.size()) + return false; + text = call.arguments[argnr].cback() ? call.arguments[argnr].cback()->str() : ""; + return true; + }); + operand.push_back(newToken((macros.find(macroName) != macros.end()) ? "1" : "0", macro, last->whitespaceahead)); + tok = last; + return; + } const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; // # is an operator in a function-like macro when it is followed by a parameter const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; @@ -3288,8 +3304,6 @@ namespace simplecpp { tok = tok->next; continue; } - if (tok->str() == DEFINED) - throw unsupported(loc, "defined"); const Token * const operandTok = tok; const bool parameter = getParameter(macro, tok) < call.arguments.size(); // an argument is not expanded when it is an operand of ## diff --git a/test.cpp b/test.cpp index fcff596e..ce55ef5a 100644 --- a/test.cpp +++ b/test.cpp @@ -2996,7 +2996,6 @@ static void ifDefinedNoPar() static void ifDefinedNested() { - HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define FOODEF defined(FOO)\n" "#if FOODEF\n" "X\n" @@ -3009,7 +3008,6 @@ static void ifDefinedNested() static void ifDefinedNestedNoPar() { - HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define FOODEF defined FOO\n" "#if FOODEF\n" "X\n" @@ -3038,7 +3036,6 @@ static void ifDefinedInvalid2() static void ifDefinedHashHash() { - HIDESET_EXPANSION_TODO(); // defined in a replacement list const char code[] = "#define ENABLE(FEATURE) defined ENABLE_##FEATURE\n" "#define ENABLE_FOO 1\n" "#if ENABLE(FOO)\n" @@ -3053,7 +3050,6 @@ static void ifDefinedHashHash() static void ifDefinedHashHash2() { - HIDESET_EXPANSION_TODO(); // defined in a replacement list // #409 // do not crash when expanding P() (as ## rhs is "null") // note: gcc outputs "defined E" @@ -3065,7 +3061,6 @@ static void ifDefinedHashHash2() static void definedAtEndOfFile() { - HIDESET_EXPANSION_TODO(); // defined in a replacement list // "defined" from a macro expansion is undefined behavior; gcc outputs "defined" here. // The operand is the last token of the file. const char code[] = "#define X\n" From 25e79f26aaec98e8fd43ce2cd7d46494461af52e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:45:49 +0100 Subject: [PATCH 26/28] HideSetExpander: add substituteRange() for a range of the replacement list __VA_OPT__ will use it for its content. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index eaaea433..31b9d87b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3286,17 +3286,18 @@ namespace simplecpp { } /** - * The replacement list of the macro, with the parameters replaced by the arguments. + * Tokens of the replacement list of the macro, with the parameters replaced by the + * arguments and ## done. * @param result destination tokenlist * @param macro the macro * @param call the macro call + * @param begin first token in the replacement list + * @param end the token after the last one */ - void substitute(TokenList &result, const Macro ¯o, const MacroCall &call) const { - if (macro.variadicOpt) - throw unsupported(loc, "__VA_OPT__"); + void substituteRange(TokenList &result, const Macro ¯o, const MacroCall &call, const Token *begin, const Token *end) const { const Token *hashHash = nullptr; // the ## before the current operand bool previousEmpty = false; // the previous operand was empty, it is a placemarker - for (const Token *tok = macro.valueToken; tok != macro.endToken; tok = tok->next) { + for (const Token *tok = begin; tok != end; tok = tok->next) { if (isHashHash(macro, tok)) { if (!result.back() && !previousEmpty) throw Macro::invalidHashHash(tok->location, macro.name(), "Missing first argument"); @@ -3323,6 +3324,18 @@ namespace simplecpp { } if (hashHash) throw Macro::invalidHashHash::unexpectedNewline(hashHash->location, macro.name()); + } + + /** + * The replacement list of the macro, with the parameters replaced by the arguments. + * @param result destination tokenlist + * @param macro the macro + * @param call the macro call + */ + void substitute(TokenList &result, const Macro ¯o, const MacroCall &call) const { + if (macro.variadicOpt) + throw unsupported(loc, "__VA_OPT__"); + substituteRange(result, macro, call, macro.valueToken, macro.endToken); for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; tok->addToHideSet(call.hideSet); From 751542287b58ea42c766befa952009610acca254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 15:48:56 +0100 Subject: [PATCH 27/28] HideSetExpander: handle __VA_OPT__ As in C23 6.10.5.1: __VA_OPT__(content) is replaced by nothing when the variable arguments are macro expanded to nothing, otherwise by its content, which is substituted like the replacement list. #__VA_OPT__(...) gives a string literal, and __VA_OPT__ can be an operand of ##. These give the same result as gcc with the hide-set expansion, but not with the old one: F(E) where E expands to nothing, #__VA_OPT__(a b), and a ## __VA_OPT__(b c) ## d with no variable arguments. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 51 +++++++++++++++++++++++++---- test.cpp | 90 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 11 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 31b9d87b..a64e2c18 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2988,7 +2988,7 @@ namespace simplecpp { * The result of an expansion is pushed back onto the input and scanned again, so a * function-like macro whose arguments follow the expansion in the source is found by * normal rescanning. - * Work in progress: __VA_OPT__ is not handled yet. + * Work in progress: the old Token::macro and location rules are not followed yet. */ class HideSetExpander { public: @@ -3073,10 +3073,6 @@ namespace simplecpp { bool whitespaceAfter{}; }; - static Macro::Error unsupported(const Location &loc, const std::string &what) { - return {loc, what + " is not supported by the hide-set expansion yet"}; - } - /** Expand the pending tokens of the input. Source tokens are only read for macro arguments. */ void expandInput(TokenList &output, Input &input) { while (!input.pending.empty()) { @@ -3234,6 +3230,37 @@ namespace simplecpp { result.takeTokens(operand); } + /** tok is __VA_OPT__ in the replacement list of a variadic macro */ + static bool isVaOpt(const Macro ¯o, const Token *tok) { + return macro.variadicOpt && tok && tok != macro.endToken && tok->str() == "__VA_OPT__"; + } + + /** + * __VA_OPT__(content) is replaced by the content, or by nothing (a placemarker) + * when the variable arguments are macro expanded to nothing (C23 6.10.5.1). + * @param operand destination tokenlist + * @param macro the macro + * @param tok the __VA_OPT__ token, it is moved to the closing ) + * @param call the macro call + */ + void getVaOpt(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { + // the parentheses were checked when the macro was defined + const Token * const lpar = tok->next; + const Token *rpar = lpar->next; + for (int par = 1; rpar != macro.endToken; rpar = rpar->next) { + if (rpar->op == '(') + ++par; + else if (rpar->op == ')' && --par == 0) + break; + } + tok = rpar; + const bool noVariableArguments = call.expandedArguments.empty() || call.expandedArguments.back().empty(); + if (!noVariableArguments) + substituteRange(operand, macro, call, lpar->next, rpar); + if (operand.back()) + operand.back()->whitespaceahead = rpar->whitespaceahead; + } + /** A new token from the replacement list of macro, Token::macro is the macro name */ Token *newToken(const TokenString &str, const Macro ¯o, bool whitespaceahead) const { auto *tok = new Token(str, loc, whitespaceahead); @@ -3267,7 +3294,19 @@ namespace simplecpp { tok = last; return; } + if (isVaOpt(macro, tok)) { + getVaOpt(operand, macro, tok, call); + return; + } const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; + if (tok->op == '#' && isVaOpt(macro, next)) { + // #__VA_OPT__(content): the content as a string literal + tok = next; + TokenList content(files); + getVaOpt(content, macro, tok, call); + operand.push_back(newToken(stringify(content, loc, macro.name(), outputList), macro, tok->whitespaceahead)); + return; + } // # is an operator in a function-like macro when it is followed by a parameter const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; if (hashArg < call.arguments.size()) { @@ -3333,8 +3372,6 @@ namespace simplecpp { * @param call the macro call */ void substitute(TokenList &result, const Macro ¯o, const MacroCall &call) const { - if (macro.variadicOpt) - throw unsupported(loc, "__VA_OPT__"); substituteRange(result, macro, call, macro.valueToken, macro.endToken); for (Token *tok = result.front(); tok; tok = tok->next) { tok->location = loc; diff --git a/test.cpp b/test.cpp index ce55ef5a..476509c5 100644 --- a/test.cpp +++ b/test.cpp @@ -1514,7 +1514,6 @@ static void define_va_args_4() // cppcheck trac #9754 static void define_va_opt_1() { - HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define p1(fmt, args...) printf(fmt __VA_OPT__(,) args)\n" "p1(\"hello\");\n" "p1(\"%s\", \"hello\");\n"; @@ -1526,7 +1525,6 @@ static void define_va_opt_1() static void define_va_opt_2() { - HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define err(...)\\\n" "__VA_OPT__(\\\n" "printf(__VA_ARGS__);\\\n" @@ -1637,7 +1635,6 @@ static void define_va_opt_7() static void define_va_opt_8() { - HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define f(...) #__VA_OPT__(x)\n" "const char* v1 = f();\n"; @@ -2540,7 +2537,6 @@ static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE { - HIDESET_EXPANSION_TODO(); // __VA_OPT__ const char code[] = "#define F(...) f(0 __VA_OPT__(,) __VA_ARGS__)\n" "#define G(X, ...) f(0, X __VA_OPT__(,) __VA_ARGS__)\n" @@ -5005,6 +5001,85 @@ static void hideSetExpansion_empty() "< > < >", preprocess(code)); } +// __VA_OPT__ (C23 6.10.5.1). The results are the same as gcc gives. +static void hideSetExpansion_va_opt_empty_after_expansion() +{ + const char code[] = "#define F(...) <__VA_OPT__(x)>\n" + "#define E\n" + "F() F(E) F(1)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "< > < > < x >", + "\n" + "\n" + "< > < x > < x >", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hash() +{ + const char code[] = "#define X 1\n" + "#define F(a,...) #__VA_OPT__(a b)\n" + "F(X,2) F(X)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "\"1 b\" \"\"", + "\n" + "\n" + "\"X\" b \"\"", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hashhash() +{ + const char code[] = "#define F(a,...) a ## __VA_OPT__(b c) ## d\n" + "F(x,1) F(x)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "xb cd xd", + "\n" + "xb cd x# \"d\"", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hash_in_content() +{ + const char code[] = "#define F(a,...) __VA_OPT__(a##a) __VA_OPT__(#a)\n" + "F(x,1)\n"; + ASSERT_EQUALS("\n" + "xx \"x\"", preprocess(code)); +} + +static void hideSetExpansion_va_opt_rescan() +{ + const char code[] = "#define LP (\n" + "#define F(...) __VA_OPT__(LP)\n" + "F(1)\n"; + ASSERT_EQUALS("\n" + "\n" + "(", preprocess(code)); +} + +static void hideSetExpansion_va_opt_comma() +{ + const char code[] = "#define F(a,...) f(a __VA_OPT__(,) __VA_ARGS__)\n" + "#define X 1\n" + "F(X) F(X,X)\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( 1 ) f ( 1 , 1 )", preprocess(code)); +} + +static void hideSetExpansion_va_opt_parentheses() +{ + const char code[] = "#define F(...) __VA_OPT__((__VA_ARGS__))\n" + "F(a,(b)) F()\n"; + ASSERT_EQUALS("\n" + "( a , ( b ) )", preprocess(code)); +} + static void undef() { const char code[] = "#define A\n" @@ -6099,6 +6174,13 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hideSetExpansion_deep); TEST_CASE(hideSetExpansion_q_example); TEST_CASE(hideSetExpansion_empty); + TEST_CASE(hideSetExpansion_va_opt_empty_after_expansion); + TEST_CASE(hideSetExpansion_va_opt_hash); + TEST_CASE(hideSetExpansion_va_opt_hashhash); + TEST_CASE(hideSetExpansion_va_opt_hash_in_content); + TEST_CASE(hideSetExpansion_va_opt_rescan); + TEST_CASE(hideSetExpansion_va_opt_comma); + TEST_CASE(hideSetExpansion_va_opt_parentheses); TEST_CASE(undef); From 07f5f84b70266abf8b0537400eb5fbab8836596b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 8 Nov 2026 16:25:42 +0100 Subject: [PATCH 28/28] HideSetExpander: add getDefined() and getStringified() getOperand() was getting long. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 91 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a64e2c18..10826178 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3269,49 +3269,74 @@ namespace simplecpp { } /** - * The tokens that a token in the replacement list is replaced by: a parameter by its - * argument, # and a parameter by a string literal, "defined X" by 1 or 0, and other tokens by a copy. - * @param operand destination tokenlist - * @param macro the macro - * @param tok the token in the replacement list; for # it is moved to the parameter - * @param call the macro call - * @param expand use the expanded argument for a parameter + * "defined X" in the replacement list is replaced by 1 or 0. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; for "defined X" it is moved to the last token of X + * @param call the macro call + * @return true if tok is "defined" with an operand */ - void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call, bool expand) const { - // "defined X" is replaced by 1 or 0 + bool getDefined(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { const Token *first; const Token *second; const Token *last; - if (tok->str() == DEFINED && Macro::findDefinedOperand(tok, first, second, last)) { - const std::string macroName = Macro::definedMacroName(first, second, [&](const Token *argtok, std::string &text) { - const unsigned int argnr = getParameter(macro, argtok); - if (argnr >= call.arguments.size()) - return false; - text = call.arguments[argnr].cback() ? call.arguments[argnr].cback()->str() : ""; - return true; - }); - operand.push_back(newToken((macros.find(macroName) != macros.end()) ? "1" : "0", macro, last->whitespaceahead)); - tok = last; - return; - } - if (isVaOpt(macro, tok)) { - getVaOpt(operand, macro, tok, call); - return; - } + if (tok->str() != DEFINED || !Macro::findDefinedOperand(tok, first, second, last)) + return false; + const std::string macroName = Macro::definedMacroName(first, second, [&](const Token *argtok, std::string &text) { + const unsigned int argnr = getParameter(macro, argtok); + if (argnr >= call.arguments.size()) + return false; + text = call.arguments[argnr].cback() ? call.arguments[argnr].cback()->str() : ""; + return true; + }); + operand.push_back(newToken((macros.find(macroName) != macros.end()) ? "1" : "0", macro, last->whitespaceahead)); + tok = last; + return true; + } + + /** + * # followed by a parameter or by __VA_OPT__(content) is replaced by a string literal. + * # is only an operator in a function-like macro. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; for # it is moved to the last token of the operand + * @param call the macro call + * @return true if tok is the # operator + */ + bool getStringified(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { + if (tok->op != '#') + return false; const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; - if (tok->op == '#' && isVaOpt(macro, next)) { - // #__VA_OPT__(content): the content as a string literal + if (isVaOpt(macro, next)) { tok = next; TokenList content(files); getVaOpt(content, macro, tok, call); operand.push_back(newToken(stringify(content, loc, macro.name(), outputList), macro, tok->whitespaceahead)); - return; + return true; } - // # is an operator in a function-like macro when it is followed by a parameter - const unsigned int hashArg = (tok->op == '#') ? getParameter(macro, next) : ~0U; - if (hashArg < call.arguments.size()) { - operand.push_back(newToken(stringify(call.arguments[hashArg], loc, macro.name(), outputList), macro, next->whitespaceahead)); - tok = next; + const unsigned int argnr = getParameter(macro, next); + if (argnr >= call.arguments.size()) + return false; + operand.push_back(newToken(stringify(call.arguments[argnr], loc, macro.name(), outputList), macro, next->whitespaceahead)); + tok = next; + return true; + } + + /** + * The tokens that a token in the replacement list is replaced by: a parameter by its + * argument, # and a parameter by a string literal, "defined X" by 1 or 0, + * __VA_OPT__(content) by its content or nothing, and other tokens by a copy. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; it is moved to the last token that is used + * @param call the macro call + * @param expand use the expanded argument for a parameter + */ + void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call, bool expand) const { + if (getDefined(operand, macro, tok, call) || getStringified(operand, macro, tok, call)) + return; + if (isVaOpt(macro, tok)) { + getVaOpt(operand, macro, tok, call); return; } const unsigned int argnr = getParameter(macro, tok);