diff --git a/main.cpp b/main.cpp index ec0fc5c7..1aef98db 100644 --- a/main.cpp +++ b/main.cpp @@ -153,6 +153,12 @@ int main(int argc, char **argv) found = true; } break; + case 'h': + if (std::strcmp(arg, "-hideset-expansion")==0) { + dui.hideSetExpansion = true; + found = true; + } + break; } if (!found) { std::cout << "error: option '" << arg << "' is unknown." << std::endl; @@ -187,6 +193,7 @@ int main(int argc, char **argv) std::cout << " -e Output errors only." << std::endl; std::cout << " -f Fail when errors were encountered (exitcode 1)." << std::endl; std::cout << " -l Print lines numbers." << std::endl; + std::cout << " -hideset-expansion Use the new macro expansion with hide sets (work in progress)." << std::endl; return 0; } diff --git a/simplecpp.cpp b/simplecpp.cpp index a6adddad..10826178 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -27,12 +27,15 @@ #include #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -668,7 +671,9 @@ std::string simplecpp::TokenList::stringify(bool linenrs) const ret << loc.line << ": "; } - if (sameline(tok->previous, tok)) + // a # from a macro expansion at the start of a line gets a space + // before it, so the output is not read as a directive (as gcc and clang do) + if (sameline(tok->previous, tok) || (tok->op == '#' && !tok->macro.empty())) ret << ' '; ret << tok->str(); @@ -733,6 +738,41 @@ static std::string stringifyToken(const std::string &str) return ret; } +/** + * The result of the # operator: the tokens as a string literal (C 6.10.3.2). + * The tokens are separated by a space where there was whitespace between them. + * @param tokens the argument of #, not macro expanded + * @param loc location for the warning + * @param macroName the macro that is expanded, for the warning + * @param outputList if not null, a warning is added when a final backslash is dropped + * @return the string literal, including the quotes + */ +static std::string stringify(const simplecpp::TokenList &tokens, const simplecpp::Location &loc, const std::string ¯oName, simplecpp::OutputList *outputList) +{ + std::string str; + for (const simplecpp::Token *tok = tokens.cfront(), *next; tok; tok = next) { + next = tok->next; + str += stringifyToken(tok->str()); + if (next && tok->whitespaceahead) + str += ' '; + } + // An odd number of trailing backslashes would escape the closing quote and + // yield an unterminated string literal (undefined behavior, C 6.10.3.2p2). + // Drop the final backslash, as gcc and clang do. + std::size_t trailing = 0; + while (trailing < str.size() && str[str.size() - 1U - trailing] == '\\') + ++trailing; + if (trailing % 2U == 1U) { + str.pop_back(); + if (outputList) { + outputList->emplace_back(simplecpp::Output::PORTABILITY_STRINGIFY, loc, + "Invalid string literal when expanding \'" + macroName + "\', ignoring final \'\\\'. " + "Undefined behavior according to C standard chapter 6.10.3.2, paragraph 2."); + } + } + return '\"' + str + '\"'; +} + void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location) { if (fileIndex_ != location.fileIndex || line >= location.line) { @@ -1317,6 +1357,14 @@ static bool isAnonymousReferenceParameter(const simplecpp::Token *tok) return false; } +/** Append the next token to tok, and delete the next token. tok gets the whitespace after the next token. */ +static void combineWithNext(simplecpp::TokenList &tokens, simplecpp::Token *tok) +{ + tok->setstr(tok->str() + tok->next->str()); + tok->whitespaceahead = tok->next->whitespaceahead; + tokens.deleteToken(tok->next); +} + void simplecpp::TokenList::combineOperators() { std::stack> executableScope{{false}}; @@ -1328,24 +1376,19 @@ void simplecpp::TokenList::combineOperators() // ellipsis ... if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) && tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) { - tok->setstr("..."); - deleteToken(tok->next); - deleteToken(tok->next); + combineWithNext(*this, tok); + combineWithNext(*this, tok); continue; } // float literals.. if (tok->previous && tok->previous->number && sameline(tok->previous, tok) && tok->previous->str().find_first_of("._") == std::string::npos) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); - if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp") && !isAlternativeAndBitandBitor(tok->next)))) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); - } - } - if (tok->next && tok->next->number) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp") && !isAlternativeAndBitandBitor(tok->next)))) + combineWithNext(*this, tok); } + if (tok->next && tok->next->number) + combineWithNext(*this, tok); } // match: [0-9.]+E [+-] [0-9]+ const char lastChar = tok->str()[tok->str().size() - 1]; @@ -1353,9 +1396,8 @@ void simplecpp::TokenList::combineOperators() ((!isHex(tok->str()) && (lastChar == 'E' || lastChar == 'e')) || (isHex(tok->str()) && (lastChar == 'P' || lastChar == 'p'))) && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { - tok->setstr(tok->str() + tok->next->op + tok->next->next->str()); - deleteToken(tok->next); - deleteToken(tok->next); + combineWithNext(*this, tok); + combineWithNext(*this, tok); } if (tok->op == '\0' || !tok->next || tok->next->op == '\0') @@ -1370,23 +1412,17 @@ void simplecpp::TokenList::combineOperators() // void f(x&=2) if (tok->op == '&' && !executableScope.top() && isAnonymousReferenceParameter(tok)) continue; - tok->setstr(tok->str() + "="); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if ((tok->op == '|' || tok->op == '&') && tok->op == tok->next->op) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if (tok->op == ':' && tok->next->op == ':') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if (tok->op == '-' && tok->next->op == '>') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } else if ((tok->op == '<' || tok->op == '>') && tok->op == tok->next->op) { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); if (tok->next && tok->next->op == '=' && tok->next->next && tok->next->next->op != '=') { - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } } else if ((tok->op == '+' || tok->op == '-') && tok->op == tok->next->op) { if (tok->location.col + 1U != tok->next->location.col) @@ -1395,8 +1431,7 @@ void simplecpp::TokenList::combineOperators() continue; if (tok->next->next && tok->next->next->number) continue; - tok->setstr(tok->str() + tok->next->str()); - deleteToken(tok->next); + combineWithNext(*this, tok); } } } @@ -1815,7 +1850,17 @@ namespace simplecpp { class Macro; using MacroMap = std::unordered_map; + /** Set the location of all tokens */ + static void setLocation(TokenList &tokens, const Location &loc) + { + for (Token *tok = tokens.front(); tok; tok = tok->next) + tok->location = loc; + } + + class HideSetExpander; + class Macro { + friend class HideSetExpander; public: explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), valueDefinedInCode_(false) {} @@ -1883,6 +1928,34 @@ namespace simplecpp { return valueDefinedInCode_; } + /** + * Copy a macro call "NAME( ... )" to a tokenlist with no line breaks: + * all tokens get the location of the name. + * @param dest destination tokenlist + * @param nameTok the macro name, followed by '(' + * @return the token after the ')', or nullptr if the call is not terminated + * @throws invalidDirectiveAsMacroParameter if there is a preprocessor directive in the arguments + */ + static const Token *copyMacroCall(TokenList &dest, const Token *nameTok) { + const Location &loc = nameTok->location; + dest.push_back(new Token(nameTok->str(), loc, nameTok->whitespaceahead)); + const Token *tok = nameTok->next; + dest.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + tok = tok->next; + int par = 1; + while (tok && par > 0) { + if (tok->op == '(') + ++par; + else if (tok->op == ')') + --par; + else if (tok->op == '#' && !sameline(tok->previous, tok)) + throw invalidDirectiveAsMacroParameter(tok->location); + dest.push_back(new Token(tok->str(), loc, tok->whitespaceahead)); + tok = tok->next; + } + return tok; + } + /** * Expand macro. This will recursively expand inner macros. * @param output destination tokenlist @@ -1909,24 +1982,9 @@ namespace simplecpp { TokenList output2(inputFiles); if (functionLike() && rawtok->next && rawtok->next->op == '(') { - // Copy macro call to a new tokenlist with no linebreaks const Token * const rawtok1 = rawtok; TokenList rawtokens2(inputFiles); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - int par = 1; - while (rawtok && par > 0) { - if (rawtok->op == '(') - ++par; - else if (rawtok->op == ')') - --par; - else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) - throw invalidDirectiveAsMacroParameter(rawtok->location); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); - rawtok = rawtok->next; - } + rawtok = copyMacroCall(rawtokens2, rawtok); if (expand(output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros, outputList)) rawtok = rawtok1->next; } else { @@ -1956,7 +2014,7 @@ namespace simplecpp { if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; const MacroMap::const_iterator macro = macros.find(macro2tok->str()); - if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isExpandedFrom(¯o->second)) + if (macro == macros.end() || !macro->second.functionLike() || macro2tok->isInHideSet(macro->first)) break; TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); @@ -2055,13 +2113,22 @@ namespace simplecpp { } }; private: - /** Create new token where Token::macro is set for replaced tokens */ + /** + * Create a new token in the expansion of this macro. + * @param str the token text + * @param loc the location + * @param replaced Token::macro is set to the name of this macro + * @param expandedFromToken if not null, the new token copies its hide set and + * its whitespace, and this macro is added to the hide set + */ Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=nullptr) const { auto *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); - if (expandedFromToken) - tok->setExpandedFrom(expandedFromToken, this); + if (expandedFromToken) { + tok->setHideSet(expandedFromToken, name()); + tok->whitespaceahead = expandedFromToken->whitespaceahead; + } return tok; } @@ -2169,10 +2236,33 @@ namespace simplecpp { return ~0U; } + /** + * Find the arguments of a call of this macro. + * @param nameTokInst the macro name in the call + * @param calledInDefine the call is in a #define, so the arguments must be on the same line + * @return the tokens that delimit the arguments: the '(', each ',' between two arguments and the ')'. + * Empty when the macro is not function-like or the name is not followed by '('. + * @throws wrongNumberOfParameters if a function-like macro has the wrong number of arguments + */ std::vector getMacroParameters(const Token *nameTokInst, bool calledInDefine) const { - if (!nameTokInst->next || nameTokInst->next->op != '(' || !functionLike()) + if (!functionLike()) return {}; + std::vector parametertokens; + if (nameTokInst->next && nameTokInst->next->op == '(') + parametertokens = findArgumentDelimiters(nameTokInst, calledInDefine); + if (variadic) { + if (parametertokens.size() < args.size()) + throw wrongNumberOfParameters(nameTokInst->location, name()); + } else { + if (parametertokens.size() != args.size() + (args.empty() ? 2U : 1U)) + throw wrongNumberOfParameters(nameTokInst->location, name()); + } + return parametertokens; + } + + /** The '(', each ',' between two arguments and the ')' of the call. nameTokInst is followed by '('. */ + std::vector findArgumentDelimiters(const Token *nameTokInst, bool calledInDefine) const { std::vector parametertokens; parametertokens.emplace_back(nameTokInst->next); unsigned int par = 0U; @@ -2228,11 +2318,33 @@ namespace simplecpp { tok = tok->next; } } - for (Token *tok2 = tokens.front(); tok2; tok2 = tok2->next) - tok2->location = lpar->location; + setLocation(tokens, lpar->location); return sameline(lpar,tok) ? tok : nullptr; } + /** + * Expand the built-in macros __FILE__, __LINE__ and __COUNTER__. + * The usage must already be added to usageList, __COUNTER__ is the number of earlier usages. + * @param output destination tokenlist + * @param loc location of the usage + * @param nameTokInst the macro name + * @return true if the macro is built-in and its value is added to output + */ + bool expandBuiltin(TokenList &output, const Location &loc, const Token *nameTokInst) const { + std::string value; + if (nameTokInst->str() == "__FILE__") + value = '\"' + output.file(loc) + '\"'; + else if (nameTokInst->str() == "__LINE__") + value = toString(loc.line); + else if (nameTokInst->str() == "__COUNTER__") + value = toString(usageList.size() - 1U); + else + return false; + output.push_back(new Token(value, loc)); + output.back()->macro = nameTokInst->str(); + return true; + } + const Token * expand(TokenList & output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros, OutputList *outputList) const { expandedmacros.insert(nameTokInst->str()); @@ -2242,45 +2354,20 @@ namespace simplecpp { usageList.emplace_back(loc); - if (nameTokInst->str() == "__FILE__") { - output.push_back(new Token('\"'+output.file(loc)+'\"', loc)); - output.back()->macro = "__FILE__"; - return nameTokInst->next; - } - if (nameTokInst->str() == "__LINE__") { - output.push_back(new Token(toString(loc.line), loc)); - output.back()->macro = "__LINE__"; + if (expandBuiltin(output, loc, nameTokInst)) return nameTokInst->next; - } - if (nameTokInst->str() == "__COUNTER__") { - output.push_back(new Token(toString(usageList.size()-1U), loc)); - output.back()->macro = "__COUNTER__"; - return nameTokInst->next; - } const bool calledInDefine = (loc.fileIndex != nameTokInst->location.fileIndex || loc.line < nameTokInst->location.line); - std::vector parametertokens1(getMacroParameters(nameTokInst, calledInDefine)); - - if (functionLike()) { - // No arguments => not macro expansion - if (nameTokInst->next && nameTokInst->next->op != '(') { - output.push_back(new Token(nameTokInst->str(), loc)); - return nameTokInst->next; - } - - // Parse macro-call - if (variadic) { - if (parametertokens1.size() < args.size()) { - throw wrongNumberOfParameters(nameTokInst->location, name()); - } - } else { - if (parametertokens1.size() != args.size() + (args.empty() ? 2U : 1U)) - throw wrongNumberOfParameters(nameTokInst->location, name()); - } + // No arguments => not macro expansion + if (functionLike() && nameTokInst->next && nameTokInst->next->op != '(') { + output.push_back(new Token(nameTokInst->str(), loc)); + return nameTokInst->next; } + std::vector parametertokens1(getMacroParameters(nameTokInst, calledInDefine)); + // If macro call uses __COUNTER__ then expand that first TokenList tokensparams(files); std::vector parametertokens2; @@ -2429,7 +2516,7 @@ namespace simplecpp { const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) return nullptr; - if (!it->second.functionLike() || temp.cback()->isExpandedFrom(&it->second)) + if (!it->second.functionLike() || temp.cback()->isInHideSet(it->first)) return nullptr; return &it->second; } @@ -2519,42 +2606,95 @@ namespace simplecpp { return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens, outputList); } - if (tok->str() == DEFINED) { - const Token * const tok2 = tok->next; - const Token * const tok3 = tok2 ? tok2->next : nullptr; - const Token * const tok4 = tok3 ? tok3->next : nullptr; - const Token *defToken = nullptr; - const Token *lastToken = nullptr; - if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { - defToken = tok3; - lastToken = tok4; - } else if (sameline(tok,tok2) && tok2->name) { - defToken = lastToken = tok2; - } - if (defToken) { - std::string macroName = defToken->str(); - if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { - TokenList temp(files); - if (expandArg(temp, defToken, parametertokens)) - macroName = temp.cback()->str(); - if (expandArg(temp, defToken->next->next->next, parametertokens)) - macroName += temp.cback() ? temp.cback()->str() : ""; - else - macroName += defToken->next->next->next->str(); - lastToken = defToken->next->next->next; - } - const bool def = (macros.find(macroName) != macros.end()); - output.push_back(newMacroToken(def ? "1" : "0", loc, true)); - return lastToken->next; - } - } + const Token *next = nullptr; + if (tok->str() == DEFINED && expandDefined(output, loc, tok, macros, parametertokens, next)) + return next; output.push_back(newMacroToken(tok->str(), loc, true, tok)); if (it != macros.end()) - output.back()->markExpandedFrom(&it->second); + output.back()->addToHideSet(it->first); return tok->next; } + /** + * Find the operand of "defined" in a replacement list: "defined X", "defined(X)" or "defined X ## Y". + * @param tok the "defined" token + * @param first output: X + * @param second output: Y for "X ## Y", otherwise nullptr + * @param last output: the last token of the operand + * @return false if there is no operand + */ + static bool findDefinedOperand(const Token *tok, const Token *&first, const Token *&second, const Token *&last) { + const Token * const tok2 = tok->next; + const Token * const tok3 = tok2 ? tok2->next : nullptr; + const Token * const tok4 = tok3 ? tok3->next : nullptr; + first = last = second = nullptr; + if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { + first = tok3; + last = tok4; + } else if (sameline(tok,tok2) && tok2->name) { + first = last = tok2; + } + if (!first) + return false; + if (first->next && first->next->op == '#' && first->next->next && first->next->next->op == '#' && first->next->next->next && first->next->next->next->name && sameline(first,first->next->next->next)) { + second = first->next->next->next; + last = second; + } + return true; + } + + /** + * The name of the macro that "defined" checks. + * @param first X in "defined X" + * @param second Y in "defined X ## Y", or nullptr + * @param argText for "X ## Y": gives the text of a parameter, and returns false if the token is not a parameter + */ + template + static std::string definedMacroName(const Token *first, const Token *second, ArgText argText) { + std::string macroName = first->str(); + if (second) { + std::string text; + if (argText(first, text)) + macroName = text; + if (argText(second, text)) + macroName += text; + else + macroName += second->str(); + } + return macroName; + } + + /** + * Expand "defined X" or "defined(X)" in the replacement list to 1 or 0. + * The operand can be a parameter, or two names pasted with ##. + * @param output destination tokenlist + * @param loc location for the result + * @param tok the "defined" token + * @param macros all macros + * @param parametertokens arguments of this macro call + * @param next output: the token after the operand + * @return false if there is no operand, then nothing is added to output + */ + bool expandDefined(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::vector ¶metertokens, const Token *&next) const { + const Token *first; + const Token *second; + const Token *last; + if (!findDefinedOperand(tok, first, second, last)) + return false; + TokenList temp(files); + const std::string macroName = definedMacroName(first, second, [&](const Token *argtok, std::string &text) { + if (!expandArg(temp, argtok, parametertokens)) + return false; + text = temp.cback() ? temp.cback()->str() : ""; + return true; + }); + const bool def = (macros.find(macroName) != macros.end()); + output.push_back(newMacroToken(def ? "1" : "0", loc, true)); + next = last->next; + return true; + } + bool expandArg(TokenList &output, const Token *tok, const std::vector ¶metertokens) const { if (!tok->name) return false; @@ -2584,7 +2724,7 @@ namespace simplecpp { const Token * const argEnd = parametertokens[argnr + 1U]; for (const Token *partok = parametertokens[argnr]->next; partok != argEnd;) { const MacroMap::const_iterator it = macros.find(partok->str()); - if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { + if (it != macros.end() && !partok->isInHideSet(it->first) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation expandedmacros2.erase(name()); TokenList temp(files); @@ -2634,28 +2774,7 @@ namespace simplecpp { TokenList tokenListHash(files); const MacroMap macros2; // temporarily bypass macro expansion tok = expandToken(tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens, outputList); - std::string str; - for (const Token *hashtok = tokenListHash.cfront(), *next; hashtok; hashtok = next) { - next = hashtok->next; - str += stringifyToken(hashtok->str()); - if (next && hashtok->whitespaceahead) - str += ' '; - } - // An odd number of trailing backslashes would escape the closing quote and - // yield an unterminated string literal (undefined behavior, C 6.10.3.2p2). - // Drop the final backslash, as gcc and clang do. - std::size_t trailing = 0; - while (trailing < str.size() && str[str.size() - 1U - trailing] == '\\') - ++trailing; - if (trailing % 2U == 1U) { - str.pop_back(); - if (outputList) { - outputList->emplace_back(Output::PORTABILITY_STRINGIFY, loc, - "Invalid string literal when expanding \'" + name() + "\', ignoring final \'\\\'. " - "Undefined behavior according to C standard chapter 6.10.3.2, paragraph 2."); - } - } - output.push_back(newMacroToken('\"' + str + '\"', loc, isReplaced(expandedmacros))); + output.push_back(newMacroToken(stringify(tokenListHash, loc, name(), outputList), loc, isReplaced(expandedmacros))); return tok; } @@ -2679,20 +2798,9 @@ namespace simplecpp { throw invalidHashHash::unexpectedNewline(tok->location, name()); const bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); - // A is a punctuator; the result is checked when B has been expanded - const bool punctuatorA = (!A->name && !A->number && !A->str().empty() && !canBeConcatenatedStringOrChar); const Token * const B = tok->next->next; - if (!punctuatorA) { - if (!B->name && !B->number && B->op && !B->isOneOf("#=")) - throw invalidHashHash::unexpectedToken(tok->location, name(), B); - if (B->op == '=') - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); - } - - // Superficial check; more in-depth would in theory be possible _after_ expandArg - if (canBeConcatenatedStringOrChar && (B->number || !B->name)) - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); + checkHashHashOperands(tok->location, name(), A, B); TokenList tokensB(files); const Token *nextTok = B->next; @@ -2701,8 +2809,7 @@ namespace simplecpp { // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. // TODO The question is whether the ## or varargs may still apply, and how to provoke? if (expandArg(tokensB, B, parametertokens)) { - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); } else { tokensB.push_back(new Token(*B)); tokensB.back()->location = loc; @@ -2721,32 +2828,23 @@ namespace simplecpp { strAB = ","; } else { - if (punctuatorA && !isValidPunctuatorPaste(A->str() + tokensB.cfront()->str())) - throw invalidHashHash::cannotCombine(tok->location, name(), A, tokensB.cfront()); + checkPaste(tok->location, name(), A, tokensB.cfront()); strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); } } else { - if (punctuatorA && !isValidPunctuatorPaste(A->str() + B->str())) - throw invalidHashHash::cannotCombine(tok->location, name(), A, B); + checkPaste(tok->location, name(), A, B); strAB = A->str() + B->str(); } - // producing universal character is undefined behavior - if (A->previous && A->previous->str() == "\\") { - if (strAB[0] == 'u' && strAB.size() == 5) - throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); - if (strAB[0] == 'U' && strAB.size() == 9) - throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); - } + checkUniversalCharacterPaste(tok->location, name(), A, strAB); if (varargs && tokensB.empty() && tok->previous->str() == ",") { output.deleteToken(A); } else if (strAB != "," && macros.find(strAB) == macros.end()) { A->setstr(strAB); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); output.takeTokens(tokensB); } else if (sameline(B, nextTok) && sameline(B, nextTok->next) && nextTok->op == '#' && nextTok->next->op == '#') { TokenList output2(files); @@ -2771,8 +2869,7 @@ namespace simplecpp { expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens, outputList); else output.takeTokens(tokens); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; + setLocation(tokensB, loc); output.takeTokens(tokensB); } } @@ -2780,6 +2877,65 @@ namespace simplecpp { return nextTok; } + /** A is a punctuator, not a name, number or literal. Pasting it is checked after B is expanded. */ + static bool isPunctuatorOperand(const Token *A) { + return !A->name && !A->number && !A->str().empty() && !isStringLiteral(A->str()) && !isCharLiteral(A->str()); + } + + /** + * Check the operands of A ## B before B is expanded. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ##, already expanded + * @param B the token after ## in the macro definition + * @throws invalidHashHash if A and B can not be pasted + */ + static void checkHashHashOperands(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { + if (!isPunctuatorOperand(A)) { + if (!B->name && !B->number && B->op && !B->isOneOf("#=")) + throw invalidHashHash::unexpectedToken(loc, macroName, B); + if (B->op == '=') + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + // Superficial check; more in-depth would in theory be possible _after_ expandArg + if ((isStringLiteral(A->str()) || isCharLiteral(A->str())) && (B->number || !B->name)) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + /** + * Check that pasting A and B gives a valid token, when A is a punctuator + * or B is a string or character literal. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ## + * @param B the first token after ##, after expansion + * @throws invalidHashHash if the result is not a valid token + */ + static void checkPaste(const Location &loc, const std::string ¯oName, const Token *A, const Token *B) { + if (isPunctuatorOperand(A) && !isValidPunctuatorPaste(A->str() + B->str())) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + // a name or number followed by a literal: only an encoding prefix such as L or u8 is valid + if ((A->name || A->number) && (isStringLiteral(B->str()) || isCharLiteral(B->str())) && + !(A->name && isStringLiteralPrefix(A->str()))) + throw invalidHashHash::cannotCombine(loc, macroName, A, B); + } + + /** + * Producing a universal character name with ## is undefined behavior. + * @param loc the location of the ## + * @param macroName the macro that is expanded + * @param A the token before ## + * @param strAB the pasted text + * @throws invalidHashHash if A follows a backslash and strAB is a universal character name + */ + static void checkUniversalCharacterPaste(const Location &loc, const std::string ¯oName, const Token *A, const std::string &strAB) { + if (!A->previous || A->previous->str() != "\\") + return; + if ((strAB[0] == 'u' && strAB.size() == 5) || (strAB[0] == 'U' && strAB.size() == 9)) + throw invalidHashHash::universalCharacterUB(loc, macroName, A, strAB); + } + static bool isReplaced(const std::set &expandedmacros) { // return true if size > 1 auto it = expandedmacros.cbegin(); @@ -2825,6 +2981,441 @@ namespace simplecpp { }; } +namespace simplecpp { + /** + * Macro expansion with hide sets (Prosser's algorithm, used by DUI::hideSetExpansion). + * Each token has a hide set: the macros that it must not be expanded into. + * The result of an expansion is pushed back onto the input and scanned again, so a + * function-like macro whose arguments follow the expansion in the source is found by + * normal rescanning. + * Work in progress: the old Token::macro and location rules are not followed yet. + */ + class HideSetExpander { + public: + HideSetExpander(const MacroMap ¯os, std::vector &files, OutputList *outputList) + : macros(macros), files(files), outputList(outputList) {} + + HideSetExpander(const HideSetExpander&) = delete; + HideSetExpander &operator=(const HideSetExpander&) = delete; + + /** + * Expand the macro at rawtok. + * @param output destination tokenlist + * @param rawtok the macro name in the source + * @return the first source token after the expansion + * @throws Macro::Error if the macro can not be expanded + */ + const Token *expand(TokenList &output, const Token *rawtok) { + loc = rawtok->location; + Input input(rawtok->next); + input.pending.push_back(new Token(*rawtok)); + expandInput(output, input); + return input.raw; + } + + private: + /** The input of an expansion: tokens to scan again, then the source */ + struct Input { + explicit Input(const Token *raw) : raw(raw) {} + Input(const Input&) = delete; + Input &operator=(const Input&) = delete; + ~Input() { + for (const Token * const tok : pending) + delete tok; + } + + /** The next token, read from the source if there are no pending tokens; nullptr at the end */ + Token *read() { + if (!pending.empty()) { + std::unique_ptr tok(pending.front()); + pending.pop_front(); + return tok.release(); + } + if (!raw) + return nullptr; + if (raw->op == '#' && !sameline(raw->previous, raw)) + throw Macro::invalidDirectiveAsMacroParameter(raw->location); + const Token * const tok = raw; + raw = raw->next; + return new Token(*tok); + } + + /** The next token, without reading it */ + const Token *peek() const { + return pending.empty() ? raw : pending.front(); + } + + /** Put tokens in front of the pending tokens */ + void pushFront(TokenList &tokens) { + std::deque front; + while (tokens.front()) { + front.push_back(new Token(*tokens.front())); + tokens.deleteToken(tokens.front()); + } + pending.insert(pending.begin(), front.begin(), front.end()); + } + + /** tokens to scan again, the next one is at the front */ + std::deque pending; + /** next token in the source, it is only read for the arguments of a macro call */ + const Token *raw; + }; + + /** What a macro call is replaced by depends on */ + struct MacroCall { + /** the hide set of the result */ + std::set hideSet; + /** the arguments as written in the call */ + std::vector arguments; + /** the arguments, macro expanded */ + std::vector expandedArguments; + /** there is whitespace after the call */ + bool whitespaceAfter{}; + }; + + /** Expand the pending tokens of the input. Source tokens are only read for macro arguments. */ + void expandInput(TokenList &output, Input &input) { + while (!input.pending.empty()) { + Token * const tok = input.read(); + const MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); + if (it == macros.end() || tok->isInHideSet(it->first)) { + output.push_back(tok); + continue; + } + const Macro ¯o = it->second; + + if (macro.functionLike()) { + const Token * const next = input.peek(); + if (!next || next->op != '(') { + output.push_back(tok); + continue; + } + } + + macro.usageList.emplace_back(loc); + if (macro.expandBuiltin(output, loc, tok)) { + delete tok; + continue; + } + + const MacroCall call = readMacroCall(macro, tok, input); + TokenList result(files); + substitute(result, macro, call); + input.pushFront(result); + } + } + + /** + * Read a macro call: the name, and "( ... )" for a function-like macro. + * @param macro the macro + * @param nameTok the macro name, it is deleted + * @param input the input after the name + * @throws wrongNumberOfParameters if a function-like macro has the wrong number of arguments + */ + MacroCall readMacroCall(const Macro ¯o, Token *nameTok, Input &input) { + MacroCall call; + if (!macro.functionLike()) { + call.hideSet = nameTok->hideSet(); + call.whitespaceAfter = nameTok->whitespaceahead; + delete nameTok; + } else { + TokenList tokens(files); + tokens.push_back(nameTok); + readCall(tokens, input); + const std::vector delimiters = macro.getMacroParameters(tokens.cfront(), false); + // the name and the ) have the hide sets of the macro calls that this call + // was produced by, the arguments are not affected by them + const std::set &hs1 = tokens.cfront()->hideSet(); + const std::set &hs2 = delimiters.back()->hideSet(); + std::set_intersection(hs1.begin(), hs1.end(), hs2.begin(), hs2.end(), std::inserter(call.hideSet, call.hideSet.begin())); + call.arguments = getArguments(macro, delimiters); + call.expandedArguments = expandArguments(call.arguments); + call.whitespaceAfter = delimiters.back()->whitespaceahead; + } + call.hideSet.insert(macro.name()); + return call; + } + + /** Read "( ... )" after the macro name in call */ + static void readCall(TokenList &call, Input &input) { + int par = 0; + while (Token * const tok = input.read()) { + call.push_back(tok); + if (tok->op == '(') + ++par; + else if (tok->op == ')' && --par == 0) + return; + } + } + + /** Each argument of the call, as written in the call */ + std::vector getArguments(const Macro ¯o, const std::vector &delimiters) const { + std::vector arguments; + for (std::size_t i = 0; i < macro.args.size(); ++i) { + arguments.emplace_back(files); + if (i + 1U >= delimiters.size()) + continue; // empty variadic argument + for (const Token *tok = delimiters[i]->next; tok != delimiters[i + 1U]; tok = tok->next) + arguments.back().push_back(new Token(*tok)); + } + return arguments; + } + + /** Each argument fully macro expanded, on its own */ + std::vector expandArguments(const std::vector &arguments) { + std::vector expanded; + for (const TokenList &argument : arguments) { + expanded.emplace_back(files); + Input input(nullptr); + for (const Token *tok = argument.cfront(); tok; tok = tok->next) + input.pending.push_back(new Token(*tok)); + expandInput(expanded.back(), input); + } + return expanded; + } + + /** The parameter number of tok in the replacement list, or ~0U if it is not a parameter */ + static unsigned int getParameter(const Macro ¯o, const Token *tok) { + return (tok && tok != macro.endToken && tok->name) ? macro.getArgNum(tok->str()) : ~0U; + } + + /** tok and the token after it are ## in the replacement list of macro */ + static bool isHashHash(const Macro ¯o, const Token *tok) { + if (!tok || tok == macro.endToken || tok->op != '#') + return false; + const Token * const next = tok->next; + return next && next != macro.endToken && next->op == '#' && + sameline(tok, next) && next->location.col == tok->location.col + 1U; + } + + /** + * A ## B: paste the last token of result with the first token of operand. + * An empty operand is a placemarker: pasting it gives the other operand. + * @param result the replacement so far, it ends with A + * @param operand B, its tokens are moved to result + * @param previousEmpty A is a placemarker + * @param macro the macro + * @param hashHash the ## in the replacement list + * @param operandTok the token after ## in the replacement list + * @throws invalidHashHash if the result is not a valid token + */ + static void paste(TokenList &result, TokenList &operand, bool previousEmpty, const Macro ¯o, const Token *hashHash, const Token *operandTok) { + Token * const A = result.back(); + const bool varargs = macro.variadic && !macro.args.empty() && operandTok->str() == macro.args.back(); + if (operand.empty()) { + // gcc extension: in ", ## __VA_ARGS__" the comma is removed when there are no variable arguments + if (varargs && A && !previousEmpty && A->op == ',') + result.deleteToken(A); + return; + } + if (previousEmpty || !A) { + result.takeTokens(operand); + return; + } + Macro::checkHashHashOperands(hashHash->location, macro.name(), A, operandTok); + // ", ## __VA_ARGS__" with variable arguments: nothing is pasted + // a string or character literal followed by a name (user-defined literal) is not pasted either + if ((varargs && A->op == ',') || isStringLiteral(A->str()) || isCharLiteral(A->str())) { + result.takeTokens(operand); + return; + } + const Token * const B = operand.cfront(); + Macro::checkPaste(hashHash->location, macro.name(), A, B); + const std::string strAB = A->str() + B->str(); + Macro::checkUniversalCharacterPaste(hashHash->location, macro.name(), A, strAB); + A->setstr(strAB); + A->whitespaceahead = B->whitespaceahead; + A->macro = macro.name(); + operand.deleteToken(operand.front()); + result.takeTokens(operand); + } + + /** tok is __VA_OPT__ in the replacement list of a variadic macro */ + static bool isVaOpt(const Macro ¯o, const Token *tok) { + return macro.variadicOpt && tok && tok != macro.endToken && tok->str() == "__VA_OPT__"; + } + + /** + * __VA_OPT__(content) is replaced by the content, or by nothing (a placemarker) + * when the variable arguments are macro expanded to nothing (C23 6.10.5.1). + * @param operand destination tokenlist + * @param macro the macro + * @param tok the __VA_OPT__ token, it is moved to the closing ) + * @param call the macro call + */ + void getVaOpt(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { + // the parentheses were checked when the macro was defined + const Token * const lpar = tok->next; + const Token *rpar = lpar->next; + for (int par = 1; rpar != macro.endToken; rpar = rpar->next) { + if (rpar->op == '(') + ++par; + else if (rpar->op == ')' && --par == 0) + break; + } + tok = rpar; + const bool noVariableArguments = call.expandedArguments.empty() || call.expandedArguments.back().empty(); + if (!noVariableArguments) + substituteRange(operand, macro, call, lpar->next, rpar); + if (operand.back()) + operand.back()->whitespaceahead = rpar->whitespaceahead; + } + + /** A new token from the replacement list of macro, Token::macro is the macro name */ + Token *newToken(const TokenString &str, const Macro ¯o, bool whitespaceahead) const { + auto *tok = new Token(str, loc, whitespaceahead); + tok->macro = macro.name(); + return tok; + } + + /** + * "defined X" in the replacement list is replaced by 1 or 0. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; for "defined X" it is moved to the last token of X + * @param call the macro call + * @return true if tok is "defined" with an operand + */ + bool getDefined(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { + const Token *first; + const Token *second; + const Token *last; + if (tok->str() != DEFINED || !Macro::findDefinedOperand(tok, first, second, last)) + return false; + const std::string macroName = Macro::definedMacroName(first, second, [&](const Token *argtok, std::string &text) { + const unsigned int argnr = getParameter(macro, argtok); + if (argnr >= call.arguments.size()) + return false; + text = call.arguments[argnr].cback() ? call.arguments[argnr].cback()->str() : ""; + return true; + }); + operand.push_back(newToken((macros.find(macroName) != macros.end()) ? "1" : "0", macro, last->whitespaceahead)); + tok = last; + return true; + } + + /** + * # followed by a parameter or by __VA_OPT__(content) is replaced by a string literal. + * # is only an operator in a function-like macro. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; for # it is moved to the last token of the operand + * @param call the macro call + * @return true if tok is the # operator + */ + bool getStringified(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call) const { + if (tok->op != '#') + return false; + const Token * const next = (tok->next != macro.endToken) ? tok->next : nullptr; + if (isVaOpt(macro, next)) { + tok = next; + TokenList content(files); + getVaOpt(content, macro, tok, call); + operand.push_back(newToken(stringify(content, loc, macro.name(), outputList), macro, tok->whitespaceahead)); + return true; + } + const unsigned int argnr = getParameter(macro, next); + if (argnr >= call.arguments.size()) + return false; + operand.push_back(newToken(stringify(call.arguments[argnr], loc, macro.name(), outputList), macro, next->whitespaceahead)); + tok = next; + return true; + } + + /** + * The tokens that a token in the replacement list is replaced by: a parameter by its + * argument, # and a parameter by a string literal, "defined X" by 1 or 0, + * __VA_OPT__(content) by its content or nothing, and other tokens by a copy. + * @param operand destination tokenlist + * @param macro the macro + * @param tok the token in the replacement list; it is moved to the last token that is used + * @param call the macro call + * @param expand use the expanded argument for a parameter + */ + void getOperand(TokenList &operand, const Macro ¯o, const Token *&tok, const MacroCall &call, bool expand) const { + if (getDefined(operand, macro, tok, call) || getStringified(operand, macro, tok, call)) + return; + if (isVaOpt(macro, tok)) { + getVaOpt(operand, macro, tok, call); + return; + } + const unsigned int argnr = getParameter(macro, tok); + if (argnr < call.arguments.size()) { + const TokenList &argument = expand ? call.expandedArguments[argnr] : call.arguments[argnr]; + for (const Token *argtok = argument.cfront(); argtok; argtok = argtok->next) + operand.push_back(new Token(*argtok)); + return; + } + operand.push_back(newToken(tok->str(), macro, tok->whitespaceahead)); + } + + /** + * Tokens of the replacement list of the macro, with the parameters replaced by the + * arguments and ## done. + * @param result destination tokenlist + * @param macro the macro + * @param call the macro call + * @param begin first token in the replacement list + * @param end the token after the last one + */ + void substituteRange(TokenList &result, const Macro ¯o, const MacroCall &call, const Token *begin, const Token *end) const { + const Token *hashHash = nullptr; // the ## before the current operand + bool previousEmpty = false; // the previous operand was empty, it is a placemarker + for (const Token *tok = begin; tok != end; tok = tok->next) { + if (isHashHash(macro, tok)) { + if (!result.back() && !previousEmpty) + throw Macro::invalidHashHash(tok->location, macro.name(), "Missing first argument"); + hashHash = tok; + tok = tok->next; + continue; + } + const Token * const operandTok = tok; + const bool parameter = getParameter(macro, tok) < call.arguments.size(); + // an argument is not expanded when it is an operand of ## + const bool expand = !hashHash && !isHashHash(macro, tok->next); + TokenList operand(files); + getOperand(operand, macro, tok, call, expand); + const bool empty = operand.empty(); + if (hashHash) + paste(result, operand, previousEmpty, macro, hashHash, operandTok); + else + result.takeTokens(operand); + // the whitespace after a parameter comes after its argument + if (parameter && tok->whitespaceahead && result.back()) + result.back()->whitespaceahead = true; + previousEmpty = empty && (hashHash ? previousEmpty : true); + hashHash = nullptr; + } + if (hashHash) + throw Macro::invalidHashHash::unexpectedNewline(hashHash->location, macro.name()); + } + + /** + * The replacement list of the macro, with the parameters replaced by the arguments. + * @param result destination tokenlist + * @param macro the macro + * @param call the macro call + */ + void substitute(TokenList &result, const Macro ¯o, const MacroCall &call) const { + substituteRange(result, macro, call, macro.valueToken, macro.endToken); + for (Token *tok = result.front(); tok; tok = tok->next) { + tok->location = loc; + tok->addToHideSet(call.hideSet); + } + // whitespace at the end of the replacement list does not count, the + // expansion is followed by the whitespace after the macro call + if (result.back()) + result.back()->whitespaceahead = call.whitespaceAfter; + } + + const MacroMap ¯os; + std::vector &files; + OutputList *outputList; + /** location of the macro name in the source, all tokens of the expansion get it */ + Location loc; + }; +} + namespace simplecpp { #ifdef __CYGWIN__ @@ -3753,7 +4344,12 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token if (it != macros.end()) { simplecpp::TokenList value(files); try { - tok1 = it->second.expand(value, tok, macros, files, outputList); + if (dui.hideSetExpansion) { + simplecpp::HideSetExpander expander(macros, files, outputList); + tok1 = expander.expand(value, tok); + } else { + tok1 = it->second.expand(value, tok, macros, files, outputList); + } } catch (const simplecpp::Macro::invalidDirectiveAsMacroParameter &err) { if (outputList) { simplecpp::Output out{ diff --git a/simplecpp.h b/simplecpp.h index 2b560d27..4f119efd 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -117,8 +117,6 @@ namespace simplecpp { }; #endif // defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) - class Macro; - /** * Location in source code */ @@ -165,7 +163,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mHideSet(tok.mHideSet) {} Token &operator=(const Token &tok) = delete; @@ -210,17 +208,28 @@ namespace simplecpp { return tok; } - void setExpandedFrom(const Token *tok, const Macro* m) { - mExpandedFrom = tok->mExpandedFrom; - mExpandedFrom.insert(m); - if (tok->whitespaceahead) - whitespaceahead = true; + /** + * The hide set holds the macros that this token must not be expanded into. + * Set it to the hide set of tok, with the macro macroName added. + */ + void setHideSet(const Token *tok, const TokenString ¯oName) { + mHideSet = tok->mHideSet; + mHideSet.insert(macroName); + } + bool isInHideSet(const TokenString ¯oName) const { + return mHideSet.find(macroName) != mHideSet.end(); + } + void addToHideSet(const TokenString ¯oName) { + mHideSet.insert(macroName); + } + void addToHideSet(const std::set &hideSet) { + mHideSet.insert(hideSet.cbegin(), hideSet.cend()); } - bool isExpandedFrom(const Macro* m) const { - return mExpandedFrom.find(m) != mExpandedFrom.end(); + const std::set &hideSet() const { + return mHideSet; } - void markExpandedFrom(const Macro* m) { - mExpandedFrom.insert(m); + void setHideSet(std::set hideSet) { + mHideSet = std::move(hideSet); } void printAll() const; @@ -236,7 +245,7 @@ namespace simplecpp { TokenString string; - std::set mExpandedFrom; + std::set mHideSet; }; /** Output from preprocessor */ @@ -277,6 +286,7 @@ namespace simplecpp { std::string std; bool clearIncludeCache{}; bool removeComments{}; /** remove comment tokens from included files */ + bool hideSetExpansion{}; /** use the new macro expansion with hide sets (work in progress) */ }; /** List of tokens. */ diff --git a/test.cpp b/test.cpp index a7db4c3c..476509c5 100644 --- a/test.cpp +++ b/test.cpp @@ -5,6 +5,8 @@ #include "simplecpp.h" +#include + #include #include #include @@ -41,10 +43,19 @@ namespace { } static Input USE_INPUT = Input::Stringstream; +/** the tests are run with the old macro expansion and with the new one with hide sets */ +static bool USE_HIDESET_EXPANSION = false; static int numberOfFailedAssertions = 0; +static int numberOfHideSetTodos = 0; + +/** Skip the rest of the test when it is run with the new macro expansion, which can not handle it yet */ +#define HIDESET_EXPANSION_TODO() do { if (USE_HIDESET_EXPANSION) { ++numberOfHideSetTodos; return; } } while (false) #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) #define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while (false) +#define TODO_ASSERT_EQUALS(wanted, current, actual) (todoAssertEquals((wanted), (current), (actual), __LINE__)) +/** A known bug of the old macro expansion that the hide-set expansion does not have */ +#define TODO_OLD_EXPANSION_ASSERT_EQUALS(wanted, current, actual) (todoOldExpansionAssertEquals((wanted), (current), (actual), __LINE__)) static std::string pprint(const std::string &in) { @@ -57,12 +68,13 @@ static std::string pprint(const std::string &in) return ret; } -static const char* inputString(Input input) { +static std::string inputString(Input input) { + const std::string expansion = USE_HIDESET_EXPANSION ? ", hide-set expansion" : ""; switch (input) { case Input::Stringstream: - return "Stringstream"; + return "Stringstream" + expansion; case Input::CharBuffer: - return "CharBuffer"; + return "CharBuffer" + expansion; } return ""; // unreachable - needed for GCC and Visual Studio } @@ -84,6 +96,31 @@ static int assertEquals(const long long &expected, const long long &actual, int return assertEquals(std::to_string(expected), std::to_string(actual), line); } +/** + * For a known bug: actual must be the current (wrong) value. When the bug is + * fixed and actual is the wanted value, the assertion fails so that it is + * changed to ASSERT_EQUALS. + */ +static void todoAssertEquals(const std::string &wanted, const std::string ¤t, const std::string &actual, int line) +{ + if (wanted == actual) { + numberOfFailedAssertions++; + std::cerr << "------ TODO assertion fixed (" << inputString(USE_INPUT) << ")---------" << std::endl; + std::cerr << "line test.cpp:" << line << std::endl; + std::cerr << "change TODO_ASSERT_EQUALS to ASSERT_EQUALS" << std::endl; + return; + } + assertEquals(current, actual, line); +} + +static void todoOldExpansionAssertEquals(const std::string &wanted, const std::string ¤t, const std::string &actual, int line) +{ + if (USE_HIDESET_EXPANSION) + assertEquals(wanted, actual, line); + else + todoAssertEquals(wanted, current, actual, line); +} + static void assertThrowFailed(int line) { numberOfFailedAssertions++; @@ -107,6 +144,14 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) +/** simplecpp::preprocess() with the macro expansion selected by USE_HIDESET_EXPANSION */ +static void runPreprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, simplecpp::FileDataCache &cache, const simplecpp::DUI &dui, simplecpp::OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr) +{ + simplecpp::DUI dui2(dui); + dui2.hideSetExpansion = USE_HIDESET_EXPANSION; + simplecpp::preprocess(output, rawtokens, files, cache, dui2, outputList, macroUsage, ifCond); +} + static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), const simplecpp::DUI &dui = {}, simplecpp::OutputList *outputList = nullptr) { switch (USE_INPUT) { @@ -166,7 +211,7 @@ static std::string preprocess(const char code[], std::size_t size, const simplec if (dui.removeComments) tokens.removeComments(); simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList, macroUsage, ifCond); + runPreprocess(tokens2, tokens, files, cache, dui, outputList, macroUsage, ifCond); simplecpp::cleanup(cache); return tokens2.stringify(); } @@ -267,6 +312,48 @@ static std::string toString(const simplecpp::OutputList &outputList) return ostr.str(); } +// Preprocess the code. Each output token is written as str(line:col), or +// str@macro(line:col) when it comes from a macro expansion. +static std::string expandedTokens(const char code[], const simplecpp::DUI &dui = simplecpp::DUI(), std::list *macroUsage = nullptr) +{ + std::vector files; + simplecpp::FileDataCache cache; + simplecpp::TokenList tokens(files); + const simplecpp::TokenList rawtokens = makeTokenList(code, files); + runPreprocess(tokens, rawtokens, files, cache, dui, nullptr, macroUsage); + std::string ret; + for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) { + if (!ret.empty()) + ret += ' '; + ret += tok->str(); + if (!tok->macro.empty()) + ret += '@' + tok->macro; + ret += '(' + std::to_string(tok->location.line) + ':' + std::to_string(tok->location.col) + ')'; + } + return ret; +} + +/** + * Each macro usage as "name line:col use line:col", with " unknown" if the value is not known. + * The lines are sorted, the order of the macro usage depends on the standard library. + */ +static std::string toString(const std::list ¯oUsage) +{ + std::vector lines; + for (const simplecpp::MacroUsage &mu : macroUsage) { + std::string line = mu.macroName + ' ' + std::to_string(mu.macroLocation.line) + ':' + std::to_string(mu.macroLocation.col) + + " use " + std::to_string(mu.useLocation.line) + ':' + std::to_string(mu.useLocation.col); + if (!mu.macroValueKnown) + line += " unknown"; + lines.push_back(std::move(line)); + } + std::sort(lines.begin(), lines.end()); + std::string ret; + for (const std::string &line : lines) + ret += line + '\n'; + return ret; +} + static void backslash() { // preprocessed differently @@ -1095,8 +1182,46 @@ static void define_invalid_3() const char code[] = "#define R()\n" "R\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); + // a function-like macro name at the end of the file is not a macro call, gcc outputs R. + // TODO: the old macro expansion reports an error + TODO_OLD_EXPANSION_ASSERT_EQUALS("\nR", "", preprocess(code, &outputList)); + TODO_OLD_EXPANSION_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'R', Wrong number of parameters for macro 'R'.\n", toString(outputList)); +} + +static void define_wrong_number_of_arguments() +{ + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b) a+b\nf(1)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b) a+b\nf(1,2,3)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("#define f(a,b,...) a+b\nf(1)\n", &outputList)); + ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", toString(outputList)); + } + { + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n1", preprocess("#define f(a,...) a __VA_ARGS__\nf(1)\n", &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // a function-like macro name that is not followed by ( is not expanded + simplecpp::OutputList outputList; + ASSERT_EQUALS("\nf ;", preprocess("#define f(a) a\nf;\n", &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // TODO: f takes no arguments, gcc reports an error + simplecpp::OutputList outputList; + TODO_ASSERT_EQUALS("", "\n1", preprocess("#define f() 1\nf(1)\n", &outputList)); + TODO_ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'f', Wrong number of parameters for macro 'f'.\n", "", toString(outputList)); + } } static void define_invalid_4() @@ -1104,8 +1229,10 @@ static void define_invalid_4() const char code[] = "#define X(...)\n" "X\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); + // a function-like macro name at the end of the file is not a macro call, gcc outputs X. + // TODO: the old macro expansion reports an error + TODO_OLD_EXPANSION_ASSERT_EQUALS("\nX", "", preprocess(code, &outputList)); + TODO_OLD_EXPANSION_ASSERT_EQUALS("", "file0,2,syntax_error,failed to expand 'X', Wrong number of parameters for macro 'X'.\n", toString(outputList)); } static void define_define_1() @@ -1243,6 +1370,7 @@ static void define_define_13() // issue #49 - empty macro static void define_define_14() // issue #58 - endless recursion { + HIDESET_EXPANSION_TODO(); // unterminated macro call const char code[] = "#define z f(w\n" "#define f()\n" "#define w f(z\n" @@ -1378,6 +1506,7 @@ static void define_va_args_3() // min number of arguments static void define_va_args_4() // cppcheck trac #9754 { + HIDESET_EXPANSION_TODO(); // comma before an empty __VA_ARGS__ const char code[] = "#define A(x, y, ...) printf(x, y, __VA_ARGS__)\n" "A(1, 2)\n"; ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); @@ -1626,7 +1755,7 @@ static void error4() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtoken = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtoken, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtoken, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1639,7 +1768,7 @@ static void error5() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1652,7 +1781,7 @@ static void error6() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error \n", toString(outputList)); } @@ -1664,7 +1793,7 @@ static void error7() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error blabla\n", toString(outputList)); } @@ -1676,7 +1805,7 @@ static void error8() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code)-1,files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error blabla\n", toString(outputList)); } @@ -1812,6 +1941,14 @@ static void hashMacroTrailingWhitespace() "\"a b\"", preprocess(code, dui)); } +static void hashCombinedOperator() +{ + // the whitespace after an operator of two characters is kept + const char code[] = "#define str(x) #x\n" + "str(a == 0) str(a -> b) str(x... y)\n"; + ASSERT_EQUALS("\n\"a == 0\" \"a -> b\" \"x... y\"", preprocess(code)); +} + static void hashEscapeTrailingBackslash() { // an odd number of trailing backslashes would escape the closing quote, @@ -2005,14 +2142,26 @@ static void hashhash10() { const char code[] = "#define x # #\n" "x"; - ASSERT_EQUALS("# #", preprocess(code)); + // a space is added before the #, as gcc and clang do, so the output is not read as a directive. + // TODO: the old macro expansion gives the tokens the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # #", " # #", preprocess(code)); +} + +static void hashFromMacroAtLineStart() +{ + // clang testsuite hash_space.c: a space before the # so that the output is not read as a directive + const char code[] = "#define HASH #\n" + "HASH define foo bar\n"; + // TODO: the old macro expansion gives the # the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # define foo bar", " #\ndefine foo bar", preprocess(code)); } static void hashhash11() { const char code[] = "#define x # # #\n" "x"; - ASSERT_EQUALS("# # #", preprocess(code)); + // TODO: the old macro expansion gives the tokens the location of the #define + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n # # #", " # # #", preprocess(code)); } static void hashhash12() @@ -2139,7 +2288,9 @@ static void hashhash_string_name() // TODO is this correct? clang fails because that's not really a valid thing but gcc seems to accept it // see https://gist.github.com/patrickdowling/877a25294f069bf059f3b07f9b5b7039 - ASSERT_EQUALS("\n\n\"string\" LIT ;", preprocess(code)); + // "string" and LIT are not pasted, so the result is rescanned and LIT is expanded. + // TODO: the old macro expansion does not rescan LIT + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n\n\"string\" _literal ;", "\n\n\"string\" LIT ;", preprocess(code)); } static void hashhashhash_int_literal() @@ -2243,6 +2394,235 @@ static void hashhash_punctuator_invalid() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '+' and 'x' yields an invalid token.\n", toString(outputList)); } +static void macro_c11_example3() // C11 6.10.3.5 EXAMPLE 3 +{ + const char code[] = + "#define x 3\n" + "#define f(a) f(x * (a))\n" + "#undef x\n" + "#define x 2\n" + "#define g f\n" + "#define z z[0]\n" + "#define h g(~\n" + "#define m(a) a(w)\n" + "#define w 0,1\n" + "#define t(a) a\n" + "#define p() int\n" + "#define q(x) x\n" + "#define r(x,y) x ## y\n" + "#define str(x) # x\n" + "f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);\n" + "g(x+(3,4)-w) | h 5) & m\n" + "(f)^m(m);\n" + "p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };\n" + "char c[2][6] = { str(hello), str() };\n"; + // TODO: the old macro expansion leaves an empty token for r(,) + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "f ( 2 * ( y + 1 ) ) + f ( 2 * ( f ( 2 * ( z [ 0 ] ) ) ) ) % f ( 2 * ( 0 ) ) + t ( 1 ) ;\n" + "f ( 2 * ( 2 + ( 3 , 4 ) - 0 , 1 ) ) | f ( 2 * ( ~ 5 ) ) & f ( 2 * ( 0 , 1 ) )\n" + "^ m ( 0 , 1 ) ;\n" + "int i [ ] = { 1 , 23 , 4 , 5 , } ;\n" + "char c [ 2 ] [ 6 ] = { \"hello\" , \"\" } ;", + preprocess(code)); +} + +static void macro_c11_example4() // C11 6.10.3.5 EXAMPLE 4 +{ + const char code[] = + "#define str(s) # s\n" + "#define xstr(s) str(s)\n" + "#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n" + " x ## s, x ## t)\n" + "#define INCFILE(n) vers ## n\n" + "#define glue(a, b) a ## b\n" + "#define xglue(a, b) glue(a, b)\n" + "#define HIGHLOW \"hello\"\n" + "#define LOW LOW \", world\"\n" + "debug(1, 2);\n" + "fputs(str(strncmp(\"abc\\0d\", \"abc\", '\\4') // this goes away\n" + " == 0) str(: @\\n), s);\n" + "xstr(INCFILE(2).h)\n" + "glue(HIGH, LOW);\n" + "xglue(HIGH, LOW)\n"; + // the comments are removed before preprocessing, like cppcheck and the simplecpp program do + simplecpp::DUI dui; + dui.removeComments = true; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "printf ( \"x\" \"1\" \"= %d, x\" \"2\" \"= %s\" , x1 , x2 ) ;\n" + "fputs ( \"strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0\"\n" + "\": @\\n\" , s ) ;\n" + "\"vers2.h\"\n" + "\"hello\" ;\n" + "\"hello\" \", world\"", + preprocess(code, dui)); +} + +static void macro_c11_example5() // C11 6.10.3.5 EXAMPLE 5 +{ + const char code[] = + "#define t(x,y,z) x ## y ## z\n" + "int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),\n" + " t(10,,), t(,11,), t(,,12), t(,,) };\n"; + // TODO: the old macro expansion leaves an empty token for t(,,) + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + "\n" + "int j [ ] = { 123 , 45 , 67 , 89 ,\n" + "10 , 11 , 12 , } ;", + preprocess(code)); +} + +static void macro_c11_example7() // C11 6.10.3.5 EXAMPLE 7 +{ + const char code[] = + "#define debug(...) fprintf(stderr, __VA_ARGS__)\n" + "#define showlist(...) puts(#__VA_ARGS__)\n" + "#define report(test, ...) ((test)?puts(#test):\\\n" + " printf(__VA_ARGS__))\n" + "debug(\"Flag\");\n" + "debug(\"X = %d\\n\", x);\n" + "showlist(The first, second, and third items.);\n" + "report(x>y, \"x is %d but y is %d\", x, y);\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "fprintf ( stderr , \"Flag\" ) ;\n" + "fprintf ( stderr , \"X = %d\\n\" , x ) ;\n" + "puts ( \"The first, second, and third items.\" ) ;\n" + "( ( x > y ) ? puts ( \"x>y\" ) : printf ( \"x is %d but y is %d\" , x , y ) ) ;", + preprocess(code)); +} + +static void macro_c23_va_opt_examples() // C23 6.10.5.1 EXAMPLE +{ + const char code[] = + "#define F(...) f(0 __VA_OPT__(,) __VA_ARGS__)\n" + "#define G(X, ...) f(0, X __VA_OPT__(,) __VA_ARGS__)\n" + "#define SDEF(sname, ...) S sname __VA_OPT__(= { __VA_ARGS__ })\n" + "#define EMP\n" + "F(a,b,c)\n" + "F()\n" + "F(EMP)\n" + "G(a,b,c)\n" + "G(a,)\n" + "G(a)\n" + "SDEF(foo);\n" + "SDEF(bar, 1, 2);\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "f ( 0 , a , b , c )\n" + "f ( 0 )\n" + "f ( 0 )\n" + "f ( 0 , a , b , c )\n" + "f ( 0 , a )\n" + "f ( 0 , a )\n" + "S foo ;\n" + "S bar = { 1 , 2 } ;", + preprocess(code)); +} + +static void hashhash_empty_placemarker() +{ + const char code[] = + "#define r(x,y) x ## y\n" + "{ r(,) }\n"; + std::vector files; + simplecpp::FileDataCache cache; + simplecpp::TokenList tokens(files); + const simplecpp::TokenList rawtokens = makeTokenList(code, files); + runPreprocess(tokens, rawtokens, files, cache, simplecpp::DUI()); + std::string str; + for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) + str += "[" + tok->str() + "]"; + // TODO: with the old macro expansion, pasting two empty arguments gives an empty token + TODO_OLD_EXPANSION_ASSERT_EQUALS("[{][}]", "[{][][}]", str); +} + +static void hashhash_invalid_dotdot() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(.,.)\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '.' and '.' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_invalid_name_string() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(a,\"s\")\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining 'a' and '\"s\"' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_invalid_number_string() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "cat(1,\"s\")\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'cat', Invalid ## usage when expanding 'cat': Combining '1' and '\"s\"' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_encoding_prefix() +{ + const char code[] = + "#define cat(a,b) a##b\n" + "#define WIDE(s) L ## s\n" + "cat(L,\"s\") cat(u8,\"s\") cat(R,\"(x)\") cat(L,'c') WIDE(\"s\")\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n\nL\"s\" u8\"s\" R\"(x)\" L'c' L\"s\"", preprocess(code, &outputList)); + ASSERT_EQUALS("", toString(outputList)); +} + static void hashhash_va_args_unexpected() { const char code[] = @@ -2675,6 +3055,16 @@ static void ifDefinedHashHash2() ASSERT_EQUALS("\n0", preprocess(code, &outputList)); } +static void definedAtEndOfFile() +{ + // "defined" from a macro expansion is undefined behavior; gcc outputs "defined" here. + // The operand is the last token of the file. + const char code[] = "#define X\n" + "#define D defined X\n" + "D"; + ASSERT_EQUALS("\n\n1", preprocess(code)); +} + static void ifLogical() { const char code[] = "#if defined(A) || defined(B)\n" @@ -3264,7 +3654,7 @@ static void missingHeader2() const simplecpp::TokenList rawtokens = makeTokenList(code,files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -3298,7 +3688,7 @@ static void nestedInclude() simplecpp::TokenList tokens2(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } @@ -3316,7 +3706,7 @@ static void systemInclude() simplecpp::TokenList tokens2(files); simplecpp::DUI dui; dui.includePaths.emplace_back("include"); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -3358,7 +3748,7 @@ static void circularInclude() const simplecpp::TokenList rawtokens = makeTokenList(code, files, "test.cpp"); cache = simplecpp::load(rawtokens, filenames, dui, &outputList, std::move(cache)); - simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + runPreprocess(tokens2, rawtokens, files, cache, dui, &outputList); } ASSERT_EQUALS("", toString(outputList)); @@ -3383,7 +3773,7 @@ static void multiline2() rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -3398,7 +3788,7 @@ static void multiline3() // #28 - macro with multiline comment rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -3414,7 +3804,7 @@ static void multiline4() // #28 - macro with multiline comment rawtokens.removeComments(); simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n\n1", tokens2.stringify()); } @@ -3757,7 +4147,7 @@ static void include3() // #16 - crash when expanding macro from header simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n1234", out.stringify()); } @@ -3785,7 +4175,7 @@ static void include4() // #27 - -include simplecpp::DUI dui; dui.includePaths.emplace_back("."); dui.includes.emplace_back("27.h"); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("123", out.stringify()); } @@ -3811,7 +4201,7 @@ static void include5() // #3 - handle #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -3831,7 +4221,7 @@ static void include6() // #57 - incomplete macro #include MACRO(,) cache.insert({"57.c", rawtokens}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens, files, cache, simplecpp::DUI()); + runPreprocess(out, rawtokens, files, cache, simplecpp::DUI()); } @@ -3857,7 +4247,7 @@ static void include7() // #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -3896,7 +4286,7 @@ static void include9() simplecpp::DUI dui; dui.includePaths.emplace_back("."); dui.removeComments = true; - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify()); } @@ -3952,7 +4342,7 @@ static void include11() // #669 - -include with preprocess() simplecpp::DUI dui; dui.includes.emplace_back("inc.h"); dui.includes.emplace_back("missing.h"); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui, &outputList); + runPreprocess(out, rawtokens_c, files, cache, dui, &outputList); ASSERT_EQUALS("123", out.stringify()); ASSERT_EQUALS(1, outputList.size()); @@ -4279,41 +4669,24 @@ static void stringify1() simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.emplace_back("."); - simplecpp::preprocess(out, rawtokens_c, files, cache, dui); + runPreprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); } +// Token::macro, the locations of expanded tokens and the macro usage are used by cppcheck static void tokenMacro1() { const char code[] = "#define A 123\n" "A"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - ASSERT_EQUALS("A", tokenList.cback()->macro); + ASSERT_EQUALS("123@A(2:1)", expandedTokens(code)); } static void tokenMacro2() { const char code[] = "#define ADD(X,Y) X+Y\n" "ADD(1,2)"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("", tok->macro); - tok = tok->next; - ASSERT_EQUALS("+", tok->str()); - ASSERT_EQUALS("ADD", tok->macro); - tok = tok->next; - ASSERT_EQUALS("2", tok->str()); - ASSERT_EQUALS("", tok->macro); + ASSERT_EQUALS("1(2:1) +@ADD(2:1) 2(2:1)", expandedTokens(code)); } static void tokenMacro3() @@ -4321,35 +4694,20 @@ static void tokenMacro3() const char code[] = "#define ADD(X,Y) X+Y\n" "#define FRED 1\n" "ADD(FRED,2)"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("FRED", tok->macro); - tok = tok->next; - ASSERT_EQUALS("+", tok->str()); - ASSERT_EQUALS("ADD", tok->macro); - tok = tok->next; - ASSERT_EQUALS("2", tok->str()); - ASSERT_EQUALS("", tok->macro); + std::list macroUsage; + ASSERT_EQUALS("1@FRED(3:1) +@ADD(3:1) 2(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("ADD 1:9 use 3:1\nFRED 2:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro4() { + HIDESET_EXPANSION_TODO(); // Token::macro and location rules const char code[] = "#define A B\n" "#define B 1\n" "A"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token * const tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str()); - ASSERT_EQUALS("A", tok->macro); + std::list macroUsage; + ASSERT_EQUALS("1@A(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 3:1\nB 2:9 use 3:1\n", toString(macroUsage)); } static void tokenMacro5() @@ -4357,14 +4715,7 @@ static void tokenMacro5() const char code[] = "#define SET_BPF(code) (code)\n" "#define SET_BPF_JUMP(code) SET_BPF(D | code)\n" "SET_BPF_JUMP(A | B | C);"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token * const tok = tokenList.cfront()->next; - ASSERT_EQUALS("D", tok->str()); - ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); + ASSERT_EQUALS("(@SET_BPF(3:1) D@SET_BPF_JUMP(3:1) |@SET_BPF_JUMP(3:1) A(3:1) |(3:1) B(3:1) |(3:1) C(3:1) )@SET_BPF(3:1) ;(3:24)", expandedTokens(code)); } static void tokenMacro6() @@ -4375,21 +4726,358 @@ static void tokenMacro6() "FILE()\n" "LINE()\n" "COUNTER()\n"; - std::vector files; - simplecpp::FileDataCache cache; - simplecpp::TokenList tokenList(files); - const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); - const simplecpp::Token *tok; + ASSERT_EQUALS("\"\"@__FILE__(4:1) 5@__LINE__(5:1) 0@__COUNTER__(6:1)", expandedTokens(code)); +} + +static void tokenMacro7() +{ + const char code[] = "#define A 1+2\n" + "x A y\n"; + std::list macroUsage; + ASSERT_EQUALS("x(2:1) 1@A(2:3) +@A(2:3) 2@A(2:3) y(2:5)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 2:3\n", toString(macroUsage)); +} + +static void tokenMacro8() +{ + HIDESET_EXPANSION_TODO(); // Token::macro and location rules + // the call g(2) is completed with tokens after the expansion of f. + // The tokens get the location of the ( and the macro g + const char code[] = "#define f g\n" + "#define g(x) x+1\n" + "f(2)\n"; + std::list macroUsage; + ASSERT_EQUALS("2(3:2) +@g(3:2) 1@g(3:2)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("f 1:9 use 3:1\ng 2:9 use 3:2\n", toString(macroUsage)); +} + +static void tokenMacro9() +{ + HIDESET_EXPANSION_TODO(); // Token::macro and location rules + // the arguments of g( come after the expansion of f + const char code[] = "#define g(x) x+1\n" + "#define f g(\n" + "f 2)\n"; + std::list macroUsage; + ASSERT_EQUALS("2(3:3) +@g(3:3) 1@g(3:3)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("f 2:9 use 3:1\ng 1:9 use 3:3\n", toString(macroUsage)); +} + +static void tokenMacro10() +{ + HIDESET_EXPANSION_TODO(); // Token::macro and location rules + // the results of # and ## have no macro name + const char code[] = "#define str(x) #x\n" + "#define cat(a,b) a##b\n" + "str(a b) cat(x,y)\n"; + std::list macroUsage; + ASSERT_EQUALS("\"a b\"(3:1) xy(3:10)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("cat 2:9 use 3:10\nstr 1:9 use 3:1\n", toString(macroUsage)); +} + +static void tokenMacro11() +{ + const char code[] = "#define F(...) f(__VA_ARGS__)\n" + "F(1,2)\n"; + std::list macroUsage; + ASSERT_EQUALS("f@F(2:1) (@F(2:1) 1(2:1) ,(2:1) 2(2:1) )@F(2:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("F 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro12() +{ + const char code[] = "#define f f\n" + "f\n"; + std::list macroUsage; + ASSERT_EQUALS("f@f(2:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("f 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro13() +{ + // all tokens of the expansion get the location of the macro name + const char code[] = "#define ADD(X,Y) X+Y\n" + "ADD(1,\n" + " 2)\n" + "z\n"; + std::list macroUsage; + ASSERT_EQUALS("1(2:1) +@ADD(2:1) 2(2:1) z(4:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("ADD 1:9 use 2:1\n", toString(macroUsage)); +} + +static void tokenMacro14() +{ + // A is used in the argument and in the body, so it is listed twice + const char code[] = "#define A 1\n" + "#define B(x) x A\n" + "B(A)\n"; + std::list macroUsage; + ASSERT_EQUALS("1@A(3:1) 1@A(3:1)", expandedTokens(code, simplecpp::DUI(), ¯oUsage)); + ASSERT_EQUALS("A 1:9 use 3:1\nA 1:9 use 3:1\nB 2:9 use 3:1\n", toString(macroUsage)); +} + +static void tokenMacro15() +{ + simplecpp::DUI dui; + dui.defines.emplace_back("D=1+2"); + std::list macroUsage; + ASSERT_EQUALS("x(1:1) 1@D(1:3) +@D(1:3) 2@D(1:3)", expandedTokens("x D\n", dui, ¯oUsage)); + ASSERT_EQUALS("D 1:1 use 1:3 unknown\n", toString(macroUsage)); +} + +// Rescanning and hide sets. The results are the same as gcc gives. +static void hideSetExpansion_recursive() +{ + const char code[] = "#define f f\n" + "f\n"; + ASSERT_EQUALS("\n" + "f", preprocess(code)); +} + +static void hideSetExpansion_mutual() +{ + const char code[] = "#define a b\n" + "#define b a\n" + "a b\n"; + ASSERT_EQUALS("\n" + "\n" + "a b", preprocess(code)); +} + +static void hideSetExpansion_std_example() +{ + const char code[] = "#define f(a) a*g\n" + "#define g(a) f(a)\n" + "f(2)(9)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 * 9 * g", preprocess(code)); +} + +static void hideSetExpansion_trailing() +{ + const char code[] = "#define f g\n" + "#define g(x) x+1\n" + "f(2)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 + 1", preprocess(code)); +} + +static void hideSetExpansion_trailing_args() +{ + const char code[] = "#define g(x) x+1\n" + "#define f g(\n" + "f 2)\n"; + ASSERT_EQUALS("\n" + "\n" + "2 + 1", preprocess(code)); +} + +static void hideSetExpansion_args_lines() +{ + const char code[] = "#define ADD(X,Y) X+Y\n" + "ADD(1,\n" + " 2)\n" + "z\n"; + ASSERT_EQUALS("\n" + "1 + 2\n" + "\n" + "z", preprocess(code)); +} + +static void hideSetExpansion_nested() +{ + const char code[] = "#define A(x) [x]\n" + "#define B(x) A(x) A(A(x))\n" + "B(1)\n"; + ASSERT_EQUALS("\n" + "\n" + "[ 1 ] [ [ 1 ] ]", preprocess(code)); +} + +static void hideSetExpansion_arg_self() +{ + const char code[] = "#define f(x) x\n" + "f(f)(1)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "f ( 1 )", + "\n" + "1", + preprocess(code)); +} + +static void hideSetExpansion_blue_in_arg() +{ + const char code[] = "#define f(x) g(x)\n" + "#define g(x) f(x)\n" + "f(f(1))\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( f ( 1 ) )", preprocess(code)); +} + +static void hideSetExpansion_paren_in_arg() +{ + const char code[] = "#define f(x,y) (x)(y)\n" + "f((1,2),3)\n"; + ASSERT_EQUALS("\n" + "( ( 1 , 2 ) ) ( 3 )", preprocess(code)); +} + +static void hideSetExpansion_not_call() +{ + const char code[] = "#define f(x) x\n" + "f ; f\n" + "(1)\n"; + ASSERT_EQUALS("\n" + "f ; 1", preprocess(code)); +} + +static void hideSetExpansion_obj_to_fn() +{ + const char code[] = "#define LPAREN (\n" + "#define f(x) [x]\n" + "f LPAREN 1)\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( 1 )", preprocess(code)); +} - tok = tokenList.cfront(); - ASSERT_EQUALS("__FILE__", tok->macro); +static void hideSetExpansion_va_args() +{ + const char code[] = "#define F(...) f(__VA_ARGS__)\n" + "F(1,2) F()\n"; + ASSERT_EQUALS("\n" + "f ( 1 , 2 ) f ( )", preprocess(code)); +} - tok = tok->next; - ASSERT_EQUALS("__LINE__", tok->macro); +static void hideSetExpansion_counter() +{ + const char code[] = "#define C __COUNTER__\n" + "C C __COUNTER__\n"; + ASSERT_EQUALS("\n" + "0 1 2", preprocess(code)); +} - tok = tok->next; - ASSERT_EQUALS("__COUNTER__", tok->macro); +static void hideSetExpansion_line() +{ + const char code[] = "#define L __LINE__\n" + "L\n"; + ASSERT_EQUALS("\n" + "2", preprocess(code)); +} + +static void hideSetExpansion_deep() +{ + const char code[] = "#define a(x) b(x)\n" + "#define b(x) c(x)\n" + "#define c(x) x x\n" + "a(a(1))\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "1 1 1 1", preprocess(code)); +} + +static void hideSetExpansion_q_example() +{ + const char code[] = "#define q(x) x\n" + "#define r(x,y) x y\n" + "q(r)(q,1)\n"; + ASSERT_EQUALS("\n" + "\n" + "q 1", preprocess(code)); +} + +static void hideSetExpansion_empty() +{ + const char code[] = "#define E\n" + "#define f(x) \n" + "f(E) f()\n"; + ASSERT_EQUALS("\n" + "\n" + "< > < >", preprocess(code)); +} + +// __VA_OPT__ (C23 6.10.5.1). The results are the same as gcc gives. +static void hideSetExpansion_va_opt_empty_after_expansion() +{ + const char code[] = "#define F(...) <__VA_OPT__(x)>\n" + "#define E\n" + "F() F(E) F(1)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "< > < > < x >", + "\n" + "\n" + "< > < x > < x >", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hash() +{ + const char code[] = "#define X 1\n" + "#define F(a,...) #__VA_OPT__(a b)\n" + "F(X,2) F(X)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "\n" + "\"1 b\" \"\"", + "\n" + "\n" + "\"X\" b \"\"", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hashhash() +{ + const char code[] = "#define F(a,...) a ## __VA_OPT__(b c) ## d\n" + "F(x,1) F(x)\n"; + // TODO: the old macro expansion is wrong + TODO_OLD_EXPANSION_ASSERT_EQUALS("\n" + "xb cd xd", + "\n" + "xb cd x# \"d\"", + preprocess(code)); +} + +static void hideSetExpansion_va_opt_hash_in_content() +{ + const char code[] = "#define F(a,...) __VA_OPT__(a##a) __VA_OPT__(#a)\n" + "F(x,1)\n"; + ASSERT_EQUALS("\n" + "xx \"x\"", preprocess(code)); +} + +static void hideSetExpansion_va_opt_rescan() +{ + const char code[] = "#define LP (\n" + "#define F(...) __VA_OPT__(LP)\n" + "F(1)\n"; + ASSERT_EQUALS("\n" + "\n" + "(", preprocess(code)); +} + +static void hideSetExpansion_va_opt_comma() +{ + const char code[] = "#define F(a,...) f(a __VA_OPT__(,) __VA_ARGS__)\n" + "#define X 1\n" + "F(X) F(X,X)\n"; + ASSERT_EQUALS("\n" + "\n" + "f ( 1 ) f ( 1 , 1 )", preprocess(code)); +} + +static void hideSetExpansion_va_opt_parentheses() +{ + const char code[] = "#define F(...) __VA_OPT__((__VA_ARGS__))\n" + "F(a,(b)) F()\n"; + ASSERT_EQUALS("\n" + "( a , ( b ) )", preprocess(code)); } static void undef() @@ -4835,7 +5523,7 @@ static void preprocess_files() ASSERT_EQUALS("", *files.cbegin()); simplecpp::FileDataCache cache; - simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); + runPreprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("", *files.cbegin()); } @@ -4852,7 +5540,7 @@ static void preprocess_files() ASSERT_EQUALS("test.cpp", *files.cbegin()); simplecpp::FileDataCache cache; - simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); + runPreprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("test.cpp", *files.cbegin()); } @@ -5191,6 +5879,11 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(define6); TEST_CASE(define7); TEST_CASE(define8); + TEST_CASE(macro_c11_example3); + TEST_CASE(macro_c11_example4); + TEST_CASE(macro_c11_example5); + TEST_CASE(macro_c11_example7); + TEST_CASE(macro_c23_va_opt_examples); TEST_CASE(define9); TEST_CASE(define10); TEST_CASE(define11); @@ -5214,6 +5907,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(define_invalid_2); TEST_CASE(define_invalid_3); TEST_CASE(define_invalid_4); + TEST_CASE(define_wrong_number_of_arguments); TEST_CASE(define_define_1); TEST_CASE(define_define_2); TEST_CASE(define_define_3); @@ -5281,6 +5975,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashEscape); TEST_CASE(hashComment); TEST_CASE(hashMacroTrailingWhitespace); + TEST_CASE(hashCombinedOperator); TEST_CASE(hashEscapeTrailingBackslash); TEST_CASE(hashhash1); TEST_CASE(hashhash2); @@ -5294,6 +5989,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash9); TEST_CASE(hashhash10); // #108 : #define x # # TEST_CASE(hashhash11); // #60: #define x # # # + TEST_CASE(hashFromMacroAtLineStart); TEST_CASE(hashhash12); TEST_CASE(hashhash13); TEST_CASE(hashhash_string_literal); @@ -5314,6 +6010,11 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash_empty_va_args); TEST_CASE(hashhash_punctuator); TEST_CASE(hashhash_punctuator_invalid); + TEST_CASE(hashhash_empty_placemarker); + TEST_CASE(hashhash_invalid_dotdot); + TEST_CASE(hashhash_invalid_name_string); + TEST_CASE(hashhash_invalid_number_string); + TEST_CASE(hashhash_encoding_prefix); TEST_CASE(hashhash_va_args_unexpected); // C standard, 5.1.1.2, paragraph 4: @@ -5355,6 +6056,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(ifDefinedInvalid2); TEST_CASE(ifDefinedHashHash); TEST_CASE(ifDefinedHashHash2); + TEST_CASE(definedAtEndOfFile); TEST_CASE(ifLogical); TEST_CASE(ifLogicalPrecedence); TEST_CASE(ifSizeof); @@ -5445,6 +6147,40 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(tokenMacro4); TEST_CASE(tokenMacro5); TEST_CASE(tokenMacro6); + TEST_CASE(tokenMacro7); + TEST_CASE(tokenMacro8); + TEST_CASE(tokenMacro9); + TEST_CASE(tokenMacro10); + TEST_CASE(tokenMacro11); + TEST_CASE(tokenMacro12); + TEST_CASE(tokenMacro13); + TEST_CASE(tokenMacro14); + TEST_CASE(tokenMacro15); + TEST_CASE(hideSetExpansion_recursive); + TEST_CASE(hideSetExpansion_mutual); + TEST_CASE(hideSetExpansion_std_example); + TEST_CASE(hideSetExpansion_trailing); + TEST_CASE(hideSetExpansion_trailing_args); + TEST_CASE(hideSetExpansion_args_lines); + TEST_CASE(hideSetExpansion_nested); + TEST_CASE(hideSetExpansion_arg_self); + TEST_CASE(hideSetExpansion_blue_in_arg); + TEST_CASE(hideSetExpansion_paren_in_arg); + TEST_CASE(hideSetExpansion_not_call); + TEST_CASE(hideSetExpansion_obj_to_fn); + TEST_CASE(hideSetExpansion_va_args); + TEST_CASE(hideSetExpansion_counter); + TEST_CASE(hideSetExpansion_line); + TEST_CASE(hideSetExpansion_deep); + TEST_CASE(hideSetExpansion_q_example); + TEST_CASE(hideSetExpansion_empty); + TEST_CASE(hideSetExpansion_va_opt_empty_after_expansion); + TEST_CASE(hideSetExpansion_va_opt_hash); + TEST_CASE(hideSetExpansion_va_opt_hashhash); + TEST_CASE(hideSetExpansion_va_opt_hash_in_content); + TEST_CASE(hideSetExpansion_va_opt_rescan); + TEST_CASE(hideSetExpansion_va_opt_comma); + TEST_CASE(hideSetExpansion_va_opt_parentheses); TEST_CASE(undef); @@ -5496,5 +6232,9 @@ int main(int argc, char **argv) { runTests(argc, argv, Input::Stringstream); runTests(argc, argv, Input::CharBuffer); + USE_HIDESET_EXPANSION = true; + runTests(argc, argv, Input::Stringstream); + if (numberOfHideSetTodos > 0) + std::cout << "Tests skipped with the hide-set expansion: " << numberOfHideSetTodos << std::endl; return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; }