Skip to content

Fix #735 (Refactoring: split TokenList::readfile() into smaller functions) #736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danmar merged 2 commits into from
Nov 7, 2026
Merged

Fix #735 (Refactoring: split TokenList::readfile() into smaller functions) #736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e7f7c3
Fix #735 (Refactoring: split TokenList::readfile() into smaller funct…
dmcppcheck Nov 7, 2026
5215cbf
comments
dmcppcheck Nov 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 131 additions & 85 deletions simplecpp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,133 @@ void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int l

static const std::string COMMENT_END("*/");

/**
* Read a name or a number.
* In a number, a ' digit separator is removed, so 1'000 gives "1000".
* The stream is left after the last character of the token.
* @param stream the stream to read from
* @param ch the first character, already read from the stream
* @return the name or number
*/
static simplecpp::TokenString readNameOrNumber(simplecpp::TokenList::Stream &stream, unsigned char ch)
{
simplecpp::TokenString currentToken;
const bool num = !!std::isdigit(ch);
while (stream.good() && isNameChar(ch)) {
currentToken += ch;
ch = stream.readChar();
if (num && ch=='\'' && isNameChar(stream.peekChar()))
ch = stream.readChar();
}
stream.ungetChar();
return currentToken;
}

/**
* Read a // comment, up to but not including the newline.
* A backslash followed by a newline continues the comment on the next line.
* The backslash, and any spaces or tabs after it, are then replaced by a
* newline in the comment text.
* @param stream the stream to read from
* @param ch the first '/', already read from the stream
* @param location the location of the comment, used for warnings
* @param multiline incremented for each continued line
* @param outputList if not null, a portability warning is added when there is
* whitespace between the backslash and the newline
* @return the comment text, including the leading //
*/
static simplecpp::TokenString readLineComment(simplecpp::TokenList::Stream &stream, unsigned char ch, const simplecpp::Location &location, unsigned int &multiline, simplecpp::OutputList *outputList)
{
simplecpp::TokenString currentToken;
while (stream.good() && ch != '\n') {
currentToken += ch;
ch = stream.readChar();
if (ch == '\\') {
simplecpp::TokenString tmp;
char tmp_ch = ch;
while ((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) {
tmp += tmp_ch;
tmp_ch = stream.readChar();
}
if (!stream.good()) {
break;
}

if (tmp_ch != '\n') {
currentToken += tmp;
} else {
const simplecpp::TokenString check_portability = currentToken + tmp;
const std::string::size_type pos = check_portability.find_last_not_of(" \t");
if (pos < check_portability.size() - 1U && check_portability[pos] == '\\')
portabilityBackslash(outputList, location);
++multiline;
tmp_ch = stream.readChar();
currentToken += '\n';
}
ch = tmp_ch;
}
}
if (ch == '\n') {
stream.ungetChar();
}
return currentToken;
}

simplecpp::TokenString simplecpp::TokenList::readBlockComment(Stream &stream, unsigned int &multiline)
{
TokenString currentToken = "/*";
(void)stream.readChar();
unsigned char ch = stream.readChar();
while (stream.good()) {
currentToken += ch;
if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END))
break;
ch = stream.readChar();
}
// multiline..

std::string::size_type pos = 0;
while ((pos = currentToken.find("\\\n",pos)) != std::string::npos) {
currentToken.erase(pos,2);
++multiline;
}
if (multiline || isLastLinePreprocessor()) {
pos = 0;
while ((pos = currentToken.find('\n',pos)) != std::string::npos) {
currentToken.erase(pos,1);
++multiline;
}
}
return currentToken;
}

void simplecpp::TokenList::readErrorText(Stream &stream, unsigned char ch, Location &location)
{
TokenString currentToken;
char prev = ' ';
while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) {
currentToken += ch;
prev = ch;
ch = stream.readChar();
}
stream.ungetChar();
std::string::size_type pos = 0;
unsigned int spliced = 0;
while ((pos = currentToken.find('\\', pos)) != std::string::npos) {
if (pos + 1 < currentToken.size() && currentToken[pos + 1] == '\n') {
currentToken.erase(pos, 2);
++spliced;
} else {
++pos;
}
}
if (!currentToken.empty()) {
push_back(new Token(currentToken, location));
location.adjust(currentToken);
}
location.line += spliced;
}

void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const DUI &dui, OutputList *outputList)
{
unsigned int multiline = 0U;
Expand Down Expand Up @@ -880,105 +1007,24 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#') {
const Token* const ppTok = cback()->previous;
if (ppTok->next && (ppTok->next->str() == "error" || ppTok->next->str() == "warning")) {
char prev = ' ';
while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) {
currentToken += ch;
prev = ch;
ch = stream.readChar();
}
stream.ungetChar();
std::string::size_type pos = 0;
unsigned int spliced = 0;
while ((pos = currentToken.find('\\', pos)) != std::string::npos) {
if (pos + 1 < currentToken.size() && currentToken[pos + 1] == '\n') {
currentToken.erase(pos, 2);
++spliced;
} else {
++pos;
}
}
if (!currentToken.empty()) {
push_back(new Token(currentToken, location));
location.adjust(currentToken);
}
location.line += spliced;
readErrorText(stream, ch, location);
continue;
}
}

// number or name
if (isNameChar(ch)) {
const bool num = !!std::isdigit(ch);
while (stream.good() && isNameChar(ch)) {
currentToken += ch;
ch = stream.readChar();
if (num && ch=='\'' && isNameChar(stream.peekChar()))
ch = stream.readChar();
}

stream.ungetChar();
currentToken = readNameOrNumber(stream, ch);
}

// comment
else if (ch == '/' && stream.peekChar() == '/') {
while (stream.good() && ch != '\n') {
currentToken += ch;
ch = stream.readChar();
if (ch == '\\') {
TokenString tmp;
char tmp_ch = ch;
while ((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) {
tmp += tmp_ch;
tmp_ch = stream.readChar();
}
if (!stream.good()) {
break;
}

if (tmp_ch != '\n') {
currentToken += tmp;
} else {
const TokenString check_portability = currentToken + tmp;
const std::string::size_type pos = check_portability.find_last_not_of(" \t");
if (pos < check_portability.size() - 1U && check_portability[pos] == '\\')
portabilityBackslash(outputList, location);
++multiline;
tmp_ch = stream.readChar();
currentToken += '\n';
}
ch = tmp_ch;
}
}
if (ch == '\n') {
stream.ungetChar();
}
currentToken = readLineComment(stream, ch, location, multiline, outputList);
}

// comment
else if (ch == '/' && stream.peekChar() == '*') {
currentToken = "/*";
(void)stream.readChar();
ch = stream.readChar();
while (stream.good()) {
currentToken += ch;
if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END))
break;
ch = stream.readChar();
}
// multiline..

std::string::size_type pos = 0;
while ((pos = currentToken.find("\\\n",pos)) != std::string::npos) {
currentToken.erase(pos,2);
++multiline;
}
if (multiline || isLastLinePreprocessor()) {
pos = 0;
while ((pos = currentToken.find('\n',pos)) != std::string::npos) {
currentToken.erase(pos,1);
++multiline;
}
}
currentToken = readBlockComment(stream, multiline);
}

// string / char literal
Expand Down
21 changes: 21 additions & 0 deletions simplecpp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,27 @@ namespace simplecpp {
const Token* lastLineTok(int maxsize=1000) const;
const Token* isLastLinePreprocessor(int maxsize=1000) const;

/**
* Read a block comment, up to and including the closing star-slash.
* Each backslash-newline is removed from the text. If the comment is
* on a continued line or in a preprocessor directive, its newlines are
* removed too, so that the directive keeps going after the comment.
* @param stream the stream to read from. The first '/' is already
* read, and the next character is '*'.
* @param multiline incremented for each newline that is removed
* @return the comment text
*/
TokenString readBlockComment(Stream &stream, unsigned int &multiline);
/**
* Read the text of a #error or #warning directive, up to but not
* including the newline, and add it as one token. A backslash-newline
* continues the text on the next line and is removed from it.
* @param stream the stream to read from
* @param ch the first character of the text, already read from the stream
* @param location the location of the text. Updated to point after it.
*/
void readErrorText(Stream &stream, unsigned char ch, Location &location);

unsigned int fileIndex(const std::string &filename);

Token *frontToken;
Expand Down
Loading