From a871e99f711c1679fabeb9e2c8b6f104ebe72755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 20:10:51 +0100 Subject: [PATCH 1/7] Fix #739 (Add a lookahead buffer to TokenList::Stream) Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 142 ++++++++++++++++++++++++++++++-------------------- test.cpp | 23 ++++++++ 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5ddd8d99..f9916ff9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -295,68 +295,58 @@ class simplecpp::TokenList::Stream { public: virtual ~Stream() = default; - virtual int get() = 0; - virtual int peek() = 0; - virtual void unget() = 0; - virtual bool good() = 0; - + /** + * Read the next character. CR and CR LF are returned as LF. + * At the end of the input, EOF converted to unsigned char is returned + * and isGood() becomes false. + */ unsigned char readChar() { - auto ch = static_cast(get()); - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (isUtf16) { - const auto ch2 = static_cast(get()); - const int ch16 = makeUtf16Char(ch, ch2); - ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); + int ch = getUnit(); + if (ch == EOF) { + atEnd = true; + lastChar = EOF; + return static_cast(EOF); } // Handling of newlines.. if (ch == '\r') { ch = '\n'; - - int ch2 = get(); - if (isUtf16) { - const int c2 = get(); - ch2 = makeUtf16Char(ch2, c2); - } - - if (ch2 != '\n') - ungetChar(); + const int ch2 = getUnit(); + if (ch2 != '\n' && ch2 != EOF) + buffer.push_back(static_cast(ch2)); } - return ch; + lastChar = ch; + return static_cast(ch); } + /** The next character, without reading it. EOF at the end of the input. */ int peekChar() { - int ch = peek(); + const int ch = getUnit(); if (ch == EOF) - return ch; - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (isUtf16) { - (void)get(); - const auto ch2 = static_cast(peek()); - unget(); - const int ch16 = makeUtf16Char(ch, ch2); - ch = (ch16 >= 0x80) ? 0xff : ch16; - } - - // Handling of newlines.. - if (ch == '\r') - ch = '\n'; - - return ch; + return EOF; + buffer.push_back(static_cast(ch)); + return (ch == '\r') ? '\n' : ch; } + /** Put back the character that was last read with readChar() */ void ungetChar() { - unget(); - if (isUtf16) - unget(); + if (lastChar != EOF) + buffer.push_back(static_cast(lastChar)); + lastChar = EOF; + } + + /** false when readChar() has reached the end of the input */ + bool isGood() const { + return !atEnd; } protected: + virtual int get() = 0; + virtual int peek() = 0; + virtual void unget() = 0; + virtual bool good() = 0; + void init() { // initialize since we use peek() in getAndSkipBOM() isUtf16 = false; @@ -365,6 +355,35 @@ class simplecpp::TokenList::Stream { } private: + /** + * Read the next character from the input, or take it from the buffer. + * Newlines are not converted. + */ + int getUnit() { + if (!buffer.empty()) { + const auto ch = static_cast(buffer.back()); + buffer.pop_back(); + return ch; + } + if (inputEnd) + return EOF; + const int ch = get(); + if (ch == EOF) { + inputEnd = true; + return EOF; + } + if (!isUtf16) + return ch; + + // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the + // character is non-ASCII character then replace it with 0xff + const int ch2 = get(); + if (ch2 == EOF) + inputEnd = true; + const int ch16 = makeUtf16Char(static_cast(ch), static_cast(ch2)); + return (ch16 >= 0x80) ? 0xff : ch16; + } + inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const { return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); } @@ -400,6 +419,15 @@ class simplecpp::TokenList::Stream { } unsigned short bom; + + /** characters that are read but not used yet. The next character is at the back. */ + std::string buffer; + /** the last character returned by readChar(), EOF if ungetChar() has nothing to put back */ + int lastChar{EOF}; + /** readChar() has returned EOF */ + bool atEnd{false}; + /** get() has returned EOF */ + bool inputEnd{false}; protected: bool isUtf16; }; @@ -738,7 +766,7 @@ static simplecpp::TokenString readNameOrNumber(simplecpp::TokenList::Stream &str { simplecpp::TokenString currentToken; const bool num = !!std::isdigit(ch); - while (stream.good() && isNameChar(ch)) { + while (stream.isGood() && isNameChar(ch)) { currentToken += ch; ch = stream.readChar(); if (num && ch=='\'' && isNameChar(stream.peekChar())) @@ -764,17 +792,17 @@ static simplecpp::TokenString readNameOrNumber(simplecpp::TokenList::Stream &str static simplecpp::TokenString readLineComment(simplecpp::TokenList::Stream &stream, unsigned char ch, const simplecpp::Location &location, unsigned int &multiline, simplecpp::OutputList *outputList) { simplecpp::TokenString currentToken; - while (stream.good() && ch != '\n') { + while (stream.isGood() && ch != '\n') { currentToken += ch; ch = stream.readChar(); if (ch == '\\') { simplecpp::TokenString tmp; char tmp_ch = ch; - while ((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { + while ((stream.isGood()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { tmp += tmp_ch; tmp_ch = stream.readChar(); } - if (!stream.good()) { + if (!stream.isGood()) { break; } @@ -803,7 +831,7 @@ simplecpp::TokenString simplecpp::TokenList::readBlockComment(Stream &stream, un TokenString currentToken = "/*"; (void)stream.readChar(); unsigned char ch = stream.readChar(); - while (stream.good()) { + while (stream.isGood()) { currentToken += ch; if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END)) break; @@ -830,7 +858,7 @@ void simplecpp::TokenList::readErrorText(Stream &stream, unsigned char ch, Locat { TokenString currentToken; char prev = ' '; - while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { + while (stream.isGood() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; prev = ch; ch = stream.readChar(); @@ -864,9 +892,9 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const cppstd_t cppstd = getCppStd(dui.std); Location location(fileIndex(filename), 1, 1); - while (stream.good()) { + while (stream.isGood()) { unsigned char ch = stream.readChar(); - if (!stream.good()) + if (!stream.isGood()) break; trailing_nl = false; @@ -1041,11 +1069,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, currentToken = ch; prefix.resize(prefix.size() - 1); ch = stream.readChar(); - while (stream.good() && ch != '(' && ch != '\n') { + while (stream.isGood() && ch != '(' && ch != '\n') { delim += ch; ch = stream.readChar(); } - if (!stream.good() || ch == '\n') { + if (!stream.isGood() || ch == '\n') { if (outputList) { Output err{ Output::SYNTAX_ERROR, @@ -1057,7 +1085,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, return; } const std::string endOfRawString(')' + delim + currentToken); - while (stream.good() && (!endsWith(currentToken, endOfRawString) || currentToken.size() <= 1)) + while (stream.isGood() && (!endsWith(currentToken, endOfRawString) || currentToken.size() <= 1)) currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { @@ -1700,7 +1728,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca bool backslash = false; char ch = 0; - while (ch != end && ch != '\r' && ch != '\n' && stream.good()) { + while (ch != end && ch != '\r' && ch != '\n' && stream.isGood()) { ch = stream.readChar(); if (backslash && ch == '\n') { ch = 0; @@ -1728,7 +1756,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca } } - if (!stream.good() || ch != end) { + if (!stream.isGood() || ch != end) { clear(); if (outputList) { Output err{ diff --git a/test.cpp b/test.cpp index 9b30794b..d17f7c45 100644 --- a/test.cpp +++ b/test.cpp @@ -4168,6 +4168,28 @@ static void readfile_no_eof_newline() readfile(code, sizeof(code)-1, {}, &outputList); ASSERT_EQUALS("", toString(outputList)); } + { + // CR is a newline + const char code[] = "x\r"; + simplecpp::OutputList outputList; + readfile(code, sizeof(code)-1, {}, &outputList); + ASSERT_EQUALS("", toString(outputList)); + } + { + const char code[] = "x\r\n"; + simplecpp::OutputList outputList; + readfile(code, sizeof(code)-1, {}, &outputList); + ASSERT_EQUALS("", toString(outputList)); + } +} + +static void readfile_cr_newlines() +{ + // CR, CR LF and LF are all newlines + ASSERT_EQUALS("a\nb\nc", readfile("a\rb\rc\r")); + ASSERT_EQUALS("a\nb\nc", readfile("a\r\nb\r\nc\r\n")); + ASSERT_EQUALS("a\nb\nc\n\n\nd", readfile("a\rb\r\nc\n\r\rd\n")); + ASSERT_EQUALS("# define A 1\nA", readfile("#define A 1\rA\r")); } static void stringify1() @@ -5335,6 +5357,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(readfile_error); TEST_CASE(readfile_file_not_found); TEST_CASE(readfile_no_eof_newline); + TEST_CASE(readfile_cr_newlines); TEST_CASE(stringify1); From 3530830f95de2f957895cbb36ff7cc02a14d3f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 20:21:32 +0100 Subject: [PATCH 2/7] Fix clang-tidy misc-override-with-different-visibility warnings Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f9916ff9..5cfbf33c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -295,6 +295,11 @@ class simplecpp::TokenList::Stream { public: virtual ~Stream() = default; + virtual int get() = 0; + virtual int peek() = 0; + virtual void unget() = 0; + virtual bool good() = 0; + /** * Read the next character. CR and CR LF are returned as LF. * At the end of the input, EOF converted to unsigned char is returned @@ -342,11 +347,6 @@ class simplecpp::TokenList::Stream { } protected: - virtual int get() = 0; - virtual int peek() = 0; - virtual void unget() = 0; - virtual bool good() = 0; - void init() { // initialize since we use peek() in getAndSkipBOM() isUtf16 = false; From 333a821507f2d7fe7a5f26cacc534ea4af221374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 20:31:11 +0100 Subject: [PATCH 3/7] Drop a dangling last byte in UTF-16 input, add tests from review Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 5 ++++- test.cpp | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5cfbf33c..927d59dc 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -378,8 +378,11 @@ class simplecpp::TokenList::Stream { // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the // character is non-ASCII character then replace it with 0xff const int ch2 = get(); - if (ch2 == EOF) + if (ch2 == EOF) { + // a dangling last byte is dropped inputEnd = true; + return EOF; + } const int ch16 = makeUtf16Char(static_cast(ch), static_cast(ch2)); return (ch16 >= 0x80) ? 0xff : ch16; } diff --git a/test.cpp b/test.cpp index d17f7c45..9329feb6 100644 --- a/test.cpp +++ b/test.cpp @@ -3981,6 +3981,11 @@ static void readfile_char_error() readfile("A = 's\n'", {}, &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + // the ' at the end of the file is not dropped after the peek for a digit separator + readfile("A = 1'", {}, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); } static void readfile_string() @@ -4386,6 +4391,18 @@ static void unicode() const char code[] = "\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00"; ASSERT_EQUALS("\n//1", readfile(code, sizeof(code)-1)); } + { + // CR newlines + const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00"; + ASSERT_EQUALS("x\ny", readfile(code, sizeof(code)-1)); + } + { + // odd number of bytes, the dangling last byte is dropped + const char code[] = "\xff\xfe\x31\x00\x0a"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("1", readfile(code, sizeof(code)-1, {}, &outputList)); + ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList)); + } } static void unicode_invalid() From b812a85a9bf779665decd2b50ff088741fb441db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 20:38:46 +0100 Subject: [PATCH 4/7] Remove Stream::good(), it is no longer used Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 927d59dc..c02d69dd 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -298,7 +298,6 @@ class simplecpp::TokenList::Stream { virtual int get() = 0; virtual int peek() = 0; virtual void unget() = 0; - virtual bool good() = 0; /** * Read the next character. CR and CR LF are returned as LF. @@ -454,9 +453,6 @@ namespace { void unget() override { istr.unget(); } - bool good() override { - return istr.good(); - } private: std::istream &istr; @@ -474,26 +470,22 @@ namespace { int get() override { if (pos >= size) - return lastStatus = EOF; + return EOF; return str[pos++]; } int peek() override { if (pos >= size) - return lastStatus = EOF; + return EOF; return str[pos]; } void unget() override { --pos; } - bool good() override { - return lastStatus != EOF; - } private: const unsigned char *str; const std::size_t size; std::size_t pos{}; - int lastStatus{}; }; class FileStream : public simplecpp::TokenList::Stream { @@ -521,7 +513,7 @@ namespace { } int get() override { - lastStatus = lastCh = fgetc(file); + lastCh = fgetc(file); return lastCh; } int peek() override { @@ -533,9 +525,6 @@ namespace { void unget() override { unget_internal(lastCh); } - bool good() override { - return lastStatus != EOF; - } private: void unget_internal(int ch) { @@ -550,7 +539,6 @@ namespace { FILE *file; int lastCh{}; - int lastStatus{}; }; } From 7d1b7c0d4b8135f090e8ee62aefa1952f232629d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 21:52:07 +0100 Subject: [PATCH 5/7] Make Stream::get(), peek() and unget() protected Calling them from outside Stream would skip the lookahead buffer, UTF-16 decoding and CR handling. Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c02d69dd..8598968b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -295,10 +295,6 @@ class simplecpp::TokenList::Stream { public: virtual ~Stream() = default; - virtual int get() = 0; - virtual int peek() = 0; - virtual void unget() = 0; - /** * Read the next character. CR and CR LF are returned as LF. * At the end of the input, EOF converted to unsigned char is returned @@ -346,6 +342,11 @@ class simplecpp::TokenList::Stream { } protected: + /** Raw input, used only by the Stream class. Call readChar(), peekChar() and ungetChar() instead. */ + virtual int get() = 0; + virtual int peek() = 0; + virtual void unget() = 0; + void init() { // initialize since we use peek() in getAndSkipBOM() isUtf16 = false; @@ -444,6 +445,7 @@ namespace { init(); } + protected: int get() override { return istr.get(); } @@ -468,6 +470,7 @@ namespace { init(); } + protected: int get() override { if (pos >= size) return EOF; @@ -512,6 +515,7 @@ namespace { file = nullptr; } + protected: int get() override { lastCh = fgetc(file); return lastCh; From 2a305a3b4b83909dc5a877ad51a44cafe98cf1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 22:00:44 +0100 Subject: [PATCH 6/7] Rename Stream::isGood() to good() Co-Authored-By: Claude Opus 5.5 --- simplecpp.cpp | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 8598968b..a6adddad 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -298,7 +298,7 @@ class simplecpp::TokenList::Stream { /** * Read the next character. CR and CR LF are returned as LF. * At the end of the input, EOF converted to unsigned char is returned - * and isGood() becomes false. + * and good() becomes false. */ unsigned char readChar() { int ch = getUnit(); @@ -336,8 +336,11 @@ class simplecpp::TokenList::Stream { lastChar = EOF; } - /** false when readChar() has reached the end of the input */ - bool isGood() const { + /** + * false when readChar() has reached the end of the input. + * Unlike std::istream::good(), peekChar() at the end of the input does not make it false. + */ + bool good() const { return !atEnd; } @@ -761,7 +764,7 @@ static simplecpp::TokenString readNameOrNumber(simplecpp::TokenList::Stream &str { simplecpp::TokenString currentToken; const bool num = !!std::isdigit(ch); - while (stream.isGood() && isNameChar(ch)) { + while (stream.good() && isNameChar(ch)) { currentToken += ch; ch = stream.readChar(); if (num && ch=='\'' && isNameChar(stream.peekChar())) @@ -787,17 +790,17 @@ static simplecpp::TokenString readNameOrNumber(simplecpp::TokenList::Stream &str static simplecpp::TokenString readLineComment(simplecpp::TokenList::Stream &stream, unsigned char ch, const simplecpp::Location &location, unsigned int &multiline, simplecpp::OutputList *outputList) { simplecpp::TokenString currentToken; - while (stream.isGood() && ch != '\n') { + while (stream.good() && ch != '\n') { currentToken += ch; ch = stream.readChar(); if (ch == '\\') { simplecpp::TokenString tmp; char tmp_ch = ch; - while ((stream.isGood()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { + while ((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { tmp += tmp_ch; tmp_ch = stream.readChar(); } - if (!stream.isGood()) { + if (!stream.good()) { break; } @@ -826,7 +829,7 @@ simplecpp::TokenString simplecpp::TokenList::readBlockComment(Stream &stream, un TokenString currentToken = "/*"; (void)stream.readChar(); unsigned char ch = stream.readChar(); - while (stream.isGood()) { + while (stream.good()) { currentToken += ch; if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END)) break; @@ -853,7 +856,7 @@ void simplecpp::TokenList::readErrorText(Stream &stream, unsigned char ch, Locat { TokenString currentToken; char prev = ' '; - while (stream.isGood() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { + while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; prev = ch; ch = stream.readChar(); @@ -887,9 +890,9 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const cppstd_t cppstd = getCppStd(dui.std); Location location(fileIndex(filename), 1, 1); - while (stream.isGood()) { + while (stream.good()) { unsigned char ch = stream.readChar(); - if (!stream.isGood()) + if (!stream.good()) break; trailing_nl = false; @@ -1064,11 +1067,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, currentToken = ch; prefix.resize(prefix.size() - 1); ch = stream.readChar(); - while (stream.isGood() && ch != '(' && ch != '\n') { + while (stream.good() && ch != '(' && ch != '\n') { delim += ch; ch = stream.readChar(); } - if (!stream.isGood() || ch == '\n') { + if (!stream.good() || ch == '\n') { if (outputList) { Output err{ Output::SYNTAX_ERROR, @@ -1080,7 +1083,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, return; } const std::string endOfRawString(')' + delim + currentToken); - while (stream.isGood() && (!endsWith(currentToken, endOfRawString) || currentToken.size() <= 1)) + while (stream.good() && (!endsWith(currentToken, endOfRawString) || currentToken.size() <= 1)) currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { @@ -1723,7 +1726,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca bool backslash = false; char ch = 0; - while (ch != end && ch != '\r' && ch != '\n' && stream.isGood()) { + while (ch != end && ch != '\r' && ch != '\n' && stream.good()) { ch = stream.readChar(); if (backslash && ch == '\n') { ch = 0; @@ -1751,7 +1754,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca } } - if (!stream.isGood() || ch != end) { + if (!stream.good() || ch != end) { clear(); if (outputList) { Output err{ From 454efc0718a23040f7bfbce9b0f8c6a4fd0606f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 7 Nov 2026 22:27:54 +0100 Subject: [PATCH 7/7] Test FileStream with the readfile edge cases Co-Authored-By: Claude Opus 5.5 --- test.cpp | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test.cpp b/test.cpp index 9329feb6..a7db4c3c 100644 --- a/test.cpp +++ b/test.cpp @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -142,6 +144,20 @@ static std::string readfile(const char code[], std::size_t size, const simplecpp return makeTokenList(code,size,files,std::string(),dui,outputList).stringify(); } +/** Write the code to a temporary file and read it with TokenList(filename), so FileStream is used */ +static std::string readfileFromFile(const char code[], std::size_t size, simplecpp::OutputList *outputList = nullptr) +{ + const std::string filename = "testrunner_filestream.tmp"; + { + std::ofstream f(filename, std::ios::binary); + f.write(code, static_cast(size)); + } + std::vector files; + std::string ret = simplecpp::TokenList(filename, files, {}, outputList).stringify(); + std::remove(filename.c_str()); + return ret; +} + static std::string preprocess(const char code[], std::size_t size, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage = nullptr, std::list *ifCond = nullptr, const std::string &file = std::string()) { std::vector files; @@ -4197,6 +4213,50 @@ static void readfile_cr_newlines() ASSERT_EQUALS("# define A 1\nA", readfile("#define A 1\rA\r")); } +static void readfile_filestream() +{ + // the edge cases from the other readfile tests, read from a file + { + const char code[] = "a\rb\rc\r"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + const char code[] = "a\r\nb\r\nc\r\n"; + ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1)); + } + { + const char code[] = "a\rb\r\nc\n\r\rd\n"; + ASSERT_EQUALS("a\nb\nc\n\n\nd", readfileFromFile(code, sizeof(code)-1)); + } + { + const char code[] = "1'"; + simplecpp::OutputList outputList; + (void)readfileFromFile(code, sizeof(code)-1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + } + { + // UTF-16LE with CR newlines + const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // UTF-16BE with CR newlines + const char code[] = "\xfe\xff\x00\x78\x00\x0d\x00\x79\x00\x0d"; + ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1)); + } + { + // UTF-16 with an odd number of bytes, the dangling last byte is dropped + const char code[] = "\xff\xfe\x31\x00\x0a"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("1", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList)); + } +} + static void stringify1() { const char code_c[] = "#include \"A.h\"\n" @@ -5375,6 +5435,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(readfile_file_not_found); TEST_CASE(readfile_no_eof_newline); TEST_CASE(readfile_cr_newlines); + TEST_CASE(readfile_filestream); TEST_CASE(stringify1);