diff --git a/simplecpp.cpp b/simplecpp.cpp index 5ddd8d99..a6adddad 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -295,68 +295,61 @@ class simplecpp::TokenList::Stream { public: virtual ~Stream() = default; - virtual int get() = 0; - virtual int peek() = 0; - virtual void unget() = 0; - virtual bool good() = 0; - + /** + * Read the next character. CR and CR LF are returned as LF. + * At the end of the input, EOF converted to unsigned char is returned + * and good() becomes false. + */ unsigned char readChar() { - auto ch = static_cast(get()); - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (isUtf16) { - const auto ch2 = static_cast(get()); - const int ch16 = makeUtf16Char(ch, ch2); - ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); + int ch = getUnit(); + if (ch == EOF) { + atEnd = true; + lastChar = EOF; + return static_cast(EOF); } // Handling of newlines.. if (ch == '\r') { ch = '\n'; - - int ch2 = get(); - if (isUtf16) { - const int c2 = get(); - ch2 = makeUtf16Char(ch2, c2); - } - - if (ch2 != '\n') - ungetChar(); + const int ch2 = getUnit(); + if (ch2 != '\n' && ch2 != EOF) + buffer.push_back(static_cast(ch2)); } - return ch; + lastChar = ch; + return static_cast(ch); } + /** The next character, without reading it. EOF at the end of the input. */ int peekChar() { - int ch = peek(); + const int ch = getUnit(); if (ch == EOF) - return ch; - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (isUtf16) { - (void)get(); - const auto ch2 = static_cast(peek()); - unget(); - const int ch16 = makeUtf16Char(ch, ch2); - ch = (ch16 >= 0x80) ? 0xff : ch16; - } - - // Handling of newlines.. - if (ch == '\r') - ch = '\n'; - - return ch; + return EOF; + buffer.push_back(static_cast(ch)); + return (ch == '\r') ? '\n' : ch; } + /** Put back the character that was last read with readChar() */ void ungetChar() { - unget(); - if (isUtf16) - unget(); + if (lastChar != EOF) + buffer.push_back(static_cast(lastChar)); + lastChar = EOF; + } + + /** + * false when readChar() has reached the end of the input. + * Unlike std::istream::good(), peekChar() at the end of the input does not make it false. + */ + bool good() const { + return !atEnd; } protected: + /** Raw input, used only by the Stream class. Call readChar(), peekChar() and ungetChar() instead. */ + virtual int get() = 0; + virtual int peek() = 0; + virtual void unget() = 0; + void init() { // initialize since we use peek() in getAndSkipBOM() isUtf16 = false; @@ -365,6 +358,38 @@ class simplecpp::TokenList::Stream { } private: + /** + * Read the next character from the input, or take it from the buffer. + * Newlines are not converted. + */ + int getUnit() { + if (!buffer.empty()) { + const auto ch = static_cast(buffer.back()); + buffer.pop_back(); + return ch; + } + if (inputEnd) + return EOF; + const int ch = get(); + if (ch == EOF) { + inputEnd = true; + return EOF; + } + if (!isUtf16) + return ch; + + // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the + // character is non-ASCII character then replace it with 0xff + const int ch2 = get(); + if (ch2 == EOF) { + // a dangling last byte is dropped + inputEnd = true; + return EOF; + } + const int ch16 = makeUtf16Char(static_cast(ch), static_cast(ch2)); + return (ch16 >= 0x80) ? 0xff : ch16; + } + inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const { return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); } @@ -400,6 +425,15 @@ class simplecpp::TokenList::Stream { } unsigned short bom; + + /** characters that are read but not used yet. The next character is at the back. */ + std::string buffer; + /** the last character returned by readChar(), EOF if ungetChar() has nothing to put back */ + int lastChar{EOF}; + /** readChar() has returned EOF */ + bool atEnd{false}; + /** get() has returned EOF */ + bool inputEnd{false}; protected: bool isUtf16; }; @@ -414,6 +448,7 @@ namespace { init(); } + protected: int get() override { return istr.get(); } @@ -423,9 +458,6 @@ namespace { void unget() override { istr.unget(); } - bool good() override { - return istr.good(); - } private: std::istream &istr; @@ -441,28 +473,25 @@ namespace { init(); } + protected: int get() override { if (pos >= size) - return lastStatus = EOF; + return EOF; return str[pos++]; } int peek() override { if (pos >= size) - return lastStatus = EOF; + return EOF; return str[pos]; } void unget() override { --pos; } - bool good() override { - return lastStatus != EOF; - } private: const unsigned char *str; const std::size_t size; std::size_t pos{}; - int lastStatus{}; }; class FileStream : public simplecpp::TokenList::Stream { @@ -489,8 +518,9 @@ namespace { file = nullptr; } + protected: int get() override { - lastStatus = lastCh = fgetc(file); + lastCh = fgetc(file); return lastCh; } int peek() override { @@ -502,9 +532,6 @@ namespace { void unget() override { unget_internal(lastCh); } - bool good() override { - return lastStatus != EOF; - } private: void unget_internal(int ch) { @@ -519,7 +546,6 @@ namespace { FILE *file; int lastCh{}; - int lastStatus{}; }; } diff --git a/test.cpp b/test.cpp index 9b30794b..a7db4c3c 100644 --- a/test.cpp +++ b/test.cpp @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -142,6 +144,20 @@ static std::string readfile(const char code[], std::size_t size, const simplecpp return makeTokenList(code,size,files,std::string(),dui,outputList).stringify(); } +/** Write the code to a temporary file and read it with TokenList(filename), so FileStream is used */ +static std::string readfileFromFile(const char code[], std::size_t size, simplecpp::OutputList *outputList = nullptr) +{ + const std::string filename = "testrunner_filestream.tmp"; + { + std::ofstream f(filename, std::ios::binary); + f.write(code, static_cast(size)); + } + std::vector files; + std::string ret = simplecpp::TokenList(filename, files, {}, outputList).stringify(); + std::remove(filename.c_str()); + return ret; +} + static std::string preprocess(const char code[], std::size_t size, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage = nullptr, std::list *ifCond = nullptr, const std::string &file = std::string()) { std::vector files; @@ -3981,6 +3997,11 @@ static void readfile_char_error() readfile("A = 's\n'", {}, &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + // the ' at the end of the file is not dropped after the peek for a digit separator + readfile("A = 1'", {}, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); } static void readfile_string() @@ -4168,6 +4189,72 @@ static void readfile_no_eof_newline() readfile(code, sizeof(code)-1, {}, &outputList); ASSERT_EQUALS("", toString(outputList)); } + { + // CR is a newline + const char code[] = "x\r"; + simplecpp::OutputList outputList; + readfile(code, sizeof(code)-1, {}, &outputList); + ASSERT_EQUALS("", toString(outputList)); + } + { + const char code[] = "x\r\n"; + simplecpp::OutputList outputList; + readfile(code, sizeof(code)-1, {}, &outputList); + ASSERT_EQUALS("", toString(outputList)); + } +} + +static void readfile_cr_newlines() +{ + // CR, CR LF and LF are all newlines + ASSERT_EQUALS("a\nb\nc", readfile("a\rb\rc\r")); + ASSERT_EQUALS("a\nb\nc", readfile("a\r\nb\r\nc\r\n")); + ASSERT_EQUALS("a\nb\nc\n\n\nd", readfile("a\rb\r\nc\n\r\rd\n")); + ASSERT_EQUALS("# define A 1\nA", readfile("#define A 1\rA\r")); +} + +static void readfile_filestream() +{ + // the edge cases from the other readfile tests, read from a file + { + const char code[] = "a\rb\rc\r"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + const char code[] = "a\r\nb\r\nc\r\n"; + ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1)); + } + { + const char code[] = "a\rb\r\nc\n\r\rd\n"; + ASSERT_EQUALS("a\nb\nc\n\n\nd", readfileFromFile(code, sizeof(code)-1)); + } + { + const char code[] = "1'"; + simplecpp::OutputList outputList; + (void)readfileFromFile(code, sizeof(code)-1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + } + { + // UTF-16LE with CR newlines + const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("", toString(outputList)); + } + { + // UTF-16BE with CR newlines + const char code[] = "\xfe\xff\x00\x78\x00\x0d\x00\x79\x00\x0d"; + ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1)); + } + { + // UTF-16 with an odd number of bytes, the dangling last byte is dropped + const char code[] = "\xff\xfe\x31\x00\x0a"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("1", readfileFromFile(code, sizeof(code)-1, &outputList)); + ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList)); + } } static void stringify1() @@ -4364,6 +4451,18 @@ static void unicode() const char code[] = "\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00"; ASSERT_EQUALS("\n//1", readfile(code, sizeof(code)-1)); } + { + // CR newlines + const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00"; + ASSERT_EQUALS("x\ny", readfile(code, sizeof(code)-1)); + } + { + // odd number of bytes, the dangling last byte is dropped + const char code[] = "\xff\xfe\x31\x00\x0a"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("1", readfile(code, sizeof(code)-1, {}, &outputList)); + ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList)); + } } static void unicode_invalid() @@ -5335,6 +5434,8 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(readfile_error); TEST_CASE(readfile_file_not_found); TEST_CASE(readfile_no_eof_newline); + TEST_CASE(readfile_cr_newlines); + TEST_CASE(readfile_filestream); TEST_CASE(stringify1);