Fix #739 (Add a lookahead buffer to TokenList::Stream)

simplecpp.cpp

@@ -295,68 +295,61 @@ class simplecpp::TokenList::Stream {
 public:
     virtual ~Stream() = default;
 
-    virtual int get() = 0;
-    virtual int peek() = 0;
-    virtual void unget() = 0;
-    virtual bool good() = 0;
-
+    /**
+     * Read the next character. CR and CR LF are returned as LF.
+     * At the end of the input, EOF converted to unsigned char is returned
+     * and good() becomes false.
+     */
     unsigned char readChar() {
-        auto ch = static_cast<unsigned char>(get());
-
-        // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
-        // character is non-ASCII character then replace it with 0xff
-        if (isUtf16) {
-            const auto ch2 = static_cast<unsigned char>(get());
-            const int ch16 = makeUtf16Char(ch, ch2);
-            ch = static_cast<unsigned char>(((ch16 >= 0x80) ? 0xff : ch16));
+        int ch = getUnit();
+        if (ch == EOF) {
+            atEnd = true;
+            lastChar = EOF;
+            return static_cast<unsigned char>(EOF);
         }
 
         // Handling of newlines..
         if (ch == '\r') {
             ch = '\n';
-
-            int ch2 = get();
-            if (isUtf16) {
-                const int c2 = get();
-                ch2 = makeUtf16Char(ch2, c2);
-            }
-
-            if (ch2 != '\n')
-                ungetChar();
+            const int ch2 = getUnit();
+            if (ch2 != '\n' && ch2 != EOF)
+                buffer.push_back(static_cast<char>(ch2));
         }
 
-        return ch;
+        lastChar = ch;
+        return static_cast<unsigned char>(ch);
     }
 
+    /** The next character, without reading it. EOF at the end of the input. */
     int peekChar() {
-        int ch = peek();
+        const int ch = getUnit();
         if (ch == EOF)
-            return ch;
-
-        // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
-        // character is non-ASCII character then replace it with 0xff
-        if (isUtf16) {
-            (void)get();
-            const auto ch2 = static_cast<unsigned char>(peek());
-            unget();
-            const int ch16 = makeUtf16Char(ch, ch2);
-            ch = (ch16 >= 0x80) ? 0xff : ch16;
-        }
-
-        // Handling of newlines..
-        if (ch == '\r')
-            ch = '\n';
-
-        return ch;
+            return EOF;
+        buffer.push_back(static_cast<char>(ch));
+        return (ch == '\r') ? '\n' : ch;
     }
 
+    /** Put back the character that was last read with readChar() */
     void ungetChar() {
-        unget();
-        if (isUtf16)
-            unget();
+        if (lastChar != EOF)
+            buffer.push_back(static_cast<char>(lastChar));
+        lastChar = EOF;
+    }
+
+    /**
+     * false when readChar() has reached the end of the input.
+     * Unlike std::istream::good(), peekChar() at the end of the input does not make it false.
+     */
+    bool good() const {
+        return !atEnd;
     }
 
 protected:
+    /** Raw input, used only by the Stream class. Call readChar(), peekChar() and ungetChar() instead. */
+    virtual int get() = 0;
+    virtual int peek() = 0;
+    virtual void unget() = 0;
+
     void init() {
         // initialize since we use peek() in getAndSkipBOM()
         isUtf16 = false;
@@ -365,6 +358,38 @@ class simplecpp::TokenList::Stream {
     }
 
 private:
+    /**
+     * Read the next character from the input, or take it from the buffer.
+     * Newlines are not converted.
+     */
+    int getUnit() {
+        if (!buffer.empty()) {
+            const auto ch = static_cast<unsigned char>(buffer.back());
+            buffer.pop_back();
+            return ch;
+        }
+        if (inputEnd)
+            return EOF;
+        const int ch = get();
+        if (ch == EOF) {
+            inputEnd = true;
+            return EOF;
+        }
+        if (!isUtf16)
+            return ch;
+
+        // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
+        // character is non-ASCII character then replace it with 0xff
+        const int ch2 = get();
+        if (ch2 == EOF) {
+            // a dangling last byte is dropped
+            inputEnd = true;
+            return EOF;
+        }
+        const int ch16 = makeUtf16Char(static_cast<unsigned char>(ch), static_cast<unsigned char>(ch2));
+        return (ch16 >= 0x80) ? 0xff : ch16;
+    }
+
     inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const {
         return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch);
     }
@@ -400,6 +425,15 @@ class simplecpp::TokenList::Stream {
     }
 
     unsigned short bom;
+
+    /** characters that are read but not used yet. The next character is at the back. */
+    std::string buffer;
+    /** the last character returned by readChar(), EOF if ungetChar() has nothing to put back */
+    int lastChar{EOF};
+    /** readChar() has returned EOF */
+    bool atEnd{false};
+    /** get() has returned EOF */
+    bool inputEnd{false};
 protected:
     bool isUtf16;
 };
@@ -414,6 +448,7 @@ namespace {
             init();
         }
 
+    protected:
         int get() override {
             return istr.get();
         }
@@ -423,9 +458,6 @@ namespace {
         void unget() override {
             istr.unget();
         }
-        bool good() override {
-            return istr.good();
-        }
 
     private:
         std::istream &istr;
@@ -441,28 +473,25 @@ namespace {
             init();
         }
 
+    protected:
         int get() override {
             if (pos >= size)
-                return lastStatus = EOF;
+                return EOF;
             return str[pos++];
         }
         int peek() override {
             if (pos >= size)
-                return lastStatus = EOF;
+                return EOF;
             return str[pos];
         }
         void unget() override {
             --pos;
         }
-        bool good() override {
-            return lastStatus != EOF;
-        }
 
     private:
         const unsigned char *str;
         const std::size_t size;
         std::size_t pos{};
-        int lastStatus{};
     };
 
     class FileStream : public simplecpp::TokenList::Stream {
@@ -489,8 +518,9 @@ namespace {
             file = nullptr;
         }
 
+    protected:
         int get() override {
-            lastStatus = lastCh = fgetc(file);
+            lastCh = fgetc(file);
             return lastCh;
         }
         int peek() override {
@@ -502,9 +532,6 @@ namespace {
         void unget() override {
             unget_internal(lastCh);
         }
-        bool good() override {
-            return lastStatus != EOF;
-        }
 
     private:
         void unget_internal(int ch) {
@@ -519,7 +546,6 @@ namespace {
 
         FILE *file;
         int lastCh{};
-        int lastStatus{};
     };
 }
 

test.cpp

@@ -9,7 +9,9 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
+#include <cstdio>
 #include <exception>
+#include <fstream>
 #include <iostream>
 #include <list>
 #include <sstream>
@@ -142,6 +144,20 @@ static std::string readfile(const char code[], std::size_t size, const simplecpp
     return makeTokenList(code,size,files,std::string(),dui,outputList).stringify();
 }
 
+/** Write the code to a temporary file and read it with TokenList(filename), so FileStream is used */
+static std::string readfileFromFile(const char code[], std::size_t size, simplecpp::OutputList *outputList = nullptr)
+{
+    const std::string filename = "testrunner_filestream.tmp";
+    {
+        std::ofstream f(filename, std::ios::binary);
+        f.write(code, static_cast<std::streamsize>(size));
+    }
+    std::vector<std::string> files;
+    std::string ret = simplecpp::TokenList(filename, files, {}, outputList).stringify();
+    std::remove(filename.c_str());
+    return ret;
+}
+
 static std::string preprocess(const char code[], std::size_t size, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list<simplecpp::MacroUsage> *macroUsage = nullptr, std::list<simplecpp::IfCond> *ifCond = nullptr, const std::string &file = std::string())
 {
     std::vector<std::string> files;
@@ -3981,6 +3997,11 @@ static void readfile_char_error()
 
     readfile("A = 's\n'", {}, &outputList);
     ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
+    outputList.clear();
+
+    // the ' at the end of the file is not dropped after the peek for a digit separator
+    readfile("A = 1'", {}, &outputList);
+    ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
 }
 
 static void readfile_string()
@@ -4168,6 +4189,72 @@ static void readfile_no_eof_newline()
         readfile(code, sizeof(code)-1, {}, &outputList);
         ASSERT_EQUALS("", toString(outputList));
     }
+    {
+        // CR is a newline
+        const char code[] = "x\r";
+        simplecpp::OutputList outputList;
+        readfile(code, sizeof(code)-1, {}, &outputList);
+        ASSERT_EQUALS("", toString(outputList));
+    }
+    {
+        const char code[] = "x\r\n";
+        simplecpp::OutputList outputList;
+        readfile(code, sizeof(code)-1, {}, &outputList);
+        ASSERT_EQUALS("", toString(outputList));
+    }
+}
+
+static void readfile_cr_newlines()
+{
+    // CR, CR LF and LF are all newlines
+    ASSERT_EQUALS("a\nb\nc", readfile("a\rb\rc\r"));
+    ASSERT_EQUALS("a\nb\nc", readfile("a\r\nb\r\nc\r\n"));
+    ASSERT_EQUALS("a\nb\nc\n\n\nd", readfile("a\rb\r\nc\n\r\rd\n"));
+    ASSERT_EQUALS("# define A 1\nA", readfile("#define A 1\rA\r"));
+}
+
+static void readfile_filestream()
+{
+    // the edge cases from the other readfile tests, read from a file
+    {
+        const char code[] = "a\rb\rc\r";
+        simplecpp::OutputList outputList;
+        ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1, &outputList));
+        ASSERT_EQUALS("", toString(outputList));
+    }
+    {
+        const char code[] = "a\r\nb\r\nc\r\n";
+        ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1));
+    }
+    {
+        const char code[] = "a\rb\r\nc\n\r\rd\n";
+        ASSERT_EQUALS("a\nb\nc\n\n\nd", readfileFromFile(code, sizeof(code)-1));
+    }
+    {
+        const char code[] = "1'";
+        simplecpp::OutputList outputList;
+        (void)readfileFromFile(code, sizeof(code)-1, &outputList);
+        ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
+    }
+    {
+        // UTF-16LE with CR newlines
+        const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00";
+        simplecpp::OutputList outputList;
+        ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1, &outputList));
+        ASSERT_EQUALS("", toString(outputList));
+    }
+    {
+        // UTF-16BE with CR newlines
+        const char code[] = "\xfe\xff\x00\x78\x00\x0d\x00\x79\x00\x0d";
+        ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1));
+    }
+    {
+        // UTF-16 with an odd number of bytes, the dangling last byte is dropped
+        const char code[] = "\xff\xfe\x31\x00\x0a";
+        simplecpp::OutputList outputList;
+        ASSERT_EQUALS("1", readfileFromFile(code, sizeof(code)-1, &outputList));
+        ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList));
+    }
 }
 
 static void stringify1()
@@ -4364,6 +4451,18 @@ static void unicode()
         const char code[] = "\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00";
         ASSERT_EQUALS("\n//1", readfile(code, sizeof(code)-1));
     }
+    {
+        // CR newlines
+        const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00";
+        ASSERT_EQUALS("x\ny", readfile(code, sizeof(code)-1));
+    }
+    {
+        // odd number of bytes, the dangling last byte is dropped
+        const char code[] = "\xff\xfe\x31\x00\x0a";
+        simplecpp::OutputList outputList;
+        ASSERT_EQUALS("1", readfile(code, sizeof(code)-1, {}, &outputList));
+        ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList));
+    }
 }
 
 static void unicode_invalid()
@@ -5335,6 +5434,8 @@ static void runTests(int argc, char **argv, Input input)
     TEST_CASE(readfile_error);
     TEST_CASE(readfile_file_not_found);
     TEST_CASE(readfile_no_eof_newline);
+    TEST_CASE(readfile_cr_newlines);
+    TEST_CASE(readfile_filestream);
 
     TEST_CASE(stringify1);