make LINE_READERs have dynamic buffer sizes

CHANGELOG.txt

@@ -4,6 +4,14 @@ KiCad ChangeLog 2010
 Please add newer entries at the top, list the date and your name with
 email address.
 
+2010-Oct-20 UPDATE Dick Hollenbeck <dick@softplc.com>
+================================================================================
+++richio:
+    LINE_READERs will now allocate a smaller initial size buffer, say 5000 bytes,
+    and then resize their buffers up to some provided maximum, after which an
+    exception is thrown should a line exceed that maximum line length.
+
+
 2010-oct-15, UPDATE Jean-Pierre Charras <jean-pierre.charras@gipsa-lab.inpg.fr>
 ================================================================================
 ++gerbview:

common/dsnlexer.cpp

@@ -86,11 +86,10 @@ void DSNLEXER::PushReader( LINE_READER* aLineReader )
 {
     readerStack.push_back( aLineReader );
     reader = aLineReader;
-    start = (char*) (*aLineReader);
 
     // force a new readLine() as first thing.
-    limit = start;
+    limit = start();
-    next  = start;
+    next  = start();
 }
 
 
@@ -103,11 +102,9 @@ bool DSNLEXER::PopReader()
 
         reader = &readerStack.back();
 
-        start = (char*) (*reader);
-    
         // force a new readLine() as first thing.
-        limit = start;
+        limit = start();
-        next  = start;
+        next  = start();
         return true;
     }
     return false;
@@ -334,7 +331,7 @@ L_read:
                 goto exit;
             }
 
-            cur = start;
+            cur = start();
 
             // skip leading whitespace
             while( cur<limit && isSpace(*cur) )
@@ -348,8 +345,8 @@ L_read:
                 {
                     // save the entire line, including new line as the current token.
                     // the '#' character may not be at offset zero.
-                    curText = start;        // entire line is the token
+                    curText = start();      // entire line is the token
-                    cur     = start;        // ensure a good curOffset below
+                    cur     = start();      // ensure a good curOffset below
                     curTok  = DSN_COMMENT;
                     head    = limit;        // do a readLine() on next call in here.
                     goto exit;
@@ -417,7 +414,7 @@ L_read:
             like:  U2-14 or "U2"-"14"
             This is detectable by a non-space immediately preceeding the dash.
         */
-        if( *cur == '-' && cur>start && !isSpace( cur[-1] ) )
+        if( *cur == '-' && cur>start() && !isSpace( cur[-1] ) )
         {
             curText = '-';
             curTok = DSN_DASH;
@@ -538,7 +535,7 @@ L_read:
 
 exit:   // single point of exit, no returns elsewhere please.
 
-    curOffset = cur - start;
+    curOffset = cur - start();
 
     next = head;
 

common/richio.cpp

@@ -37,10 +37,18 @@
 LINE_READER::LINE_READER( unsigned aMaxLineLength )
 {
     lineNum = 0;
+
+    if( aMaxLineLength == 0 )       // caller is goofed up.
+        aMaxLineLength = LINE_READER_LINE_DEFAULT_MAX;
+
     maxLineLength = aMaxLineLength;
 
-    // the real capacity is 10 bytes larger than requested.
+    // start at the INITIAL size, expand as needed up to the MAX size in maxLineLength
-    capacity = aMaxLineLength + 10;
+    capacity = LINE_READER_LINE_INITIAL_SIZE;
+
+    // but never go above user's aMaxLineLength, and leave space for trailing nul
+    if( capacity > aMaxLineLength-1 )
+        capacity = aMaxLineLength-1;
 
     line = new char[capacity];
 
@@ -49,6 +57,28 @@ LINE_READER::LINE_READER( unsigned aMaxLineLength )
 }
 
 
+void LINE_READER::expandCapacity( unsigned newsize )
+{
+    // length can equal maxLineLength and nothing breaks, there's room for
+    // the terminating nul. cannot go over this.
+    if( newsize > maxLineLength+1 )
+        newsize = maxLineLength+1;
+
+    if( newsize > capacity )
+    {
+        capacity = newsize;
+
+        // resize the buffer, and copy the original data
+        char* bigger = new char[capacity];
+
+        memcpy( bigger, line, length );
+
+        delete line;
+        line = bigger;
+    }
+}
+
+
 FILE_LINE_READER::FILE_LINE_READER( FILE* aFile, const wxString& aFileName, unsigned aMaxLineLength ) :
     LINE_READER( aMaxLineLength ),
     fp( aFile )
@@ -57,56 +87,61 @@ FILE_LINE_READER::FILE_LINE_READER( FILE* aFile, const wxString& aFileName, unsi
 }
 
 
-int FILE_LINE_READER::ReadLine() throw (IOError)
+unsigned FILE_LINE_READER::ReadLine() throw (IOError)
 {
-    const char* p = fgets( line, capacity, fp );
-
-    if( !p )
-    {
-        line[0] = 0;
     length  = 0;
-    }
+    line[0] = 0;
-    else
-    {
-        length = strlen( line );
 
-        if( length > maxLineLength )
+    // fgets always put a terminating nul at end of its read.
+    while( fgets( line + length, capacity - length, fp ) )
+    {
+        length += strlen( line + length );
+
+        if( length == maxLineLength )
             throw IOError( _("Line length exceeded") );
 
-        ++lineNum;
+        // a normal line breaks here, once through
+        if( length < capacity-1 || line[length-1] == '\n' )
+            break;
+
+        expandCapacity( capacity * 2 );
     }
 
+    if( length )
+        ++lineNum;
+
     return length;
 }
 
 
-int STRING_LINE_READER::ReadLine() throw (IOError)
+unsigned STRING_LINE_READER::ReadLine() throw (IOError)
 {
     size_t      nlOffset = lines.find( '\n', ndx );
-    size_t      advance;
 
     if( nlOffset == std::string::npos )
-        advance = lines.length() - ndx;
+        length = lines.length() - ndx;
     else
-        advance = nlOffset - ndx + 1;     // include the newline, so +1
+        length = nlOffset - ndx + 1;     // include the newline, so +1
 
-    if( advance )
+    if( length )
     {
-        if( advance > maxLineLength )
+        if( length >= maxLineLength )
             throw IOError( _("Line length exceeded") );
 
-        wxASSERT( ndx + advance <= lines.length() );
+        if( length > capacity )
+            expandCapacity( length );
 
-        memcpy( line, &source[ndx], advance );
+        wxASSERT( ndx + length <= lines.length() );
+
+        memcpy( line, &source[ndx], length );
 
         ++lineNum;
-        ndx += advance;
+        ndx += length;
     }
 
-    length = advance;
+    line[length] = 0;
-    line[advance] = 0;
 
-    return advance;
+    return length;
 }
 
 

include/dsnlexer.h

@@ -79,7 +79,6 @@ enum DSN_SYNTAX_T {
 class DSNLEXER
 {
     char*               next;
-    char*               start;
     char*               limit;
 
     typedef boost::ptr_vector<LINE_READER>  READER_STACK;
@@ -100,14 +99,21 @@ class DSNLEXER
     const KEYWORD*      keywords;
     unsigned            keywordCount;
 
+    /// Use casting char* operator to get start of line, which is dynamic since reader
+    /// can be resizing its buffer at each reader->ReadLine() only.
+    char*               start() const { return (char*) (*reader); }
+
     void init();
 
     int readLine() throw (IOError)
     {
-        int len = reader->ReadLine();
+        unsigned len = reader->ReadLine();
 
-        next  = start;
+        // set next and limit to start() and start() + len.
-        limit = start + len;
+        // start() is constant until the next ReadLine(), which could resize and
+        // relocate reader's line buffer.
+        next  = start();
+        limit = next + len;
 
         return len;
     }

include/richio.h

@@ -61,6 +61,8 @@ struct IOError
     }
 };
 
+#define LINE_READER_LINE_DEFAULT_MAX        100000
+#define LINE_READER_LINE_INITIAL_SIZE       5000
 
 /**
  * Class LINE_READER
@@ -70,13 +72,18 @@ struct IOError
 class LINE_READER
 {
 protected:
-    unsigned            length;
+    unsigned    length;         ///< no. bytes in line before trailing nul.
     int         lineNum;
-    char*               line;
+
-    unsigned            maxLineLength;
+    char*       line;           ///< the read line of UTF8 text
-    unsigned            capacity;
+    unsigned    capacity;       ///< no. bytes allocated for line.
+
+    unsigned    maxLineLength;  ///< maximum allowed capacity using resizing.
+
     wxString    source;         ///< origin of text lines, e.g. filename or "clipboard"
 
+    void        expandCapacity( unsigned newsize );
+
 public:
 
     /**
@@ -84,7 +91,7 @@ public:
      * builds a line reader and fixes the length of the maximum supported
      * line length to @a aMaxLineLength.
      */
-    LINE_READER( unsigned aMaxLineLength );
+    LINE_READER( unsigned aMaxLineLength = LINE_READER_LINE_DEFAULT_MAX );
 
     virtual ~LINE_READER()
     {
@@ -96,10 +103,10 @@ public:
      * reads a line of text into the buffer and increments the line number
      * counter.  If the line is larger than the buffer size, then an exception
      * is thrown.
-     * @return int - The number of bytes read, 0 at end of file.
+     * @return unsigned - The number of bytes read, 0 at end of file.
      * @throw IOError only when a line is too long.
      */
-    virtual int ReadLine() throw (IOError) = 0;
+    virtual unsigned ReadLine() throw (IOError) = 0;
 
     /**
      * Function GetSource
@@ -118,7 +125,7 @@ public:
      * is a casting operator that returns a char* pointer to the start of the
      * line buffer.
      */
-    operator char* ()
+    operator char* () const
     {
         return line;
     }
@@ -166,7 +173,7 @@ public:
      * @param aFileName is the name of the file for error reporting purposes.
      * @param aMaxLineLength is the number of bytes to use in the line buffer.
      */
-    FILE_LINE_READER( FILE* aFile,  const wxString& aFileName, unsigned aMaxLineLength );
+    FILE_LINE_READER( FILE* aFile,  const wxString& aFileName, unsigned aMaxLineLength = LINE_READER_LINE_DEFAULT_MAX );
 
     ~FILE_LINE_READER()
     {
@@ -179,10 +186,10 @@ public:
      * reads a line of text into the buffer and increments the line number
      * counter.  If the line is larger than the buffer size, then an exception
      * is thrown.
-     * @return int - The number of bytes read, 0 at end of file.
+     * @return unsigned - The number of bytes read, 0 at end of file.
      * @throw IOError only when a line is too long.
      */
-    int ReadLine() throw (IOError);
+    unsigned ReadLine() throw (IOError);
 
     /**
      * Function Rewind
@@ -234,10 +241,10 @@ public:
      * reads a line of text into the buffer and increments the line number
      * counter.  If the line is larger than the buffer size, then an exception
      * is thrown.
-     * @return int - The number of bytes read, 0 at end of file.
+     * @return unsigned - The number of bytes read, 0 at end of file.
      * @throw IOError only when a line is too long.
      */
-    int ReadLine() throw (IOError);
+    unsigned ReadLine() throw (IOError);
 };