From 6700c604aa78a58b5912b09752acf88656dde359 Mon Sep 17 00:00:00 2001
From: dickelbeck <Unknown>
Date: Sat, 20 Feb 2010 19:13:30 +0000
Subject: [PATCH]     DSNLEXER now supports:     1)  nested quotes.  This is in
 anticipation of broader usage of the         file type/syntax.  A string like
 this in the file:             "my ""favorate"" string"         can be
 returned as             my "favorite" string     2)  CommentsAsTokens is
 implemented, so you can ask the lexer to return         comments as tokens,
 so they can be preserved.  The default is to ignore         them.  A comment
 is defined as any line that has # as its first         non-blank character. 
 (This means comments cannot follow anything else         on a line.)

---
 CHANGELOG.txt       | 15 ++++++++++
 TODO.txt            |  4 ---
 common/dsnlexer.cpp | 68 ++++++++++++++++++++++++++++++++++++++++++---
 include/dsnlexer.h  |  6 ++--
 4 files changed, 82 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 24120afbb9..67b31af15c 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -4,6 +4,21 @@ KiCad ChangeLog 2010
 Please add newer entries at the top, list the date and your name with
 email address.
 
+2010-Feb-20 UPDATE Dick Hollenbeck <dick@softplc.com>
+================================================================================
+++common
+    DSNLEXER now supports:
+    1)  nested quotes.  This is in anticipation of broader usage of the
+        file type/syntax.  A string like this in the file:
+            "my ""favorate"" string"
+        can be returned as
+            my "favorite" string
+    2)  CommentsAsTokens is implemented, so you can ask the lexer to return
+        comments as tokens, so they can be preserved.  The default is to ignore
+        them.  A comment is defined as any line that has # as its first
+        non-blank character.  (This means comments cannot follow anything else
+        on a line.)
+
 
 2010-Feb-19 UPDATE Jean-Pierre Charras <jean-pierre.charras@gipsa-lab.inpg.fr>
 ================================================================================
diff --git a/TODO.txt b/TODO.txt
index 805b977a61..1de9aefd40 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -96,10 +96,6 @@ P2) Write accessors for all items in PCB_VISIBLE such as grid control, so that
     Then example 2 in RS274xrevd_e.pdf will draw properly.
 
 
-Dick:
-D1) Get the nested quote support for DSNLEXER fixed up and committed.
-
-
 LAYER_WIDGET for PCBNEW (Dick)
 -----------------------
 L6) Test, and fix up any remaining issues with the PCB_VISIBLE support after P2)
diff --git a/common/dsnlexer.cpp b/common/dsnlexer.cpp
index 5e5527d05b..187a237454 100644
--- a/common/dsnlexer.cpp
+++ b/common/dsnlexer.cpp
@@ -51,7 +51,6 @@ static int compare( const void* a1, const void* a2 )
 DSNLEXER::DSNLEXER( FILE* aFile, const wxString& aFilename,
     const KEYWORD* aKeywordTable, unsigned aKeywordCount ) :
         reader( aFile, 4096 )
-
 {
     keywords = aKeywordTable;
     keywordCount = aKeywordCount;
@@ -62,6 +61,8 @@ DSNLEXER::DSNLEXER( FILE* aFile, const wxString& aFilename,
 
     space_in_quoted_tokens = true;
 
+    commentsAreTokens = false;
+
     // "start" should never change until we change the reader.  The DSN
     // format spec supports an include file mechanism but we can add that later
     // using a std::stack to hold a stack of LINE_READERs to track nesting.
@@ -211,9 +212,23 @@ L_read:
             while( cur<limit && isSpace(*cur) )
                 ++cur;
 
-            // if the first non-blank character is #, this line is a comment.
+            // If the first non-blank character is #, this line is a comment.
+            // Comments cannot follow any other token on the same line.
             if( cur<limit && *cur=='#' )
-                goto L_read;
+            {
+                if( commentsAreTokens )
+                {
+                    // save the entire line, including new line as the current token.
+                    // the '#' character may not be at offset zero.
+                    curText = start;        // entire line is the token
+                    cur     = start;        // ensure a good curOffset below
+                    curTok  = DSN_COMMENT;
+                    head    = limit;        // do a readLine() on next call in here.
+                    goto exit;
+                }
+                else
+                    goto L_read;
+            }
         }
         else
         {
@@ -303,11 +318,55 @@ L_read:
         // a quoted string
         if( *cur == stringDelimiter )
         {
+            // New code, understands nested quotes, and deliberately restricts
+            // strings to a single line. Still strips off leading and trailing
+            // quotes, and now removes internal doubled up quotes
+#if 1
+            head = cur;
+
+            // copy the token, character by character so we can remove doubled up quotes.
+            curText.clear();
+
+            while( head < limit )
+            {
+                if( *head==stringDelimiter )
+                {
+                    if( head+1<limit && head[1]==stringDelimiter )
+                    {
+                        // include only one of the doubled-up stringDelimiters
+                        curText += *head;
+                        head    += 2;
+                        continue;
+                    }
+                    else if( head == cur )
+                    {
+                        ++head;     // skip the leading quote
+                        continue;
+                    }
+
+                    // fall thru
+                }
+
+                // check for a terminator
+                if( isStringTerminator( *head ) )
+                {
+                    curTok = DSN_STRING;
+                    ++head;
+                    goto exit;
+                }
+
+                curText += *head++;
+            }
+
+            wxString errtxt(_("Un-terminated delimited string") );
+            ThrowIOError( errtxt, CurOffset() );
+
+#else   // old code, did not understand nested quotes
             ++cur;  // skip over the leading delimiter: ",', or $
 
             head = cur;
 
-            while( head<limit &&  !isStringTerminator( *head ) )
+            while( head<limit  &&  !isStringTerminator( *head ) )
                 ++head;
 
             if( head >= limit )
@@ -323,6 +382,7 @@ L_read:
 
             curTok  = DSN_STRING;
             goto exit;
+#endif
         }
 
         // Maybe it is a token we will find in the token table.
diff --git a/include/dsnlexer.h b/include/dsnlexer.h
index 9f5e8f3561..8c679f5a24 100644
--- a/include/dsnlexer.h
+++ b/include/dsnlexer.h
@@ -85,7 +85,7 @@ class DSNLEXER
     LINE_READER         reader;
     int                 stringDelimiter;
     bool                space_in_quoted_tokens; ///< blank spaces within quoted strings
-    bool                comments_are_tokens;    ///< true if should return comments as tokens
+    bool                commentsAreTokens;      ///< true if should return comments as tokens
 
     wxString            filename;
     int                 prevTok;        ///< curTok from previous NextTok() call.
@@ -194,8 +194,8 @@ public:
      */
     bool SetCommentsAreTokens( bool val )
     {
-        bool old = comments_are_tokens;
-        comments_are_tokens = val;
+        bool old = commentsAreTokens;
+        commentsAreTokens = val;
         return old;
     }