More work on Gerber pick and place files

In P&P files, user strings are now quoted, and use UTF8 encoding

common/gbr_metadata.cpp

@@ -30,6 +30,7 @@
 
 #include <fctsys.h>
 #include <gbr_metadata.h>
+#include <utf8.h>
 
 wxString GbrMakeCreationDateAttributeString( GBR_NC_STRING_FORMAT aFormat )
 {
@@ -314,19 +315,53 @@ std::string GBR_APERTURE_METADATA::FormatAttribute( GBR_APERTURE_ATTRIB aAttribu
     return full_attribute_string;
 }
 
+
+// Helper function to convert a ascii hex char to its integer value
+// If the char is not a hexa char, return -1
+int char2Hex( unsigned aCode )
+{
+    if( aCode >= '0' && aCode <= '9' )
+        return aCode - '0';
+
+    if( aCode >= 'A' && aCode <= 'F' )
+        return aCode - 'A' + 10;
+
+    if( aCode >= 'a' && aCode <= 'f' )
+        return aCode - 'a' + 10;
+
+    return -1;
+}
+
+
 wxString FormatStringFromGerber( const wxString& aString )
 {
-    // make the inverse conversion of formatStringToGerber()
+    // make the inverse conversion of FormatStringToGerber()
     // It converts a "normalized" gerber string and convert it to a 16 bits sequence unicode
     // and return a wxString (unicode 16) from the gerber string
-    wxString txt;
+    wxString txt;           // The string converted from Gerber string
+    wxString uniString;     // the unicode string from UTF8 Gerber string but without converted escape sequence
 
-    for( unsigned ii = 0; ii < aString.Length(); ++ii )
+    unsigned count = aString.Length();
+
+    for( unsigned ii = 0; ii < count; ++ii )
     {
         unsigned code = aString[ii];
 
         if( code == '\\' )
         {
+            // If next char is not a hexadecimal char, just skip the '\'
+            // It is perhaps a escape sequence like \\ or \" or ...
+            if( ii < count-1 )
+            {
+                code = aString[ii+1];
+
+                if( char2Hex( code ) < 0 )
+                {
+                    ++ii;
+                    txt.Append( aString[ii] );
+                    continue;
+                }
+            }
             // Convert 4 hexadecimal digits to a 16 bit unicode
             // (Gerber allows only 4 hexadecimal digits)
             long value = 0;
@@ -335,8 +370,8 @@ wxString FormatStringFromGerber( const wxString& aString )
             {
                 value <<= 4;
                 code = aString[++ii];
-                // Very basic conversion, but it expects a valid gerber file
+                // Basic conversion (with no control), but it expects a valid gerber file
-                int hexa = (code <= '9' ? code - '0' : code - 'A' + 10) & 0xF;
+                int hexa = char2Hex( code );
                 value += hexa;
             }
 
@@ -350,19 +385,20 @@ wxString FormatStringFromGerber( const wxString& aString )
 }
 
 
-std::string formatStringToGerber( const wxString& aString )
+wxString ConvertNotAllowedCharsInGerber( const wxString& aString, bool aAllowUtf8Chars, bool aQuoteString )
 {
-    /* format string means convert any code > 0x7F and unautorized code to a hexadecimal
+    /* format string means convert any code > 0x7E and unautorized codes to a hexadecimal
      * 16 bits sequence unicode
-     * unautorized codes are ',' '*' '%' '\'
+     * unautorized codes are ',' '*' '%' '\' and are used as separators in Gerber files
      */
-    std::string txt;
+    wxString txt;
 
-    txt.reserve( aString.Length() );
+    if( aQuoteString )
+        txt << "\"";
 
     for( unsigned ii = 0; ii < aString.Length(); ++ii )
     {
-        unsigned code = aString[ii];
+        wxChar code = aString[ii];
         bool convert = false;
 
         switch( code )
@@ -378,7 +414,10 @@ std::string formatStringToGerber( const wxString& aString )
             break;
         }
 
-        if( convert || code > 0x7F )
+        if( !aAllowUtf8Chars && code > 0x7F )
+            convert = true;
+
+        if( convert )
         {
             txt += '\\';
 
@@ -389,9 +428,35 @@ std::string formatStringToGerber( const wxString& aString )
             txt += hexa;
         }
         else
-            txt += char( code );
+            txt += code;
     }
 
+    if( aQuoteString )
+        txt << "\"";
+
+    return txt;
+}
+
+
+std::string FormatStringToGerber( const wxString& aString )
+{
+    wxString converted;
+    /* format string means convert any code > 0x7E and unautorized codes to a hexadecimal
+     * 16 bits sequence unicode
+     * unautorized codes are ',' '*' '%' '\'
+     * This conversion is not made for quoted strings, because if the string is
+     * quoted, the conversion is expected to be already made, and the returned string must use
+     * UTF8 encoding
+     */
+    if( aString[0] != '\"' || aString[aString.Len()-1] != '\"' )
+        converted = ConvertNotAllowedCharsInGerber( aString, false, false );
+    else
+        converted = aString;
+
+    // Convert the char string to std::string. Be carefull when converting awxString to
+    // a std::string: using static_cast<const char*> is mandatory
+    std::string txt = static_cast<const char*>( converted.utf8_str() );
+
     return txt;
 }
 
@@ -437,13 +502,13 @@ bool FormatNetAttribute( std::string& aPrintedText, std::string& aLastNetAttribu
         // print info associated to a flashed pad (cmpref, pad name)
         // example: %TO.P,R5,3*%
         pad_attribute_string = prepend_string + "TO.P,";
-        pad_attribute_string += formatStringToGerber( aData->m_Cmpref ) + ",";
+        pad_attribute_string += FormatStringToGerber( aData->m_Cmpref ) + ",";
 
         if( aData->m_Padname.IsEmpty() )
             // Happens for "mechanical" or never connected pads
-            pad_attribute_string += formatStringToGerber( NO_PAD_NAME );
+            pad_attribute_string += FormatStringToGerber( NO_PAD_NAME );
         else
-            pad_attribute_string += formatStringToGerber( aData->m_Padname );
+            pad_attribute_string += FormatStringToGerber( aData->m_Padname );
 
         pad_attribute_string += eol_string;
     }
@@ -466,11 +531,11 @@ bool FormatNetAttribute( std::string& aPrintedText, std::string& aLastNetAttribu
             {
                 // Happens for not connected pads: use a normalized
                 // dummy name
-                net_attribute_string += formatStringToGerber( NO_NET_NAME );
+                net_attribute_string += FormatStringToGerber( NO_NET_NAME );
             }
         }
         else
-            net_attribute_string += formatStringToGerber( aData->m_Netname );
+            net_attribute_string += FormatStringToGerber( aData->m_Netname );
 
         net_attribute_string += eol_string;
     }
@@ -483,7 +548,7 @@ bool FormatNetAttribute( std::string& aPrintedText, std::string& aLastNetAttribu
         // Because GBR_NETINFO_PAD option already contains this info, it is not
         // created here for a GBR_NETINFO_PAD attribute
         cmp_attribute_string = prepend_string + "TO.C,";
-        cmp_attribute_string += formatStringToGerber( aData->m_Cmpref ) + eol_string;
+        cmp_attribute_string += FormatStringToGerber( aData->m_Cmpref ) + eol_string;
     }
 
     // the full list of requested attributes:
@@ -590,7 +655,7 @@ void GBR_CMP_PNP_METADATA::ClearData()
 /**
  * @return a string containing the formated metadata in X2 syntax.
  * one line by non empty data
- * the orientation is always generated
+ * the orientation (.CRot) and mount type (.CMnt) are always generated
  */
 wxString GBR_CMP_PNP_METADATA::FormatCmpPnPMetadata()
 {

gerbview/X2_gerber_attributes.cpp

@@ -43,6 +43,7 @@
 
 #include <wx/log.h>
 #include <X2_gerber_attributes.h>
+#include <macros.h>
 
 /*
  * class X2_ATTRIBUTE
@@ -92,11 +93,11 @@ void X2_ATTRIBUTE::DbgListPrms()
 bool X2_ATTRIBUTE::ParseAttribCmd( FILE* aFile, char *aBuffer, int aBuffSize, char* &aText,
                                    int& aLineNum )
 {
-    // parse a TF command and fill m_Prms by the parameters found.
+    // parse a TF, TA, TO ... command and fill m_Prms by the parameters found.
     // the "%TF" (start of command) is already read by the caller
 
     bool ok = true;
-    wxString data;
+    std::string data;
 
     for( ; ; )
     {
@@ -114,19 +115,19 @@ bool X2_ATTRIBUTE::ParseAttribCmd( FILE* aFile, char *aBuffer, int aBuffSize, ch
                 break;
 
             case '*':       // End of block
-                m_Prms.Add( data );
+                m_Prms.Add( FROM_UTF8( data.c_str() ) );
-                data.Empty();
+                data.clear();
                 aText++;
                 break;
 
             case ',':       // End of parameter (separator)
                 aText++;
-                m_Prms.Add( data );
+                m_Prms.Add( FROM_UTF8( data.c_str() ) );
-                data.Empty();
+                data.clear();
                 break;
 
             default:
-                data.Append( *aText );
+                data += *aText;
                 aText++;
                 break;
             }

gerbview/rs274x.cpp

@@ -34,6 +34,7 @@
 #include <gerbview.h>
 #include <gerber_file_image.h>
 #include <X2_gerber_attributes.h>
+#include <gbr_metadata.h>
 
 extern int ReadInt( char*& text, bool aSkipSeparator = true );
 extern double ReadDouble( char*& text, bool aSkipSeparator = true );
@@ -138,50 +139,6 @@ int GERBER_FILE_IMAGE::ReadXCommandID( char*& text )
     return result;
 }
 
-/**
- * Convert a string read from a gerber file to an unicode string
- * Usual chars (ASCII7 values) are the only values allowed in Gerber files,
- * and are just copied.
- * However Gerber format allows using non ASCII7 values by coding them in a
- * sequence of 4 hexadecimal chars (16 bits hexadecimal value)
- * Hexadecimal coded values ("\hhhh") are converted to
- * the unicode char value
- */
-static const wxString fromGerberString( const wxString& aGbrString )
-{
-    wxString text;
-
-    for( unsigned ii = 0; ii < aGbrString.size(); ++ii )
-    {
-        if( aGbrString[ii] == '\\' )
-        {
-            unsigned value = 0;
-
-            for( int jj = 0; jj < 4; jj++ )
-            {   // Convert 4 hexa digits to binary value:
-                ii++;
-                value <<= 4;
-                int digit = aGbrString[ii];
-
-                if( digit >= '0' && digit <= '9' )
-                    digit -= '0';
-                else if( digit >= 'A' && digit <= 'F' )
-                    digit -= 'A' - 10;
-                else if( digit >= 'a' && digit <= 'f' )
-                    digit -= 'a' - 10;
-                else digit = 0;
-
-                value += digit & 0xF;
-            }
-
-            text.Append( wxUniChar( value ) );
-        }
-        else
-            text.Append( aGbrString[ii] );
-    }
-
-    return text;
-}
 
 bool GERBER_FILE_IMAGE::ReadRS274XCommand( char *aBuff, unsigned int aBuffSize, char*& aText )
 {
@@ -451,18 +408,18 @@ bool GERBER_FILE_IMAGE::ExecuteRS274XCommand( int aCommand, char* aBuff,
         if( dummy.GetAttribute() == ".N" )
         {
             m_NetAttributeDict.m_NetAttribType |= GBR_NETLIST_METADATA::GBR_NETINFO_NET;
-            m_NetAttributeDict.m_Netname = fromGerberString( dummy.GetPrm( 1 ) );
+            m_NetAttributeDict.m_Netname = FormatStringFromGerber( dummy.GetPrm( 1 ) );
         }
         else if( dummy.GetAttribute() == ".C" )
         {
             m_NetAttributeDict.m_NetAttribType |= GBR_NETLIST_METADATA::GBR_NETINFO_CMP;
-            m_NetAttributeDict.m_Cmpref = fromGerberString( dummy.GetPrm( 1 ) );
+            m_NetAttributeDict.m_Cmpref = FormatStringFromGerber( dummy.GetPrm( 1 ) );
         }
         else if( dummy.GetAttribute() == ".P" )
         {
             m_NetAttributeDict.m_NetAttribType |= GBR_NETLIST_METADATA::GBR_NETINFO_PAD;
-            m_NetAttributeDict.m_Cmpref = fromGerberString( dummy.GetPrm( 1 ) );
+            m_NetAttributeDict.m_Cmpref = FormatStringFromGerber( dummy.GetPrm( 1 ) );
-            m_NetAttributeDict.m_Padname = fromGerberString( dummy.GetPrm( 2 ) );
+            m_NetAttributeDict.m_Padname = FormatStringFromGerber( dummy.GetPrm( 2 ) );
         }
         }
         break;

include/gbr_metadata.h

@@ -210,10 +210,24 @@ private:
  * @param aString = the wxString to convert
  * @return a std::string (ASCII7 coded) compliant with a gerber string
  */
-std::string formatStringToGerber( const wxString& aString );
+std::string FormatStringToGerber( const wxString& aString );
+
 
 /**
- * This helper function make the inverse conversion of formatStringToGerber()
+ * Similar to FormatStringToGerber.
+ * "normalize" aString and convert it to a Gerber compatible wxString
+ * Normalisation means unautorized code to a hexadecimal 16 bits sequence unicode
+ * and, on request convert any code > 0x7F.
+ * unautorized codes are ',' '*' '%' '\'
+ * @param aString = the wxString to convert
+ * @param aAllowUtf8Chars = false to convert non ASCII7 values to unicode sequence
+ * @param aQuoteString = true to double quote the returned string
+ * @return a wxString without unautorized chars (and converted non ASCII7 chars on request)
+ */
+wxString ConvertNotAllowedCharsInGerber( const wxString& aString, bool aAllowUtf8Chars, bool aQuoteString );
+
+/**
+ * This helper function make the inverse conversion of FormatStringToGerber()
  * It converts a "normalized" gerber string and convert it to a 16 bits sequence unicode
  * @param aString = the wxString compliant with a gerber string format
  * @return a wxString (unicode 16) from the gerber string

pcbnew/exporters/gerber_placefile_writer.cpp

@@ -111,6 +111,7 @@ int PLACEFILE_GERBER_WRITER::CreatePlaceFile( wxString& aFullFilename,
     plotter.StartPlot();
 
     int cmp_count = 0;
+    bool allowUtf8 = true;
 
     for( MODULE* footprint : fp_list )
     {
@@ -119,7 +120,9 @@ int PLACEFILE_GERBER_WRITER::CreatePlaceFile( wxString& aFullFilename,
         gbr_metadata.SetApertureAttrib( GBR_APERTURE_METADATA::GBR_APERTURE_ATTRIB_CMP_POSITION );
 
         // Add object attribute: component reference to flash (mainly usefull for users)
-        wxString ref = footprint->GetReference();
+        // using quoted UTF8 string
+        wxString ref = ConvertNotAllowedCharsInGerber( footprint->GetReference(),
+                                                       allowUtf8, true );
 
         gbr_metadata.SetCmpReference( ref );
         gbr_metadata.SetNetAttribType( GBR_NETLIST_METADATA::GBR_NETINFO_CMP );
@@ -143,15 +146,15 @@ int PLACEFILE_GERBER_WRITER::CreatePlaceFile( wxString& aFullFilename,
                                 : GBR_CMP_PNP_METADATA::MOUNT_TYPE_TH;
 
         // Add component value info:
-        pnpAttrib.m_Value = FormatStringFromGerber( footprint->GetValue() );
+        pnpAttrib.m_Value = ConvertNotAllowedCharsInGerber( footprint->GetValue(), allowUtf8, true );
 
         // Add component footprint info:
         wxString fp_info = FROM_UTF8( footprint->GetFPID().GetLibItemName().c_str() );
-        pnpAttrib.m_Footprint = FormatStringFromGerber( fp_info );
+        pnpAttrib.m_Footprint = ConvertNotAllowedCharsInGerber( fp_info, allowUtf8, true );
 
         // Add footprint lib name:
         fp_info = FROM_UTF8( footprint->GetFPID().GetLibNickname().c_str() );
-        pnpAttrib.m_LibraryName = FormatStringFromGerber( fp_info );
+        pnpAttrib.m_LibraryName = ConvertNotAllowedCharsInGerber( fp_info, allowUtf8, true );
 
         gbr_metadata.m_NetlistMetadata.SetExtraData( pnpAttrib.FormatCmpPnPMetadata() );