Altium import: Use UTF16 string table for PCB texts

Fixes https://gitlab.com/kicad/code/kicad/-/issues/7948

common/plugins/altium/altium_parser.h

@@ -71,8 +71,8 @@ public:
         uint8_t len = Read<uint8_t>();
         if( GetRemainingBytes() >= len )
         {
-
+            // TODO: Identify where the actual code page is stored. For now, this default code page
-            //altium uses LATIN1/ISO 8859-1, convert it
+            //       has limited impact, because recent Altium files come with a UTF16 string table
             wxString val = wxString( m_pos, wxConvISO8859_1, len );
             m_pos += len;
             return val;
@@ -84,6 +84,36 @@ public:
         }
     }
 
+    std::map<uint32_t, wxString> ReadWideStringTable()
+    {
+        std::map<uint32_t, wxString> table;
+        size_t                       remaining = GetRemainingBytes();
+
+        while( remaining >= 8 )
+        {
+            uint32_t index = Read<uint32_t>();
+            uint32_t length = Read<uint32_t>();
+            wxString str;
+            remaining -= 8;
+
+            if( length <= 2 )
+                length = 0; // for empty strings, not even the null bytes are present
+            else
+            {
+                if( length > remaining )
+                    break;
+
+                str = wxString( m_pos, wxMBConvUTF16LE(), length - 2 );
+            }
+
+            table.emplace( index, str );
+            m_pos += length;
+            remaining -= length;
+        }
+
+        return table;
+    }
+
     std::vector<char> ReadVector( size_t aSize )
     {
         if( aSize > GetRemainingBytes() )

pcbnew/plugins/altium/altium_circuit_maker_plugin.cpp

@@ -95,7 +95,8 @@ BOARD* ALTIUM_CIRCUIT_MAKER_PLUGIN::Load( const wxString& aFileName, BOARD* aApp
             { ALTIUM_PCB_DIR::SHAPEBASEDREGIONS6,"BDAA2C70289849078C8EBEEC7F0848\\" },
             { ALTIUM_PCB_DIR::TEXTS6, "A34BC67C2A5F408D8F377378C5C5E2\\" },
             { ALTIUM_PCB_DIR::TRACKS6, "412A754DBB864645BF01CD6A80C358\\" },
-            { ALTIUM_PCB_DIR::VIAS6, "C87A685A0EFA4A90BEEFD666198B56\\" }
+            { ALTIUM_PCB_DIR::VIAS6, "C87A685A0EFA4A90BEEFD666198B56\\" },
+            { ALTIUM_PCB_DIR::WIDESTRINGS6, "C1C6540EA23C48D3BF8F9A4ABB9D6D\\" }
     };
     // clang-format on
 

pcbnew/plugins/altium/altium_circuit_studio_plugin.cpp

@@ -95,7 +95,8 @@ BOARD* ALTIUM_CIRCUIT_STUDIO_PLUGIN::Load( const wxString& aFileName, BOARD* aAp
             { ALTIUM_PCB_DIR::SHAPEBASEDREGIONS6, "D5F54B536E124FB89E2D51B1121508\\" },
             { ALTIUM_PCB_DIR::TEXTS6, "349ABBB211DB4F5B8AE41B1B49555A\\" },
             { ALTIUM_PCB_DIR::TRACKS6, "530C20C225354B858B2578CAB8C08D\\" },
-            { ALTIUM_PCB_DIR::VIAS6, "CA5F5989BCDB404DA70A9D1D3D5758\\" }
+            { ALTIUM_PCB_DIR::VIAS6, "CA5F5989BCDB404DA70A9D1D3D5758\\" },
+            { ALTIUM_PCB_DIR::WIDESTRINGS6, "87FBF0C5BC194B909FF42199450A76\\" }
     };
     // clang-format on
 

pcbnew/plugins/altium/altium_designer_plugin.cpp

@@ -95,7 +95,8 @@ BOARD* ALTIUM_DESIGNER_PLUGIN::Load( const wxString& aFileName, BOARD* aAppendTo
             { ALTIUM_PCB_DIR::SHAPEBASEDREGIONS6, "ShapeBasedRegions6\\" },
             { ALTIUM_PCB_DIR::TEXTS6, "Texts6\\" },
             { ALTIUM_PCB_DIR::TRACKS6, "Tracks6\\" },
-            { ALTIUM_PCB_DIR::VIAS6, "Vias6\\" }
+            { ALTIUM_PCB_DIR::VIAS6, "Vias6\\" },
+            { ALTIUM_PCB_DIR::WIDESTRINGS6, "WideStrings6\\" }
     };
     // clang-format on
 

pcbnew/plugins/altium/altium_parser_pcb.cpp

@@ -779,7 +779,7 @@ ATRACK6::ATRACK6( ALTIUM_PARSER& aReader )
         THROW_IO_ERROR( "Tracks6 stream was not parsed correctly" );
 }
 
-ATEXT6::ATEXT6( ALTIUM_PARSER& aReader )
+ATEXT6::ATEXT6( ALTIUM_PARSER& aReader, std::map<uint32_t, wxString>& aStringTable )
 {
     ALTIUM_RECORD recordtype = static_cast<ALTIUM_RECORD>( aReader.Read<uint8_t>() );
 
@@ -806,7 +806,9 @@ ATEXT6::ATEXT6( ALTIUM_PARSER& aReader )
     isItalic = aReader.Read<uint8_t>() != 0;
     aReader.Skip( 64 ); // font_name
     isInverted = aReader.Read<uint8_t>() != 0;
-    aReader.Skip( 21 );
+    aReader.Skip( 4 );
+    uint32_t stringIndex = aReader.Read<uint32_t>();
+    aReader.Skip( 13 );
     textposition = static_cast<ALTIUM_TEXT_POSITION>( aReader.Read<uint8_t>() );
     /**
      * In Altium 14 (subrecord1 == 230) only left bottom is valid? I think there is a bit missing.
@@ -820,10 +822,15 @@ ATEXT6::ATEXT6( ALTIUM_PARSER& aReader )
 
     aReader.SkipSubrecord();
 
-    // Subrecord 2 - String
+    // Subrecord 2 - Legacy 8bit string, max 255 chars, unknown codepage
     aReader.ReadAndSetSubrecordLength();
 
-    text = aReader.ReadWxString(); // TODO: what about strings with length > 255?
+    auto entry = aStringTable.find( stringIndex );
+
+    if( entry != aStringTable.end() )
+        text = entry->second;
+    else
+        text = aReader.ReadWxString();
 
     // Normalize Windows line endings
     text.Replace( "\r\n", "\n" );

pcbnew/plugins/altium/altium_parser_pcb.h

@@ -661,7 +661,7 @@ struct ATEXT6
 
     wxString text;
 
-    explicit ATEXT6( ALTIUM_PARSER& aReader );
+    explicit ATEXT6( ALTIUM_PARSER& aReader, std::map<uint32_t, wxString>& aStringTable );
 };
 
 struct AFILL6

pcbnew/plugins/altium/altium_pcb.cpp

@@ -421,6 +421,11 @@ void ALTIUM_PCB::Parse( const CFB::CompoundFileReader& aReader,
                 {
                     this->ParseTracks6Data( aReader, fileHeader );
                 } },
+        { false, ALTIUM_PCB_DIR::WIDESTRINGS6,
+                [this]( auto aReader, auto fileHeader )
+                {
+                    this->ParseWideStrings6Data( aReader, fileHeader );
+                } },
         { true, ALTIUM_PCB_DIR::TEXTS6,
                 [this]( auto aReader, auto fileHeader )
                 {
@@ -2636,6 +2641,20 @@ void ALTIUM_PCB::ParseTracks6Data( const CFB::CompoundFileReader& aReader,
     }
 }
 
+void ALTIUM_PCB::ParseWideStrings6Data( const CFB::CompoundFileReader&  aReader,
+                                        const CFB::COMPOUND_FILE_ENTRY* aEntry )
+{
+    if( m_progressReporter )
+        m_progressReporter->Report( _( "Loading unicode strings..." ) );
+
+    ALTIUM_PARSER reader( aReader, aEntry );
+    
+    m_unicodeStrings = reader.ReadWideStringTable();
+
+    if( reader.GetRemainingBytes() != 0 )
+        THROW_IO_ERROR( "WideStrings6 stream is not fully parsed" );
+}
+
 void ALTIUM_PCB::ParseTexts6Data( const CFB::CompoundFileReader& aReader,
                                   const CFB::COMPOUND_FILE_ENTRY* aEntry )
 {
@@ -2647,7 +2666,7 @@ void ALTIUM_PCB::ParseTexts6Data( const CFB::CompoundFileReader& aReader,
     while( reader.GetRemainingBytes() >= 4 /* TODO: use Header section of file */ )
     {
         checkpoint();
-        ATEXT6 elem( reader );
+        ATEXT6 elem( reader, m_unicodeStrings );
 
         if( elem.fonttype == ALTIUM_TEXT_TYPE::BARCODE )
         {

pcbnew/plugins/altium/altium_pcb.h

@@ -173,6 +173,8 @@ private:
             const CFB::CompoundFileReader& aReader, const CFB::COMPOUND_FILE_ENTRY* aEntry );
     void ParseRegions6Data(
             const CFB::CompoundFileReader& aReader, const CFB::COMPOUND_FILE_ENTRY* aEntry );
+    void ParseWideStrings6Data(
+            const CFB::CompoundFileReader& aReader, const CFB::COMPOUND_FILE_ENTRY* aEntry );
 
     // Helper Functions
     void HelperParseDimensions6Linear( const ADIMENSION6& aElem );
@@ -193,6 +195,7 @@ private:
     std::vector<ZONE*>                   m_polygons;
     std::vector<PCB_DIMENSION_BASE*>     m_radialDimensions;
     std::map<wxString, wxString>         m_models;
+    std::map<uint32_t, wxString>         m_unicodeStrings;
     size_t                               m_num_nets;
     std::map<ALTIUM_LAYER, PCB_LAYER_ID> m_layermap; // used to correctly map copper layers
     std::map<ALTIUM_RULE_KIND, std::vector<ARULE6>> m_rules;