haskal
/
kicad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize library load time for Altium plugin 

Load times were >20min for moderate sized libraries as EnumFiles
iterated over the entire list for each operation.  The update modifies
our third-party lib to allow a return value, stopping the iteration when
we find our desired entry.  This also provides a short-circuit for
ASCII-based names, allowing single-level parsing if available
This commit is contained in:
Seth Hillbrand 2023-08-18 13:42:39 -07:00
parent 5131dae568
commit 57ba38560c
5 changed files with 153 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
common/plugins/altium/altium_parser.cpp
View File

@ -103,7 +103,7 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
m_reader->EnumFiles( root, 2, m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void int level ) -> int
{ {
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() ); std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen ); std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
@ -122,48 +122,126 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
patternMap.emplace( key, fpName ); patternMap.emplace( key, fpName );
} }
return 0;
} ); } );
return patternMap; return patternMap;
} }
wxString ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>
ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
{ {
if( !m_reader ) if( !m_reader )
return wxEmptyString; return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry(); const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry();
if( !root ) if( !root )
return wxEmptyString; return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
wxString ret; wxString retStr;
const CFB::COMPOUND_FILE_ENTRY* retEntry = nullptr;
m_reader->EnumFiles( root, 2, // Cheap and easy check first as most ASCII-coded libs are under the same dir name
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, m_reader->EnumFiles( root, 1,
int level ) -> void [&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string&, int ) -> int
{ {
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() ); // We are only looking for one string, so if we found it, break the loop
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen ); if( retStr != wxEmptyString )
return 1;
if( m_reader->IsStream( entry ) && fileName == L"Parameters" ) std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString( if( aFpUnicodeName.ToStdWstring().compare( 0, 10, dirName, 0, 10 ) )
parameterProperties, wxT( "PATTERN" ), wxT( "" ) ); return 0;
if( fpName == aFpUnicodeName ) m_reader->EnumFiles( tentry, 1,
{ [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
ret = dirName; {
} std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
}
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} ); } );
return ret; if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
// Now do the expensive check, iterating through each directory in the library and reading the files
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string& dir,
int level ) -> int
{
if( retStr != wxEmptyString )
return 1;
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
} }
@ -178,41 +256,52 @@ ALTIUM_COMPOUND_FILE::FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEn
m_reader->EnumFiles( aEntry, 1, m_reader->EnumFiles( aEntry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void int level ) -> int
{ {
if( ret != nullptr )
return 1;
if( m_reader->IsStream( entry ) == aIsStream ) if( m_reader->IsStream( entry ) == aIsStream )
{ {
std::string name = UTF16ToUTF8( entry->name ); std::string name = UTF16ToUTF8( entry->name );
if( name == aName.c_str() ) if( name == aName.c_str() )
{ {
ret = entry; ret = entry;
return 1;
} }
} }
return 0;
} ); } );
return ret; return ret;
} }
const CFB::COMPOUND_FILE_ENTRY* const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const ALTIUM_COMPOUND_FILE::FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart,
const std::vector<std::string>& aStreamPath ) const
{ {
if( !m_reader ) if( !m_reader )
return nullptr; return nullptr;
const CFB::COMPOUND_FILE_ENTRY* currentDirEntry = m_reader->GetRootEntry(); if( !aStart )
aStart = m_reader->GetRootEntry();
auto it = aStreamPath.cbegin(); auto it = aStreamPath.cbegin();
while( currentDirEntry != nullptr )
while( aStart != nullptr )
{ {
const std::string& name = *it; const std::string& name = *it;
if( ++it == aStreamPath.cend() ) if( ++it == aStreamPath.cend() )
{ {
return FindStreamSingleLevel( currentDirEntry, name, true ); const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, true );
return ret;
} }
else else
{ {
currentDirEntry = FindStreamSingleLevel( currentDirEntry, name, false ); const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, false );
aStart = ret;
} }
} }
@ -220,6 +309,13 @@ ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath )
} }
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
{
return FindStream( nullptr, aStreamPath );
}
ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile, ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile,
const CFB::COMPOUND_FILE_ENTRY* aEntry ) const CFB::COMPOUND_FILE_ENTRY* aEntry )
{ {

4
common/plugins/altium/altium_parser.h
View File

@ -65,10 +65,12 @@ public:
std::map<wxString, wxString> ListLibFootprints() const; std::map<wxString, wxString> ListLibFootprints() const;
wxString FindLibFootprintDirName( const wxString& aFpUnicodeName ) const; std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const; const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart, const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry, const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry,
const std::string aName, const std::string aName,
const bool aIsStream ) const; const bool aIsStream ) const;

2
pcbnew/plugins/altium/altium_designer_plugin.cpp
View File

@ -162,6 +162,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
const std::vector<std::string> streamName = { "Library", "Data" }; const std::vector<std::string> streamName = { "Library", "Data" };
const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName ); const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName );
if( libraryData == nullptr ) if( libraryData == nullptr )
{ {
THROW_IO_ERROR( THROW_IO_ERROR(
@ -174,6 +175,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
uint32_t numberOfFootprints = parser.Read<uint32_t>(); uint32_t numberOfFootprints = parser.Read<uint32_t>();
aFootprintNames.Alloc( numberOfFootprints ); aFootprintNames.Alloc( numberOfFootprints );
for( size_t i = 0; i < numberOfFootprints; i++ ) for( size_t i = 0; i < numberOfFootprints; i++ )
{ {
parser.ReadAndSetSubrecordLength(); parser.ReadAndSetSubrecordLength();

14
pcbnew/plugins/altium/altium_pcb.cpp
View File

@ -34,6 +34,7 @@
#include <pcb_shape.h> #include <pcb_shape.h>
#include <pcb_text.h> #include <pcb_text.h>
#include <pcb_track.h> #include <pcb_track.h>
#include <profile.h>
#include <string_utils.h> #include <string_utils.h>
#include <zone.h> #include <zone.h>
@ -606,7 +607,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
// ParseWideStrings6Data( altiumLibFile, unicodeStringsData ); // ParseWideStrings6Data( altiumLibFile, unicodeStringsData );
// } // }
wxString fpDirName = altiumLibFile.FindLibFootprintDirName(aFootprintName); std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> ret = altiumLibFile.FindLibFootprintDirName(aFootprintName);
wxString fpDirName = std::get<0>( ret );
const CFB::COMPOUND_FILE_ENTRY* footprintStream = std::get<1>( ret );
if( fpDirName.IsEmpty() ) if( fpDirName.IsEmpty() )
{ {
@ -615,7 +619,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
} }
const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" }; const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" };
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( streamName ); const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( footprintStream, { "Data" } );
if( footprintData == nullptr ) if( footprintData == nullptr )
{ {
@ -635,7 +639,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(), const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(),
"Parameters" }; "Parameters" };
const CFB::COMPOUND_FILE_ENTRY* parametersData = const CFB::COMPOUND_FILE_ENTRY* parametersData =
altiumLibFile.FindStream( parametersStreamName ); altiumLibFile.FindStream( footprintStream, { "Parameters" } );
if( parametersData != nullptr ) if( parametersData != nullptr )
{ {
@ -653,10 +657,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
} }
const std::vector<std::string> extendedPrimitiveInformationStreamName{ const std::vector<std::string> extendedPrimitiveInformationStreamName{
fpDirName.ToStdString(), "ExtendedPrimitiveInformation", "Data" "ExtendedPrimitiveInformation", "Data"
}; };
const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData = const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData =
altiumLibFile.FindStream( extendedPrimitiveInformationStreamName ); altiumLibFile.FindStream( footprintStream, extendedPrimitiveInformationStreamName );
if( extendedPrimitiveInformationData != nullptr ) if( extendedPrimitiveInformationData != nullptr )
ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData ); ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData );

5
thirdparty/compoundfilereader/compoundfilereader.h vendored
View File

@ -130,7 +130,7 @@ struct helper
}; };
typedef std::basic_string<uint16_t> utf16string; typedef std::basic_string<uint16_t> utf16string;
typedef std::function<void(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)> typedef std::function<int(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
EnumFilesCallback; EnumFilesCallback;
class CompoundFileReader class CompoundFileReader
@ -238,7 +238,8 @@ private:
if (entry == nullptr) if (entry == nullptr)
return; return;
callback(entry, dir, currentLevel + 1); if( callback(entry, dir, currentLevel + 1) != 0 )
return;
const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID); const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID);
if (child != nullptr) if (child != nullptr)