haskal
/
kicad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize library load time for Altium plugin 

Load times were >20min for moderate sized libraries as EnumFiles
iterated over the entire list for each operation.  The update modifies
our third-party lib to allow a return value, stopping the iteration when
we find our desired entry.  This also provides a short-circuit for
ASCII-based names, allowing single-level parsing if available
This commit is contained in:
Seth Hillbrand 2023-08-18 13:42:39 -07:00
parent 5131dae568
commit 57ba38560c
5 changed files with 153 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
common/plugins/altium/altium_parser.cpp
View File

@ -103,7 +103,7 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
int level ) -> int
{
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
@ -122,29 +122,44 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
patternMap.emplace( key, fpName );
}
return 0;
} );
return patternMap;
}
wxString ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>
ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
{
if( !m_reader )
return wxEmptyString;
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry();
if( !root )
return wxEmptyString;
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
wxString ret;
wxString retStr;
const CFB::COMPOUND_FILE_ENTRY* retEntry = nullptr;
m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
// Cheap and easy check first as most ASCII-coded libs are under the same dir name
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string&, int ) -> int
{
// We are only looking for one string, so if we found it, break the loop
if( retStr != wxEmptyString )
return 1;
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
if( aFpUnicodeName.ToStdWstring().compare( 0, 10, dirName, 0, 10 ) )
return 0;
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
@ -158,12 +173,75 @@ wxString ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnico
if( fpName == aFpUnicodeName )
{
ret = dirName;
retStr = dirName;
return 1;
}
}
return 0;
} );
return ret;
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
// Now do the expensive check, iterating through each directory in the library and reading the files
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string& dir,
int level ) -> int
{
if( retStr != wxEmptyString )
return 1;
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
}
@ -178,41 +256,52 @@ ALTIUM_COMPOUND_FILE::FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEn
m_reader->EnumFiles( aEntry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
int level ) -> int
{
if( ret != nullptr )
return 1;
if( m_reader->IsStream( entry ) == aIsStream )
{
std::string name = UTF16ToUTF8( entry->name );
if( name == aName.c_str() )
{
ret = entry;
return 1;
}
}
return 0;
} );
return ret;
}
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
ALTIUM_COMPOUND_FILE::FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart,
const std::vector<std::string>& aStreamPath ) const
{
if( !m_reader )
return nullptr;
const CFB::COMPOUND_FILE_ENTRY* currentDirEntry = m_reader->GetRootEntry();
if( !aStart )
aStart = m_reader->GetRootEntry();
auto it = aStreamPath.cbegin();
while( currentDirEntry != nullptr )
while( aStart != nullptr )
{
const std::string& name = *it;
if( ++it == aStreamPath.cend() )
{
return FindStreamSingleLevel( currentDirEntry, name, true );
const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, true );
return ret;
}
else
{
currentDirEntry = FindStreamSingleLevel( currentDirEntry, name, false );
const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, false );
aStart = ret;
}
}
@ -220,6 +309,13 @@ ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath )
}
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
{
return FindStream( nullptr, aStreamPath );
}
ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile,
const CFB::COMPOUND_FILE_ENTRY* aEntry )
{

4
common/plugins/altium/altium_parser.h
View File

@ -65,10 +65,12 @@ public:
std::map<wxString, wxString> ListLibFootprints() const;
wxString FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart, const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry,
const std::string aName,
const bool aIsStream ) const;

2
pcbnew/plugins/altium/altium_designer_plugin.cpp
View File

@ -162,6 +162,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
const std::vector<std::string> streamName = { "Library", "Data" };
const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName );
if( libraryData == nullptr )
{
THROW_IO_ERROR(
@ -174,6 +175,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
uint32_t numberOfFootprints = parser.Read<uint32_t>();
aFootprintNames.Alloc( numberOfFootprints );
for( size_t i = 0; i < numberOfFootprints; i++ )
{
parser.ReadAndSetSubrecordLength();

14
pcbnew/plugins/altium/altium_pcb.cpp
View File

@ -34,6 +34,7 @@
#include <pcb_shape.h>
#include <pcb_text.h>
#include <pcb_track.h>
#include <profile.h>
#include <string_utils.h>
#include <zone.h>
@ -606,7 +607,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
// ParseWideStrings6Data( altiumLibFile, unicodeStringsData );
// }
wxString fpDirName = altiumLibFile.FindLibFootprintDirName(aFootprintName);
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> ret = altiumLibFile.FindLibFootprintDirName(aFootprintName);
wxString fpDirName = std::get<0>( ret );
const CFB::COMPOUND_FILE_ENTRY* footprintStream = std::get<1>( ret );
if( fpDirName.IsEmpty() )
{
@ -615,7 +619,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
}
const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" };
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( streamName );
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( footprintStream, { "Data" } );
if( footprintData == nullptr )
{
@ -635,7 +639,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(),
"Parameters" };
const CFB::COMPOUND_FILE_ENTRY* parametersData =
altiumLibFile.FindStream( parametersStreamName );
altiumLibFile.FindStream( footprintStream, { "Parameters" } );
if( parametersData != nullptr )
{
@ -653,10 +657,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
}
const std::vector<std::string> extendedPrimitiveInformationStreamName{
fpDirName.ToStdString(), "ExtendedPrimitiveInformation", "Data"
"ExtendedPrimitiveInformation", "Data"
};
const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData =
altiumLibFile.FindStream( extendedPrimitiveInformationStreamName );
altiumLibFile.FindStream( footprintStream, extendedPrimitiveInformationStreamName );
if( extendedPrimitiveInformationData != nullptr )
ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData );

5
thirdparty/compoundfilereader/compoundfilereader.h vendored
View File

@ -130,7 +130,7 @@ struct helper
};
typedef std::basic_string<uint16_t> utf16string;
typedef std::function<void(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
typedef std::function<int(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
EnumFilesCallback;
class CompoundFileReader
@ -238,7 +238,8 @@ private:
if (entry == nullptr)
return;
callback(entry, dir, currentLevel + 1);
if( callback(entry, dir, currentLevel + 1) != 0 )
return;
const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID);
if (child != nullptr)