haskal
/
kicad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize library load time for Altium plugin 

Load times were >20min for moderate sized libraries as EnumFiles
iterated over the entire list for each operation.  The update modifies
our third-party lib to allow a return value, stopping the iteration when
we find our desired entry.  This also provides a short-circuit for
ASCII-based names, allowing single-level parsing if available
This commit is contained in:
Seth Hillbrand 2023-08-18 13:42:39 -07:00
parent 5131dae568
commit 57ba38560c
5 changed files with 153 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
common/plugins/altium/altium_parser.cpp
View File

@ -103,7 +103,7 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
m_reader->EnumFiles( root, 2, m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void int level ) -> int
{ {
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() ); std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen ); std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
@ -122,48 +122,126 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
patternMap.emplace( key, fpName ); patternMap.emplace( key, fpName );
} }
return 0;
} ); } );
return patternMap; return patternMap;
} }
wxString ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>
ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
{ {
if( !m_reader ) if( !m_reader )
return wxEmptyString; return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry(); const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry();
if( !root ) if( !root )
return wxEmptyString; return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
wxString ret; wxString retStr;
const CFB::COMPOUND_FILE_ENTRY* retEntry = nullptr;
m_reader->EnumFiles( root, 2, // Cheap and easy check first as most ASCII-coded libs are under the same dir name
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, m_reader->EnumFiles( root, 1,
int level ) -> void [&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string&, int ) -> int
{ {
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() ); // We are only looking for one string, so if we found it, break the loop
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen ); if( retStr != wxEmptyString )
return 1;
if( m_reader->IsStream( entry ) && fileName == L"Parameters" ) std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString( if( aFpUnicodeName.ToStdWstring().compare( 0, 10, dirName, 0, 10 ) )
parameterProperties, wxT( "PATTERN" ), wxT( "" ) ); return 0;
if( fpName == aFpUnicodeName ) m_reader->EnumFiles( tentry, 1,
{ [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
ret = dirName; {
} std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
}
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} ); } );
return ret; if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
// Now do the expensive check, iterating through each directory in the library and reading the files
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string& dir,
int level ) -> int
{
if( retStr != wxEmptyString )
return 1;
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
} }
@ -178,41 +256,52 @@ ALTIUM_COMPOUND_FILE::FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEn
m_reader->EnumFiles( aEntry, 1, m_reader->EnumFiles( aEntry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void int level ) -> int
{ {
if( ret != nullptr )
return 1;
if( m_reader->IsStream( entry ) == aIsStream ) if( m_reader->IsStream( entry ) == aIsStream )
{ {
std::string name = UTF16ToUTF8( entry->name ); std::string name = UTF16ToUTF8( entry->name );
if( name == aName.c_str() ) if( name == aName.c_str() )
{ {
ret = entry; ret = entry;
return 1;
} }
} }
return 0;
} ); } );
return ret; return ret;
} }
const CFB::COMPOUND_FILE_ENTRY* const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const ALTIUM_COMPOUND_FILE::FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart,
const std::vector<std::string>& aStreamPath ) const
{ {
if( !m_reader ) if( !m_reader )
return nullptr; return nullptr;
const CFB::COMPOUND_FILE_ENTRY* currentDirEntry = m_reader->GetRootEntry(); if( !aStart )
aStart = m_reader->GetRootEntry();
auto it = aStreamPath.cbegin(); auto it = aStreamPath.cbegin();
while( currentDirEntry != nullptr )
while( aStart != nullptr )
{ {
const std::string& name = *it; const std::string& name = *it;
if( ++it == aStreamPath.cend() ) if( ++it == aStreamPath.cend() )
{ {
return FindStreamSingleLevel( currentDirEntry, name, true ); const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, true );
return ret;
} }
else else
{ {
currentDirEntry = FindStreamSingleLevel( currentDirEntry, name, false ); const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, false );
aStart = ret;
} }
} }
@ -220,6 +309,13 @@ ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath )
} }
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
{
return FindStream( nullptr, aStreamPath );
}
ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile, ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile,
const CFB::COMPOUND_FILE_ENTRY* aEntry ) const CFB::COMPOUND_FILE_ENTRY* aEntry )
{ {

4
common/plugins/altium/altium_parser.h
View File

@ -65,10 +65,12 @@ public:
std::map<wxString, wxString> ListLibFootprints() const; std::map<wxString, wxString> ListLibFootprints() const;
wxString FindLibFootprintDirName( const wxString& aFpUnicodeName ) const; std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const; const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart, const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry, const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry,
const std::string aName, const std::string aName,
const bool aIsStream ) const; const bool aIsStream ) const;

2
pcbnew/plugins/altium/altium_designer_plugin.cpp
View File

@ -162,6 +162,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
const std::vector<std::string> streamName = { "Library", "Data" }; const std::vector<std::string> streamName = { "Library", "Data" };
const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName ); const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName );
if( libraryData == nullptr ) if( libraryData == nullptr )
{ {
THROW_IO_ERROR( THROW_IO_ERROR(
@ -174,6 +175,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
uint32_t numberOfFootprints = parser.Read<uint32_t>(); uint32_t numberOfFootprints = parser.Read<uint32_t>();
aFootprintNames.Alloc( numberOfFootprints ); aFootprintNames.Alloc( numberOfFootprints );
for( size_t i = 0; i < numberOfFootprints; i++ ) for( size_t i = 0; i < numberOfFootprints; i++ )
{ {
parser.ReadAndSetSubrecordLength(); parser.ReadAndSetSubrecordLength();

14
pcbnew/plugins/altium/altium_pcb.cpp
View File

@ -34,6 +34,7 @@
#include <pcb_shape.h> #include <pcb_shape.h>
#include <pcb_text.h> #include <pcb_text.h>
#include <pcb_track.h> #include <pcb_track.h>
#include <profile.h>
#include <string_utils.h> #include <string_utils.h>
#include <zone.h> #include <zone.h>
@ -606,7 +607,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
// ParseWideStrings6Data( altiumLibFile, unicodeStringsData ); // ParseWideStrings6Data( altiumLibFile, unicodeStringsData );
// } // }
wxString fpDirName = altiumLibFile.FindLibFootprintDirName(aFootprintName); std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> ret = altiumLibFile.FindLibFootprintDirName(aFootprintName);
wxString fpDirName = std::get<0>( ret );
const CFB::COMPOUND_FILE_ENTRY* footprintStream = std::get<1>( ret );
if( fpDirName.IsEmpty() ) if( fpDirName.IsEmpty() )
{ {
@ -615,7 +619,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
} }
const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" }; const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" };
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( streamName ); const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( footprintStream, { "Data" } );
if( footprintData == nullptr ) if( footprintData == nullptr )
{ {
@ -635,7 +639,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(), const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(),
"Parameters" }; "Parameters" };
const CFB::COMPOUND_FILE_ENTRY* parametersData = const CFB::COMPOUND_FILE_ENTRY* parametersData =
altiumLibFile.FindStream( parametersStreamName ); altiumLibFile.FindStream( footprintStream, { "Parameters" } );
if( parametersData != nullptr ) if( parametersData != nullptr )
{ {
@ -653,10 +657,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
} }
const std::vector<std::string> extendedPrimitiveInformationStreamName{ const std::vector<std::string> extendedPrimitiveInformationStreamName{
fpDirName.ToStdString(), "ExtendedPrimitiveInformation", "Data" "ExtendedPrimitiveInformation", "Data"
}; };
const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData = const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData =
altiumLibFile.FindStream( extendedPrimitiveInformationStreamName ); altiumLibFile.FindStream( footprintStream, extendedPrimitiveInformationStreamName );
if( extendedPrimitiveInformationData != nullptr ) if( extendedPrimitiveInformationData != nullptr )
ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData ); ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData );

25
thirdparty/compoundfilereader/compoundfilereader.h vendored
View File

@ -1,23 +1,23 @@
/** /**
Microsoft Compound File (and Property Set) Reader Microsoft Compound File (and Property Set) Reader
http://en.wikipedia.org/wiki/Compound_File_Binary_Format http://en.wikipedia.org/wiki/Compound_File_Binary_Format
Format specification: Format specification:
MS-CFB: https://msdn.microsoft.com/en-us/library/dd942138.aspx MS-CFB: https://msdn.microsoft.com/en-us/library/dd942138.aspx
MS-OLEPS: https://msdn.microsoft.com/en-us/library/dd942421.aspx MS-OLEPS: https://msdn.microsoft.com/en-us/library/dd942421.aspx
Note: Note:
1. For simplification, the code assumes that the target system is little-endian. 1. For simplification, the code assumes that the target system is little-endian.
2. The reader operates the passed buffer in-place. 2. The reader operates the passed buffer in-place.
You must keep the input buffer valid when you are using the reader. You must keep the input buffer valid when you are using the reader.
3. Single thread usage. 3. Single thread usage.
Example 1: print all streams in a compound file Example 1: print all streams in a compound file
\code \code
CFB::CompoundFileReader reader(buffer, len); CFB::CompoundFileReader reader(buffer, len);
reader.EnumFiles(reader.GetRootEntry(), -1, reader.EnumFiles(reader.GetRootEntry(), -1,
[&](const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, int level)->void [&](const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, int level)->void
{ {
bool isDirectory = !reader.IsStream(entry); bool isDirectory = !reader.IsStream(entry);
@ -99,7 +99,7 @@ struct PROPERTY_SET_STREAM_HDR
uint32_t systemIdentifier; uint32_t systemIdentifier;
unsigned char clsid[16]; unsigned char clsid[16];
uint32_t numPropertySets; uint32_t numPropertySets;
struct struct
{ {
char fmtid[16]; char fmtid[16];
uint32_t offset; uint32_t offset;
@ -130,7 +130,7 @@ struct helper
}; };
typedef std::basic_string<uint16_t> utf16string; typedef std::basic_string<uint16_t> utf16string;
typedef std::function<void(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)> typedef std::function<int(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
EnumFilesCallback; EnumFilesCallback;
class CompoundFileReader class CompoundFileReader
@ -163,7 +163,7 @@ public:
m_miniStreamStartSector = root->startSectorLocation; m_miniStreamStartSector = root->startSectorLocation;
} }
/// Get entry (directory or file) by its ID. /// Get entry (directory or file) by its ID.
/// Pass "0" to get the root directory entry. -- This is the start point to navigate the compound file. /// Pass "0" to get the root directory entry. -- This is the start point to navigate the compound file.
/// Use the returned object to access child entries. /// Use the returned object to access child entries.
const COMPOUND_FILE_ENTRY* GetEntry(size_t entryID) const const COMPOUND_FILE_ENTRY* GetEntry(size_t entryID) const
@ -230,7 +230,7 @@ public:
private: private:
// Enum entries with same level, including 'entry' itself // Enum entries with same level, including 'entry' itself
void EnumNodes(const COMPOUND_FILE_ENTRY* entry, int currentLevel, int maxLevel, void EnumNodes(const COMPOUND_FILE_ENTRY* entry, int currentLevel, int maxLevel,
const utf16string& dir, EnumFilesCallback callback) const const utf16string& dir, EnumFilesCallback callback) const
{ {
if (maxLevel > 0 && currentLevel >= maxLevel) if (maxLevel > 0 && currentLevel >= maxLevel)
@ -238,7 +238,8 @@ private:
if (entry == nullptr) if (entry == nullptr)
return; return;
callback(entry, dir, currentLevel + 1); if( callback(entry, dir, currentLevel + 1) != 0 )
return;
const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID); const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID);
if (child != nullptr) if (child != nullptr)
@ -332,8 +333,8 @@ private:
{ {
throw FileCorrupted(); throw FileCorrupted();
} }
LocateFinalSector(m_miniStreamStartSector, sector * m_minisectorSize + offset, &sector, &offset); LocateFinalSector(m_miniStreamStartSector, sector * m_minisectorSize + offset, &sector, &offset);
return SectorOffsetToAddress(sector, offset); return SectorOffsetToAddress(sector, offset);
} }