haskal
/
kicad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize library load time for Altium plugin 

Load times were >20min for moderate sized libraries as EnumFiles
iterated over the entire list for each operation.  The update modifies
our third-party lib to allow a return value, stopping the iteration when
we find our desired entry.  This also provides a short-circuit for
ASCII-based names, allowing single-level parsing if available
This commit is contained in:
Seth Hillbrand 2023-08-18 13:42:39 -07:00
parent 5131dae568
commit 57ba38560c
5 changed files with 153 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
common/plugins/altium/altium_parser.cpp
View File

@ -103,7 +103,7 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
int level ) -> int
{
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
@ -122,48 +122,126 @@ std::map<wxString, wxString> ALTIUM_COMPOUND_FILE::ListLibFootprints() const
patternMap.emplace( key, fpName );
}
return 0;
} );
return patternMap;
}
wxString ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>
ALTIUM_COMPOUND_FILE::FindLibFootprintDirName( const wxString& aFpUnicodeName ) const
{
if( !m_reader )
return wxEmptyString;
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry();
if( !root )
return wxEmptyString;
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
wxString ret;
wxString retStr;
const CFB::COMPOUND_FILE_ENTRY* retEntry = nullptr;
m_reader->EnumFiles( root, 2,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
{
std::wstring dirName = UTF16ToWstring( dir.data(), dir.size() );
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
// Cheap and easy check first as most ASCII-coded libs are under the same dir name
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string&, int ) -> int
{
// We are only looking for one string, so if we found it, break the loop
if( retStr != wxEmptyString )
return 1;
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( aFpUnicodeName.ToStdWstring().compare( 0, 10, dirName, 0, 10 ) )
return 0;
if( fpName == aFpUnicodeName )
{
ret = dirName;
}
}
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
return ret;
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
// Now do the expensive check, iterating through each directory in the library and reading the files
m_reader->EnumFiles( root, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string& dir,
int level ) -> int
{
if( retStr != wxEmptyString )
return 1;
std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
m_reader->EnumFiles( tentry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&, int ) -> int
{
std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
if( m_reader->IsStream( entry ) && fileName == L"Parameters" )
{
ALTIUM_PARSER parametersReader( *this, entry );
std::map<wxString, wxString> parameterProperties =
parametersReader.ReadProperties();
wxString fpName = ALTIUM_PARSER::ReadUnicodeString(
parameterProperties, wxT( "PATTERN" ), wxT( "" ) );
if( fpName == aFpUnicodeName )
{
retStr = dirName;
return 1;
}
}
return 0;
} );
if( retStr != wxEmptyString )
{
retEntry = tentry;
return 1;
}
return 0;
} );
if( retStr != wxEmptyString )
return std::make_tuple( retStr, retEntry );
return std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*>( wxEmptyString, nullptr );
}
@ -178,41 +256,52 @@ ALTIUM_COMPOUND_FILE::FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEn
m_reader->EnumFiles( aEntry, 1,
[&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
int level ) -> void
int level ) -> int
{
if( ret != nullptr )
return 1;
if( m_reader->IsStream( entry ) == aIsStream )
{
std::string name = UTF16ToUTF8( entry->name );
if( name == aName.c_str() )
{
ret = entry;
return 1;
}
}
return 0;
} );
return ret;
}
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
ALTIUM_COMPOUND_FILE::FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart,
const std::vector<std::string>& aStreamPath ) const
{
if( !m_reader )
return nullptr;
const CFB::COMPOUND_FILE_ENTRY* currentDirEntry = m_reader->GetRootEntry();
if( !aStart )
aStart = m_reader->GetRootEntry();
auto it = aStreamPath.cbegin();
while( currentDirEntry != nullptr )
while( aStart != nullptr )
{
const std::string& name = *it;
if( ++it == aStreamPath.cend() )
{
return FindStreamSingleLevel( currentDirEntry, name, true );
const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, true );
return ret;
}
else
{
currentDirEntry = FindStreamSingleLevel( currentDirEntry, name, false );
const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, false );
aStart = ret;
}
}
@ -220,6 +309,13 @@ ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath )
}
const CFB::COMPOUND_FILE_ENTRY*
ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
{
return FindStream( nullptr, aStreamPath );
}
ALTIUM_PARSER::ALTIUM_PARSER( const ALTIUM_COMPOUND_FILE& aFile,
const CFB::COMPOUND_FILE_ENTRY* aEntry )
{

4
common/plugins/altium/altium_parser.h
View File

@ -65,10 +65,12 @@ public:
std::map<wxString, wxString> ListLibFootprints() const;
wxString FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> FindLibFootprintDirName( const wxString& aFpUnicodeName ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart, const std::vector<std::string>& aStreamPath ) const;
const CFB::COMPOUND_FILE_ENTRY* FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry,
const std::string aName,
const bool aIsStream ) const;

2
pcbnew/plugins/altium/altium_designer_plugin.cpp
View File

@ -162,6 +162,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
const std::vector<std::string> streamName = { "Library", "Data" };
const CFB::COMPOUND_FILE_ENTRY* libraryData = altiumLibFile.FindStream( streamName );
if( libraryData == nullptr )
{
THROW_IO_ERROR(
@ -174,6 +175,7 @@ void ALTIUM_DESIGNER_PLUGIN::FootprintEnumerate( wxArrayString& aFootprintNames
uint32_t numberOfFootprints = parser.Read<uint32_t>();
aFootprintNames.Alloc( numberOfFootprints );
for( size_t i = 0; i < numberOfFootprints; i++ )
{
parser.ReadAndSetSubrecordLength();

14
pcbnew/plugins/altium/altium_pcb.cpp
View File

@ -34,6 +34,7 @@
#include <pcb_shape.h>
#include <pcb_text.h>
#include <pcb_track.h>
#include <profile.h>
#include <string_utils.h>
#include <zone.h>
@ -606,7 +607,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
// ParseWideStrings6Data( altiumLibFile, unicodeStringsData );
// }
wxString fpDirName = altiumLibFile.FindLibFootprintDirName(aFootprintName);
std::tuple<wxString, const CFB::COMPOUND_FILE_ENTRY*> ret = altiumLibFile.FindLibFootprintDirName(aFootprintName);
wxString fpDirName = std::get<0>( ret );
const CFB::COMPOUND_FILE_ENTRY* footprintStream = std::get<1>( ret );
if( fpDirName.IsEmpty() )
{
@ -615,7 +619,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
}
const std::vector<std::string> streamName{ fpDirName.ToStdString(), "Data" };
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( streamName );
const CFB::COMPOUND_FILE_ENTRY* footprintData = altiumLibFile.FindStream( footprintStream, { "Data" } );
if( footprintData == nullptr )
{
@ -635,7 +639,7 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
const std::vector<std::string> parametersStreamName{ fpDirName.ToStdString(),
"Parameters" };
const CFB::COMPOUND_FILE_ENTRY* parametersData =
altiumLibFile.FindStream( parametersStreamName );
altiumLibFile.FindStream( footprintStream, { "Parameters" } );
if( parametersData != nullptr )
{
@ -653,10 +657,10 @@ FOOTPRINT* ALTIUM_PCB::ParseFootprint( const ALTIUM_COMPOUND_FILE& altiumLibFile
}
const std::vector<std::string> extendedPrimitiveInformationStreamName{
fpDirName.ToStdString(), "ExtendedPrimitiveInformation", "Data"
"ExtendedPrimitiveInformation", "Data"
};
const CFB::COMPOUND_FILE_ENTRY* extendedPrimitiveInformationData =
altiumLibFile.FindStream( extendedPrimitiveInformationStreamName );
altiumLibFile.FindStream( footprintStream, extendedPrimitiveInformationStreamName );
if( extendedPrimitiveInformationData != nullptr )
ParseExtendedPrimitiveInformationData( altiumLibFile, extendedPrimitiveInformationData );

25
thirdparty/compoundfilereader/compoundfilereader.h vendored
View File

@ -1,23 +1,23 @@
/**
Microsoft Compound File (and Property Set) Reader
http://en.wikipedia.org/wiki/Compound_File_Binary_Format
Format specification:
MS-CFB: https://msdn.microsoft.com/en-us/library/dd942138.aspx
MS-OLEPS: https://msdn.microsoft.com/en-us/library/dd942421.aspx
Note:
Note:
1. For simplification, the code assumes that the target system is little-endian.
2. The reader operates the passed buffer in-place.
You must keep the input buffer valid when you are using the reader.
3. Single thread usage.
Example 1: print all streams in a compound file
\code
CFB::CompoundFileReader reader(buffer, len);
reader.EnumFiles(reader.GetRootEntry(), -1,
reader.EnumFiles(reader.GetRootEntry(), -1,
[&](const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir, int level)->void
{
bool isDirectory = !reader.IsStream(entry);
@ -99,7 +99,7 @@ struct PROPERTY_SET_STREAM_HDR
uint32_t systemIdentifier;
unsigned char clsid[16];
uint32_t numPropertySets;
struct
struct
{
char fmtid[16];
uint32_t offset;
@ -130,7 +130,7 @@ struct helper
};
typedef std::basic_string<uint16_t> utf16string;
typedef std::function<void(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
typedef std::function<int(const COMPOUND_FILE_ENTRY*, const utf16string& dir, int level)>
EnumFilesCallback;
class CompoundFileReader
@ -163,7 +163,7 @@ public:
m_miniStreamStartSector = root->startSectorLocation;
}
/// Get entry (directory or file) by its ID.
/// Get entry (directory or file) by its ID.
/// Pass "0" to get the root directory entry. -- This is the start point to navigate the compound file.
/// Use the returned object to access child entries.
const COMPOUND_FILE_ENTRY* GetEntry(size_t entryID) const
@ -230,7 +230,7 @@ public:
private:
// Enum entries with same level, including 'entry' itself
void EnumNodes(const COMPOUND_FILE_ENTRY* entry, int currentLevel, int maxLevel,
void EnumNodes(const COMPOUND_FILE_ENTRY* entry, int currentLevel, int maxLevel,
const utf16string& dir, EnumFilesCallback callback) const
{
if (maxLevel > 0 && currentLevel >= maxLevel)
@ -238,7 +238,8 @@ private:
if (entry == nullptr)
return;
callback(entry, dir, currentLevel + 1);
if( callback(entry, dir, currentLevel + 1) != 0 )
return;
const COMPOUND_FILE_ENTRY* child = GetEntry(entry->childID);
if (child != nullptr)
@ -332,8 +333,8 @@ private:
{
throw FileCorrupted();
}
LocateFinalSector(m_miniStreamStartSector, sector * m_minisectorSize + offset, &sector, &offset);
return SectorOffsetToAddress(sector, offset);
}