Use threading for loading symbol libraries

eeschema/CMakeLists.txt

@@ -232,6 +232,7 @@ set( EESCHEMA_SRCS
     schematic_undo_redo.cpp
     sch_edit_frame.cpp
     sheet.cpp
+    symbol_async_loader.cpp
     symbol_lib_table.cpp
     symbol_tree_model_adapter.cpp
     symbol_tree_synchronizing_adapter.cpp

eeschema/symbol_async_loader.cpp

@@ -0,0 +1,130 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2021 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2021 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <thread>
+
+#include <symbol_async_loader.h>
+#include <symbol_lib_table.h>
+#include <symbol_tree_model_adapter.h>
+#include <widgets/progress_reporter.h>
+
+
+SYMBOL_ASYNC_LOADER::SYMBOL_ASYNC_LOADER( const std::vector<wxString>& aNicknames,
+        SYMBOL_TREE_MODEL_ADAPTER* aAdapter,
+        std::unordered_map<wxString, std::vector<LIB_PART*>>& aOutput,
+        PROGRESS_REPORTER* aReporter ) :
+        m_nicknames( aNicknames ),
+        m_adapter( aAdapter ),
+        m_output( aOutput ),
+        m_reporter( aReporter )
+{
+    m_onlyPowerSymbols = ( m_adapter->GetFilter() == LIB_TREE_MODEL_ADAPTER::CMP_FILTER_POWER );
+    m_threadCount      = std::max<size_t>( 1, std::thread::hardware_concurrency() - 1 );
+
+    m_canceled.store( false );
+    m_nextLibrary.store( 0 );
+    m_returns.resize( m_threadCount );
+}
+
+
+
+SYMBOL_ASYNC_LOADER::~SYMBOL_ASYNC_LOADER()
+{
+    Abort();
+}
+
+
+void SYMBOL_ASYNC_LOADER::Start()
+{
+    for( size_t ii = 0; ii < m_threadCount; ++ii )
+        m_returns[ii] = std::async( std::launch::async, &SYMBOL_ASYNC_LOADER::worker, this );
+}
+
+
+bool SYMBOL_ASYNC_LOADER::Join()
+{
+    for( size_t ii = 0; ii < m_threadCount; ++ii )
+    {
+        if( !m_returns[ii].valid() )
+            continue;
+
+        m_returns[ii].wait();
+
+        const std::vector<LOADED_PAIR>& ret = m_returns[ii].get();
+
+        if( !ret.empty() )
+        {
+            for( const LOADED_PAIR& pair : ret )
+                m_output[pair.first] = pair.second;
+        }
+    }
+
+    return true;
+}
+
+
+void SYMBOL_ASYNC_LOADER::Abort()
+{
+    m_canceled.store( true );
+    Join();
+}
+
+
+bool SYMBOL_ASYNC_LOADER::Done()
+{
+    return m_nextLibrary.load() >= m_nicknames.size();
+}
+
+
+std::vector<SYMBOL_ASYNC_LOADER::LOADED_PAIR> SYMBOL_ASYNC_LOADER::worker()
+{
+    std::vector<LOADED_PAIR> ret;
+
+    bool onlyPower = m_onlyPowerSymbols;
+
+    for( size_t libraryIndex = m_nextLibrary++; libraryIndex < m_nicknames.size();
+         libraryIndex = m_nextLibrary++ )
+    {
+        if( m_canceled.load() )
+            break;
+
+        const wxString& nickname = m_nicknames[libraryIndex];
+        LOADED_PAIR     pair( nickname, {} );
+
+        try
+        {
+            m_adapter->m_libs->LoadSymbolLib( pair.second, nickname, onlyPower );
+            ret.emplace_back( std::move( pair ) );
+        }
+        catch( const IO_ERROR& ioe )
+        {
+            wxString msg = wxString::Format( _( "Error loading symbol library %s.\n\n%s\n" ),
+                                             nickname, ioe.What() );
+
+            std::lock_guard<std::mutex> lock( m_errorMutex );
+            m_errors += msg;
+        }
+
+        if( m_reporter )
+            m_reporter->AdvancePhase( wxString::Format( _( "Loading library \"%s\"" ), nickname ) );
+    }
+
+    return ret;
+}

eeschema/symbol_async_loader.h

@@ -0,0 +1,104 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2021 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2021 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KICAD_SYMBOL_ASYNC_LOADER_H
+#define KICAD_SYMBOL_ASYNC_LOADER_H
+
+#include <atomic>
+#include <future>
+#include <mutex>
+#include <vector>
+
+#include <symbol_tree_model_adapter.h>
+
+class LIB_PART;
+class PROGRESS_REPORTER;
+
+
+class SYMBOL_ASYNC_LOADER
+{
+public:
+    /**
+     * Constructs a loader for symbol libraries
+     * @param aNicknames is a list of library nicknames to load
+     * @param aAdapter is a pointer to the parent symbol tree model adapter
+     * @param aOutput will be filled with the loaded parts
+     * @param aReporter will be used to repord progress, of not null
+     */
+    SYMBOL_ASYNC_LOADER( const std::vector<wxString>& aNicknames,
+                         SYMBOL_TREE_MODEL_ADAPTER* aAdapter,
+                         std::unordered_map<wxString, std::vector<LIB_PART*>>& aOutput,
+                         PROGRESS_REPORTER* aReporter = nullptr );
+
+    ~SYMBOL_ASYNC_LOADER();
+
+    /**
+     * Spins up threads to load all the libraries in m_nicknames
+     */
+    void Start();
+
+    /**
+     * Finalizes the threads and combines the output into the target output map
+     */
+    bool Join();
+
+    /**
+     * Cancels a load in-progress
+     */
+    void Abort();
+
+    ///< Return true if loading is done
+    bool Done();
+
+    ///< Returns a string containing any errors generated during the load
+    const wxString& GetErrors() const { return m_errors; }
+
+    ///< Represents a pair of <nickname, loaded parts list>
+    typedef std::pair<wxString, std::vector<LIB_PART*>> LOADED_PAIR;
+
+private:
+    ///< Worker job that loads libraries and returns a list of pairs of <nickname, loaded parts>
+    std::vector<LOADED_PAIR> worker();
+
+    ///<  list of libraries to load
+    std::vector<wxString> m_nicknames;
+
+    ///< Handle to the adapter that owns this loader
+    SYMBOL_TREE_MODEL_ADAPTER* m_adapter;
+
+    ///< True if we are loading only power symbols
+    bool m_onlyPowerSymbols;
+
+    ///< Handle to map that will be filled with the loaded parts per library
+    std::unordered_map<wxString, std::vector<LIB_PART*>>& m_output;
+
+    ///< Progress reporter (may be null)
+    PROGRESS_REPORTER* m_reporter;
+
+    size_t              m_threadCount;
+    std::atomic<size_t> m_nextLibrary;
+    std::atomic_bool    m_canceled;
+    wxString            m_errors;
+    std::mutex          m_errorMutex;
+
+    std::vector<std::future<std::vector<LOADED_PAIR>>> m_returns;
+};
+
+#endif

eeschema/symbol_tree_model_adapter.cpp

@@ -21,13 +21,15 @@
 
 #include <wx/tokenzr.h>
 #include <wx/window.h>
-#include <widgets/app_progress_dialog.h>
+#include <widgets/progress_reporter.h>
 
 #include <eda_pattern_match.h>
 #include <symbol_lib_table.h>
 #include <lib_symbol.h>
+#include <locale_io.h>
 #include <generate_alias_info.h>
 #include <symbol_tree_model_adapter.h>
+#include <symbol_async_loader.h>
 
 
 bool SYMBOL_TREE_MODEL_ADAPTER::m_show_progress = true;
@@ -56,31 +58,54 @@ SYMBOL_TREE_MODEL_ADAPTER::~SYMBOL_TREE_MODEL_ADAPTER()
 void SYMBOL_TREE_MODEL_ADAPTER::AddLibraries( const std::vector<wxString>& aNicknames,
                                               wxWindow* aParent )
 {
-    APP_PROGRESS_DIALOG* prg = nullptr;
+    std::unique_ptr<WX_PROGRESS_REPORTER> prg = nullptr;
-    wxLongLong        nextUpdate = wxGetUTCTimeMillis() + (PROGRESS_INTERVAL_MILLIS / 2);
 
     if( m_show_progress )
     {
-        prg = new APP_PROGRESS_DIALOG( _( "Loading Symbol Libraries" ), wxEmptyString,
+        prg = std::make_unique<WX_PROGRESS_REPORTER>( aParent, _( "Loading Symbol Libraries" ),
-                                       aNicknames.size(), aParent );
+                                                      aNicknames.size(), true );
     }
 
     // Disable KIID generation: not needed for library parts; sometimes very slow
     KIID::CreateNilUuids( true );
 
-    unsigned int ii = 0;
+    std::unordered_map<wxString, std::vector<LIB_PART*>> loadedSymbols;
 
-    for( const auto& nickname : aNicknames )
+    SYMBOL_ASYNC_LOADER loader( aNicknames, this, loadedSymbols, prg.get() );
-    {
-        if( prg && wxGetUTCTimeMillis() > nextUpdate )
-        {
-            prg->Update( ii, wxString::Format( _( "Loading library \"%s\"" ), nickname ) );
 
-            nextUpdate = wxGetUTCTimeMillis() + PROGRESS_INTERVAL_MILLIS;
+    LOCALE_IO toggle;
+
+    loader.Start();
+
+    while( !loader.Done() )
+    {
+        if( prg )
+            prg->KeepRefreshing();
+
+        wxMilliSleep( PROGRESS_INTERVAL_MILLIS );
     }
 
-        AddLibrary( nickname );
+    if( prg && prg->IsCancelled() )
-        ii++;
+    {
+        loader.Abort();
+    }
+    else
+    {
+        loader.Join();
+    }
+
+    if( !loader.GetErrors().IsEmpty() )
+    {
+        wxLogError( loader.GetErrors() );
+    }
+
+    if( loadedSymbols.size() > 0 )
+    {
+        for( const std::pair<wxString, std::vector<LIB_PART*>>& pair : loadedSymbols )
+        {
+            std::vector<LIB_TREE_ITEM*> treeItems( pair.second.begin(), pair.second.end() );
+            DoAddLibrary( pair.first, m_libs->GetDescription( pair.first ), treeItems, false );
+        }
     }
 
     KIID::CreateNilUuids( false );
@@ -95,7 +120,7 @@ void SYMBOL_TREE_MODEL_ADAPTER::AddLibraries( const std::vector<wxString>& aNick
         // manager. A side effect is the call of ShowModal() of a dialog following
         // the use of SYMBOL_TREE_MODEL_ADAPTER creating a APP_PROGRESS_DIALOG
         // has a broken behavior (incorrect modal behavior).
-        delete prg;
+        prg.reset();
         m_show_progress = false;
     }
 }

eeschema/symbol_tree_model_adapter.h

@@ -63,6 +63,7 @@ protected:
     SYMBOL_TREE_MODEL_ADAPTER( EDA_BASE_FRAME* aParent, LIB_TABLE* aLibs );
 
 private:
+    friend class SYMBOL_ASYNC_LOADER;
     /**
      * Flag to only show the symbol library table load progress dialog the first time.
      */