implemented better caching for categories and parts overview to reduce processing time

common/http_lib/http_lib_connection.cpp

@@ -39,7 +39,7 @@ HTTP_LIB_CONNECTION::HTTP_LIB_CONNECTION( const HTTP_LIB_SOURCE& aSource, bool a
     m_rootURL = aSource.root_url;
     m_token = aSource.token;
     m_sourceType = aSource.type;
-    m_timeout = aSource.timeout;
+    m_timeout = aSource.timeout_parts;
 
     if( aTestConnectionNow )
     {

common/http_lib/http_lib_settings.cpp

@@ -43,7 +43,11 @@ HTTP_LIB_SETTINGS::HTTP_LIB_SETTINGS( const std::string& aFilename ) :
 
     m_params.emplace_back( new PARAM<std::string>( "source.token", &m_Source.token, "" ) );
 
-    m_params.emplace_back( new PARAM<int>( "source.timeout_seconds", &m_Source.timeout, 2 ) );
+    m_params.emplace_back( new PARAM<int>( "source.timeout_parts_seconds", &m_Source.timeout_parts, 30 ) );
+
+    m_params.emplace_back( new PARAM<int>( "source.timeout_categories_seconds",
+                                           &m_Source.timeout_categories, 600 ) );
+
 }
 
 

eeschema/sch_io/http_lib/sch_io_http_lib.cpp

@@ -69,29 +69,48 @@ void SCH_IO_HTTP_LIB::EnumerateSymbolLib( std::vector<LIB_SYMBOL*>& aSymbolList,
             ( aProperties
               && aProperties->find( SYMBOL_LIB_TABLE::PropPowerSymsOnly ) != aProperties->end() );
 
-    // clear buffer
-    m_cachedCategoryParts.clear();
-
     for(const HTTP_LIB_CATEGORY& category : m_conn->getCategories() )
     {
+        bool refresh_cache = true;
+
         std::vector<HTTP_LIB_PART> found_parts;
 
-        if( !m_conn->SelectAll( category, found_parts ) )
+         // Check if there is already a part in our cache, if not fetch it
+        if( m_cachedCategories.find( category.id ) != m_cachedCategories.end() )
         {
-            if( !m_conn->GetLastError().empty() )
+            // check if it's outdated, if so re-fetch
+            if( std::difftime( std::time( nullptr ), m_cachedCategories[category.id].lastCached )
+                < m_settings->m_Source.timeout_categories )
             {
-                wxString msg = wxString::Format( _( "Error retriving data from HTTP library %s: %s" ),
+                refresh_cache = false;
-                                                 category.name, m_conn->GetLastError() );
-                THROW_IO_ERROR( msg );
             }
-
-            continue;
         }
 
-        // cache information for later use in LoadSymbol()
+        if( refresh_cache )
-        m_cachedCategoryParts.emplace( category.id, found_parts );
+        {
+            if( !m_conn->SelectAll( category, found_parts ) )
+            {
+                if( !m_conn->GetLastError().empty() )
+                {
+                    wxString msg =
+                            wxString::Format( _( "Error retriving data from HTTP library %s: %s" ),
+                                              category.name, m_conn->GetLastError() );
+                    THROW_IO_ERROR( msg );
+                }
 
-        for( const HTTP_LIB_PART& part : found_parts )
+                continue;
+            }
+
+            // remove cached parts
+            m_cachedCategories[category.id].cachedParts.clear();
+
+            // Copy newly cached data across
+            m_cachedCategories[category.id].cachedParts = found_parts;
+            m_cachedCategories[category.id].lastCached = std::time( nullptr );
+
+        }
+
+        for( const HTTP_LIB_PART& part : m_cachedCategories[category.id].cachedParts )
         {
             wxString libIDString( part.name );
 
@@ -147,7 +166,7 @@ LIB_SYMBOL* SCH_IO_HTTP_LIB::LoadSymbol( const wxString&        aLibraryPath,
     }
 
     // get the matching query ID
-    for( const HTTP_LIB_PART& part : m_cachedCategoryParts[foundCategory->id] )
+    for( const HTTP_LIB_PART& part : m_cachedCategories[foundCategory->id].cachedParts )
     {
         if( part.id == std::get<0>( relations ) )
         {

eeschema/sch_io/http_lib/sch_io_http_lib.h

@@ -100,8 +100,6 @@ private:
 
     SYMBOL_LIB_TABLE* m_libTable;
 
-    std::map<std::string, std::vector<HTTP_LIB_PART>> m_cachedCategoryParts;
-
      /// Generally will be null if no valid connection is established
     std::unique_ptr<HTTP_LIB_CONNECTION> m_conn;
 
@@ -121,6 +119,9 @@ private:
     wxString datasheet_field = "datasheet";
     wxString reference_field = "reference";
 
+     //     category.id       category
+    std::map<std::string, HTTP_LIB_CATEGORY> m_cachedCategories;
+
 };
 
 #endif // SCH_IO_HTTP_LIB_H_

include/http_lib/http_lib_settings.h

@@ -37,16 +37,11 @@ struct HTTP_LIB_SOURCE
     std::string          root_url;
     std::string          api_version;
     std::string          token;
-    int                  timeout;
+    int                  timeout_parts;
+    int                  timeout_categories;
 };
 
 
-struct HTTP_LIB_CATEGORY
-{
-    std::string id;   ///< id of category
-    std::string name; ///< name of category
-};
-
 struct HTTP_LIB_PART
 {
     std::string id;
@@ -62,6 +57,18 @@ struct HTTP_LIB_PART
     std::map<std::string, std::tuple<std::string, bool>> fields; ///< additional generic fields
 };
 
+
+struct HTTP_LIB_CATEGORY
+{
+    std::string id;   ///< id of category
+    std::string name; ///< name of category
+
+    std::time_t lastCached = 0;
+
+    std::vector<HTTP_LIB_PART> cachedParts;
+};
+
+
 class HTTP_LIB_SETTINGS : public JSON_SETTINGS
 {
 public: