Add ability to update subgraphs based on changeset

Previously, almost all connectivity updates were full updates, meaning
that the entire connectivity graph would be rebuilt each time a change
was made to the schematic.  This update modifies the update to only
correct the subgraphs that are directly affected by the change.

It uses the existing connection graph to find all affected subgraphs as
well as connections to the changes based on the visible schematic.
These elements are removed from the existing connectivity graph and
marked dirty.  They then have a new connectivity graph built only around
their changes.  The resulting graph is merged into the original.

Currently, this ability is behind an advanced config flag
`IncrementalConnectivity` while testing.

Fixes https://gitlab.com/kicad/code/kicad/issues/10846

Fixes https://gitlab.com/kicad/code/kicad/issues/1794

(cherry picked from commit a154571438)

common/advanced_config.cpp

@@ -62,6 +62,11 @@ namespace AC_STACK
 namespace AC_KEYS
 {
 
+/**
+ * Should the schematic use the new incremental connectivity algorithm
+ */
+static const wxChar IncrementalConnectivity[] = wxT( "IncrementalConnectivity" );
+
 /**
  * Decide whether to attempt usage of the 3DConnexion mouse
  */
@@ -326,6 +331,7 @@ ADVANCED_CFG::ADVANCED_CFG()
 
     m_UseClipper2               = true;
     m_Use3DConnexionDriver      = false;
+    m_IncrementalConnectivity   = false;
 
     loadFromConfigFile();
 }
@@ -472,6 +478,9 @@ void ADVANCED_CFG::loadSettings( wxConfigBase& aCfg )
     configParams.push_back( new PARAM_CFG_BOOL( true, AC_KEYS::Use3DConnexionDriver,
                                                 &m_Use3DConnexionDriver, m_Use3DConnexionDriver ) );
 
+    configParams.push_back( new PARAM_CFG_BOOL( true, AC_KEYS::IncrementalConnectivity,
+                                                &m_IncrementalConnectivity, m_IncrementalConnectivity ) );
+
 
 
     // Special case for trace mask setting...we just grab them and set them immediately

eeschema/connection_graph.cpp

@@ -228,6 +228,24 @@ bool CONNECTION_SUBGRAPH::ResolveDrivers( bool aCheckMultipleDrivers )
 }
 
 
+void CONNECTION_SUBGRAPH::getAllConnectedItems( std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>>& aItems, std::set<CONNECTION_SUBGRAPH*>& aSubgraphs )
+{
+    CONNECTION_SUBGRAPH* sg = this;
+
+    while( sg->m_absorbed_by )
+        sg = sg->m_absorbed_by;
+
+    aSubgraphs.insert( sg );
+    aSubgraphs.insert( sg->m_absorbed_subgraphs.begin(), sg->m_absorbed_subgraphs.end() );
+
+    for( SCH_ITEM* item : sg->m_items )
+        aItems.emplace( m_sheet, item );
+
+    for( CONNECTION_SUBGRAPH* child_sg : sg->m_hier_children )
+        child_sg->getAllConnectedItems( aItems, aSubgraphs );
+}
+
+
 wxString CONNECTION_SUBGRAPH::GetNetName() const
 {
     if( !m_driver || m_dirty )
@@ -377,6 +395,10 @@ void CONNECTION_SUBGRAPH::Absorb( CONNECTION_SUBGRAPH* aOther )
         AddItem( item );
     }
 
+    m_absorbed_subgraphs.insert( aOther );
+    m_absorbed_subgraphs.insert( aOther->m_absorbed_subgraphs.begin(),
+            aOther->m_absorbed_subgraphs.end() );
+
     m_bus_neighbors.insert( aOther->m_bus_neighbors.begin(), aOther->m_bus_neighbors.end() );
     m_bus_parents.insert( aOther->m_bus_parents.begin(), aOther->m_bus_parents.end() );
 
@@ -457,10 +479,80 @@ CONNECTION_SUBGRAPH::PRIORITY CONNECTION_SUBGRAPH::GetDriverPriority( SCH_ITEM*
 }
 
 
+void CONNECTION_GRAPH::Merge( CONNECTION_GRAPH& aGraph )
+{
+    std::copy( aGraph.m_items.begin(), aGraph.m_items.end(),
+            std::back_inserter( m_items ) );
+
+    for( SCH_ITEM* item : aGraph.m_items )
+        item->SetConnectionGraph( this );
+
+    std::copy( aGraph.m_subgraphs.begin(), aGraph.m_subgraphs.end(),
+            std::back_inserter( m_subgraphs ) );
+
+    for( CONNECTION_SUBGRAPH* sg : aGraph.m_subgraphs )
+        sg->m_graph = this;
+
+    std::copy( aGraph.m_driver_subgraphs.begin(),
+            aGraph.m_driver_subgraphs.end(),
+            std::back_inserter( m_driver_subgraphs ) );
+
+    std::copy( aGraph.m_sheet_to_subgraphs_map.begin(),
+            aGraph.m_sheet_to_subgraphs_map.end(),
+            std::inserter( m_sheet_to_subgraphs_map,
+                    m_sheet_to_subgraphs_map.begin() ) );
+
+    std::copy( aGraph.m_invisible_power_pins.begin(),
+            aGraph.m_invisible_power_pins.end(),
+            std::back_inserter( m_invisible_power_pins ) );
+
+    std::copy( aGraph.m_net_name_to_code_map.begin(),
+            aGraph.m_net_name_to_code_map.end(),
+            std::inserter( m_net_name_to_code_map,
+                    m_net_name_to_code_map.begin() ) );
+
+    std::copy( aGraph.m_bus_name_to_code_map.begin(),
+            aGraph.m_bus_name_to_code_map.end(),
+            std::inserter( m_bus_name_to_code_map,
+                    m_net_name_to_code_map.begin() ) );
+
+    std::copy( aGraph.m_net_code_to_subgraphs_map.begin(),
+            aGraph.m_net_code_to_subgraphs_map.end(),
+            std::inserter( m_net_code_to_subgraphs_map,
+                    m_net_code_to_subgraphs_map.begin() ) );
+
+    std::copy( aGraph.m_net_name_to_subgraphs_map.begin(),
+            aGraph.m_net_name_to_subgraphs_map.end(),
+            std::inserter( m_net_name_to_subgraphs_map,
+                    m_net_name_to_subgraphs_map.begin() ) );
+
+    std::copy( aGraph.m_item_to_subgraph_map.begin(),
+            aGraph.m_item_to_subgraph_map.end(),
+            std::inserter( m_item_to_subgraph_map,
+                    m_item_to_subgraph_map.begin() ) );
+
+    std::copy( aGraph.m_local_label_cache.begin(),
+            aGraph.m_local_label_cache.end(),
+            std::inserter( m_local_label_cache, m_local_label_cache.begin() ) );
+
+    std::copy( aGraph.m_global_label_cache.begin(),
+            aGraph.m_global_label_cache.end(),
+            std::inserter( m_global_label_cache, m_global_label_cache.begin() ) );
+
+    m_last_bus_code = std::max( m_last_bus_code, aGraph.m_last_bus_code );
+    m_last_net_code = std::max( m_last_net_code, aGraph.m_last_net_code );
+    m_last_subgraph_code = std::max( m_last_subgraph_code, aGraph.m_last_subgraph_code );
+}
+
+
 void CONNECTION_GRAPH::Reset()
 {
     for( auto& subgraph : m_subgraphs )
-        delete subgraph;
+    {
+        /// Only delete subgraphs of which we are the owner
+        if( subgraph->m_graph == this )
+            delete subgraph;
+    }
 
     m_items.clear();
     m_subgraphs.clear();
@@ -552,6 +644,169 @@ void CONNECTION_GRAPH::Recalculate( const SCH_SHEET_LIST& aSheetList, bool aUnco
         recalc_time.Show();
 }
 
+std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>> CONNECTION_GRAPH::ExtractAffectedItems(
+        const std::set<SCH_ITEM*> &aItems )
+{
+    std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>> retvals;
+    std::set<CONNECTION_SUBGRAPH*> subgraphs;
+
+    auto traverse_subgraph = [&retvals, &subgraphs]( CONNECTION_SUBGRAPH* aSubgraph )
+    {
+        // Find the primary subgraph on this sheet
+        while( aSubgraph->m_absorbed_by )
+            aSubgraph = aSubgraph->m_absorbed_by;
+
+        // Find the top most connected subgraph on all sheets
+        while( aSubgraph->m_hier_parent )
+            aSubgraph = aSubgraph->m_hier_parent;
+
+        // Recurse through all subsheets to collect connected items
+        aSubgraph->getAllConnectedItems( retvals, subgraphs );
+    };
+
+    for( SCH_ITEM* item : aItems )
+    {
+        auto it = m_item_to_subgraph_map.find( item );
+
+        if( it == m_item_to_subgraph_map.end() )
+            continue;
+
+        CONNECTION_SUBGRAPH* sg = it->second;
+
+        traverse_subgraph( sg );
+
+        for( auto& bus_it : sg->m_bus_neighbors )
+        {
+            for( CONNECTION_SUBGRAPH* bus_sg : bus_it.second )
+                traverse_subgraph( bus_sg );
+        }
+
+        for( auto& bus_it : sg->m_bus_parents )
+        {
+            for( CONNECTION_SUBGRAPH* bus_sg : bus_it.second )
+                traverse_subgraph( bus_sg );
+        }
+    }
+
+    removeSubgraphs( subgraphs );
+
+    return retvals;
+}
+
+
+void CONNECTION_GRAPH::removeSubgraphs( std::set<CONNECTION_SUBGRAPH*>& aSubgraphs )
+{
+    std::sort( m_driver_subgraphs.begin(), m_driver_subgraphs.end() );
+    std::sort( m_subgraphs.begin(), m_subgraphs.end() );
+    std::set<int> codes_to_remove;
+
+    for( auto el : m_sheet_to_subgraphs_map )
+    {
+        std::sort( el.second.begin(), el.second.end() );
+    }
+
+    for( CONNECTION_SUBGRAPH* sg : aSubgraphs )
+    {
+        {
+            auto it = std::lower_bound( m_driver_subgraphs.begin(), m_driver_subgraphs.end(), sg );
+
+            if( it != m_driver_subgraphs.end() )
+                m_driver_subgraphs.erase( it );
+        }
+
+        {
+            auto it = std::lower_bound( m_subgraphs.begin(), m_subgraphs.end(), sg );
+
+            if( it != m_subgraphs.end() )
+                m_subgraphs.erase( it );
+        }
+
+        for( auto el : m_sheet_to_subgraphs_map )
+        {
+            auto it = std::lower_bound( el.second.begin(), el.second.end(), sg );
+
+            if( it != el.second.end() )
+                el.second.erase( it );
+        }
+
+        auto remove_sg = [sg]( auto it ) -> bool
+                        {
+                            for( const CONNECTION_SUBGRAPH* test_sg : it->second )
+                            {
+                                if( sg == test_sg )
+                                    return true;
+                            }
+
+                            return false;
+                        };
+
+        for( auto it = m_global_label_cache.begin(); it != m_global_label_cache.end(); )
+        {
+            if( remove_sg( it ) )
+                it = m_global_label_cache.erase( it );
+            else
+                ++it;
+        }
+
+        for( auto it = m_local_label_cache.begin(); it != m_local_label_cache.end(); )
+        {
+            if( remove_sg( it ) )
+                it = m_local_label_cache.erase( it );
+            else
+                ++it;
+        }
+
+        for( auto it = m_net_code_to_subgraphs_map.begin(); it != m_net_code_to_subgraphs_map.end(); )
+        {
+            if( remove_sg( it ) )
+            {
+                codes_to_remove.insert( it->first.Netcode );
+                it = m_net_code_to_subgraphs_map.erase( it );
+            }
+            else
+                ++it;
+        }
+
+        for( auto it = m_net_name_to_subgraphs_map.begin(); it != m_net_name_to_subgraphs_map.end(); )
+        {
+            if( remove_sg( it ) )
+                it = m_net_name_to_subgraphs_map.erase( it );
+            else
+                ++it;
+        }
+
+        for( auto it = m_item_to_subgraph_map.begin(); it != m_item_to_subgraph_map.end(); )
+        {
+            if( it->second == sg )
+                it = m_item_to_subgraph_map.erase( it );
+            else
+                ++it;
+        }
+    }
+
+    for( auto it = m_net_name_to_code_map.begin(); it != m_net_name_to_code_map.end(); )
+    {
+        if( codes_to_remove.find( it->second ) != codes_to_remove.end() )
+            it = m_net_name_to_code_map.erase( it );
+        else
+            ++it;
+    }
+
+    for( auto it = m_bus_name_to_code_map.begin(); it != m_bus_name_to_code_map.end(); )
+    {
+        if( codes_to_remove.find( it->second ) != codes_to_remove.end() )
+            it = m_bus_name_to_code_map.erase( it );
+        else
+            ++it;
+    }
+
+    for( CONNECTION_SUBGRAPH* sg : aSubgraphs )
+    {
+        sg->m_code = -1;
+        delete sg;
+    }
+}
+
 
 void CONNECTION_GRAPH::updateItemConnectivity( const SCH_SHEET_PATH& aSheet,
                                                const std::vector<SCH_ITEM*>& aItemList )
@@ -1355,7 +1610,7 @@ void CONNECTION_GRAPH::processSubGraphs()
 
             for( CONNECTION_SUBGRAPH* candidate : candidate_subgraphs )
             {
-                if( candidate->m_absorbed )
+                if( candidate->m_absorbed || candidate == subgraph )
                     continue;
 
                 bool match = false;
@@ -1912,7 +2167,6 @@ void CONNECTION_GRAPH::propagateToNeighbors( CONNECTION_SUBGRAPH* aSubgraph, boo
                                             candidate->m_code, candidate->m_driver_connection->Name() );
 
                                 candidate->m_hier_parent = aParent;
-
                                 search_list.push_back( candidate );
                                 break;
                             }
@@ -1959,6 +2213,7 @@ void CONNECTION_GRAPH::propagateToNeighbors( CONNECTION_SUBGRAPH* aSubgraph, boo
                                     aParent->m_code, candidate->m_code,
                                     candidate->m_driver_connection->Name() );
 
+                        aParent->m_hier_children.insert( candidate );
                         search_list.push_back( candidate );
                         break;
                     }

eeschema/connection_graph.h

@@ -129,6 +129,15 @@ public:
     /// Updates all items to match the driver connection
     void UpdateItemConnections();
 
+    /// Provides a read-only reference to the items in the subgraph
+    const std::vector<SCH_ITEM*>& GetItems() const
+    {
+        return m_items;
+    }
+
+    /// Finds all items in the subgraph as well as child subgraphs recursively
+    void getAllConnectedItems( std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>>& aItems, std::set<CONNECTION_SUBGRAPH*>& aSubgraphs );
+
     /**
      * Return the priority (higher is more important) of a candidate driver
      *
@@ -173,6 +182,9 @@ public:
     /// If this subgraph is absorbed, points to the absorbing (and valid) subgraph
     CONNECTION_SUBGRAPH* m_absorbed_by;
 
+    /// Set of subgraphs that have been absorbed by this subgraph
+    std::set<CONNECTION_SUBGRAPH*> m_absorbed_subgraphs;
+
     long m_code;
 
     /**
@@ -211,7 +223,7 @@ public:
      *
      * For example, if this subgraph is a bus D[7..0], and on the same sheet there is
      * a net with label D7, this map will contain an entry for the D7 bus member, and
-     * the vector will contain a pointer to the D7 net subgraph.
+     * the set will contain a pointer to the D7 net subgraph.
      */
     std::unordered_map< std::shared_ptr<SCH_CONNECTION>,
                         std::unordered_set<CONNECTION_SUBGRAPH*> > m_bus_neighbors;
@@ -233,6 +245,9 @@ public:
     // If not null, this indicates the subgraph on a higher level sheet that is linked to this one
     CONNECTION_SUBGRAPH* m_hier_parent;
 
+    // If not null, this indicates the subgraph(s) on a lower level sheet that are linked to this one
+    std::unordered_set<CONNECTION_SUBGRAPH*> m_hier_children;
+
     /// A cache of escaped netnames from schematic items
     mutable std::unordered_map<SCH_ITEM*, wxString> m_driver_name_cache;
 
@@ -299,6 +314,13 @@ public:
         m_schematic = aSchematic;
     }
 
+    void SetLastCodes( const CONNECTION_GRAPH* aOther )
+    {
+        m_last_net_code = aOther->m_last_net_code;
+        m_last_bus_code = aOther->m_last_bus_code;
+        m_last_subgraph_code = aOther->m_last_subgraph_code;
+    }
+
     /**
      * Updates the connection graph for the given list of sheets.
      *
@@ -363,6 +385,24 @@ public:
      */
     wxString GetResolvedSubgraphName( const CONNECTION_SUBGRAPH* aSubGraph ) const;
 
+    /**
+     * For a set of items, this will remove the connected items and their
+     * associated data including subgraphs and generated codes from the connection graph.
+     *
+     * @param aItems A vector of items whose presence should be removed from the graph
+     * @return The full set of all items associated with the input items that were removed
+     */
+    std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>> ExtractAffectedItems(
+            const std::set<SCH_ITEM*> &aItems );
+
+    /**
+     * Combines the input graph contents into the current graph.  After merging, the
+     * original graph is invalid.
+     *
+     * @param aGraph Input graph reference to add to the current graph
+     */
+    void Merge( CONNECTION_GRAPH& aGraph );
+
 private:
     /**
      * Updates the graphical connectivity between items (i.e. where they touch)
@@ -472,6 +512,14 @@ private:
      */
     void propagateToNeighbors( CONNECTION_SUBGRAPH* aSubgraph, bool aForce );
 
+    /**
+     * Removes references to the given subgraphs from all structures in the
+     * connection graph.
+     *
+     * @param aSubgraphs set of unique subgraphs to find/remove
+     */
+    void removeSubgraphs( std::set<CONNECTION_SUBGRAPH*>& aSubgraphs );
+
     /**
      * Search for a matching bus member inside a bus connection
      *

eeschema/dialogs/dialog_label_properties.cpp

@@ -595,7 +595,6 @@ bool DIALOG_LABEL_PROPERTIES::TransferDataFromWindow()
 
     m_Parent->UpdateItem( m_currentLabel );
     m_Parent->GetCanvas()->Refresh();
-    m_Parent->OnModify();
 
     return true;
 }

eeschema/sch_edit_frame.cpp

@@ -1479,7 +1479,73 @@ void SCH_EDIT_FRAME::RecalculateConnections( SCH_CLEANUP_FLAGS aCleanupFlags )
                 GetCanvas()->GetView()->Update( aChangedItem, KIGFX::REPAINT );
             };
 
-    Schematic().ConnectionGraph()->Recalculate( list, true, &changeHandler );
+    if( !ADVANCED_CFG::GetCfg().m_IncrementalConnectivity || aCleanupFlags == GLOBAL_CLEANUP
+            || m_undoList.m_CommandsList.empty() )
+    {
+        Schematic().ConnectionGraph()->Recalculate( list, true, &changeHandler );
+    }
+    else
+    {
+        auto& changed_list = m_undoList.m_CommandsList.back();
+        std::set<SCH_ITEM*> changed_items;
+        std::vector<VECTOR2I> pts;
+
+        for( unsigned ii = 0; ii < changed_list->GetCount(); ++ii )
+        {
+            SCH_ITEM* item = static_cast<SCH_ITEM*>( changed_list->GetPickedItem( ii ) );
+
+            std::vector<VECTOR2I> tmp_pts = item->GetConnectionPoints();
+            pts.insert( pts.end(), tmp_pts.begin(), tmp_pts.end() );
+            changed_items.insert( item );
+        }
+
+        for( VECTOR2I& pt: pts )
+        {
+            for( SCH_ITEM* item : GetScreen()->Items().Overlapping(pt ) )
+            {
+                if( !item->IsConnectable() )
+                    continue;
+
+                if( SCH_LINE* line = dyn_cast<SCH_LINE*>( item ) )
+                {
+                    if( line->HitTest( pt ) )
+                    {
+                        changed_items.insert( item );
+                        continue;
+                    }
+                }
+
+                if( item->IsConnected( pt ) )
+                    changed_items.insert( item );
+            }
+        }
+
+        std::set<std::pair<SCH_SHEET_PATH, SCH_ITEM*>> all_items =
+                Schematic().ConnectionGraph()->ExtractAffectedItems(
+                        changed_items );
+
+        CONNECTION_GRAPH new_graph( &Schematic() );
+
+        new_graph.SetLastCodes( Schematic().ConnectionGraph() );
+
+        for( auto&[ path, item ] : all_items )
+        {
+            switch( item->Type() )
+            {
+            case SCH_FIELD_T:
+            case SCH_PIN_T:
+            case SCH_SHEET_PIN_T:
+                static_cast<SCH_ITEM*>( item->GetParent() )->SetConnectivityDirty();
+                break;
+
+            default:
+                item->SetConnectivityDirty();
+            }
+        }
+
+        new_graph.Recalculate( list, false, &changeHandler );
+        Schematic().ConnectionGraph()->Merge( new_graph );
+    }
 
     GetCanvas()->GetView()->UpdateAllItemsConditionally(
             [&]( KIGFX::VIEW_ITEM* aItem ) -> int

eeschema/sch_item.cpp

@@ -26,13 +26,14 @@
 #include <settings/settings_manager.h>
 #include <eeschema_settings.h>
 #include <eda_item.h>
-#include <trace_helpers.h>
+#include <sch_connection.h>
 #include <sch_item.h>
 #include <sch_screen.h>
 #include <sch_sheet_path.h>
 #include <sch_draw_panel.h>
 #include <sch_edit_frame.h>
 #include <schematic.h>
+#include <trace_helpers.h>
 #include <general.h>
 #include <netclass.h>
 #include <project/project_file.h>
@@ -163,6 +164,13 @@ SCH_CONNECTION* SCH_ITEM::Connection( const SCH_SHEET_PATH* aSheet ) const
 }
 
 
+void SCH_ITEM::SetConnectionGraph( CONNECTION_GRAPH* aGraph )
+{
+    for( auto& [path, conn] : m_connection_map )
+        conn->SetGraph( aGraph );
+}
+
+
 std::shared_ptr<NETCLASS> SCH_ITEM::GetEffectiveNetClass( const SCH_SHEET_PATH* aSheet ) const
 {
     static std::shared_ptr<NETCLASS> nullNetclass = std::make_shared<NETCLASS>( wxEmptyString );

eeschema/sch_item.h

@@ -414,6 +414,9 @@ public:
 
     void SetConnectivityDirty( bool aDirty = true ) { m_connectivity_dirty = aDirty; }
 
+    /// Updates the connection graph for all connections in this item
+    void SetConnectionGraph( CONNECTION_GRAPH* aGraph );
+
     virtual void SetLastResolvedState( const SCH_ITEM* aItem ) { }
 
     std::shared_ptr<NETCLASS> GetEffectiveNetClass( const SCH_SHEET_PATH* aSheet = nullptr ) const;

include/advanced_config.h

@@ -244,6 +244,11 @@ public:
      */
     bool m_Use3DConnexionDriver;
 
+    /**
+     * Use the new incremental netlister for realtime jobs
+     */
+    bool m_IncrementalConnectivity;
+
 ///@}