diff --git a/eeschema/tools/ee_selection_tool.cpp b/eeschema/tools/ee_selection_tool.cpp
index 226e33f85f..0ed06e70ac 100644
--- a/eeschema/tools/ee_selection_tool.cpp
+++ b/eeschema/tools/ee_selection_tool.cpp
@@ -818,7 +818,7 @@ bool EE_SELECTION_TOOL::CollectHits( EE_COLLECTOR& aCollector, const VECTOR2I& a
 
 
 void EE_SELECTION_TOOL::narrowSelection( EE_COLLECTOR& collector, const VECTOR2I& aWhere,
-                                         bool aCheckLocked )
+                                         bool aCheckLocked, bool aSelectedOnly )
 {
     for( int i = collector.GetCount() - 1; i >= 0; --i )
     {
@@ -833,6 +833,12 @@ void EE_SELECTION_TOOL::narrowSelection( EE_COLLECTOR& collector, const VECTOR2I
             collector.Remove( i );
             continue;
         }
+
+        if( aSelectedOnly && !collector[i]->IsSelected() )
+        {
+            collector.Remove( i );
+            continue;
+        }
     }
 
     // Apply some ugly heuristics to avoid disambiguation menus whenever possible
@@ -947,7 +953,7 @@ bool EE_SELECTION_TOOL::SelectPoint( const VECTOR2I& aWhere,
     if( !CollectHits( collector, aWhere, aScanTypes ) )
         return false;
 
-    narrowSelection( collector, aWhere, aCheckLocked );
+    narrowSelection( collector, aWhere, aCheckLocked, aSubtract );
 
     return selectPoint( collector, aWhere, aItem, aSelectionCancelledFlag, aAdd, aSubtract,
                         aExclusiveOr );
diff --git a/eeschema/tools/ee_selection_tool.h b/eeschema/tools/ee_selection_tool.h
index 66d072d9e6..9b3a4ec20a 100644
--- a/eeschema/tools/ee_selection_tool.h
+++ b/eeschema/tools/ee_selection_tool.h
@@ -191,8 +191,10 @@ private:
      * @param aCollector [in, out] Provides collection conditions and stores collected items.
      * @param aWhere point where we should narrow (if relevant)
      * @param aCheckLocked If false, remove locked elements from #collector
+     * @param aSelectedOnly If true, remove non-selected items from #collector
      */
-    void narrowSelection( EE_COLLECTOR& collector, const VECTOR2I& aWhere, bool aCheckLocked );
+    void narrowSelection( EE_COLLECTOR& collector, const VECTOR2I& aWhere, bool aCheckLocked,
+                          bool aSelectedOnly = false );
 
     /**
      * Perform a click-type selection at a point (usually the cursor position).
diff --git a/pcbnew/tools/pcb_selection_tool.cpp b/pcbnew/tools/pcb_selection_tool.cpp
index 334d80dbcf..0e022044b6 100644
--- a/pcbnew/tools/pcb_selection_tool.cpp
+++ b/pcbnew/tools/pcb_selection_tool.cpp
@@ -722,6 +722,16 @@ bool PCB_SELECTION_TOOL::selectPoint( const VECTOR2I& aWhere, bool aOnDrag,
     // Apply the stateful filter
     FilterCollectedItems( collector, false );
 
+    // For subtracting, we only want items that are selected
+    if( m_subtractive )
+    {
+        for( int i = collector.GetCount() - 1; i >= 0; --i )
+        {
+            if( !collector[i]->IsSelected() )
+                collector.Remove( i );
+        }
+    }
+
     // Apply some ugly heuristics to avoid disambiguation menus whenever possible
     if( collector.GetCount() > 1 && !m_skip_heuristics )
         GuessSelectionCandidates( collector, aWhere );