diff --git a/eeschema/netlist_exporters/netlist_exporter_orcadpcb2.cpp b/eeschema/netlist_exporters/netlist_exporter_orcadpcb2.cpp
index 7b58ca204b..01076e8692 100644
--- a/eeschema/netlist_exporters/netlist_exporter_orcadpcb2.cpp
+++ b/eeschema/netlist_exporters/netlist_exporter_orcadpcb2.cpp
@@ -84,8 +84,7 @@ bool NETLIST_EXPORTER_ORCADPCB2::WriteNetlist( const wxString& aOutFileName,
             if( symbol->GetLibSymbolRef()
                   && symbol->GetLibSymbolRef()->GetFPFilters().GetCount() != 0  )
             {
-                cmpList.push_back( SCH_REFERENCE( symbol, symbol->GetLibSymbolRef().get(),
-                                                  sheet ) );
+                cmpList.push_back( SCH_REFERENCE( symbol, sheet ) );
             }
 
             footprint = symbol->GetFootprintFieldText( true, &sheet, false );
diff --git a/eeschema/sch_reference_list.cpp b/eeschema/sch_reference_list.cpp
index 996ff4c897..b9939290af 100644
--- a/eeschema/sch_reference_list.cpp
+++ b/eeschema/sch_reference_list.cpp
@@ -795,14 +795,11 @@ int SCH_REFERENCE_LIST::CheckAnnotation( ANNOTATION_ERROR_HANDLER aHandler )
 }
 
 
-SCH_REFERENCE::SCH_REFERENCE( SCH_SYMBOL* aSymbol, LIB_SYMBOL* aLibSymbol,
-                              const SCH_SHEET_PATH& aSheetPath )
+SCH_REFERENCE::SCH_REFERENCE( SCH_SYMBOL* aSymbol, const SCH_SHEET_PATH& aSheetPath )
 {
     wxASSERT( aSymbol != nullptr );
 
     m_rootSymbol = aSymbol;
-    m_libPart    = aLibSymbol;     // Warning: can be nullptr for orphan symbols
-                                   // (i.e. with a symbol library not found)
     m_unit       = aSymbol->GetUnitSelection( &aSheetPath );
     m_footprint  = aSymbol->GetFootprintFieldText( true, &aSheetPath, false );
     m_sheetPath  = aSheetPath;
diff --git a/eeschema/sch_reference_list.h b/eeschema/sch_reference_list.h
index ebe44b7a16..021eef43e3 100644
--- a/eeschema/sch_reference_list.h
+++ b/eeschema/sch_reference_list.h
@@ -80,7 +80,6 @@ public:
             m_sheetPath()
     {
         m_rootSymbol      = nullptr;
-        m_libPart         = nullptr;
         m_unit            = 0;
         m_isNew           = false;
         m_numRef          = 0;
@@ -88,11 +87,11 @@ public:
         m_sheetNum        = 0;
     }
 
-    SCH_REFERENCE( SCH_SYMBOL* aSymbol, LIB_SYMBOL* aLibSymbol, const SCH_SHEET_PATH& aSheetPath );
+    SCH_REFERENCE( SCH_SYMBOL* aSymbol, const SCH_SHEET_PATH& aSheetPath );
 
     SCH_SYMBOL* GetSymbol() const           { return m_rootSymbol; }
 
-    LIB_SYMBOL* GetLibPart() const          { return m_libPart; }
+    LIB_SYMBOL* GetLibPart() const          { return m_rootSymbol->GetLibSymbolRef().get(); }
 
     const SCH_SHEET_PATH& GetSheetPath() const { return m_sheetPath; }
 
@@ -226,8 +225,8 @@ public:
 
     bool IsUnitsLocked()
     {
-        if( m_libPart )
-            return m_libPart->UnitsLocked();
+        if( GetLibPart() )
+            return GetLibPart()->UnitsLocked();
         else
             return true; // Assume units locked when we don't have a library
     }
@@ -238,7 +237,6 @@ private:
     /// Symbol reference prefix, without number (for IC1, this is IC) )
     wxString        m_ref;               // it's private, use the accessors please
     SCH_SYMBOL*     m_rootSymbol;        ///< The symbol associated the reference object.
-    LIB_SYMBOL*     m_libPart;           ///< The source symbol from a library.
     VECTOR2I        m_symbolPos;         ///< The physical position of the symbol in schematic
                                          ///< used to annotate by X or Y position
     int             m_unit;              ///< The unit number for symbol with multiple parts
diff --git a/eeschema/sch_sheet_path.cpp b/eeschema/sch_sheet_path.cpp
index 717bcd3e87..0822569777 100644
--- a/eeschema/sch_sheet_path.cpp
+++ b/eeschema/sch_sheet_path.cpp
@@ -397,11 +397,9 @@ void SCH_SHEET_PATH::AppendSymbol( SCH_REFERENCE_LIST& aReferences, SCH_SYMBOL*
     // affects power symbols.
     if( aIncludePowerSymbols || aSymbol->GetRef( this )[0] != wxT( '#' ) )
     {
-        LIB_SYMBOL* symbol = aSymbol->GetLibSymbolRef().get();
-
-        if( symbol || aForceIncludeOrphanSymbols )
+        if( aSymbol->GetLibSymbolRef() || aForceIncludeOrphanSymbols )
         {
-            SCH_REFERENCE schReference( aSymbol, symbol, *this );
+            SCH_REFERENCE schReference( aSymbol, *this );
 
             schReference.SetSheetNumber( m_virtualPageNumber );
             aReferences.AddItem( schReference );
@@ -434,7 +432,7 @@ void SCH_SHEET_PATH::AppendMultiUnitSymbol( SCH_MULTI_UNIT_REFERENCE_MAP& aRefLi
 
     if( symbol && symbol->GetUnitCount() > 1 )
     {
-        SCH_REFERENCE schReference = SCH_REFERENCE( aSymbol, symbol, *this );
+        SCH_REFERENCE schReference = SCH_REFERENCE( aSymbol, *this );
         schReference.SetSheetNumber( m_virtualPageNumber );
         wxString reference_str = schReference.GetRef();
 
@@ -909,7 +907,7 @@ void SCH_SHEET_LIST::AnnotatePowerSymbols()
 
             if( libSymbol && libSymbol->IsPower() )
             {
-                SCH_REFERENCE schReference( symbol, libSymbol, sheet );
+                SCH_REFERENCE schReference( symbol, sheet );
                 references.AddItem( schReference );
             }
         }
diff --git a/eeschema/tools/sch_drawing_tools.cpp b/eeschema/tools/sch_drawing_tools.cpp
index 318f58fc3a..0c011981fe 100644
--- a/eeschema/tools/sch_drawing_tools.cpp
+++ b/eeschema/tools/sch_drawing_tools.cpp
@@ -196,7 +196,7 @@ int SCH_DRAWING_TOOLS::PlaceSymbol( const TOOL_EVENT& aEvent )
                 // Then we need to annotate all instances by sheet
                 for( SCH_SHEET_PATH& instance : newInstances )
                 {
-                    SCH_REFERENCE newReference( symbol, symbol->GetLibSymbolRef().get(), instance );
+                    SCH_REFERENCE newReference( symbol, instance );
                     SCH_REFERENCE_LIST refs;
                     refs.AddItem( newReference );
 
@@ -373,8 +373,7 @@ int SCH_DRAWING_TOOLS::PlaceSymbol( const TOOL_EVENT& aEvent )
                 annotate();
 
                 // Update the list of references for the next symbol placement.
-                SCH_REFERENCE placedSymbolReference( symbol, symbol->GetLibSymbolRef().get(),
-                                                     m_frame->GetCurrentSheet() );
+                SCH_REFERENCE placedSymbolReference( symbol, m_frame->GetCurrentSheet() );
                 existingRefs.AddItem( placedSymbolReference );
                 existingRefs.SortByReferenceOnly();
 
@@ -436,9 +435,7 @@ int SCH_DRAWING_TOOLS::PlaceSymbol( const TOOL_EVENT& aEvent )
                         annotate();
 
                         // Update the list of references for the next symbol placement.
-                        SCH_REFERENCE placedSymbolReference( symbol,
-                                                             symbol->GetLibSymbolRef().get(),
-                                                             m_frame->GetCurrentSheet() );
+                        SCH_REFERENCE placedSymbolReference( symbol, m_frame->GetCurrentSheet() );
                         existingRefs.AddItem( placedSymbolReference );
                         existingRefs.SortByReferenceOnly();
                     }
diff --git a/eeschema/tools/sch_editor_control.cpp b/eeschema/tools/sch_editor_control.cpp
index da787fd88a..a29883e0e3 100644
--- a/eeschema/tools/sch_editor_control.cpp
+++ b/eeschema/tools/sch_editor_control.cpp
@@ -1809,7 +1809,7 @@ int SCH_EDITOR_CONTROL::Paste( const TOOL_EVENT& aEvent )
                 // Ignore symbols from a non-existant library.
                 if( libSymbol )
                 {
-                    SCH_REFERENCE schReference( symbol, libSymbol, sheetPath );
+                    SCH_REFERENCE schReference( symbol, sheetPath );
                     schReference.SetSheetNumber( sheetPath.GetVirtualPageNumber() );
                     pastedSymbols[sheetPath].AddItem( schReference );
                 }