Update macOS to Python 3, wx 3.1, and Phoenix.

This is a work-in-progress.  It could use testing while I continue to fix
the remaining pieces.

There are some changes that will be needed for signing and notarization.

This currently relies on a Python tool I wrote (dyldstyle) to fixup
KiCad.app correctly. I would like any bundle fixing necessary to use a
built KiCad on macOS to live inside KiCad, rather than in
kicad-mac-builder or elsewhere.  While I was experimenting, I found this
worked, however, and I would love to get extra hands testing.

I added a CMake argument, MACOS_EXTRA_BUNDLE_FIX_DIRS, for devs and
packagers who have extra directories they need to add to
fixup_bundle on KiCad.app.

There's an issue about differing behavior when KiCad is opened via
the command line or via Finder/launchd.

CMakeLists.txt

@@ -844,10 +844,10 @@ if( KICAD_SCRIPTING OR KICAD_SCRIPTING_MODULES )
     endif()
 
     if( APPLE )
-        # relative path for python in bundle
+        # change to add_compile_definitions() after minimum required CMake version is 3.12
-        set( PYTHON_LIB_DIR "python/site-packages" )
+        set_property( DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}")
-        # install into bundle Frameworks folder
+        #TODO put this into Resources/site-packages or something that's notarizable, but as it is we pull it in during bundling
-        set( PYTHON_DEST "${OSX_BUNDLE_BUILD_LIB_DIR}/${PYTHON_LIB_DIR}"
+        set( PYTHON_DEST "${PYTHON_SITE_PACKAGE_PATH}"
             CACHE PATH "Python module install path."
             )
     elseif( VCPKG_TOOLCHAIN )

kicad/CMakeLists.txt

@@ -110,12 +110,24 @@ if( APPLE )
         set( SPICE_HELPER "0" )
     endif()
 
+    if( PYTHON_FRAMEWORK )
+        set( PYTHON_FRAMEWORK_HELPER "1" )
+    else()
+        set( PYTHON_FRAMEWORK_HELPER "0" )
+    endif()
+
+    if( MACOS_EXTRA_BUNDLE_FIX_DIRS )
+        set( BUNDLE_FIX_DIRS ${BUNDLE_FIX_DIRS} ${MACOS_EXTRA_BUNDLE_FIX_DIRS} ) # TODO: where should we document MACOS_EXTRA_BUNDLE_FIX_DIRS?
+    endif()
+
     if( KICAD_USE_OCC )
-        set( KICAD_BUNDLE_LIBS ${OCC_LIBRARY_DIR} )
+        set( BUNDLE_FIX_DIRS ${BUNDLE_FIX_DIRS} ${OCC_LIBRARY_DIR} )
     endif()
 
     # make main bundle relocatable
     install( CODE "
+        set( BUNDLE_FIX_DIRS ${BUNDLE_FIX_DIRS} ) # pull in the variable from above, so we can append if needed
+
         # find all kicad libs and modules
         file( GLOB PLUGINS_KIFACE ${OSX_BUNDLE_INSTALL_KIFACE_DIR}/*.kiface )
         file( GLOB_RECURSE PLUGINS_SO ${OSX_BUNDLE_INSTALL_PLUGIN_DIR}/*.so )
@@ -141,12 +153,66 @@ if( APPLE )
         ${OSX_BUNDLE_OVERRIDE_PATHS}
 
         # do all the work
-        include( BundleUtilities )
+        include( ${CMAKE_MODULE_PATH}/BundleUtilities.cmake )
 
-        fixup_bundle( ${OSX_BUNDLE_INSTALL_BIN_DIR}/kicad
+        if ( ${PYTHON_FRAMEWORK_HELPER} )
-            \"\${BUNDLE_FIX_LIBS}\"
+            # This idea here is to (eventually) repair anything that fixup_bundle doesn't handle
-            \" ${KICAD_BUNDLE_LIBS}\"
+            # properly for our setup with both Python.framework *and* symlinked subapps
+            # that's needed for *running* here
+
+            # Anything that's needed strictly for packaging and making redistributable
+            # macOS builds can be defined in kicad-mac-builder
+
+            # Of course, making it all work right here would be even slicker,
+            # but if wishes were horses...
+
+            # At this point, the above is not true.  The bottom comes from what we were doing in KiCad patches
+            # in kicad-mac-builder for Python 2, but currently the effectual fixing for Python3 is happening in dyldstyle.
+
+            execute_process( COMMAND cp -RP ${PYTHON_FRAMEWORK} ${OSX_BUNDLE_INSTALL_LIB_DIR}/)
+            # We're using cp -RP because CMake's COPY_RESOLVED_BUNDLE... and COPY_DIRECTORY don't handle symlinks correctly
+
+            file( REMOVE GLOB ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/3.8/lib/python3.8/site-packages/wx/*.dylib)
+
+            # Add any .so files in the site-packages directory to the list of things to fixup during fixup_bundle
+            set( PYTHON_SITE_PACKAGES_PATH ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/3.8/lib/python3.8/site-packages)
+            file( GLOB_RECURSE PYTHON_SITE_PACKAGES_LIBS \${PYTHON_SITE_PACKAGES_PATH}/*.so )
+            set( BUNDLE_FIX_LIBS \${BUNDLE_FIX_LIBS} \${PYTHON_SITE_PACKAGES_LIBS} )
+
+            fixup_bundle( ${OSX_BUNDLE_INSTALL_BIN_DIR}/kicad
+                \"\${BUNDLE_FIX_LIBS}\"
+                \"\${BUNDLE_FIX_DIRS}\"
+                IGNORE_ITEM \"Python;python;python3;python3.8;pythonw;pythonw3;pythonw3.8\"
+                )
+
+            # BundleUtilities clobbers the rpaths and install_names that we carefully setup in Python.framework, even if we mark Python things as IGNORE_ITEMs.
+
+            # python/site-packages/_pcbnew.so
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python.framework/Versions/3.8/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/3.8/lib/python3.8/site-packages/_pcbnew.so )
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python.framework/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/3.8/lib/python3.8/site-packages/_pcbnew.so )
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python.framework/Versions/3.8/Resources/Python.app/Contents/MacOS/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/3.8/lib/python3.8/site-packages/_pcbnew.so )
+
+            # _pcbnew.kiface
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_KIFACE_DIR}/_pcbnew.kiface )
+
+            execute_process( COMMAND install_name_tool -add_rpath @executable_path/../Frameworks ${OSX_BUNDLE_INSTALL_KIFACE_DIR}/_pcbnew.kiface )
+
+            # Python.framework/Versions/Current/bin/python3
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python.framework/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/Current/bin/python3 )
+            execute_process( COMMAND install_name_tool -add_rpath @executable_path/../../../../ ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/Current/bin/python3 )
+
+            # Python.framework/Versions/Current/Resources/Python.app/Contents/MacOS/Python
+            execute_process( COMMAND install_name_tool -change @executable_path/../../Contents/MacOS/Python.framework/Versions/3.8/Resources/Python.app/Contents/MacOS/Python @rpath/Python.framework/Python ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Versions/Current/Resources/Python.app/Contents/MacOS/Python )
+            execute_process( COMMAND install_name_tool -add_rpath @executable_path/../Frameworks ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Resources/Python.app/Contents/MacOS/Python )
+            execute_process( COMMAND install_name_tool -add_rpath @executable_path/../../../../../../../ ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Resources/Python.app/Contents/MacOS/Python )
+            execute_process( COMMAND install_name_tool -add_rpath @executable_path/../../../../../ ${OSX_BUNDLE_INSTALL_LIB_DIR}/Python.framework/Resources/Python.app/Contents/MacOS/Python )
+
+        else()
+            fixup_bundle( ${OSX_BUNDLE_INSTALL_BIN_DIR}/kicad
+                \"\${BUNDLE_FIX_LIBS}\"
+                \"\${BUNDLE_FIX_DIRS}\"
             )
+        endif()
 
         if( ${SPICE_HELPER} )
             execute_process( COMMAND install_name_tool -id @executable_path/../PlugIns/sim/libngspice.0.dylib libngspice.0.dylib

pcbnew/CMakeLists.txt

@@ -882,16 +882,21 @@ if( APPLE )
 
     if( KICAD_SCRIPTING_WXPYTHON )
         # find wx-X.Y-osx_cocoa path below PYTHON_SITE_PACKAGE_PATH
-        file( GLOB WXPYTHON_DIR RELATIVE ${PYTHON_SITE_PACKAGE_PATH} ${PYTHON_SITE_PACKAGE_PATH}/wx-?.?-osx_cocoa )
+        # TODO It does not seem that we need this, at this point, but
-        if( NOT WXPYTHON_DIR )
+        # before removing it, I'd like to take a look and figure out why
-            message( FATAL_ERROR "Could not find 'wx-?.?-osx_cocoa' in '${PYTHON_SITE_PACKAGE_PATH}'" )
+        # (and if it is needed, but it's just needed for something I'm not testing!)
-        endif()
+        # Can we leave this alone? I don't think we use this with kicad-mac-builder but
-        # copy contents
+        # that doesn't mean it's not usable...
-        add_custom_target( ScriptingWxpythonCopy ALL
+#        file( GLOB WXPYTHON_DIR RELATIVE ${PYTHON_SITE_PACKAGE_PATH} ${PYTHON_SITE_PACKAGE_PATH}/wx-?.?-osx_cocoa )
-            COMMAND ${CMAKE_COMMAND} -E copy_directory "${PYTHON_SITE_PACKAGE_PATH}/${WXPYTHON_DIR}" "${PYTHON_DEST}/${WXPYTHON_DIR}"
+#        if( NOT WXPYTHON_DIR )
-            COMMAND ${CMAKE_COMMAND} -E copy "${PYTHON_SITE_PACKAGE_PATH}/wxversion.py" "${PYTHON_DEST}"
+#            message( FATAL_ERROR "Could not find 'wx-?.?-osx_cocoa' in '${PYTHON_SITE_PACKAGE_PATH}'" )
-            COMMENT "Copying wxPython into ${PYTHON_DEST}"
+#        endif()
-            )
+#        # copy contents
-        add_dependencies( ScriptingWxpythonCopy ScriptingPythonDirectoryCreation )
+#        add_custom_target( ScriptingWxpythonCopy ALL
+#                COMMAND ${CMAKE_COMMAND} -E copy_directory "${PYTHON_SITE_PACKAGE_PATH}/${WXPYTHON_DIR}" "${PYTHON_DEST}/${WXPYTHON_DIR}"
+#                COMMAND ${CMAKE_COMMAND} -E copy "${PYTHON_SITE_PACKAGE_PATH}/wxversion.py" "${PYTHON_DEST}"
+#                COMMENT "Copying wxPython into ${PYTHON_DEST}"
+#                )
+#        add_dependencies( ScriptingWxpythonCopy ScriptingPythonDirectoryCreation )
     endif()
 endif()

pcbnew/pcbnew.cpp

@@ -256,9 +256,17 @@ static bool scriptingSetup()
         pypath += wxT( ":" ) + wxString( wxGetenv("KICAD_PATH") );
     }
 
-    // Bundle wxPython folder (<kicad.app>/Contents/Frameworks/python/site-packages)
+    // TODO: Can we just use PYTHON_DEST here once we set that correctly?
+
+    wxString pythonMajorMinorVersion;
+    pythonMajorMinorVersion += wxT( PYTHON_MAJOR_MINOR_VERSION );
+
+    // Bundle Python folder (<kicad.app>/Contents/Frameworks/Python.framework/Versions/x.y/lib/pythonx.y/site-packages)
+    //
     pypath += wxT( ":" ) + Pgm().GetExecutablePath() +
-              wxT( "Contents/Frameworks/python/site-packages" );
+              wxT( "Contents/Frameworks/Python.framework/Versions/") + pythonMajorMinorVersion +
+              wxT("/lib/python") + pythonMajorMinorVersion
+              + wxT("/site-packages" );
 
     // Original content of $PYTHONPATH
     if( wxGetenv( wxT( "PYTHONPATH" ) ) != NULL )
@@ -269,6 +277,13 @@ static bool scriptingSetup()
     // set $PYTHONPATH
     wxSetEnv( "PYTHONPATH", pypath );
 
+    wxString pyhome;
+
+    pyhome += Pgm().GetExecutablePath() +
+              wxT( "Contents/Frameworks/Python.framework/Versions/Current" );
+
+    // set $PYTHONHOME
+    wxSetEnv( "PYTHONHOME", pyhome );
 #else
     wxString pypath;