kicad/kicad/pcm/pcm_task_manager.cpp

654 lines
21 KiB
C++

/*
* This program source code file is part of KiCad, a free EDA CAD application.
*
* Copyright (C) 2021 Andrew Lutsenko, anlutsenko at gmail dot com
* Copyright (C) 1992-2021 KiCad Developers, see AUTHORS.txt for contributors.
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// kicad_curl_easy.h **must be** included before any wxWidgets header to avoid conflicts
// at least on Windows/msys2
#include "kicad_curl/kicad_curl.h"
#include "kicad_curl/kicad_curl_easy.h"
#include <paths.h>
#include "pcm_task_manager.h"
#include <reporter.h>
#include <wxstream_helper.h>
#include <fstream>
#include <thread>
#include <unordered_set>
#include <wx/dir.h>
#include <wx/filename.h>
#include <wx/msgdlg.h>
#include <wx/sstream.h>
#include <wx/wfstream.h>
#include <wx/zipstrm.h>
void compile_keep_on_update_regex( const PCM_PACKAGE& pkg, const PACKAGE_VERSION& ver,
std::forward_list<wxRegEx>& aKeepOnUpdate )
{
auto compile_regex = [&]( const wxString& regex )
{
aKeepOnUpdate.emplace_front( regex, wxRE_DEFAULT );
if( !aKeepOnUpdate.front().IsValid() )
aKeepOnUpdate.pop_front();
};
std::for_each( pkg.keep_on_update.begin(), pkg.keep_on_update.end(), compile_regex );
std::for_each( ver.keep_on_update.begin(), ver.keep_on_update.end(), compile_regex );
}
void PCM_TASK_MANAGER::DownloadAndInstall( const PCM_PACKAGE& aPackage, const wxString& aVersion,
const wxString& aRepositoryId, const bool isUpdate )
{
PCM_TASK download_task = [aPackage, aVersion, aRepositoryId, isUpdate, this]()
{
wxFileName file_path( PATHS::GetUserCachePath(), "" );
file_path.AppendDir( "pcm" );
file_path.SetFullName( wxString::Format( "%s_v%s.zip", aPackage.identifier, aVersion ) );
auto find_pkgver = std::find_if( aPackage.versions.begin(), aPackage.versions.end(),
[&aVersion]( const PACKAGE_VERSION& pv )
{
return pv.version == aVersion;
} );
if( find_pkgver == aPackage.versions.end() )
{
m_reporter->PCMReport( wxString::Format( _( "Version %s of package %s not found!" ),
aVersion, aPackage.identifier ),
RPT_SEVERITY_ERROR );
return;
}
if( !wxDirExists( file_path.GetPath() )
&& !wxFileName::Mkdir( file_path.GetPath(), wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL ) )
{
m_reporter->PCMReport( _( "Unable to create download directory!" ),
RPT_SEVERITY_ERROR );
return;
}
int code = downloadFile( file_path.GetFullPath(), *find_pkgver->download_url );
if( code != CURLE_OK )
{
// Cleanup after ourselves and exit
wxRemoveFile( file_path.GetFullPath() );
return;
}
PCM_TASK install_task = [aPackage, aVersion, aRepositoryId, file_path, isUpdate, this]()
{
installDownloadedPackage( aPackage, aVersion, aRepositoryId, file_path, isUpdate );
};
m_install_queue.push( install_task );
};
m_download_queue.push( download_task );
}
int PCM_TASK_MANAGER::downloadFile( const wxString& aFilePath, const wxString& url )
{
TRANSFER_CALLBACK callback = [&]( size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow )
{
if( dltotal > 1024 )
m_reporter->SetDownloadProgress( dlnow, dltotal );
else
m_reporter->SetDownloadProgress( 0.0, 0.0 );
return m_reporter->IsCancelled();
};
std::ofstream out( aFilePath.ToUTF8(), std::ofstream::binary );
KICAD_CURL_EASY curl;
curl.SetOutputStream( &out );
curl.SetURL( url.ToUTF8().data() );
curl.SetFollowRedirects( true );
curl.SetTransferCallback( callback, 250000L );
m_reporter->PCMReport( wxString::Format( _( "Downloading package url: '%s'" ), url ),
RPT_SEVERITY_INFO );
int code = curl.Perform();
out.close();
uint64_t download_total;
if( CURLE_OK == curl.GetTransferTotal( download_total ) )
m_reporter->SetDownloadProgress( download_total, download_total );
if( code != CURLE_OK && code != CURLE_ABORTED_BY_CALLBACK )
{
m_reporter->PCMReport( wxString::Format( _( "Failed to download url %s\n%s" ), url,
curl.GetErrorText( code ) ),
RPT_SEVERITY_ERROR );
}
return code;
}
void PCM_TASK_MANAGER::installDownloadedPackage( const PCM_PACKAGE& aPackage,
const wxString& aVersion,
const wxString& aRepositoryId,
const wxFileName& aFilePath, const bool isUpdate )
{
auto pkgver = std::find_if( aPackage.versions.begin(), aPackage.versions.end(),
[&aVersion]( const PACKAGE_VERSION& pv )
{
return pv.version == aVersion;
} );
if( pkgver == aPackage.versions.end() )
{
m_reporter->PCMReport( wxString::Format( _( "Version %s of package %s not found!" ),
aVersion, aPackage.identifier ),
RPT_SEVERITY_ERROR );
return;
}
// wxRegEx is not CopyConstructible hence the weird choice of forward_list
std::forward_list<wxRegEx> keep_on_update;
if( isUpdate )
compile_keep_on_update_regex( aPackage, *pkgver, keep_on_update );
const std::optional<wxString>& hash = pkgver->download_sha256;
bool hash_match = true;
if( hash )
{
std::ifstream stream( aFilePath.GetFullPath().ToUTF8(), std::ios::binary );
hash_match = m_pcm->VerifyHash( stream, *hash );
}
if( !hash_match )
{
m_reporter->PCMReport( wxString::Format( _( "Downloaded archive hash for package "
"%s does not match repository entry. "
"This may indicate a problem with the "
"package, if the issue persists "
"report this to repository maintainers." ),
aPackage.identifier ),
RPT_SEVERITY_ERROR );
}
else
{
if( isUpdate )
{
m_reporter->PCMReport(
wxString::Format( _( "Removing previous version of package '%s'." ),
aPackage.identifier ),
RPT_SEVERITY_INFO );
deletePackageDirectories( aPackage.identifier, keep_on_update );
}
m_reporter->PCMReport(
wxString::Format( _( "Extracting package '%s'." ), aPackage.identifier ),
RPT_SEVERITY_INFO );
if( extract( aFilePath.GetFullPath(), aPackage.identifier, true ) )
{
m_pcm->MarkInstalled( aPackage, pkgver->version, aRepositoryId );
}
else
{
// Cleanup possibly partially extracted package
deletePackageDirectories( aPackage.identifier );
}
std::unique_lock lock( m_changed_package_types_guard );
m_changed_package_types.insert( aPackage.type );
}
m_reporter->PCMReport(
wxString::Format( _( "Removing downloaded archive '%s'." ), aFilePath.GetFullName() ),
RPT_SEVERITY_INFO );
wxRemoveFile( aFilePath.GetFullPath() );
}
bool PCM_TASK_MANAGER::extract( const wxString& aFilePath, const wxString& aPackageId,
bool isMultiThreaded )
{
wxFFileInputStream stream( aFilePath );
wxZipInputStream zip( stream );
wxLogNull no_wx_logging;
int entries = zip.GetTotalEntries();
int extracted = 0;
wxArchiveEntry* entry = zip.GetNextEntry();
if( !zip.IsOk() )
{
m_reporter->PCMReport( _( "Error extracting file!" ), RPT_SEVERITY_ERROR );
return false;
}
// Namespace delimiter changed on disk to allow flat loading of Python modules
wxString clean_package_id = aPackageId;
clean_package_id.Replace( '.', '_' );
for( ; entry; entry = zip.GetNextEntry() )
{
wxArrayString path_parts =
wxSplit( entry->GetName(), wxFileName::GetPathSeparator(), (wxChar) 0 );
if( path_parts.size() < 2
|| PCM_PACKAGE_DIRECTORIES.find( path_parts[0] ) == PCM_PACKAGE_DIRECTORIES.end()
|| path_parts[path_parts.size() - 1].IsEmpty() )
{
// Ignore files in the root of the archive and files outside of package dirs.
continue;
}
// m_reporter->Report( wxString::Format( _( "Extracting file '%s'\n" ), entry->GetName() ),
// RPT_SEVERITY_INFO );
// Transform paths from
// <PackageRoot>/$folder/$contents
// To
// $KICAD7_3RD_PARTY/$folder/$package_id/$contents
path_parts.Insert( clean_package_id, 1 );
path_parts.Insert( m_pcm->Get3rdPartyPath(), 0 );
wxString fullname = wxJoin( path_parts, wxFileName::GetPathSeparator(), (wxChar) 0 );
// Ensure the target directory exists and create it if not.
wxString t_path = wxPathOnly( fullname );
if( !wxDirExists( t_path ) )
{
wxFileName::Mkdir( t_path, wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL );
}
wxTempFileOutputStream out( fullname );
if( !( CopyStreamData( zip, out, entry->GetSize() ) && out.Commit() ) )
{
m_reporter->PCMReport( _( "Error extracting file!" ), RPT_SEVERITY_ERROR );
return false;
}
extracted++;
m_reporter->SetPackageProgress( extracted, entries );
if( !isMultiThreaded )
m_reporter->KeepRefreshing( false );
if( m_reporter->IsCancelled() )
break;
}
zip.CloseEntry();
if( m_reporter->IsCancelled() )
{
m_reporter->PCMReport( _( "Aborting package installation." ), RPT_SEVERITY_INFO );
return false;
}
m_reporter->PCMReport( _( "Extracted package\n" ), RPT_SEVERITY_INFO );
m_reporter->SetPackageProgress( entries, entries );
return true;
}
void PCM_TASK_MANAGER::InstallFromFile( wxWindow* aParent, const wxString& aFilePath )
{
wxFFileInputStream stream( aFilePath );
if( !stream.IsOk() )
{
wxLogError( _( "Could not open archive file." ) );
return;
}
wxZipInputStream zip( stream );
if( !zip.IsOk() )
{
wxLogError( _( "Invalid archive file format." ) );
return;
}
nlohmann::json metadata;
for( wxArchiveEntry* entry = zip.GetNextEntry(); entry != nullptr; entry = zip.GetNextEntry() )
{
// Find and load metadata.json
if( entry->GetName() != "metadata.json" )
continue;
wxStringOutputStream strStream;
if( CopyStreamData( zip, strStream, entry->GetSize() ) )
{
try
{
metadata = nlohmann::json::parse( strStream.GetString().ToUTF8().data() );
m_pcm->ValidateJson( metadata );
}
catch( const std::exception& e )
{
wxLogError( wxString::Format( _( "Unable to parse package metadata:\n\n%s" ),
e.what() ) );
break;
}
}
}
if( metadata.empty() )
{
wxLogError( _( "Archive does not contain a valid metadata.json file" ) );
return;
}
PCM_PACKAGE package = metadata.get<PCM_PACKAGE>();
if( package.versions.size() != 1 )
{
wxLogError( _( "Archive metadata must have a single version defined" ) );
return;
}
bool isUpdate = false;
// wxRegEx is not CopyConstructible hence the weird choice of forward_list
std::forward_list<wxRegEx> keep_on_update;
const std::vector<PCM_INSTALLATION_ENTRY> installed_packages = m_pcm->GetInstalledPackages();
if( std::find_if( installed_packages.begin(), installed_packages.end(),
[&]( const PCM_INSTALLATION_ENTRY& entry )
{
return entry.package.identifier == package.identifier;
} )
!= installed_packages.end() )
{
if( wxMessageBox(
wxString::Format(
_( "Package with identifier %s is already installed. "
"Would you like to update it to the version from selected file?" ),
package.identifier ),
_( "Update package" ), wxICON_EXCLAMATION | wxYES_NO, aParent )
== wxNO )
return;
isUpdate = true;
compile_keep_on_update_regex( package, package.versions[0], keep_on_update );
}
m_reporter = std::make_unique<DIALOG_PCM_PROGRESS>( aParent, false );
#ifdef __WXMAC__
m_reporter->ShowWindowModal();
#else
m_reporter->Show();
#endif
if( isUpdate )
{
m_reporter->PCMReport( wxString::Format( _( "Removing previous version of package '%s'." ),
package.identifier ),
RPT_SEVERITY_INFO );
deletePackageDirectories( package.identifier, keep_on_update );
}
if( extract( aFilePath, package.identifier, false ) )
m_pcm->MarkInstalled( package, package.versions[0].version, "" );
m_reporter->SetFinished();
m_reporter->KeepRefreshing( false );
m_reporter->Destroy();
m_reporter.reset();
aParent->Raise();
std::unique_lock lock( m_changed_package_types_guard );
m_changed_package_types.insert( package.type );
}
class PATH_COLLECTOR : public wxDirTraverser
{
private:
std::vector<wxString>& m_files;
std::vector<wxString>& m_dirs;
public:
explicit PATH_COLLECTOR( std::vector<wxString>& aFiles, std::vector<wxString>& aDirs ) :
m_files( aFiles ), m_dirs( aDirs )
{
}
wxDirTraverseResult OnFile( const wxString& aFilePath ) override
{
m_files.push_back( aFilePath );
return wxDIR_CONTINUE;
}
wxDirTraverseResult OnDir( const wxString& dirPath ) override
{
m_dirs.push_back( dirPath );
return wxDIR_CONTINUE;
}
};
void PCM_TASK_MANAGER::deletePackageDirectories( const wxString& aPackageId,
const std::forward_list<wxRegEx>& aKeep )
{
// Namespace delimiter changed on disk to allow flat loading of Python modules
wxString clean_package_id = aPackageId;
clean_package_id.Replace( '.', '_' );
int path_prefix_len = m_pcm->Get3rdPartyPath().Length();
auto sort_func = []( const wxString& a, const wxString& b )
{
if( a.length() > b.length() )
return true;
if( a.length() < b.length() )
return false;
if( a != b )
return a < b;
return false;
};
for( const wxString& dir : PCM_PACKAGE_DIRECTORIES )
{
wxFileName d( m_pcm->Get3rdPartyPath(), "" );
d.AppendDir( dir );
d.AppendDir( clean_package_id );
if( !d.DirExists() )
continue;
m_reporter->PCMReport( wxString::Format( _( "Removing directory %s" ), d.GetPath() ),
RPT_SEVERITY_INFO );
if( aKeep.empty() )
{
if( !d.Rmdir( wxPATH_RMDIR_RECURSIVE ) )
{
m_reporter->PCMReport(
wxString::Format( _( "Failed to remove directory %s" ), d.GetPath() ),
RPT_SEVERITY_ERROR );
}
}
else
{
std::vector<wxString> files;
std::vector<wxString> dirs;
PATH_COLLECTOR collector( files, dirs );
wxDir( d.GetFullPath() )
.Traverse( collector, wxEmptyString, wxDIR_DEFAULT | wxDIR_NO_FOLLOW );
// Do a poor mans post order traversal by sorting paths in reverse length order
std::sort( files.begin(), files.end(), sort_func );
std::sort( dirs.begin(), dirs.end(), sort_func );
// Delete files that don't match any of the aKeep regexs
for( const wxString& file : files )
{
bool del = true;
for( const wxRegEx& re : aKeep )
{
wxString tmp = file.Mid( path_prefix_len );
tmp.Replace( "\\", "/" );
if( re.Matches( tmp ) )
{
// m_reporter->PCMReport( wxString::Format( _( "Keeping file '%s'." ), tmp ),
// RPT_SEVERITY_INFO );
del = false;
break;
}
}
if( del )
wxRemoveFile( file );
}
// Delete any empty dirs
for( const wxString& empty_dir : dirs )
{
wxFileName dname( empty_dir, "" );
dname.Rmdir(); // not passing any flags here will only remove empty directories
}
}
}
}
void PCM_TASK_MANAGER::Uninstall( const PCM_PACKAGE& aPackage )
{
PCM_TASK task = [aPackage, this]
{
deletePackageDirectories( aPackage.identifier );
m_pcm->MarkUninstalled( aPackage );
std::unique_lock lock( m_changed_package_types_guard );
m_changed_package_types.insert( aPackage.type );
m_reporter->PCMReport(
wxString::Format( _( "Package %s uninstalled" ), aPackage.identifier ),
RPT_SEVERITY_INFO );
};
m_install_queue.push( task );
}
void PCM_TASK_MANAGER::RunQueue( wxWindow* aParent )
{
m_reporter = std::make_unique<DIALOG_PCM_PROGRESS>( aParent );
m_reporter->SetNumPhases( m_download_queue.size() + m_install_queue.size() );
#ifdef __WXMAC__
m_reporter->ShowWindowModal();
#else
m_reporter->Show();
#endif
wxSafeYield();
std::mutex mutex;
std::condition_variable condvar;
bool download_complete = false;
std::thread download_thread(
[&]()
{
while( !m_download_queue.empty() && !m_reporter->IsCancelled() )
{
PCM_TASK task;
m_download_queue.pop( task );
task();
condvar.notify_all();
}
std::unique_lock<std::mutex> lock( mutex );
download_complete = true;
condvar.notify_all();
} );
std::thread install_thread(
[&]()
{
std::unique_lock<std::mutex> lock( mutex );
do
{
condvar.wait( lock,
[&]()
{
return download_complete || !m_install_queue.empty()
|| m_reporter->IsCancelled();
} );
lock.unlock();
while( !m_install_queue.empty() && !m_reporter->IsCancelled() )
{
PCM_TASK task;
m_install_queue.pop( task );
task();
m_reporter->AdvancePhase();
}
lock.lock();
} while( ( !m_install_queue.empty() || !download_complete )
&& !m_reporter->IsCancelled() );
m_reporter->PCMReport( _( "Done." ), RPT_SEVERITY_INFO );
m_reporter->SetFinished();
} );
m_reporter->KeepRefreshing( true );
download_thread.join();
install_thread.join();
// Destroy the reporter only after the threads joined
// Incase the reporter terminated due to cancellation
m_reporter->Destroy();
m_reporter.reset();
aParent->Raise();
}