File 0001-Add-predownload-plugin.patch of Package libzypp
From bf30279f77f56a5858a4434a6d6c1b40dc1c19a4 Mon Sep 17 00:00:00 2001
From: Andrii Nikitin <anikitin@suse.de>
Date: Mon, 16 Sep 2024 19:06:16 +0200
Subject: [PATCH] Add predownload plugin
---
zypp/PluginExecutor.cc | 6 ++
zypp/PluginExecutor.h | 3 +
zypp/PluginScript.cc | 22 +++++
zypp/PluginScript.h | 10 ++
zypp/target/CommitPackageCache.cc | 3 +
zypp/target/CommitPackageCache.h | 1 +
zypp/target/CommitPackageCacheImpl.h | 5 +
zypp/target/TargetImpl.cc | 136 +++++++++++++++++++++++++++
8 files changed, 186 insertions(+)
diff --git a/zypp/PluginExecutor.cc b/zypp/PluginExecutor.cc
index 826a915cf..3cefba596 100644
--- a/zypp/PluginExecutor.cc
+++ b/zypp/PluginExecutor.cc
@@ -101,6 +101,9 @@ namespace zypp
const std::list<PluginScript> scripts() const
{ return _scripts; }
+ PluginScript first()
+ { return *_scripts.begin(); }
+
private:
/** Launch a plugin sending PLUGINSTART message. */
void doLoad( const PathInfo & pi_r )
@@ -177,6 +180,9 @@ namespace zypp
void PluginExecutor::send( const PluginFrame & frame_r )
{ _pimpl->send( frame_r ); }
+ PluginScript PluginExecutor::first()
+ { return _pimpl->first(); }
+
std::ostream & operator<<( std::ostream & str, const PluginExecutor & obj )
{ return str << obj._pimpl->scripts(); }
diff --git a/zypp/PluginExecutor.h b/zypp/PluginExecutor.h
index 275112734..51cafd39e 100644
--- a/zypp/PluginExecutor.h
+++ b/zypp/PluginExecutor.h
@@ -61,6 +61,9 @@ namespace zypp
/** Number of open plugins */
size_t size() const;
+ /** First plugin */
+ PluginScript first();
+
public:
/** Find and launch plugins sending \c PLUGINBEGIN.
*
diff --git a/zypp/PluginScript.cc b/zypp/PluginScript.cc
index 89fcf0796..94e1bd09b 100644
--- a/zypp/PluginScript.cc
+++ b/zypp/PluginScript.cc
@@ -175,6 +175,8 @@ namespace zypp
void send( const PluginFrame & frame_r ) const;
+ Progress progress() const;
+
PluginFrame receive() const;
private:
@@ -274,6 +276,23 @@ namespace zypp
return _lastReturn;
}
+ PluginScript::Progress PluginScript::Impl::progress() const
+ {
+ const PluginFrame frame_r = PluginFrame( "PLUGIN_PROGRESS" );
+ PluginFrame resp;
+ Progress ret(-1,-1);
+ this->send( frame_r );
+ resp = this->receive();
+ if(resp.empty())
+ return ret;
+
+ ret.first = atoi( resp.getHeaderNT("PLUGIN_PROGRESS_CURRENT", "-1").c_str() );
+ ret.second = atoi( resp.getHeaderNT("PLUGIN_PROGRESS_MAX", "-1").c_str() );
+
+ return ret;
+ }
+
+
void PluginScript::Impl::send( const PluginFrame & frame_r ) const
{
if ( !_cmd )
@@ -519,6 +538,9 @@ namespace zypp
void PluginScript::send( const PluginFrame & frame_r ) const
{ _pimpl->send( frame_r ); }
+ PluginScript::Progress PluginScript::progress() const
+ { return _pimpl->progress(); }
+
PluginFrame PluginScript::receive() const
{ return _pimpl->receive(); }
diff --git a/zypp/PluginScript.h b/zypp/PluginScript.h
index fc6410cca..881bc7626 100644
--- a/zypp/PluginScript.h
+++ b/zypp/PluginScript.h
@@ -66,6 +66,7 @@ namespace zypp
public:
/** Commandline arguments passed to a script on \ref open. */
using Arguments = std::vector<std::string>;
+ using Progress = std::pair<int, int>;
/** \c pid_t(-1) constant indicating no connection. */
static const pid_t NotConnected;
@@ -171,6 +172,15 @@ namespace zypp
*
*/
void send( const PluginFrame & frame_r ) const;
+
+ /** Send PLUGIN_PROGRESS frame and return /ref PluginScript::Progress
+ * \throw PluginScriptNotConnected
+ * \throw PluginScriptSendTimeout
+ * \throw PluginScriptDiedUnexpectedly (does not \ref close)
+ * \throw PluginScriptException on error
+ *
+ */
+ Progress progress() const;
/** Receive a \ref PluginFrame.
* \throw PluginScriptNotConnected
diff --git a/zypp/target/CommitPackageCache.cc b/zypp/target/CommitPackageCache.cc
index 1a5dee8ac..d91522164 100644
--- a/zypp/target/CommitPackageCache.cc
+++ b/zypp/target/CommitPackageCache.cc
@@ -143,6 +143,9 @@ namespace zypp
ManagedFile CommitPackageCache::get( const PoolItem & citem_r )
{ return _pimpl->get( citem_r ); }
+ ManagedFile CommitPackageCache::get_from_cache( const PoolItem & citem_r )
+ { return _pimpl->get_from_cache( citem_r ); }
+
bool CommitPackageCache::preloaded() const
{ return _pimpl->preloaded(); }
diff --git a/zypp/target/CommitPackageCache.h b/zypp/target/CommitPackageCache.h
index bf50c92dd..4be69eb14 100644
--- a/zypp/target/CommitPackageCache.h
+++ b/zypp/target/CommitPackageCache.h
@@ -84,6 +84,7 @@ namespace zypp
/** Provide a package. */
ManagedFile get( const PoolItem & citem_r );
+ ManagedFile get_from_cache( const PoolItem & citem_r );
/** \overload */
ManagedFile get( sat::Solvable citem_r )
{ return get( PoolItem(citem_r) ); }
diff --git a/zypp/target/CommitPackageCacheImpl.h b/zypp/target/CommitPackageCacheImpl.h
index 95ac84936..5b5a9284d 100644
--- a/zypp/target/CommitPackageCacheImpl.h
+++ b/zypp/target/CommitPackageCacheImpl.h
@@ -59,6 +59,11 @@ namespace zypp
return sourceProvidePackage( citem_r );
}
+ virtual ManagedFile get_from_cache( const PoolItem & citem_r )
+ {
+ return sourceProvideCachedPackage( citem_r );
+ }
+
void setCommitList( std::vector<sat::Solvable> commitList_r )
{ _commitList = std::move(commitList_r); }
diff --git a/zypp/target/TargetImpl.cc b/zypp/target/TargetImpl.cc
index 690d1c907..579be59df 100644
--- a/zypp/target/TargetImpl.cc
+++ b/zypp/target/TargetImpl.cc
@@ -1301,6 +1301,8 @@ namespace zypp
MIL << "Target loaded: " << system.solvablesSize() << " resolvables" << endl;
}
+ void predownloadPluginsHook( CommitPackageCache & packageCache, const ZYppCommitResult::TransactionStepList & steps );
+
///////////////////////////////////////////////////////////////////
//
// COMMIT
@@ -1466,6 +1468,9 @@ namespace zypp
bool miss = false;
if ( policy_r.downloadMode() != DownloadAsNeeded )
{
+
+ predownloadPluginsHook(packageCache, steps);
+
// Preload the cache. Until now this means pre-loading all packages.
// Once DownloadInHeaps is fully implemented, this will change and
// we may actually have more than one heap.
@@ -3060,6 +3065,137 @@ namespace zypp
repo::SrcPackageProvider prov( access_r );
return prov.provideSrcPackage( srcPackage_r );
}
+
+
+ void predownloadPluginsHook( CommitPackageCache & packageCache, const ZYppCommitResult::TransactionStepList & steps )
+ {
+ PluginExecutor plugins;
+ plugins.load( ZConfig::instance().pluginsPath()/"predownload" );
+ if ( ! plugins )
+ return;
+ MIL << "TargetImpl::predownloadPluginsHook() start" << endl;
+
+ // first build items for download, skipping those which are already in the cache
+ ZYppCommitResult::TransactionStepList stepsNotCached;
+ for_( it, steps.begin(), steps.end() )
+ {
+ switch ( it->stepType() )
+ {
+ case sat::Transaction::TRANSACTION_INSTALL:
+ case sat::Transaction::TRANSACTION_MULTIINSTALL:
+ // proceed: only install actionas may require download.
+ break;
+
+ default:
+ // next: no download for or non-packages and delete actions.
+ continue;
+ break;
+ }
+
+ PoolItem pi( *it );
+ if ( pi->isKind<Package>() || pi->isKind<SrcPackage>() )
+ {
+ ManagedFile localfile;
+ localfile = packageCache.get_from_cache( pi );
+ if (!localfile->empty())
+ localfile.resetDispose(); // keep the package file in the cache
+ else {
+ stepsNotCached.push_back(*it);
+ }
+ }
+ }
+ using namespace std;
+ if (stepsNotCached.empty()) {
+ cout << "Nothing to download" << endl;
+ return;
+ }
+ cout << "Preparing predownload plugins..." << endl;
+
+ string message = ( ZConfig::instance().repoCachePath()/"packages" ).asString();
+ plugins.send( PluginFrame( "PREDOWNLOAD_DEST", message ) );
+ if ( plugins.empty() ) {
+ cout << "predownload plugins have died" << endl;
+ return;
+ }
+
+ map<string, Url> repoUrls;
+ map<string, list<string> > repoFiles;
+ for_(it, stepsNotCached.begin(), stepsNotCached.end()) {
+ if ( sat::Solvable solv = it->satSolvable() ) {
+ Repository repo = solv.repository();
+ string alias = repo.alias();
+ Url url = repo.info().url();
+ if (!url.isValid())
+ continue;
+ PoolItem pi( solv );
+
+ Package::constPtr p = pi->asKind<Package>();
+ string loc = p->location().filename().asString();
+
+ if (loc.size() < 2)
+ continue;
+ // let's trim leading './'
+ if (loc[0] == '.' && loc[1] == '/')
+ loc = loc.substr(2);
+
+ repoUrls[alias] = url;
+ repoFiles[alias].push_back(loc);
+ }
+ }
+
+ for_(it, repoUrls.begin(), repoUrls.end()) {
+ message = it->first + string(" ") + it->second.asString();
+ list<string>& files = repoFiles[it->first];
+ if (files.empty())
+ continue;
+ for_(i, files.begin(), files.end()) {
+ message += " " + *i;
+ }
+ message += " "; // make sure there is a delimiter after last package
+ plugins.send( PluginFrame( "PREDOWNLOAD_FROM_REPO", message ) );
+ }
+ if ( plugins.empty() ) {
+ cout << "predownload plugins all have died" << endl;
+ return;
+ }
+ // only the first survived plugin will be called for now
+ PluginScript plugin = plugins.first();
+ cout << "Starting predownload plugin " << plugin.script() << endl;
+
+ plugin.send( PluginFrame( "PREDOWNLOAD_START" ) );
+ plugin.receive();
+ typedef PluginScript::Progress Progress;
+ Progress last(0,0);
+ bool allgood = 0;
+
+ while(1) {
+ Progress pr = plugin.progress();
+ DBG << "predownload progress " << pr.first << '/' << pr.second << endl;
+ if (pr.second < 1) {
+ cout << "predownload plugins have failed to report progress" << endl;
+ break;
+ }
+ if (last.first != pr.first) {
+ cout << "Waiting predownload... Progress: " << pr.first << "/" << pr.second << endl;
+ last = pr;
+ }
+ if (pr.first < pr.second)
+ sleep(1);
+ else {
+ if (pr.first > 0)
+ allgood = 1;
+ break;
+ }
+ }
+ const string & lastError = plugin.lastExecError();
+ if (!lastError.empty()) {
+ cout << "predownload plugins error: " << lastError << endl;
+ } else if (allgood) {
+ cout << "predownload plugin finished without errors" << endl;
+ }
+
+ MIL << "TargetImpl::predownloadPluginsHook() end" << endl;
+ }
////////////////////////////////////////////////////////////////
} // namespace target
///////////////////////////////////////////////////////////////////
--
2.46.0
