File test.patch of Package libzypp
commit fb7a073381a64309218bce7e5339d36df2e8a519
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date: Tue Sep 27 04:53:46 2022 +0000
Don't do 2 HTTP requests when one is enough
TODO: if the file does not exist, we need to handle it
improves time of zypper ref from 10.7 to 9.4s
diff --git a/zypp/MediaSetAccess.cc b/zypp/MediaSetAccess.cc
index b7faf2505..80e3be169 100644
--- a/zypp/MediaSetAccess.cc
+++ b/zypp/MediaSetAccess.cc
@@ -183,11 +183,12 @@ IMPL_PTR_TYPE(MediaSetAccess);
{
try
{
- if ( doesFileExist( file, media_nr ) )
+ //if ( doesFileExist( file, media_nr ) )
+ // handle not-found condition via MediaFileNotFoundException
return provideFile( OnMediaLocation( file, media_nr ), PROVIDE_NON_INTERACTIVE );
}
catch ( const media::MediaFileNotFoundException & excpt_r )
- { ZYPP_CAUGHT( excpt_r ); }
+ { } // ignore not found
catch ( const media::MediaForbiddenException & excpt_r )
{ ZYPP_CAUGHT( excpt_r ); }
catch ( const media::MediaNotAFileException & excpt_r )
commit c55053fe13d99f3d549516de0b170e086b363b5d
Author: Benjamin Zeller <bzeller@suse.de>
Date: Tue Sep 27 10:51:35 2022 +0200
Use a dynamic fallback for BLKSIZE in downloads
When not receiving a blocklist via metalink file from the server
MediaMultiCurl used to fallback to a fixed, relatively small BLKSIZE.
This patch changes the fallback into a dynamic value based on the
filesize using a similar metric as the MirrorCache implementation on the
server side.
diff --git a/zypp/media/MediaMultiCurl.cc b/zypp/media/MediaMultiCurl.cc
index f5986441f..72cecf229 100644
--- a/zypp/media/MediaMultiCurl.cc
+++ b/zypp/media/MediaMultiCurl.cc
@@ -121,6 +121,9 @@ public:
void run(std::vector<Url> &urllist);
protected:
+
+ static size_t makeBlksize ( size_t filesize );
+
friend class multifetchworker;
const MediaMultiCurl *_context;
@@ -139,6 +142,7 @@ protected:
bool _havenewjob;
size_t _blkno;
+ size_t _defaultBlksize = 0; //< The blocksize to use if the metalink file does not specify one
off_t _blkoff;
size_t _activeworkers;
size_t _lookupworkers;
@@ -164,7 +168,6 @@ public:
};
constexpr auto MIN_REQ_MIRRS = 4;
-constexpr auto BLKSIZE = 131072;
constexpr auto MAXURLS = 10;
//////////////////////////////////////////////////////////////////////
@@ -707,18 +710,19 @@ multifetchworker::nextjob()
MediaBlockList *blklist = _request->_blklist;
if (!blklist)
{
- _blksize = BLKSIZE;
+ _blksize = _request->_defaultBlksize;
if (_request->_filesize != off_t(-1))
- {
- if (_request->_blkoff >= _request->_filesize)
- {
- stealjob();
- return;
- }
- _blksize = _request->_filesize - _request->_blkoff;
- if (_blksize > BLKSIZE)
- _blksize = BLKSIZE;
- }
+ {
+ if (_request->_blkoff >= _request->_filesize)
+ {
+ stealjob();
+ return;
+ }
+ _blksize = _request->_filesize - _request->_blkoff;
+ if (_blksize > _request->_defaultBlksize)
+ _blksize = _request->_defaultBlksize;
+ }
+ DBG << "No BLOCKLIST falling back to chunk size: " << _request->_defaultBlksize << std::endl;
}
else
{
@@ -734,8 +738,10 @@ multifetchworker::nextjob()
_request->_blkoff = blk.off;
}
_blksize = blk.off + blk.size - _request->_blkoff;
- if (_blksize > BLKSIZE && !blklist->haveChecksum(_request->_blkno))
- _blksize = BLKSIZE;
+ if (_blksize > _request->_defaultBlksize && !blklist->haveChecksum(_request->_blkno)) {
+ DBG << "Block: "<< _request->_blkno << " has no checksum falling back to default blocksize: " << _request->_defaultBlksize << std::endl;
+ _blksize = _request->_defaultBlksize;
+ }
}
_blkno = _request->_blkno;
_blkstart = _request->_blkoff;
@@ -791,6 +797,7 @@ multifetchrequest::multifetchrequest(const MediaMultiCurl *context, const Pathna
_report = report;
_blklist = blklist;
_filesize = filesize;
+ _defaultBlksize = makeBlksize( filesize );
_multi = multi;
_stealing = false;
_havenewjob = false;
@@ -1156,6 +1163,15 @@ multifetchrequest::run(std::vector<Url> &urllist)
}
}
+inline size_t multifetchrequest::makeBlksize ( size_t filesize )
+{
+ // this case should never happen because we never start a multi download if we do not know the filesize beforehand
+ if ( filesize == 0 ) return 2 * 1024 * 1024;
+ else if ( filesize < 2*256*1024 ) return filesize;
+ else if ( filesize < 8*1024*1024 ) return 256*1024;
+ else if ( filesize < 256*1024*1024 ) return 1024*1024;
+ return 4*1024*1024;
+}
//////////////////////////////////////////////////////////////////////
@@ -1610,4 +1626,3 @@ void MediaMultiCurl::toEasyPool(const std::string &host, CURL *easy) const
} // namespace media
} // namespace zypp
-
commit cf5026534730351953cbd442b197b1bb96b0d229
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date: Tue Sep 27 04:42:14 2022 +0000
Drop fetch of media.1
changes time for zypper ref from 13.4 to 10.7 seconds
diff --git a/zypp/repo/yum/Downloader.cc b/zypp/repo/yum/Downloader.cc
index 4b495a8c3..1a1158d6e 100644
--- a/zypp/repo/yum/Downloader.cc
+++ b/zypp/repo/yum/Downloader.cc
@@ -201,9 +201,11 @@ namespace yum
RepoStatus Downloader::status( MediaSetAccess & media_r )
{
RepoStatus ret { media_r.provideOptionalFile( repoInfo().path() / "/repodata/repomd.xml" ) };
+ /*
if ( !ret.empty() ) // else: mandatory master index is missing
ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
// else: mandatory master index is missing -> stay empty
+ */
return ret;
}
} // namespace yum
commit d58ffe449e8af09fff7efdf9500241b5d7e0e553
Author: Bernhard M. Wiedemann <bwiedemann@suse.de>
Date: Sat Oct 1 19:46:25 2022 +0200
Add ZYPP_NOCHUNK
diff --git a/zypp/media/MediaMultiCurl.cc b/zypp/media/MediaMultiCurl.cc
index 72cecf229..f532ca8e1 100644
--- a/zypp/media/MediaMultiCurl.cc
+++ b/zypp/media/MediaMultiCurl.cc
@@ -700,6 +700,7 @@ multifetchworker::disableCompetition()
void
multifetchworker::nextjob()
{
+ const char *nochunk = getenv("ZYPP_NOCHUNK");
_noendrange = false;
if (_request->_stealing)
{
@@ -708,11 +709,12 @@ multifetchworker::nextjob()
}
MediaBlockList *blklist = _request->_blklist;
- if (!blklist)
+ if (nochunk || !blklist)
{
_blksize = _request->_defaultBlksize;
if (_request->_filesize != off_t(-1))
{
+ if (nochunk) _blksize = _request->_filesize;
if (_request->_blkoff >= _request->_filesize)
{
stealjob();
