File createrepo-0.9.8-add-lzma-option-to-generate-pr... of Package createrepo

Overview Repositories Revisions Requests Users Attributes Meta

File createrepo-0.9.8-add-lzma-option-to-generate-primary.xml.lzma.patch of Package createrepo (Revision f8d9e8e8a445bc2704e209bc22b89886)

Currently displaying revision f8d9e8e8a445bc2704e209bc22b89886 , Show latest

--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -45,7 +45,7 @@ try:
 except ImportError:
     pass
 
-from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \
+from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, HybridFile, \
                   checksum_and_rename
 import deltarpms
 
@@ -389,7 +389,10 @@ class MetaDataGenerator:
         # setup the primary metadata file
         primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
                                        self.conf.primaryfile)
-        fo = _gzipOpen(primaryfilepath, 'w')
+        if self.conf.lzma:
+            fo = HybridFile(primaryfilepath, 'w')
+        else:
+            fo = GzipFile(primaryfilepath, 'w')
         fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
         fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
             ' xmlns:suse="http://novell.com/package/metadata/suse/common"' \
@@ -870,18 +873,30 @@ class MetaDataGenerator:
             except AttributeError:
                 dbversion = '9'
             rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None)
+        if self.conf.lzma:
+            workfiles.append((self.conf.primaryfile[:-3] + ".lzma", "primary_lzma"))
 
+        open_csums = {}
+        open_sizes = {}
         for (rpm_file, ftype) in workfiles:
             complete_path = os.path.join(repopath, rpm_file)
 
-            zfo = _gzipOpen(complete_path)
-            # This is misc.checksum() done locally so we can get the size too.
-            data = misc.Checksums([sumtype])
-            while data.read(zfo, 2**16):
-                pass
-            uncsum = data.hexdigest(sumtype)
-            unsize = len(data)
-            zfo.close()
+            if ftype.endswith("_lzma"):
+                # reuse the open-checksum computed for the .gz file
+                type = ftype[:-5]
+                uncsum = open_csums[type]
+                unsize = open_sizes[type]
+            else:
+                zfo = _gzipOpen(complete_path)
+                # This is misc.checksum() done locally so we can get the size too.
+                data = misc.Checksums([sumtype])
+                while data.read(zfo, 2**16):
+                    pass
+                uncsum = data.hexdigest(sumtype)
+                unsize = len(data)
+                zfo.close()
+                open_csums[ftype] = uncsum
+                open_sizes[ftype] = unsize
             csum = misc.checksum(sumtype, complete_path)
             timestamp = os.stat(complete_path)[8]
 
@@ -981,7 +996,7 @@ class MetaDataGenerator:
             if self.conf.baseurl is not None:
                 location.newProp('xml:base', self.conf.baseurl)
             if self.conf.unique_md_filenames:
-                res_file = '%s-%s.xml.gz' % (csum, ftype)
+                res_file = '%s-%s' % (csum, rpm_file)
                 orig_file = os.path.join(repopath, rpm_file)
                 dest_file = os.path.join(repopath, res_file)
                 os.rename(orig_file, dest_file)
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -21,6 +21,7 @@ import os.path
 import sys
 import bz2
 import gzip
+import subprocess
 from gzip import write32u, FNAME
 from yum import misc
 
@@ -50,6 +51,46 @@ class GzipFile(gzip.GzipFile):
         if fname:
             self.fileobj.write(fname + '\000')
 
+class LzmaFile:
+    def __init__(self, filename, mode, compresslevel=None):
+        if mode != "w" and mode != "wb":
+            raise NotImplementedError("Only writing of lzma files is supported")
+        self.file = open(filename, mode)
+        cmd = ["lzma"]
+        if compresslevel:
+                cmd.append("-%d" % compresslevel)
+        self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                stdout=self.file)
+        return
+
+    def write(self, text):
+        self.process.stdin.write(text)
+        return
+
+    def close(self):
+        self.process.stdin.close()
+        self.file.close()
+        return
+
+class HybridFile:
+    def __init__(self, filename, mode):
+        if filename.endswith(".gz"):
+            filename = filename[:-3]
+        if filename.endswith(".lzma"):
+            filename = filename[:-5]
+        self.lzma = GzipFile(filename + ".gz", mode)
+        self.gz = LzmaFile(filename + ".lzma", mode)
+        return
+
+    def write(self, text):
+        self.gz.write(text)
+        self.lzma.write(text)
+        return
+
+    def close(self):
+        self.gz.close()
+        self.lzma.close()
+        return
 
 def _gzipOpen(filename, mode="rb", compresslevel=9):
     return GzipFile(filename, mode, compresslevel)
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -70,6 +70,8 @@ def parse_args(args, conf):
     #parser.add_option("--database-only", default=False, action="store_true",
     #  dest='database_only',
     #  help="Only make the sqlite databases - does not work with --update, yet")
+    parser.add_option("--lzma", default=False, action="store_true",
+	help="create lzma-compressed metadata")
     parser.add_option("--update", default=False, action="store_true",
         help="use the existing repodata to speed up creation of new")
     parser.add_option("--update-md-path", default=None, dest='update_md_path',

Places

File createrepo-0.9.8-add-lzma-option-to-generate-primary.xml.lzma.patch of Package createrepo (Revision f8d9e8e8a445bc2704e209bc22b89886)

Places