Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:AndreasSchwab:13.1
python3-rpm
zstd.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File zstd.patch of Package python3-rpm
From 3684424fe297c996bb05bb64631336fa2903df12 Mon Sep 17 00:00:00 2001 From: Jeff Johnson <n3npq@mac.com> Date: Wed, 9 Aug 2017 16:42:56 +0200 Subject: [PATCH] Add support for zstd compressed payload v2 (Igor Gnatenko): * Switch off from RPM_CHECK_LIB * Reference zstd from rpm.pc * Link rpmio with zstd v3 (Florian Festi): * move changes to cvtfmode into separate patches * do not error out on wrong compression levels * ifdef out zstdio Closes: https://github.com/rpm-software-management/rpm/issues/256 Closes: https://github.com/rpm-software-management/rpm/issues/297 Signed-off-by: Igor Gnatenko <i.gnatenko.brain@gmail.com> Index: build/pack.c =================================================================== --- build/pack.c.orig +++ build/pack.c @@ -296,6 +296,12 @@ static rpmRC writeRPM(Package pkg, unsig compr = "lzma"; (void) rpmlibNeedsFeature(pkg, "PayloadIsLzma", "4.4.6-1"); #endif +#ifdef HAVE_ZSTD + } else if (rstreq(s+1, "zstdio")) { + compr = "zstd"; + /* Add prereq on rpm version that understands zstd payloads */ + (void) rpmlibNeedsFeature(pkg, "PayloadIsZstd", "5.4.18-1"); +#endif } else { rpmlog(RPMLOG_ERR, _("Unknown payload compression: %s\n"), rpmio_flags); Index: build/parsePrep.c =================================================================== --- build/parsePrep.c.orig +++ build/parsePrep.c @@ -197,6 +197,9 @@ static char *doUntar(rpmSpec spec, uint3 t = "%{__7zip} x"; needtar = 0; break; + case COMPRESSED_ZSTD: + t = "%{__zstd} -dc"; + break; } zipper = rpmGetPath(t, NULL); if (needtar) { Index: configure.ac =================================================================== --- configure.ac.orig +++ configure.ac @@ -134,6 +134,7 @@ AC_PATH_PROG(__SED, sed, /bin/sed, $MYPA AC_PATH_PROG(__SEMODULE, semodule, /usr/bin/semodule, $MYPATH) AC_PATH_PROG(__SSH, ssh, /usr/bin/ssh, $MYPATH) AC_PATH_PROG(__TAR, tar, /bin/tar, $MYPATH) +AC_PATH_PROG(__ZSTD, zstd, /usr/bin/zstd, $MYPATH) AC_PATH_PROG(__LD, ld, /usr/bin/ld, $MYPATH) AC_PATH_PROG(__NM, nm, /usr/bin/nm, $MYPATH) @@ -201,6 +202,31 @@ AC_CHECK_HEADERS([lzma.h],[ AC_SUBST(WITH_LZMA_LIB) #================= +# Check for zstd. + +AC_ARG_ENABLE([zstd], + [AS_HELP_STRING([--enable-zstd=@<:@yes/no/auto@:>@], + [build without zstd support (default=auto)])], + [enable_zstd="$enableval"], + [enable_zstd=auto]) + +AS_IF([test "x$enable_zstd" != "xno"], [ + PKG_CHECK_MODULES([ZSTD], [libzstd], [have_zstd=yes], [have_zstd=no]) + AS_IF([test "$enable_zstd" = "yes"], [ + if test "$have_zstd" = "no"; then + AC_MSG_ERROR([--enable-zstd specified, but not available]) + fi + ]) +]) + +if test "x$have_zstd" = "xyes"; then + AC_DEFINE([HAVE_ZSTD], [1], [Define if libzstd is available]) + ZSTD_REQUIRES=libzstd + AC_SUBST(ZSTD_REQUIRES) +fi +AM_CONDITIONAL([HAVE_ZSTD], [test "x$have_zstd" = "xyes"]) + +#================= dnl dnl Check for features Index: lib/rpmds.c =================================================================== --- lib/rpmds.c.orig +++ lib/rpmds.c @@ -995,6 +995,11 @@ static const struct rpmlibProvides_s rpm { "rpmlib(TildeInVersions)", "4.10.0-1", ( RPMSENSE_EQUAL), N_("dependency comparison supports versions with tilde.") }, +#ifdef HAVE_ZSTD + { "rpmlib(PayloadIsZstd)", "5.4.18-1", + (RPMSENSE_RPMLIB|RPMSENSE_EQUAL), + N_("package payload can be compressed using zstd.") }, +#endif { NULL, NULL, 0, NULL } }; Index: macros.in =================================================================== --- macros.in.orig +++ macros.in @@ -68,6 +68,7 @@ %__ssh @__SSH@ %__tar @__TAR@ %__unzip @__UNZIP@ +%__zstd @__ZSTD@ %__git @__GIT@ %__hg @__HG@ %__bzr @__BZR@ Index: rpm.pc.in =================================================================== --- rpm.pc.in.orig +++ rpm.pc.in @@ -8,7 +8,7 @@ Name: RPM Description: RPM Package Manager Version: @VERSION@ URL: http://rpm.org -# Requires: +Requires.private: @ZSTD_REQUIRES@ # Conflicts: Cflags: -I${includedir} Libs: -L${libdir} -lrpm -lrpmio Index: rpmio/Makefile.am =================================================================== --- rpmio/Makefile.am.orig +++ rpmio/Makefile.am @@ -4,6 +4,7 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_ AM_CPPFLAGS += @WITH_NSS_INCLUDE@ AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@ AM_CPPFLAGS += @WITH_POPT_INCLUDE@ +AM_CPPFLAGS += $(ZSTD_CFLAGS) AM_CPPFLAGS += -I$(top_srcdir)/misc AM_CPPFLAGS += -DRPMCONFIGDIR="\"@RPMCONFIGDIR@\"" AM_CPPFLAGS += -DLOCALSTATEDIR="\"$(localstatedir)\"" @@ -35,6 +36,7 @@ librpmio_la_LIBADD = \ @WITH_LIBELF_LIB@ \ @WITH_POPT_LIB@ \ @WITH_LZMA_LIB@ \ + $(ZSTD_LIBS) \ -lpthread if WITH_INTERNAL_BEECRYPT Index: rpmio/macro.c =================================================================== --- rpmio/macro.c.orig +++ rpmio/macro.c @@ -843,6 +843,9 @@ doFoo(MacroBuf mb, int negate, const cha case COMPRESSED_7ZIP: sprintf(be, "%%__7zip x %s", b); break; + case COMPRESSED_ZSTD: + sprintf(be, "%%__zstd -dc %s", b); + break; } b = be; } else if (STREQ("getenv", f, fn)) { Index: rpmio/rpmfileutil.c =================================================================== --- rpmio/rpmfileutil.c.orig +++ rpmio/rpmfileutil.c @@ -347,6 +347,9 @@ int rpmFileIsCompressed(const char * fil (magic[4] == 0x5a) && (magic[5] == 0x00)) { /* new style xz (lzma) with magic */ *compressed = COMPRESSED_XZ; + } else if ((magic[0] == 0x28) && (magic[1] == 0x85) && + (magic[2] == 0x2f) ) { + *compressed = COMPRESSED_ZSTD; } else if ((magic[0] == 'L') && (magic[1] == 'Z') && (magic[2] == 'I') && (magic[3] == 'P')) { *compressed = COMPRESSED_LZIP; Index: rpmio/rpmfileutil.h =================================================================== --- rpmio/rpmfileutil.h.orig +++ rpmio/rpmfileutil.h @@ -26,7 +26,8 @@ typedef enum rpmCompressedMagic_e { COMPRESSED_XZ = 5, /*!< xz can handle */ COMPRESSED_LZIP = 6, /*!< lzip can handle */ COMPRESSED_LRZIP = 7, /*!< lrzip can handle */ - COMPRESSED_7ZIP = 8 /*!< 7zip can handle */ + COMPRESSED_7ZIP = 8, /*!< 7zip can handle */ + COMPRESSED_ZSTD = 10 /*!< zstd can handle */ } rpmCompressedMagic; /** \ingroup rpmfileutil Index: rpmio/rpmio.c =================================================================== --- rpmio/rpmio.c.orig +++ rpmio/rpmio.c @@ -175,6 +175,9 @@ static const FDIO_t gzdio; static const FDIO_t bzdio; static const FDIO_t xzdio; static const FDIO_t lzdio; +#ifdef HAVE_ZSTD +static const FDIO_t zstdio; +#endif /** \ingroup rpmio * Update digest(s) attached to fd. @@ -220,6 +223,10 @@ static const char * fdbg(FD_t fd) } else if (fps->io == lzdio) { sprintf(be, "LZD %p fdno %d", fps->fp, fps->fdno); #endif +#if HAVE_ZSTD + } else if (fps->io == zstdio) { + sprintf(be, "ZSTD %p fdno %d", fps->fp, fps->fdno); +#endif } else { sprintf(be, "??? io %p fp %p fdno %d ???", fps->io, fps->fp, fps->fdno); @@ -858,6 +865,11 @@ static const char * getFdErrstr (FD_t fd errstr = fd->errcookie; } else #endif /* HAVE_LZMA_H */ +#ifdef HAVE_ZSTD + if (fdGetIo(fd) == zstdio) { + errstr = fd->errcookie; + } else +#endif /* HAVE_ZSTD */ { errstr = (fd->syserrno ? strerror(fd->syserrno) : ""); } @@ -1197,6 +1209,284 @@ static const FDIO_t lzdio = &lzdio_s; #endif /* HAVE_LZMA_H */ /* =============================================================== */ +/* Support for ZSTD library. */ +#ifdef HAVE_ZSTD + +#include <zstd.h> + +typedef struct rpmzstd_s { + int flags; /*!< open flags. */ + int fdno; + int level; /*!< compression level */ + FILE * fp; + void * _stream; /*!< ZSTD_{C,D}Stream */ + size_t nb; + void * b; + ZSTD_inBuffer zib; /*!< ZSTD_inBuffer */ + ZSTD_outBuffer zob; /*!< ZSTD_outBuffer */ +} * rpmzstd; + +static rpmzstd rpmzstdNew(const char *path, int fdno, const char *fmode) +{ + int flags = 0; + int level = 3; + const char * s = fmode; + char stdio[32]; + char *t = stdio; + char *te = t + sizeof(stdio) - 2; + int c; + + switch ((c = *s++)) { + case 'a': + *t++ = (char)c; + flags &= ~O_ACCMODE; + flags |= O_WRONLY | O_CREAT | O_APPEND; + break; + case 'w': + *t++ = (char)c; + flags &= ~O_ACCMODE; + flags |= O_WRONLY | O_CREAT | O_TRUNC; + break; + case 'r': + *t++ = (char)c; + flags &= ~O_ACCMODE; + flags |= O_RDONLY; + break; + } + + while ((c = *s++) != 0) { + switch (c) { + case '.': + break; + case '+': + if (t < te) *t++ = c; + flags &= ~O_ACCMODE; + flags |= O_RDWR; + continue; + break; + default: + if (c >= (int)'0' && c <= (int)'9') { + level = strtol(s-1, (char **)&s, 10); + if (level < 1){ + level = 1; + rpmlog(RPMLOG_WARNING, "Invalid compression level for zstd. Using %i instead.\n", 1); + } + if (level > 19) { + level = 19; + rpmlog(RPMLOG_WARNING, "Invalid compression level for zstd. Using %i instead.\n", 19); + } + } + continue; + break; + } + break; + } + *t = '\0'; + + FILE * fp = path ? fopen(path, stdio) : fdopen(fdno, stdio); + if (fp == NULL) + return NULL; + + void * _stream = NULL; + size_t nb = 0; + + if ((flags & O_ACCMODE) == O_RDONLY) { /* decompressing */ + if ((_stream = (void *) ZSTD_createDStream()) == NULL + || ZSTD_isError(ZSTD_initDStream(_stream))) { + return NULL; + } + nb = ZSTD_DStreamInSize(); + } else { /* compressing */ + if ((_stream = (void *) ZSTD_createCStream()) == NULL + || ZSTD_isError(ZSTD_initCStream(_stream, level))) { + return NULL; + } + nb = ZSTD_CStreamOutSize(); + } + + rpmzstd zstd = (rpmzstd) xcalloc(1, sizeof(*zstd)); + zstd->flags = flags; + zstd->fdno = fdno; + zstd->level = level; + zstd->fp = fp; + zstd->_stream = _stream; + zstd->nb = nb; + zstd->b = xmalloc(nb); + + return zstd; +} + +static void * zstdFileno(FD_t fd) +{ + void * rc = NULL; + + if (fd == NULL) + return NULL; + + for (int i = fd->nfps; i >= 0; i--) { + FDSTACK_t * fps = &fd->fps[i]; + if (fps->io != zstdio) + continue; + rc = fps->fp; + break; + } + return rc; +} + +static FD_t zstdOpen(const char * path, const char * fmode) +{ + FD_t fd; + rpmzstd zstd; + + zstd = rpmzstdNew(path, -1, fmode); + if (zstd == NULL) return NULL; + fd = fdNew(path); + fdPop(fd); fdPush(fd, zstdio, zstd, -1); + return fdLink(fd); +} + +static FD_t zstdFdopen(FD_t fd, const char * fmode) +{ + int fdno; + rpmzstd zstd; + + if (fd == NULL || fmode == NULL) return NULL; + fdno = fdFileno(fd); + fdSetFdno(fd, -1); /* XXX skip the fdio close */ + if (fdno < 0) return NULL; + zstd = rpmzstdNew(NULL, fdno, fmode); + if (zstd == NULL) return NULL; + fdPush(fd, zstdio, zstd, fdno); /* Push zstdio onto stack */ + return fdLink(fd); +} + +static int zstdFlush(FD_t fd) +{ + rpmzstd zstd = zstdFileno(fd); +assert(zstd); + int rc = -1; + + if ((zstd->flags & O_ACCMODE) == O_RDONLY) { /* decompressing */ + rc = 0; + } else { /* compressing */ + /* close frame */ + zstd->zob.dst = zstd->b; + zstd->zob.size = zstd->nb; + zstd->zob.pos = 0; + int xx = ZSTD_flushStream(zstd->_stream, &zstd->zob); + if (ZSTD_isError(xx)) + fd->errcookie = ZSTD_getErrorName(xx); + else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp)) + fd->errcookie = "zstdFlush fwrite failed."; + else + rc = 0; + } + return rc; +} + +static ssize_t zstdRead(FD_t fd, void * buf, size_t count) +{ + rpmzstd zstd = zstdFileno(fd); +assert(zstd); + ZSTD_outBuffer zob = { buf, count, 0 }; + + while (zob.pos < zob.size) { + /* Re-fill compressed data buffer. */ + if (zstd->zib.pos >= zstd->zib.size) { + zstd->zib.size = fread(zstd->b, 1, zstd->nb, zstd->fp); + if (zstd->zib.size == 0) + break; /* EOF */ + zstd->zib.src = zstd->b; + zstd->zib.pos = 0; + } + + /* Decompress next chunk. */ + int xx = ZSTD_decompressStream(zstd->_stream, &zob, &zstd->zib); + if (ZSTD_isError(xx)) { + fd->errcookie = ZSTD_getErrorName(xx); + return -1; + } + } + return zob.pos; +} + +static ssize_t zstdWrite(FD_t fd, const void * buf, size_t count) +{ + rpmzstd zstd = zstdFileno(fd); +assert(zstd); + ZSTD_inBuffer zib = { buf, count, 0 }; + + while (zib.pos < zib.size) { + + /* Reset to beginning of compressed data buffer. */ + zstd->zob.dst = zstd->b; + zstd->zob.size = zstd->nb; + zstd->zob.pos = 0; + + /* Compress next chunk. */ + int xx = ZSTD_compressStream(zstd->_stream, &zstd->zob, &zib); + if (ZSTD_isError(xx)) { + fd->errcookie = ZSTD_getErrorName(xx); + return -1; + } + + /* Write compressed data buffer. */ + if (zstd->zob.pos > 0) { + size_t nw = fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp); + if (nw != zstd->zob.pos) { + fd->errcookie = "zstdWrite fwrite failed."; + return -1; + } + } + } + return zib.pos; +} + +static int zstdClose(FD_t fd) +{ + rpmzstd zstd = zstdFileno(fd); +assert(zstd); + int rc = -2; + + if ((zstd->flags & O_ACCMODE) == O_RDONLY) { /* decompressing */ + rc = 0; + ZSTD_freeDStream(zstd->_stream); + } else { /* compressing */ + /* close frame */ + zstd->zob.dst = zstd->b; + zstd->zob.size = zstd->nb; + zstd->zob.pos = 0; + int xx = ZSTD_endStream(zstd->_stream, &zstd->zob); + if (ZSTD_isError(xx)) + fd->errcookie = ZSTD_getErrorName(xx); + else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp)) + fd->errcookie = "zstdClose fwrite failed."; + else + rc = 0; + ZSTD_freeCStream(zstd->_stream); + } + + if (zstd->fp && fileno(zstd->fp) > 2) + (void) fclose(zstd->fp); + + if (zstd->b) free(zstd->b); + free(zstd); + + if (_rpmio_debug || rpmIsDebug()) fdstat_print(fd, "ZSTDIO", stderr); + if (rc == 0) + fdFree(fd); + return rc; +} + +static const struct FDIO_s zstdio_s = { + zstdRead, zstdWrite, NULL, zstdClose, NULL, NULL, NULL, fdFileno, + NULL, zstdOpen, zstdFileno, zstdFlush, NULL +}; +static const FDIO_t zstdio = &zstdio_s ; + +#endif /* HAVE_ZSTD */ + +/* =============================================================== */ const char *Fstrerror(FD_t fd) { @@ -1374,7 +1664,7 @@ static void cvtfmode (const char *m, FD_t Fdopen(FD_t ofd, const char *fmode) { - char stdio[20], other[20], zstdio[40]; + char stdio[20], other[20], z_stdio[40]; const char *end = NULL; FDIO_t iof = NULL; FD_t fd = ofd; @@ -1388,9 +1678,9 @@ fprintf(stderr, "*** Fdopen(%p,%s) %s\n" cvtfmode(fmode, stdio, sizeof(stdio), other, sizeof(other), &end, NULL); if (stdio[0] == '\0') return NULL; - zstdio[0] = '\0'; - strncat(zstdio, stdio, sizeof(zstdio) - strlen(zstdio) - 1); - strncat(zstdio, other, sizeof(zstdio) - strlen(zstdio) - 1); + z_stdio[0] = '\0'; + strncat(z_stdio, stdio, sizeof(z_stdio) - strlen(z_stdio) - 1); + strncat(z_stdio, other, sizeof(z_stdio) - strlen(z_stdio) - 1); if (end == NULL && other[0] == '\0') return fd; @@ -1400,19 +1690,24 @@ fprintf(stderr, "*** Fdopen(%p,%s) %s\n" iof = fdio; } else if (rstreq(end, "gzdio") || rstreq(end, "gzip")) { iof = gzdio; - fd = gzdFdopen(fd, zstdio); + fd = gzdFdopen(fd, z_stdio); #if HAVE_BZLIB_H } else if (rstreq(end, "bzdio") || rstreq(end, "bzip2")) { iof = bzdio; - fd = bzdFdopen(fd, zstdio); + fd = bzdFdopen(fd, z_stdio); #endif #if HAVE_LZMA_H } else if (rstreq(end, "xzdio") || rstreq(end, "xz")) { iof = xzdio; - fd = xzdFdopen(fd, zstdio); + fd = xzdFdopen(fd, z_stdio); } else if (rstreq(end, "lzdio") || rstreq(end, "lzma")) { iof = lzdio; - fd = lzdFdopen(fd, zstdio); + fd = lzdFdopen(fd, z_stdio); +#endif +#if HAVE_ZSTD + } else if (rstreq(end, "zstdio") || rstreq(end, "zstd")) { + iof = zstdio; + fd = zstdFdopen(fd, z_stdio); #endif } else if (rstreq(end, "ufdio")) { iof = ufdio; @@ -1422,7 +1717,7 @@ fprintf(stderr, "*** Fdopen(%p,%s) %s\n" {}; if (*end == '\0') { iof = gzdio; - fd = gzdFdopen(fd, zstdio); + fd = gzdFdopen(fd, z_stdio); } } if (iof == NULL) @@ -1535,6 +1830,11 @@ int Ferror(FD_t fd) ec = (fd->syserrno || fd->errcookie != NULL) ? -1 : 0; i--; /* XXX fdio under xzdio/lzdio always has fdno == -1 */ #endif +#if HAVE_ZSTD + } else if (fps->io == zstdio) { + ec = (fd->syserrno || fd->errcookie != NULL) ? -1 : 0; + i--; /* XXX fdio under zstdio always has fdno == -1 */ +#endif } else { /* XXX need to check ufdio/gzdio/bzdio/fdio errors correctly. */ ec = (fdFileno(fd) < 0 ? -1 : 0);
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor