Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:12.2:Update
pigz
pigz-dictzip.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File pigz-dictzip.patch of Package pigz
diff -u -r pigz-2.1.6.orig/pigz.c pigz-2.1.6/pigz.c --- pigz-2.1.6.orig/pigz.c 2010-01-17 21:09:37.000000000 +0100 +++ pigz-2.1.6/pigz.c 2011-06-02 12:29:12.245689087 +0200 @@ -304,6 +304,8 @@ } \ } while (0) +#define DZIP_HEADER_LEN 24 + /* globals (modified by main thread only when it's the only thread) */ local int ind; /* input file descriptor */ local int outd; /* output file descriptor */ @@ -322,11 +324,12 @@ local int list; /* true to list files instead of compress */ local int first = 1; /* true if we need to print listing header */ local int decode; /* 0 to compress, 1 to decompress, 2 to test */ +local int dictzip; /* true to write random access dictionary */ local int level; /* compression level */ local int rsync; /* true for rsync blocking */ local int procs; /* maximum number of compression threads (>= 1) */ local int dict; /* true to initialize dictionary in each thread */ -local size_t size; /* uncompressed input size per thread (>= 32K) */ +size_t size; /* uncompressed input size per thread (>= 32K) */ /* saved gzip/zip header data for decompression, testing, and listing */ local time_t stamp; /* time stamp from gzip header */ @@ -560,6 +563,17 @@ return dos; } +long long get_file_size(int fd) +{ + long long old_pos = lseek(fd, 0, SEEK_CUR); + long long retval; + + retval = lseek(fd, 0, SEEK_END); + lseek(fd, old_pos, SEEK_SET); + + return retval; +} + /* put a 4-byte integer into a byte array in LSB order or MSB order */ #define PUT2L(a,b) (*(a)=(b)&0xff,(a)[1]=(b)>>8) #define PUT4L(a,b) (PUT2L(a,(b)&0xffff),PUT2L((a)+2,(b)>>16)) @@ -613,12 +627,27 @@ head[0] = 31; head[1] = 139; head[2] = 8; /* deflate */ - head[3] = name != NULL ? 8 : 0; + head[3] = 0; + if (dictzip) + head[3] |= 4; + if (name != NULL) + head[3] |= 8; PUT4L(head + 4, mtime); head[8] = level == 9 ? 2 : (level == 1 ? 4 : 0); head[9] = 3; /* unix */ writen(outd, head, 10); len = 10; + if (dictzip) { + long long file_len = get_file_size(ind); + int extra_len = DZIP_HEADER_LEN + ((file_len / size) + 2) * sizeof(int); + + char *extra = (char*)malloc(extra_len); + + memset(extra, 0, extra_len); + writen(outd, extra, extra_len); + free(extra); + len += extra_len; + } if (name != NULL) writen(outd, (unsigned char *)name, strlen(name) + 1); if (name != NULL) @@ -1165,6 +1194,10 @@ unsigned long ulen; /* total uncompressed size (overflow ok) */ unsigned long clen; /* total compressed size (overflow ok) */ unsigned long check; /* check value of uncompressed data */ + char *extra = NULL; /* dictzip extra header field */ + int dz_cur; /* dictzip current pointer to header */ + long long file_len = get_file_size(ind); + int extra_len = DZIP_HEADER_LEN + ((file_len / size) + 2) * sizeof(int); (void)dummy; @@ -1172,6 +1205,25 @@ Trace(("-- write thread running")); head = put_header(); + if (dictzip) { + extra = (char*)malloc(extra_len + 32); + memset(extra, 0, extra_len); + + if (extra_len >= ((1ULL << 16) - 1)) + bail("file too long for dictzip\n", ""); + /* extra length */ + PUT2L(extra, extra_len - 2); + /* Random Access ID */ + extra[2] = 'R'; + extra[3] = 'A'; + /* RA version 99 */ + /* XXX need to standardize! */ + PUT2L(extra + 6, 99); + // PUT2L(extra + 6, 1); + + dz_cur = 0; + } + /* process output of compress threads until end of input */ ulen = clen = 0; check = CHECK(0L, Z_NULL, 0); @@ -1191,6 +1243,15 @@ ulen += (unsigned long)len; clen += (unsigned long)(job->out->len); + if (dictzip) { + if (((dz_cur * 4) + DZIP_HEADER_LEN + sizeof(int)) > extra_len) { + printf(" %d > %d\n", dz_cur, file_len / size); + bail ("input file too large\n", ""); + } + PUT4L(extra + (dz_cur * 4) + DZIP_HEADER_LEN, job->out->len); + dz_cur++; + } + /* write the compressed data and drop the output buffer */ Trace(("-- writing #%ld", seq)); writen(outd, job->out->buf, job->out->len); @@ -1215,6 +1276,22 @@ /* write trailer */ put_trailer(ulen, clen, check, head); + if (dictzip) { + /* chunk length */ + PUT4L(extra + 8, size); + /* chunk count */ + PUT4L(extra + 12, dz_cur); +// PUT4L(extra + 12, (file_len / size) + 1); + /* Target file size (64 bit) */ + PUT4L(extra + 16, file_len); + PUT4L(extra + 20, file_len >> 32); + /* write extra header */ + if (lseek(outd, 10, SEEK_SET) != 10) + bail("couldn't seek in output file\n", ""); + writen(outd, extra, extra_len); + free(extra); + } + /* verify no more jobs, prepare for next use */ possess(compress_have); assert(compress_head == NULL && peek_lock(compress_have) == 0); @@ -2625,6 +2702,18 @@ /* prepare gzip header information for compression */ name = headis & 1 ? justname(in) : NULL; mtime = headis & 2 ? st.st_mtime : 0; + + /* Find a sane chunk size */ + if (dictzip) { + long long file_len = get_file_size(ind); + int extra_len; + + extra_len = DZIP_HEADER_LEN + ((file_len / size) + 2) * sizeof(int); + while (extra_len >= ((1 << 16) - 1)) { + size += (1 << 10ULL); + extra_len = DZIP_HEADER_LEN + ((file_len / size) + 2) * sizeof(int); + } + } } SET_BINARY_MODE(ind); @@ -2678,6 +2767,8 @@ out = malloc(strlen("<stdout>") + 1); if (out == NULL) bail("not enough memory", ""); + if (dictzip) + bail("need to have a seekable output for dictzip\n", ""); strcpy(out, "<stdout>"); outd = 1; if (!decode && !force && isatty(outd)) @@ -2796,6 +2887,7 @@ " -i, --independent Compress blocks independently for damage recovery", " -R, --rsyncable Input-determined block locations for rsync", " -d, --decompress Decompress the compressed input", +" -e, --dictzip Write dictzip random seek information in gzip header", " -t, --test Test the integrity of the compressed input", " -l, --list List the contents of the compressed input", " -f, --force Force overwrite, compress .gz, links, and to terminal", @@ -2868,17 +2960,18 @@ force = 0; /* don't overwrite, don't compress links */ recurse = 0; /* don't go into directories */ form = 0; /* use gzip format */ + dictzip = 0; /* don't write dictzip information */ } /* long options conversion to short options */ local char *longopts[][2] = { {"LZW", "Z"}, {"ascii", "a"}, {"best", "9"}, {"bits", "Z"}, - {"blocksize", "b"}, {"decompress", "d"}, {"fast", "1"}, {"force", "f"}, - {"help", "h"}, {"independent", "i"}, {"keep", "k"}, {"license", "L"}, - {"list", "l"}, {"name", "N"}, {"no-name", "n"}, {"no-time", "T"}, - {"processes", "p"}, {"quiet", "q"}, {"recursive", "r"}, {"rsyncable", "R"}, - {"silent", "q"}, {"stdout", "c"}, {"suffix", "S"}, {"test", "t"}, - {"to-stdout", "c"}, {"uncompress", "d"}, {"verbose", "v"}, + {"blocksize", "b"}, {"decompress", "d"}, {"dictzip", "e"}, {"fast", "1"}, + {"force", "f"}, {"help", "h"}, {"independent", "i"}, {"keep", "k"}, + {"license", "L"}, {"list", "l"}, {"name", "N"}, {"no-name", "n"}, + {"no-time", "T"}, {"processes", "p"}, {"quiet", "q"}, {"recursive", "r"}, + {"rsyncable", "R"}, {"silent", "q"}, {"stdout", "c"}, {"suffix", "S"}, + {"test", "t"}, {"to-stdout", "c"}, {"uncompress", "d"}, {"verbose", "v"}, {"version", "V"}, {"zip", "K"}, {"zlib", "z"}}; #define NLOPTS (sizeof(longopts) / (sizeof(char *) << 1)) @@ -2984,6 +3077,7 @@ case 'b': get = 1; break; case 'c': pipeout = 1; break; case 'd': decode = 1; headis = 0; break; + case 'e': dictzip = 1; dict = 0; break; case 'f': force = 1; break; case 'h': help(); break; case 'i': dict = 0; break; @@ -3112,6 +3206,19 @@ fprintf(stderr, "warning: output is concatenated zip files "); fprintf(stderr, "-- pigz will not be able to extract\n"); } + + /* dictzip sanity checks */ + if (dictzip && (form > 1)) { + fprintf(stderr, "warning: dictzip only works on gzip files\n"); + } + +#if 0 + if (dictzip && (size >= (64 << 10))) { + fprintf(stderr, "warning: dictzip needs chunks < 64k.\n"); + size = (63 << 10); + } +#endif + process(strcmp(argv[n], "-") ? argv[n] : NULL); done++; }
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
