41 #if defined(WITH_BZIP2)
46 #define _RPMBZ_INTERNAL
53 #define _RPMZLOG_INTERNAL
61 if (zq->verbosity > 2) { \
66 #define _RPMZQ_INTERNAL
88 static struct rpmzQueue_s __zq;
96 static void _rpmzqArgCallback(poptContext con,
97 enum poptCallbackReason reason,
98 const struct poptOption * opt,
const char *
arg,
106 if (opt->arg == NULL)
108 case 'q': zq->verbosity = 0;
break;
109 case 'v': zq->verbosity++;
break;
112 fprintf(stderr,
_(
"Unknown option -%c\n"), (
char)opt->val);
113 poptPrintUsage(con, stderr, 0);
120 Usage: pigz [
options] [files ...]
121 will compress files
in place, adding the
suffix '.gz'. If no files are
122 specified, stdin will be compressed to stdout. pigz does what gzip does,
123 but spreads the work over multiple processors and cores when compressing.
126 -0 to -9, --fast, --best Compression levels, --fast is -1, --best is -9
129 -
i, --independent Compress blocks independently
for damage recovery
130 -R, --rsyncable Input-determined block locations
for rsync
131 -
d, --decompress Decompress the compressed input
132 -t, --test Test the integrity of the compressed input
133 -l, --list List the contents of the compressed input
134 -f, --force Force overwrite, compress .gz, links, and to terminal
135 -r, --recursive Process the contents of all subdirectories
137 -z, --zlib Compress to zlib (.zz) instead of gzip format
138 -K, --zip Compress to PKWare zip (.zip) single entry format
139 -k, --keep Do not delete original
file after processing
140 -c, --stdout Write all processed
output to stdout (wont delete)
143 -T, --no-
time Do not store or restore mod
time in/from header
144 -q, --quiet Print no messages, even on error
145 -
v, --verbose Provide more verbose
output
149 Parallel BZIP2 v1.0.5 - by: Jeff Gilchrist http:
150 [Jan. 08, 2009] (uses libbzip2 by Julian Seward)
152 Usage: ./pbzip2 [-1 .. -9] [-b#cdfhklp#qrtVz] <filename> <filename2> <filenameN>
153 -b# : where # is the
file block
size in 100k (
default 9 = 900k)
157 -h : print
this help message
158 -k : keep input
file, dont
delete
159 -l : load average determines
max number processors to use
160 -p# : where # is the number of processors (
default 2)
161 -q : quiet
mode (
default)
162 -r : read entire input file into RAM and split between processors
163 -t : test compressed file integrity
165 -V : display version info
for pbzip2 then exit
166 -z : compress
file (
default)
167 -1 .. -9 :
set BWT block
size to 100k .. 900k (
default 900k)
169 Example: pbzip2 -b15vk myfile.tar
170 Example: pbzip2 -p4 -r -5 myfile.tar second*.txt
171 Example: tar cf myfile.tar.bz2 --use-compress-
prog=pbzip2 dir_to_compress/
172 Example: pbzip2 -d myfile.tar.bz2
179 { NULL,
'\0', POPT_ARG_CALLBACK | POPT_CBFLAG_INC_DATA | POPT_CBFLAG_CONTINUE,
180 (
void *)_rpmzqArgCallback, 0, NULL, NULL },
183 {
"fast",
'\0', POPT_ARG_VAL, &__zq.level, 1,
184 N_(
"fast compression"), NULL },
185 {
"best",
'\0', POPT_ARG_VAL, &__zq.level, 9,
186 N_(
"best compression"), NULL },
189 N_(
"extreme compression"), NULL },
190 { NULL,
'0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 0,
192 { NULL,
'1', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 1,
194 { NULL,
'2', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 2,
196 { NULL,
'3', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 3,
198 { NULL,
'4', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 4,
200 { NULL,
'5', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 5,
202 { NULL,
'6', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 6,
204 { NULL,
'7', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 7,
206 { NULL,
'8', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 8,
208 { NULL,
'9', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.level, 9,
212 {
"blocksize",
'b', POPT_ARG_INT|POPT_ARGFLAG_SHOW_DEFAULT, &__zq.blocksize, 0,
213 N_(
"Set compression block size to mmmK"),
N_(
"mmm") },
215 {
"processes",
'p', POPT_ARG_INT|POPT_ARGFLAG_SHOW_DEFAULT, &__zq.threads, 0,
216 N_(
"Allow up to n compression threads"),
N_(
"n") },
219 {
"blocksize",
'b', POPT_ARG_VAL|POPT_ARGFLAG_SHOW_DEFAULT, NULL,
'b',
220 N_(
"Set compression block size to mmmK"),
N_(
"mmm") },
222 {
"processes",
'p', POPT_ARG_INT|POPT_ARGFLAG_SHOW_DEFAULT, NULL,
'p',
223 N_(
"Allow up to n compression threads"),
N_(
"n") },
227 N_(
"Compress blocks independently for damage recovery"), NULL },
230 N_(
"Input-determined block locations for rsync"), NULL },
233 {
"threads",
'\0', POPT_ARG_INT|POPT_ARGFLAG_SHOW_DEFAULT, &__zq.threads, 0,
234 N_(
"Allow up to n compression threads"),
N_(
"n") },
238 {
"compress",
'z', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__zq.mode,
RPMZ_MODE_COMPRESS,
239 N_(
"force compression"), NULL },
241 N_(
"force decompression"), NULL },
243 N_(
"Decompress the compressed input"), NULL },
245 N_(
"Test the integrity of the compressed input"), NULL },
247 N_(
"List the contents of the compressed input"), NULL },
248 {
"info",
'\0', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__zq.flags,
RPMZ_FLAGS_LIST,
249 N_(
"list block sizes, total sizes, and possible metadata"), NULL },
251 N_(
"Force: --overwrite --recompress --symlinks --tty"), NULL },
254 N_(
" Permit overwrite of output files"), NULL },
257 N_(
" Permit compress of already compressed files"), NULL },
260 N_(
" Permit symlink input file to be compressed"), NULL },
263 N_(
" Permit compressed output to terminal"), NULL },
268 N_(
"Process the contents of all subdirectories"), NULL },
269 {
"suffix",
'S', POPT_ARG_STRING, &__zq.suffix, 0,
270 N_(
"Use suffix .sss instead of .gz (for compression)"),
N_(
".sss") },
271 {
"ascii",
'a', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, NULL,
'a',
272 N_(
"Compress to LZW (.Z) instead of gzip format"), NULL },
273 {
"bits",
'Z', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, NULL,
'Z',
274 N_(
"Compress to LZW (.Z) instead of gzip format"), NULL },
276 N_(
"Compress to zlib (.zz) instead of gzip format"), NULL },
278 N_(
"Compress to PKWare zip (.zip) single entry format"), NULL },
280 N_(
"Do not delete original file after processing"), NULL },
282 N_(
"Write all processed output to stdout (won't delete)"), NULL },
283 {
"to-stdout",
'c', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__zq.flags,
RPMZ_FLAGS_STDOUT,
284 N_(
"write to standard output and don't delete input files"), NULL },
289 N_(
"Store/restore file name and mod time in/from header"), NULL },
291 N_(
"Do not store or restore file name in/from header"), NULL },
294 N_(
"Do not store or restore mod time in/from header"), NULL },
301 #define zqFprintf if (_rpmzq_debug) fprintf
309 unsigned int len = job->out->len;
311 rc = BZ2_bzBuffToBuffCompress((
char *)job->out->buf, &len,
312 (
char *)job->in->buf, job->in->len, bz->B, bz->V, bz->W);
315 zqFprintf(stderr,
"==> %s(%p,%p) rc %d\n", __FUNCTION__, bz, job, rc);
325 unsigned int len = job->out->len;
327 rc = BZ2_bzBuffToBuffDecompress((
char *)job->out->buf, &len,
328 (
char *)job->in->buf, job->in->len, bz->S, bz->V);
331 zqFprintf(stderr,
"==> %s(%p,%p) rc %d\n", __FUNCTION__, bz, job, rc);
362 zqFprintf(stderr,
" ++ pool %p[%u,%d]\n", pool, (
unsigned)size, limit);
375 if (pool->limit == 0)
379 if (pool->head != NULL) {
382 pool->head = space->next;
389 assert(pool->limit != 0);
399 space->len = (pool ? pool->size :
len);
401 space->buf = (
unsigned char *)
xmalloc(space->len);
402 space->ptr = space->buf;
408 zqFprintf(stderr,
" ++ space %p[%d] buf %p[%u]\n", space, 1, space->buf, (
unsigned)space->len);
421 zqFprintf(stderr,
" ++ space %p[%d] buf %p[%u]\n", space, use+1, space->buf, (
unsigned)space->len);
435 zqFprintf(stderr,
" -- space %p[%d] buf %p[%u]\n", space, use, space->buf, (
unsigned)space->len);
440 fprintf(stderr,
"==> FIXME: %s: space %p[%d]\n", __FUNCTION__, space, use);
446 space->buf = (
unsigned char *) space->ptr;
447 space->len = pool->size;
450 space->next = pool->head;
456 space->ptr =
_free(space->ptr);
459 space =
_free(space);
477 while ((space = pool->head) != NULL) {
478 pool->head = space->next;
479 space->ptr =
_free(space->ptr);
482 space =
_free(space);
489 assert(count == pool->made);
491 if (count != pool->made)
492 fprintf(stderr,
"==> FIXME: %s: count %d pool->made %d\n", __FUNCTION__, count, pool->made);
494 zqFprintf(stderr,
" -- pool %p count %d\n", pool, count);
509 zqFprintf(stderr,
" ++ job %p[%ld] use %d\n", job, seq, 1);
516 if (job == NULL)
return NULL;
519 zqFprintf(stderr,
" ++ job %p[%ld] use %d\n", job, job->seq, use+1);
534 zqFprintf(stderr,
" -- job %p[%ld] use %d %p => %p\n", job, job->seq, use, job->in, job->out);
539 fprintf(stderr,
"==> FIXME: %s: job %p[%ld] use %d\n", __FUNCTION__, job, job->seq, use);
543 rpmzJPool jpool = job->pool;
552 if (job->use != NULL)
554 if (job->calc != NULL)
574 struct rpmzJob_s job;
577 zqFprintf(stderr,
"--> %s(%p)\n", __FUNCTION__, zq);
579 if (zq->_zc.q == NULL)
587 zq->_zc.q->head = &job;
589 zq->_zc.q->tail = &job.next;
594 Trace((zlog,
"-- joined %d compress threads", caught));
596 assert(caught == zq->_zc.cthreads);
598 if (caught != zq->_zc.cthreads)
599 fprintf(stderr,
"==> FIXME: %s: caught %d z->_zc.cthreads %d\n", __FUNCTION__, caught, zq->_zc.cthreads);
601 zq->_zc.cthreads = 0;
605 Trace((zlog,
"-- freed %d output buffers", caught));
607 Trace((zlog,
"-- freed %d input buffers", caught));
615 zqFprintf(stderr,
"--> %s(%p)\n", __FUNCTION__, zq);
617 if (zq->_zc.q != NULL)
625 zq->_zc.pool =
rpmzqNewPool(zq->iblocksize, zq->ilimit);
626 zqFprintf(stderr,
"--> in_pool: %p[%u] blocksize %u\n", zq->_zc.pool, (
unsigned)zq->ilimit, (
unsigned)zq->iblocksize);
627 zq->_zw.pool =
rpmzqNewPool(zq->oblocksize, zq->olimit);
628 zqFprintf(stderr,
"--> out_pool: %p[%u] blocksize %u\n", zq->_zw.pool, (
unsigned)zq->olimit, (
unsigned)zq->oblocksize);
643 zq->iblocksize = zq->blocksize;
645 zq->oblocksize = zq->blocksize;
658 zs->tail = &zs->head;
664 if (zs->have != NULL)
667 zs->tail = &zs->head;
676 assert(zs->head == NULL &&
yarnPeekLock(zs->have) == 0);
689 if (job->seq == -1) {
695 zs->head = job->next;
697 if (job->next == NULL)
698 zs->tail = &zs->head;
711 zs->tail = &job->next;
726 if (zs->first != NULL)
737 assert(zs->head == NULL &&
yarnPeekLock(zs->first) == -1);
751 zs->head = job->next;
753 yarnTwist(zs->first,
TO, zs->head == NULL ? -1 : zs->head->seq);
766 while ((here = *prior) != NULL) {
767 if (here->seq > job->seq)
786 job = zq->_zc.q->head;
788 if (job->seq == -1) {
794 zq->_zc.q->head = job->next;
796 if (job->next == NULL)
797 zq->_zc.q->tail = &zq->_zc.q->head;
809 *zq->_zc.q->tail = job;
810 zq->_zc.q->tail = &job->next;
822 job = zq->_zw.q->head;
825 zq->_zw.q->head = job->next;
827 yarnTwist(zq->_zw.q->first,
TO, zq->_zw.q->head == NULL ? -1 : zq->_zw.q->head->seq);
842 default: assert(0);
break;
844 pct = (100.0 * job->out->len) / job->in->len;
845 zqFprintf(stderr,
" job %p[%ld]:\t%p[%u] => %p[%u]\t(%3.1f%%)\n",
846 job, job->seq, job->in->buf, (
unsigned)job->in->len,
847 job->out->buf, (unsigned)job->out->len, pct);
848 Trace((zlog,
"-- compressed #%ld %3.1f%%%s", job->seq, pct,
849 (job->more ?
"" :
" (last)")));
852 pct = (100.0 * job->in->len) / job->out->len;
853 zqFprintf(stderr,
" job %p[%ld]:\t%p[%u] <= %p[%u]\t(%3.1f%%)\n",
854 job, job->seq, job->in->buf, (
unsigned)job->in->len,
855 job->out->buf, (unsigned)job->out->len, pct);
856 Trace((zlog,
"-- decompressed #%ld %3.1f%%%s", job->seq, pct,
857 (job->more ?
"" :
" (last)")));
862 prior = &zq->_zw.q->head;
863 while ((here = *prior) != NULL) {
864 if (here->seq > job->seq)
873 yarnTwist(zq->_zw.q->first,
TO, zq->_zw.q->head->seq);
884 default: assert(0);
break;
889 outlen = ((job->in->len*1.01)+600);
893 if (job->out->len < outlen) {
894 fprintf(stderr,
"==> FIXME: %s: job->out %p %p[%u] malloc(%u)\n", __FUNCTION__, job->out, job->out->buf, (
unsigned)job->out->len, (
unsigned)outlen);
897 job->out->len = outlen;
898 job->out->buf = (
unsigned char *)
xmalloc(job->out->len);
905 outlen = 6 * job->in->len;
907 if (job->out->len < outlen) {
908 fprintf(stderr,
"==> FIXME: %s: job->out %p %p[%u] malloc(%u)\n", __FUNCTION__, job->out, job->out->buf, (
unsigned)job->out->len, (
unsigned)outlen);
911 job->out->len = outlen;
912 job->out->buf = (
unsigned char *)
xmalloc(job->out->len);
916 static int _grow = 2;
918 outlen = job->out->len * _grow;
919 ret = rpmbzDecompressBlock(bz, job);
920 if (ret != BZ_OUTBUFF_FULL)
922 fprintf(stderr,
"==> FIXME: %s: job->out %p %p[%u] realloc(%u)\n", __FUNCTION__, job->out, job->out->buf, (
unsigned)job->out->len, (
unsigned)outlen);
923 if (job->out->use != NULL)
926 fprintf(stderr,
"==> FIXME: %s: job->out %p %p[%u] free\n", __FUNCTION__, job->out, job->out->buf, (
unsigned)job->out->len);
927 job->out->buf =
_free(job->out->buf);
928 job->out =
_free(job->out);
931 job->out->len = outlen;
932 job->out->buf = (
unsigned char *)
xmalloc(job->out->len);
934 assert(ret == BZ_OK);
945 static void rpmzqCompressThread (
void *_zq)
950 rpmbz bz = rpmbzInit(zq->level, -1, -1, zq->omode);
953 zqFprintf(stderr,
"--> %s(%p) bz %p\n", __FUNCTION__, zq, bz);
965 static void rpmzqDecompressThread(
void *_zq)
970 rpmbz bz = rpmbzInit(zq->level, -1, -1, zq->omode);
973 zqFprintf(stderr,
"--> %s(%p) bz %p\n", __FUNCTION__, zq, bz);
988 if (zq->_zc.cthreads < seq && zq->_zc.cthreads < (
int)threads) {
990 default: assert(0);
break;
991 case O_WRONLY: (void)
yarnLaunch(rpmzqCompressThread, zq);
break;
992 case O_RDONLY: (void)
yarnLaunch(rpmzqDecompressThread, zq);
break;
static const char * suffix[]
const char const double d
void yarnTwist(yarnLock bolt, yarnTwistOP op, long val)
rpmzFIFO rpmzqInitFIFO(long val)
void rpmzqAddCJob(rpmzQueue zq, rpmzJob job)
const char const char size_t len
void rpmzqLaunch(rpmzQueue zq, long seq, unsigned int threads)
start another compress/decompress thread if needed
void yarnPossess(yarnLock bolt)
struct rpmzJob_s * rpmzJob
rpmzJob rpmzqDelWJob(rpmzQueue zq, long seq)
rpmzSEQ rpmzqFiniSEQ(rpmzSEQ zs)
long yarnPeekLock(yarnLock bolt)
rpmzSpace rpmzqNewSpace(rpmzPool pool, size_t len)
Get a space from a pool (or malloc if pool == NULL).
struct rpmzQueue_s * rpmzQueue
rpmzPool rpmzqFreePool(rpmzPool pool, int *countp)
Job queue and buffer pool management.
rpmzQueue rpmzqNew(rpmzQueue zq, rpmzLog zlog, int limit)
static yarnThread threads
int rpmbzCompressBlock(void *_bz, rpmzJob job)
Yet Another syslog(3) API clone.
void yarnWaitFor(yarnLock bolt, yarnWaitOP op, long val)
void * xcalloc(size_t nmemb, size_t size)
const char const bson_bool_t v
void rpmzqUseSpace(rpmzSpace space)
Increment the use count to require one more drop before returning this space to the pool...
struct rpmzSEQ_s * rpmzSEQ
rpmzPool rpmzqNewPool(size_t size, int limit)
void rpmzqAddWJob(rpmzQueue zq, rpmzJob job)
const char const bson * data
struct rpmzLog_s * rpmzLog
trace log pointer
#define POPT_ARGFLAG_TOGGLE
yarnLock yarnNewLock(long initial)
const char const bson const bson int limit
yarnLock yarnFreeLock(yarnLock bolt)
void rpmzqVerifySEQ(rpmzSEQ zs)
rpmzJob rpmzqUseJob(rpmzJob job)
struct rpmzPool_s * rpmzPool
rpmzJob rpmzqDelCJob(rpmzQueue zq)
rpmzJob rpmzqDelFIFO(rpmzFIFO zs)
void rpmzqAddFIFO(rpmzFIFO zs, rpmzJob job)
struct rpmzSpace_s * rpmzSpace
const char const char int arg
void rpmzqAddSEQ(rpmzSEQ zs, rpmzJob job)
void yarnRelease(yarnLock bolt)
rpmzJob rpmzqNewJob(long seq)
void rpmzqFini(rpmzQueue zq)
command the compress threads to all return, then join them all (call from main thread), free all the thread-related resources
static void output(int indent, int *offset, const char *fmt,...)
rpmzQueue rpmzqFree(rpmzQueue zq)
struct poptOption rpmzqOptionsPoptTable[]
rpmzJob rpmzqDropJob(rpmzJob job)
const char const bson const bson bson * out
static void set(char *t, NODE *ip)
const char const char int int max
yarnThread yarnLaunch(void(*probe)(void *), void *payload)
rpmzSpace rpmzqDropSpace(rpmzSpace space)
Drop a space, returning to the pool (or free'ing if no pool) when the use count is zero...
const char const char size_t size
rpmzJob rpmzqDelSEQ(rpmzSEQ zs, long seq)
static void * _free(const void *p)
Wrapper to free(3), hides const compilation noise, permit NULL, return NULL.
rpmzFIFO rpmzqFiniFIFO(rpmzFIFO zs)
rpmzSEQ rpmzqInitSEQ(long val)
struct rpmzFIFO_s * rpmzFIFO
void rpmzqVerify(rpmzQueue zq)
verify no more jobs, prepare for next use
void rpmzqInit(rpmzQueue zq)
setup job lists (call from main thread)
const char const bson const bson int int int options
void rpmzqVerifyFIFO(rpmzFIFO zs)