* deflate.c: Include verify.h.
(RSYNC_WIN, RSYNC_SUM_MATCH): Define.
(rsync_sum, rsync_chunk_end): Declare file-scoped globals.
(lm_init): Initialize globals.
(fill_window): Update rsync_chunk_end.
(rsync_roll): New function.
(RSYNC_ROLL): New macro.
(FLUSH_BLOCK): Update for new "pad" parameter.
(deflate_fast): Use RSYNC_ROLL and flush/pad.
(deflate): Likewise.
* trees.c (flush_block): Add "pad" parameter.
* gzip.c (rsync): New global.
(RSYNCABLE_OPTION, longopts, help): Add the option.
(main): Set the new global.
* gzip.h (rsync): Declare new global.
(flush_block): Update prototype.
* doc/gzip.texi: Document it.
* gzip.1: Likewise.
* bootstrap.conf: Use verify module.
* NEWS (New feature): Mention it.
* Makefile.am (check-local): Add tests and use AM_V__* command-
hiding opions. Reported against Debian here:
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=118118
FILES_TO_CHECK = $(bin_SCRIPTS) \
$(top_srcdir)/ChangeLog $(top_srcdir)/configure $(top_srcdir)/gzip.c
check-local: $(FILES_TO_CHECK) $(bin_PROGRAMS) gzip.doc.gz
- { test '$(srcdir)' != . || ./zdiff --__bindir . -c gzip.doc.gz; }
- ./zdiff --__bindir . -c $(srcdir)/gzip.doc $(srcdir)/gzip.doc
- ./zdiff --__bindir . $(srcdir)/gzip.doc gzip.doc.gz
- ./zdiff --__bindir . -c - $(srcdir)/gzip.doc <gzip.doc.gz
- ./zdiff --__bindir . -c gzip.doc.gz gzip.doc.gz
- ./zgrep --__bindir . -iV >/dev/null
- for file in $(FILES_TO_CHECK); do \
- ./gzip -cv -- "$$file" | ./gzip -d | cmp - "$$file" || exit 1; \
+ $(AM_V_GEN){ test '$(srcdir)' != . \
+ || ./zdiff --__bindir . -c gzip.doc.gz; }
+ $(AM_V_at)./zdiff --__bindir . -c $(srcdir)/gzip.doc $(srcdir)/gzip.doc
+ $(AM_V_at)./zdiff --__bindir . $(srcdir)/gzip.doc gzip.doc.gz
+ $(AM_V_at)./zdiff --__bindir . -c - $(srcdir)/gzip.doc <gzip.doc.gz
+ $(AM_V_at)./zdiff --__bindir . -c gzip.doc.gz gzip.doc.gz
+ $(AM_V_at)./zgrep --__bindir . -iV >/dev/null
+ $(AM_V_at)for opt in --rsyncable '' -1 -9; do \
+ for file in $(FILES_TO_CHECK); do \
+ ./gzip $$opt -c -- "$$file" \
+ | ./gzip -d | cmp - "$$file" || exit 1; \
+ done; \
done
- @echo 'Test succeeded.'
install-exec-hook: remove-installed-links
install-exec-hook remove-installed-links:
this option makes gzip safer in the presence of system crashes, it
can make gzip considerably slower.
+ gzip now accepts the --rsyncable option. This option is accepted in
+ all modes, but has effect only when compressing: it makes the resulting
+ output more amenable to efficient use of rsync. For example, when a
+ large input file gets a small change, a gzip --rsyncable image of
+ that file will remain largely unchanged, too. Without --rsyncable,
+ even a tiny change in the input could result in a totally different
+ gzip-compressed output file.
+
** Bug fixes
gzip -k -v no longer reports that files are replaced.
unlinkat
update-copyright
utimens
+verify
xalloc
yesno
'
#include "tailor.h"
#include "gzip.h"
#include "lzw.h" /* just for consistency checking */
+#include "verify.h"
/* ===========================================================================
* Configuration parameters
#endif
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+#ifndef RSYNC_WIN
+# define RSYNC_WIN 4096
+#endif
+verify(RSYNC_WIN < MAX_DIST);
+
+#define RSYNC_SUM_MATCH(sum) ((sum) % RSYNC_WIN == 0)
+/* Whether window sum matches magic value */
+
/* ===========================================================================
* Local data used by the "longest match" routines.
*/
unsigned good_match;
/* Use a faster search when the previous match is longer than this */
+local ulg rsync_sum; /* rolling sum of rsync window */
+local ulg rsync_chunk_end; /* next rsync sequence point */
/* Values for max_lazy_match, good_match and max_chain_length, depending on
* the desired pack level (0..9). The values given below have been tuned to
#endif
/* prev will be initialized on the fly */
+ /* rsync params */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ rsync_sum = 0;
+
/* Set the default configuration parameters:
*/
max_lazy_match = configuration_table[pack_level].max_lazy;
memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
match_start -= WSIZE;
strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
+ if (rsync_chunk_end != 0xFFFFFFFFUL)
+ rsync_chunk_end -= WSIZE;
block_start -= (long) WSIZE;
}
}
+/* With an initial offset of START, advance rsync's rolling checksum
+ by NUM bytes. */
+local void rsync_roll(unsigned int start, unsigned int num)
+{
+ unsigned i;
+
+ if (start < RSYNC_WIN) {
+ /* before window fills. */
+ for (i = start; i < RSYNC_WIN; i++) {
+ if (i == start + num)
+ return;
+ rsync_sum += (ulg)window[i];
+ }
+ num -= (RSYNC_WIN - start);
+ start = RSYNC_WIN;
+ }
+
+ /* buffer after window full */
+ for (i = start; i < start+num; i++) {
+ /* New character in */
+ rsync_sum += (ulg)window[i];
+ /* Old character out */
+ rsync_sum -= (ulg)window[i - RSYNC_WIN];
+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
+ rsync_chunk_end = i;
+ }
+}
+
+/* ===========================================================================
+ * Set rsync_chunk_end if window sum matches magic value.
+ */
+#define RSYNC_ROLL(s, n) \
+ do { if (rsync) rsync_roll((s), (n)); } while(0)
+
/* ===========================================================================
* Flush the current block, with given end-of-file flag.
* IN assertion: strstart is set to the end of the current match.
*/
#define FLUSH_BLOCK(eof) \
flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
- (char*)NULL, (long)strstart - block_start, (eof))
+ (char*)NULL, (long)strstart - block_start, flush-1, (eof))
/* ===========================================================================
* Processes a new input file and return its compressed length. This
local off_t deflate_fast()
{
IPos hash_head; /* head of the hash chain */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed, 2=>and padded */
unsigned match_length = 0; /* length of best match */
prev_length = MIN_MATCH-1;
lookahead -= match_length;
+ RSYNC_ROLL(strstart, match_length);
/* Insert new strings in the hash table only if the match length
* is not too large. This saves time but degrades compression.
*/
/* No match, output a literal byte */
Tracevv((stderr,"%c",window[strstart]));
flush = ct_tally (0, window[strstart]);
+ RSYNC_ROLL(strstart, 1);
lookahead--;
strstart++;
}
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
if (flush) FLUSH_BLOCK(0), block_start = strstart;
/* Make sure that we always have enough lookahead, except
{
IPos hash_head; /* head of hash chain */
IPos prev_match; /* previous match */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed */
int match_available = 0; /* set if previous match exists */
register unsigned match_length = MIN_MATCH-1; /* length of best match */
*/
lookahead -= prev_length-1;
prev_length -= 2;
+ RSYNC_ROLL(strstart, prev_length+1);
do {
strstart++;
INSERT_STRING(strstart, hash_head);
match_available = 0;
match_length = MIN_MATCH-1;
strstart++;
- if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
} else if (match_available) {
/* If there was no match at the previous position, output a
* single literal. If there was a match but the current match
* is longer, truncate the previous match to a single literal.
*/
Tracevv((stderr,"%c",window[strstart-1]));
- if (ct_tally (0, window[strstart-1])) {
- FLUSH_BLOCK(0), block_start = strstart;
+ flush = ct_tally (0, window[strstart-1]);
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
}
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
} else {
/* There is no previous match to compare with, wait for
* the next step to decide.
*/
+ if (rsync && strstart > rsync_chunk_end) {
+ /* Reset huffman tree */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ FLUSH_BLOCK(0), block_start = strstart;
+ }
+
match_available = 1;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
}
into the directory and compress all the files it finds there (or
decompress them in the case of @command{gunzip}).
+@item --rsyncable
+Cater better to the @command{rsync} program by periodically resetting
+the internal structure of the compressed data stream. This lets the
+@code{rsync} program take advantage of similarities in the uncompressed
+input when synchronizing two files compressed with this flag. The cost:
+the compressed output is usually about one percent larger.
+
@item --suffix @var{suf}
@itemx -S @var{suf}
Use suffix @var{suf} instead of @samp{.gz}. Any suffix can be
The default compression level is
.BR \-6
(that is, biased towards high compression at expense of speed).
+.TP
+.B \-\-rsyncable
+When you synchronize a compressed file between two computers, this option allows rsync to transfer only files that were changed in the archive instead of the entire archive.
+Normally, after a change is made to any file in the archive, the compression algorithm can generate a new version of the archive that does not match the previous version of the archive. In this case, rsync transfers the entire new version of the archive to the remote computer.
+With this option, rsync can transfer only the changed files as well as a small amount of metadata that is required to update the archive structure in the area that was changed.
.SH "ADVANCED USAGE"
Multiple compressed files can be concatenated. In this case,
.I gunzip
unsigned insize; /* valid bytes in inbuf */
unsigned inptr; /* index of next byte to be processed in inbuf */
unsigned outcnt; /* bytes in output buffer */
+int rsync = 0; /* make ryncable chunks */
static int handled_sig[] =
{
enum
{
PRESUME_INPUT_TTY_OPTION = CHAR_MAX + 1,
+ RSYNCABLE_OPTION,
SYNCHRONOUS_OPTION,
/* A value greater than all valid long options, used as a flag to
{"best", 0, 0, '9'}, /* compress better */
{"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */
{"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */
-
+ {"rsyncable", 0, 0, RSYNCABLE_OPTION}, /* make rsync-friendly archive */
{ 0, 0, 0, 0 }
};
" -Z, --lzw produce output compatible with old compress",
" -b, --bits=BITS max number of bits per code (implies -Z)",
#endif
+ " --rsyncable Make rsync-friendly archive",
"",
"With no FILE, or when FILE is -, read standard input.",
"",
recursive = 1;
#endif
break;
+
+ case RSYNCABLE_OPTION:
+ rsync = 1;
+ break;
case 'S':
#ifdef NO_MULTIPLE_DOTS
if (*optarg == '.') optarg++;
extern unsigned insize; /* valid bytes in inbuf */
extern unsigned inptr; /* index of next byte to be processed in inbuf */
extern unsigned outcnt; /* bytes in output buffer */
+extern int rsync; /* deflate into rsyncable chunks */
extern off_t bytes_in; /* number of input bytes */
extern off_t bytes_out; /* number of output bytes */
/* in trees.c */
extern void ct_init (ush *attr, int *method);
extern int ct_tally (int dist, int lc);
-extern off_t flush_block (char *buf, ulg stored_len, int eof);
+extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
/* in bits.c */
extern void bi_init (file_t zipfile);
* trees or store, and output the encoded block to the zip file. This function
* returns the total compressed length for the file so far.
*/
-off_t flush_block(buf, stored_len, eof)
+off_t flush_block(buf, stored_len, pad, eof)
char *buf; /* input block, or NULL if too old */
ulg stored_len; /* length of input block */
+ int pad; /* pad output to byte boundary */
int eof; /* true if this is the last block for a file */
{
ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
Assert (input_len == bytes_in, "bad input size");
bi_windup();
compressed_len += 7; /* align on byte boundary */
+ } else if (pad && (compressed_len % 8) != 0) {
+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */
+ compressed_len = (compressed_len + 3 + 7) & ~7L;
+ copy_block(buf, 0, 1); /* with header */
}
return compressed_len >> 3;
projects / debian / gzip / commitdiff