Signed-off-by: Maxim Storchak <m.storchak@gmail.com>lilik-openwrt-22.03
@ -1,83 +0,0 @@ | |||
commit 4fc78878e01451d6465e2072f3f0653182f885c1 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Sun May 1 16:29:34 2016 -0700 | |||
Tweak indentation only. | |||
diff --git a/checksum.c b/checksum.c | |||
index 6ebb56b..bac775d 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -188,32 +188,31 @@ void sum_update(const char *p, int32 len) | |||
{ | |||
if (protocol_version >= 30) { | |||
md5_update(&md, (uchar *)p, len); | |||
- return; | |||
- } | |||
+ } else { | |||
+ if (len + sumresidue < CSUM_CHUNK) { | |||
+ memcpy(md.buffer + sumresidue, p, len); | |||
+ sumresidue += len; | |||
+ return; | |||
+ } | |||
- if (len + sumresidue < CSUM_CHUNK) { | |||
- memcpy(md.buffer + sumresidue, p, len); | |||
- sumresidue += len; | |||
- return; | |||
- } | |||
+ if (sumresidue) { | |||
+ int32 i = CSUM_CHUNK - sumresidue; | |||
+ memcpy(md.buffer + sumresidue, p, i); | |||
+ mdfour_update(&md, (uchar *)md.buffer, CSUM_CHUNK); | |||
+ len -= i; | |||
+ p += i; | |||
+ } | |||
- if (sumresidue) { | |||
- int32 i = CSUM_CHUNK - sumresidue; | |||
- memcpy(md.buffer + sumresidue, p, i); | |||
- mdfour_update(&md, (uchar *)md.buffer, CSUM_CHUNK); | |||
- len -= i; | |||
- p += i; | |||
- } | |||
+ while (len >= CSUM_CHUNK) { | |||
+ mdfour_update(&md, (uchar *)p, CSUM_CHUNK); | |||
+ len -= CSUM_CHUNK; | |||
+ p += CSUM_CHUNK; | |||
+ } | |||
- while (len >= CSUM_CHUNK) { | |||
- mdfour_update(&md, (uchar *)p, CSUM_CHUNK); | |||
- len -= CSUM_CHUNK; | |||
- p += CSUM_CHUNK; | |||
+ sumresidue = len; | |||
+ if (sumresidue) | |||
+ memcpy(md.buffer, p, sumresidue); | |||
} | |||
- | |||
- sumresidue = len; | |||
- if (sumresidue) | |||
- memcpy(md.buffer, p, sumresidue); | |||
} | |||
int sum_end(char *sum) | |||
@@ -221,12 +220,12 @@ int sum_end(char *sum) | |||
if (protocol_version >= 30) { | |||
md5_result(&md, (uchar *)sum); | |||
return MD5_DIGEST_LEN; | |||
- } | |||
- | |||
- if (sumresidue || protocol_version >= 27) | |||
- mdfour_update(&md, (uchar *)md.buffer, sumresidue); | |||
+ } else { | |||
+ if (sumresidue || protocol_version >= 27) | |||
+ mdfour_update(&md, (uchar *)md.buffer, sumresidue); | |||
- mdfour_result(&md, (uchar *)sum); | |||
+ mdfour_result(&md, (uchar *)sum); | |||
- return MD4_DIGEST_LEN; | |||
+ return MD4_DIGEST_LEN; | |||
+ } | |||
} |
@ -1,650 +0,0 @@ | |||
commit a5a7d3a297b836387b0ac677383bdddaf2ac3598 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Sun May 1 16:32:45 2016 -0700 | |||
Add --checksum-choice option to choose the checksum algorithms. | |||
diff --git a/authenticate.c b/authenticate.c | |||
index 5f125de..d60ee20 100644 | |||
--- a/authenticate.c | |||
+++ b/authenticate.c | |||
@@ -71,7 +71,7 @@ static void gen_challenge(const char *addr, char *challenge) | |||
SIVAL(input, 20, tv.tv_usec); | |||
SIVAL(input, 24, getpid()); | |||
- sum_init(0); | |||
+ sum_init(-1, 0); | |||
sum_update(input, sizeof input); | |||
len = sum_end(digest); | |||
@@ -85,7 +85,7 @@ static void generate_hash(const char *in, const char *challenge, char *out) | |||
char buf[MAX_DIGEST_LEN]; | |||
int len; | |||
- sum_init(0); | |||
+ sum_init(-1, 0); | |||
sum_update(in, strlen(in)); | |||
sum_update(challenge, strlen(challenge)); | |||
len = sum_end(buf); | |||
diff --git a/checksum.c b/checksum.c | |||
index bac775d..8b38833 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -24,6 +24,76 @@ | |||
extern int checksum_seed; | |||
extern int protocol_version; | |||
extern int proper_seed_order; | |||
+extern char *checksum_choice; | |||
+ | |||
+#define CSUM_NONE 0 | |||
+#define CSUM_ARCHAIC 1 | |||
+#define CSUM_MD4_BUSTED 2 | |||
+#define CSUM_MD4_OLD 3 | |||
+#define CSUM_MD4 4 | |||
+#define CSUM_MD5 5 | |||
+ | |||
+int xfersum_type = 0; /* used for the file transfer checksums */ | |||
+int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */ | |||
+ | |||
+/* Returns 1 if --whole-file must be enabled. */ | |||
+int parse_checksum_choice(void) | |||
+{ | |||
+ char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; | |||
+ if (cp) { | |||
+ xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice); | |||
+ checksum_type = parse_csum_name(cp+1, -1); | |||
+ } else | |||
+ xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1); | |||
+ return xfersum_type == CSUM_NONE; | |||
+} | |||
+ | |||
+int parse_csum_name(const char *name, int len) | |||
+{ | |||
+ if (len < 0 && name) | |||
+ len = strlen(name); | |||
+ | |||
+ if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) { | |||
+ if (protocol_version >= 30) | |||
+ return CSUM_MD5; | |||
+ if (protocol_version >= 27) | |||
+ return CSUM_MD4_OLD; | |||
+ if (protocol_version >= 21) | |||
+ return CSUM_MD4_BUSTED; | |||
+ return CSUM_ARCHAIC; | |||
+ } | |||
+ if (len == 3 && strncasecmp(name, "md4", 3) == 0) | |||
+ return CSUM_MD4; | |||
+ if (len == 3 && strncasecmp(name, "md5", 3) == 0) | |||
+ return CSUM_MD5; | |||
+ if (len == 4 && strncasecmp(name, "none", 4) == 0) | |||
+ return CSUM_NONE; | |||
+ | |||
+ rprintf(FERROR, "unknown checksum name: %s\n", name); | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
+} | |||
+ | |||
+int csum_len_for_type(int cst) | |||
+{ | |||
+ switch (cst) { | |||
+ case CSUM_NONE: | |||
+ return 1; | |||
+ case CSUM_ARCHAIC: | |||
+ return 2; | |||
+ case CSUM_MD4: | |||
+ case CSUM_MD4_OLD: | |||
+ case CSUM_MD4_BUSTED: | |||
+ return MD4_DIGEST_LEN; | |||
+ case CSUM_MD5: | |||
+ return MD5_DIGEST_LEN; | |||
+ } | |||
+ return 0; | |||
+} | |||
+ | |||
+int canonical_checksum(int csum_type) | |||
+{ | |||
+ return csum_type >= CSUM_MD4 ? 1 : 0; | |||
+} | |||
/* | |||
a simple 32 bit checksum that can be upadted from either end | |||
@@ -47,12 +117,12 @@ uint32 get_checksum1(char *buf1, int32 len) | |||
return (s1 & 0xffff) + (s2 << 16); | |||
} | |||
- | |||
void get_checksum2(char *buf, int32 len, char *sum) | |||
{ | |||
md_context m; | |||
- if (protocol_version >= 30) { | |||
+ switch (xfersum_type) { | |||
+ case CSUM_MD5: { | |||
uchar seedbuf[4]; | |||
md5_begin(&m); | |||
if (proper_seed_order) { | |||
@@ -69,7 +139,11 @@ void get_checksum2(char *buf, int32 len, char *sum) | |||
} | |||
} | |||
md5_result(&m, (uchar *)sum); | |||
- } else { | |||
+ break; | |||
+ } | |||
+ case CSUM_MD4: | |||
+ case CSUM_MD4_OLD: | |||
+ case CSUM_MD4_BUSTED: { | |||
int32 i; | |||
static char *buf1; | |||
static int32 len1; | |||
@@ -100,10 +174,12 @@ void get_checksum2(char *buf, int32 len, char *sum) | |||
* are multiples of 64. This is fixed by calling mdfour_update() | |||
* even when there are no more bytes. | |||
*/ | |||
- if (len - i > 0 || protocol_version >= 27) | |||
+ if (len - i > 0 || xfersum_type != CSUM_MD4_BUSTED) | |||
mdfour_update(&m, (uchar *)(buf1+i), len-i); | |||
mdfour_result(&m, (uchar *)sum); | |||
+ break; | |||
+ } | |||
} | |||
} | |||
@@ -123,7 +199,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK); | |||
- if (protocol_version >= 30) { | |||
+ switch (checksum_type) { | |||
+ case CSUM_MD5: | |||
md5_begin(&m); | |||
for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { | |||
@@ -136,7 +213,10 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); | |||
md5_result(&m, (uchar *)sum); | |||
- } else { | |||
+ break; | |||
+ case CSUM_MD4: | |||
+ case CSUM_MD4_OLD: | |||
+ case CSUM_MD4_BUSTED: | |||
mdfour_begin(&m); | |||
for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { | |||
@@ -149,10 +229,14 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
* are multiples of 64. This is fixed by calling mdfour_update() | |||
* even when there are no more bytes. */ | |||
remainder = (int32)(len - i); | |||
- if (remainder > 0 || protocol_version >= 27) | |||
+ if (remainder > 0 || checksum_type != CSUM_MD4_BUSTED) | |||
mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); | |||
mdfour_result(&m, (uchar *)sum); | |||
+ break; | |||
+ default: | |||
+ rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type); | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
close(fd); | |||
@@ -161,18 +245,33 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
static int32 sumresidue; | |||
static md_context md; | |||
+static int cursum_type; | |||
-void sum_init(int seed) | |||
+void sum_init(int csum_type, int seed) | |||
{ | |||
char s[4]; | |||
- if (protocol_version >= 30) | |||
+ if (csum_type < 0) | |||
+ csum_type = parse_csum_name(NULL, 0); | |||
+ cursum_type = csum_type; | |||
+ | |||
+ switch (csum_type) { | |||
+ case CSUM_MD5: | |||
md5_begin(&md); | |||
- else { | |||
+ break; | |||
+ case CSUM_MD4: | |||
+ mdfour_begin(&md); | |||
+ sumresidue = 0; | |||
+ break; | |||
+ case CSUM_MD4_OLD: | |||
+ case CSUM_MD4_BUSTED: | |||
mdfour_begin(&md); | |||
sumresidue = 0; | |||
SIVAL(s, 0, seed); | |||
sum_update(s, 4); | |||
+ break; | |||
+ case CSUM_NONE: | |||
+ break; | |||
} | |||
} | |||
@@ -186,13 +285,17 @@ void sum_init(int seed) | |||
**/ | |||
void sum_update(const char *p, int32 len) | |||
{ | |||
- if (protocol_version >= 30) { | |||
+ switch (cursum_type) { | |||
+ case CSUM_MD5: | |||
md5_update(&md, (uchar *)p, len); | |||
- } else { | |||
+ break; | |||
+ case CSUM_MD4: | |||
+ case CSUM_MD4_OLD: | |||
+ case CSUM_MD4_BUSTED: | |||
if (len + sumresidue < CSUM_CHUNK) { | |||
memcpy(md.buffer + sumresidue, p, len); | |||
sumresidue += len; | |||
- return; | |||
+ break; | |||
} | |||
if (sumresidue) { | |||
@@ -212,20 +315,32 @@ void sum_update(const char *p, int32 len) | |||
sumresidue = len; | |||
if (sumresidue) | |||
memcpy(md.buffer, p, sumresidue); | |||
+ break; | |||
+ case CSUM_NONE: | |||
+ break; | |||
} | |||
} | |||
int sum_end(char *sum) | |||
{ | |||
- if (protocol_version >= 30) { | |||
+ switch (cursum_type) { | |||
+ case CSUM_MD5: | |||
md5_result(&md, (uchar *)sum); | |||
- return MD5_DIGEST_LEN; | |||
- } else { | |||
- if (sumresidue || protocol_version >= 27) | |||
+ break; | |||
+ case CSUM_MD4: | |||
+ case CSUM_MD4_OLD: | |||
+ mdfour_update(&md, (uchar *)md.buffer, sumresidue); | |||
+ mdfour_result(&md, (uchar *)sum); | |||
+ break; | |||
+ case CSUM_MD4_BUSTED: | |||
+ if (sumresidue) | |||
mdfour_update(&md, (uchar *)md.buffer, sumresidue); | |||
- | |||
mdfour_result(&md, (uchar *)sum); | |||
- | |||
- return MD4_DIGEST_LEN; | |||
+ break; | |||
+ case CSUM_NONE: | |||
+ *sum = '\0'; | |||
+ break; | |||
} | |||
+ | |||
+ return csum_len_for_type(cursum_type); | |||
} | |||
diff --git a/compat.c b/compat.c | |||
index c792312..505cb7f 100644 | |||
--- a/compat.c | |||
+++ b/compat.c | |||
@@ -338,4 +338,6 @@ void setup_protocol(int f_out,int f_in) | |||
} else { | |||
checksum_seed = read_int(f_in); | |||
} | |||
+ | |||
+ init_flist(); | |||
} | |||
diff --git a/flist.c b/flist.c | |||
index c1e48b3..acb95f7 100644 | |||
--- a/flist.c | |||
+++ b/flist.c | |||
@@ -33,6 +33,7 @@ extern int am_sender; | |||
extern int am_generator; | |||
extern int inc_recurse; | |||
extern int always_checksum; | |||
+extern int checksum_type; | |||
extern int module_id; | |||
extern int ignore_errors; | |||
extern int numeric_ids; | |||
@@ -137,9 +138,8 @@ void init_flist(void) | |||
rprintf(FINFO, "FILE_STRUCT_LEN=%d, EXTRA_LEN=%d\n", | |||
(int)FILE_STRUCT_LEN, (int)EXTRA_LEN); | |||
} | |||
- checksum_len = protocol_version < 21 ? 2 | |||
- : protocol_version < 30 ? MD4_DIGEST_LEN | |||
- : MD5_DIGEST_LEN; | |||
+ parse_checksum_choice(); /* Sets checksum_type && xfersum_type */ | |||
+ checksum_len = csum_len_for_type(checksum_type); | |||
} | |||
static int show_filelist_p(void) | |||
diff --git a/log.c b/log.c | |||
index 24256de..f7da1e5 100644 | |||
--- a/log.c | |||
+++ b/log.c | |||
@@ -31,12 +31,13 @@ extern int am_generator; | |||
extern int local_server; | |||
extern int quiet; | |||
extern int module_id; | |||
-extern int checksum_len; | |||
extern int allow_8bit_chars; | |||
extern int protocol_version; | |||
extern int always_checksum; | |||
extern int preserve_times; | |||
extern int msgs2stderr; | |||
+extern int xfersum_type; | |||
+extern int checksum_type; | |||
extern int stdout_format_has_i; | |||
extern int stdout_format_has_o_or_i; | |||
extern int logfile_format_has_i; | |||
@@ -46,6 +47,7 @@ extern int64 total_data_written; | |||
extern int64 total_data_read; | |||
extern mode_t orig_umask; | |||
extern char *auth_user; | |||
+extern char *checksum_choice; | |||
extern char *stdout_format; | |||
extern char *logfile_format; | |||
extern char *logfile_name; | |||
@@ -669,13 +671,15 @@ static void log_formatted(enum logcode code, const char *format, const char *op, | |||
n = buf2; | |||
break; | |||
case 'C': | |||
- if (protocol_version >= 30 | |||
- && (iflags & ITEM_TRANSFER | |||
- || (always_checksum && S_ISREG(file->mode)))) { | |||
- const char *sum = iflags & ITEM_TRANSFER | |||
- ? sender_file_sum : F_SUM(file); | |||
- n = sum_as_hex(sum); | |||
- } else { | |||
+ n = NULL; | |||
+ if (S_ISREG(file->mode)) { | |||
+ if (always_checksum && canonical_checksum(checksum_type)) | |||
+ n = sum_as_hex(checksum_type, F_SUM(file)); | |||
+ else if (iflags & ITEM_TRANSFER && canonical_checksum(xfersum_type)) | |||
+ n = sum_as_hex(xfersum_type, sender_file_sum); | |||
+ } | |||
+ if (!n) { | |||
+ int checksum_len = csum_len_for_type(always_checksum ? checksum_type : xfersum_type); | |||
memset(buf2, ' ', checksum_len*2); | |||
buf2[checksum_len*2] = '\0'; | |||
n = buf2; | |||
diff --git a/main.c b/main.c | |||
index 3132aa9..3908ccf 100644 | |||
--- a/main.c | |||
+++ b/main.c | |||
@@ -1595,8 +1595,6 @@ int main(int argc,char *argv[]) | |||
* that implement getcwd that way "pwd" can't be found after chroot. */ | |||
change_dir(NULL, CD_NORMAL); | |||
- init_flist(); | |||
- | |||
if ((write_batch || read_batch) && !am_server) { | |||
if (write_batch) | |||
write_batch_shell_file(orig_argc, orig_argv, argc); | |||
diff --git a/match.c b/match.c | |||
index b15f2eb..ff10310 100644 | |||
--- a/match.c | |||
+++ b/match.c | |||
@@ -24,7 +24,7 @@ | |||
extern int checksum_seed; | |||
extern int append_mode; | |||
-extern int checksum_len; | |||
+extern int xfersum_type; | |||
int updating_basis_file; | |||
char sender_file_sum[MAX_DIGEST_LEN]; | |||
@@ -360,13 +360,15 @@ static void hash_search(int f,struct sum_struct *s, | |||
**/ | |||
void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) | |||
{ | |||
+ int checksum_len; | |||
+ | |||
last_match = 0; | |||
false_alarms = 0; | |||
hash_hits = 0; | |||
matches = 0; | |||
data_transfer = 0; | |||
- sum_init(checksum_seed); | |||
+ sum_init(xfersum_type, checksum_seed); | |||
if (append_mode > 0) { | |||
if (append_mode == 2) { | |||
@@ -407,8 +409,7 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) | |||
matched(f, s, buf, len, -1); | |||
} | |||
- if (sum_end(sender_file_sum) != checksum_len) | |||
- overflow_exit("checksum_len"); /* Impossible... */ | |||
+ checksum_len = sum_end(sender_file_sum); | |||
/* If we had a read error, send a bad checksum. We use all bits | |||
* off as long as the checksum doesn't happen to be that, in | |||
diff --git a/options.c b/options.c | |||
index 4a5cdc8..308443b 100644 | |||
--- a/options.c | |||
+++ b/options.c | |||
@@ -182,6 +182,7 @@ char *dest_option = NULL; | |||
static int remote_option_alloc = 0; | |||
int remote_option_cnt = 0; | |||
const char **remote_options = NULL; | |||
+const char *checksum_choice = NULL; | |||
int quiet = 0; | |||
int output_motd = 1; | |||
@@ -721,6 +722,7 @@ void usage(enum logcode F) | |||
#endif | |||
rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); | |||
rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); | |||
+ rprintf(F," --checksum-choice=STR choose the checksum algorithms\n"); | |||
rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); | |||
rprintf(F," -B, --block-size=SIZE force a fixed checksum block-size\n"); | |||
rprintf(F," -e, --rsh=COMMAND specify the remote shell to use\n"); | |||
@@ -953,6 +955,7 @@ static struct poptOption long_options[] = { | |||
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 }, | |||
{"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 }, | |||
{"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 }, | |||
+ {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 }, | |||
{"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 }, | |||
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, | |||
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |||
@@ -1814,6 +1817,15 @@ int parse_arguments(int *argc_p, const char ***argv_p) | |||
} | |||
} | |||
+ if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) { | |||
+ /* Call this early to verify the args and figure out if we need to force | |||
+ * --whole-file. Note that the parse function will get called again later, | |||
+ * just in case an "auto" choice needs to know the protocol_version. */ | |||
+ if (parse_checksum_choice()) | |||
+ whole_file = 1; | |||
+ } else | |||
+ checksum_choice = NULL; | |||
+ | |||
if (human_readable > 1 && argc == 2 && !am_server) { | |||
/* Allow the old meaning of 'h' (--help) on its own. */ | |||
usage(FINFO); | |||
@@ -2597,6 +2609,12 @@ void server_options(char **args, int *argc_p) | |||
args[ac++] = arg; | |||
} | |||
+ if (checksum_choice) { | |||
+ if (asprintf(&arg, "--checksum-choice=%s", checksum_choice) < 0) | |||
+ goto oom; | |||
+ args[ac++] = arg; | |||
+ } | |||
+ | |||
if (am_sender) { | |||
if (max_delete > 0) { | |||
if (asprintf(&arg, "--max-delete=%d", max_delete) < 0) | |||
diff --git a/receiver.c b/receiver.c | |||
index 4ea4c09..f9b97dd 100644 | |||
--- a/receiver.c | |||
+++ b/receiver.c | |||
@@ -48,11 +48,11 @@ extern int append_mode; | |||
extern int sparse_files; | |||
extern int preallocate_files; | |||
extern int keep_partial; | |||
-extern int checksum_len; | |||
extern int checksum_seed; | |||
extern int inplace; | |||
extern int allowed_lull; | |||
extern int delay_updates; | |||
+extern int xfersum_type; | |||
extern mode_t orig_umask; | |||
extern struct stats stats; | |||
extern char *tmpdir; | |||
@@ -234,6 +234,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
static char file_sum1[MAX_DIGEST_LEN]; | |||
struct map_struct *mapbuf; | |||
struct sum_struct sum; | |||
+ int checksum_len; | |||
int32 len; | |||
OFF_T offset = 0; | |||
OFF_T offset2; | |||
@@ -269,7 +270,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
} else | |||
mapbuf = NULL; | |||
- sum_init(checksum_seed); | |||
+ sum_init(xfersum_type, checksum_seed); | |||
if (append_mode > 0) { | |||
OFF_T j; | |||
@@ -393,8 +394,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
exit_cleanup(RERR_FILEIO); | |||
} | |||
- if (sum_end(file_sum1) != checksum_len) | |||
- overflow_exit("checksum_len"); /* Impossible... */ | |||
+ checksum_len = sum_end(file_sum1); | |||
if (mapbuf) | |||
unmap_file(mapbuf); | |||
diff --git a/rsync.yo b/rsync.yo | |||
index 8971828..0ec5e55 100644 | |||
--- a/rsync.yo | |||
+++ b/rsync.yo | |||
@@ -380,6 +380,7 @@ to the detailed description below for a complete description. verb( | |||
--preallocate allocate dest files before writing | |||
-n, --dry-run perform a trial run with no changes made | |||
-W, --whole-file copy files whole (w/o delta-xfer algorithm) | |||
+ --checksum-choice=STR choose the checksum algorithms | |||
-x, --one-file-system don't cross filesystem boundaries | |||
-B, --block-size=SIZE force a fixed checksum block-size | |||
-e, --rsh=COMMAND specify the remote shell to use | |||
@@ -1280,14 +1281,27 @@ the "bytes sent", "bytes received", "literal data", and "matched data" | |||
statistics are too small, and the "speedup" value is equivalent to a run | |||
where no file transfers were needed. | |||
-dit(bf(-W, --whole-file)) With this option rsync's delta-transfer algorithm | |||
-is not used and the whole file is sent as-is instead. The transfer may be | |||
+dit(bf(-W, --whole-file)) This option disables rsync's delta-transfer algorithm, | |||
+which causes all transferred files to be sent whole. The transfer may be | |||
faster if this option is used when the bandwidth between the source and | |||
destination machines is higher than the bandwidth to disk (especially when the | |||
"disk" is actually a networked filesystem). This is the default when both | |||
the source and destination are specified as local paths, but only if no | |||
batch-writing option is in effect. | |||
+dit(bf(--checksum-choice=STR)) This option overrides the checksum algoriths. | |||
+If one algorithm name is specified, it is used for both the transfer checksums | |||
+and (assuming bf(--checksum) is specifed) the pre-transfer checksumming. If two | |||
+comma-separated names are supplied, the first name affects the transfer | |||
+checksums, and the second name affects the pre-transfer checksumming. | |||
+ | |||
+The algorithm choices are "auto", "md4", "md5", and "none". If "none" is | |||
+specified for the first name, the bf(--whole-file) option is forced on and no | |||
+checksum verification is performed on the transferred data. If "none" is | |||
+specified for the second name, the bf(--checksum) option cannot be used. The | |||
+"auto" option is the default, where rsync bases its algorithm choice on the | |||
+protocol version (for backward compatibility with older rsync versions). | |||
+ | |||
dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a | |||
filesystem boundary when recursing. This does not limit the user's ability | |||
to specify items to copy from multiple filesystems, just rsync's recursion | |||
diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo | |||
index 1813354..64156ae 100644 | |||
--- a/rsyncd.conf.yo | |||
+++ b/rsyncd.conf.yo | |||
@@ -656,7 +656,7 @@ quote(itemization( | |||
it() %b the number of bytes actually transferred | |||
it() %B the permission bits of the file (e.g. rwxrwxrwt) | |||
it() %c the total size of the block checksums received for the basis file (only when sending) | |||
- it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above). | |||
+ it() %C the full-file checksum if it is known for the file. For older rsync protocols/versions, the checksum was salted, and is thus not a useful value (and is not displayed when that is the case). For the checksum to output for a file, either the bf(--checksum) option must be in-effect or the file must have been transferred without a salted checksum being used. See the bf(--checksum-choice) option for a way to choose the algorithm. | |||
it() %f the filename (long form on sender; no trailing "/") | |||
it() %G the gid of the file (decimal) or "DEFAULT" | |||
it() %h the remote host name (only available for a daemon) | |||
diff --git a/t_stub.c b/t_stub.c | |||
index 6002250..26951a6 100644 | |||
--- a/t_stub.c | |||
+++ b/t_stub.c | |||
@@ -25,7 +25,6 @@ int modify_window = 0; | |||
int preallocate_files = 0; | |||
int protect_args = 0; | |||
int module_id = -1; | |||
-int checksum_len = 0; | |||
int relative_paths = 0; | |||
int module_dirlen = 0; | |||
int preserve_acls = 0; | |||
@@ -97,3 +96,8 @@ filter_rule_list daemon_filter_list; | |||
{ | |||
return "tester"; | |||
} | |||
+ | |||
+ int csum_len_for_type(int cst) | |||
+{ | |||
+ return cst ? 16 : 1; | |||
+} | |||
diff --git a/util2.c b/util2.c | |||
index cc368af..a892e51 100644 | |||
--- a/util2.c | |||
+++ b/util2.c | |||
@@ -25,8 +25,6 @@ | |||
#include "itypes.h" | |||
#include "inums.h" | |||
-extern int checksum_len; | |||
- | |||
/** | |||
* Sleep for a specified number of milliseconds. | |||
* | |||
@@ -79,10 +77,11 @@ void *_realloc_array(void *ptr, unsigned int size, size_t num) | |||
return realloc(ptr, size * num); | |||
} | |||
-const char *sum_as_hex(const char *sum) | |||
+const char *sum_as_hex(int csum_type, const char *sum) | |||
{ | |||
static char buf[MAX_DIGEST_LEN*2+1]; | |||
int i, x1, x2; | |||
+ int checksum_len = csum_len_for_type(csum_type); | |||
char *c = buf + checksum_len*2; | |||
assert(c - buf < (int)sizeof buf); | |||
diff --git a/xattrs.c b/xattrs.c | |||
index 57833e5..6a77a0b 100644 | |||
--- a/xattrs.c | |||
+++ b/xattrs.c | |||
@@ -258,7 +258,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp) | |||
if (datum_len > MAX_FULL_DATUM) { | |||
/* For large datums, we store a flag and a checksum. */ | |||
name_offset = 1 + MAX_DIGEST_LEN; | |||
- sum_init(checksum_seed); | |||
+ sum_init(-1, checksum_seed); | |||
sum_update(ptr, datum_len); | |||
free(ptr); | |||
@@ -821,7 +821,7 @@ static int rsync_xal_set(const char *fname, item_list *xalp, | |||
goto still_abbrev; | |||
} | |||
- sum_init(checksum_seed); | |||
+ sum_init(-1, checksum_seed); | |||
sum_update(ptr, len); | |||
sum_end(sum); | |||
if (memcmp(sum, rxas[i].datum + 1, MAX_DIGEST_LEN) != 0) { |
@ -1,593 +0,0 @@ | |||
commit f3873b3d88b61167b106e7b9227a20147f8f6197 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Mon Oct 10 11:49:50 2016 -0700 | |||
Support --sparse combined with --preallocate or --inplace. | |||
The new code tries to punch holes in the destination file using newer | |||
Linux fallocate features. It also supports a --whole-file + --sparse + | |||
--inplace copy on any filesystem by truncating the destination file. | |||
diff --git a/configure.ac b/configure.ac | |||
index b5e4049..e01e124 100644 | |||
--- a/configure.ac | |||
+++ b/configure.ac | |||
@@ -614,6 +614,36 @@ if test x"$rsync_cv_have_fallocate" = x"yes"; then | |||
AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error]) | |||
fi | |||
+AC_MSG_CHECKING([for FALLOC_FL_PUNCH_HOLE]) | |||
+AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ | |||
+ #define _GNU_SOURCE 1 | |||
+ #include <linux/falloc.h> | |||
+ #ifndef FALLOC_FL_PUNCH_HOLE | |||
+ #error FALLOC_FL_PUNCH_HOLE is missing | |||
+ #endif | |||
+ ]])], [ | |||
+ AC_MSG_RESULT([yes]) | |||
+ AC_DEFINE([HAVE_FALLOC_FL_PUNCH_HOLE], [1], [Define if FALLOC_FL_PUNCH_HOLE is available.]) | |||
+ ], [ | |||
+ AC_MSG_RESULT([no]) | |||
+ ] | |||
+) | |||
+ | |||
+AC_MSG_CHECKING([for FALLOC_FL_ZERO_RANGE]) | |||
+AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ | |||
+ #define _GNU_SOURCE 1 | |||
+ #include <linux/falloc.h> | |||
+ #ifndef FALLOC_FL_ZERO_RANGE | |||
+ #error FALLOC_FL_ZERO_RANGE is missing | |||
+ #endif | |||
+ ]])], [ | |||
+ AC_MSG_RESULT([yes]) | |||
+ AC_DEFINE([HAVE_FALLOC_FL_ZERO_RANGE], [1], [Define if FALLOC_FL_ZERO_RANGE is available.]) | |||
+ ], [ | |||
+ AC_MSG_RESULT([no]) | |||
+ ] | |||
+) | |||
+ | |||
AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ | |||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/syscall.h> | |||
#include <sys/types.h>]], [[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);]])],[rsync_cv_have_sys_fallocate=yes],[rsync_cv_have_sys_fallocate=no])]) | |||
diff --git a/fileio.c b/fileio.c | |||
index 70e079d..1e8a562 100644 | |||
--- a/fileio.c | |||
+++ b/fileio.c | |||
@@ -35,7 +35,10 @@ | |||
extern int sparse_files; | |||
+OFF_T preallocated_len = 0; | |||
+ | |||
static OFF_T sparse_seek = 0; | |||
+static OFF_T sparse_past_write = 0; | |||
int sparse_end(int f, OFF_T size) | |||
{ | |||
@@ -63,8 +66,10 @@ int sparse_end(int f, OFF_T size) | |||
return ret; | |||
} | |||
- | |||
-static int write_sparse(int f, char *buf, int len) | |||
+/* Note that the offset is just the caller letting us know where | |||
+ * the current file position is in the file. The use_seek arg tells | |||
+ * us that we should seek over matching data instead of writing it. */ | |||
+static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int len) | |||
{ | |||
int l1 = 0, l2 = 0; | |||
int ret; | |||
@@ -77,9 +82,24 @@ static int write_sparse(int f, char *buf, int len) | |||
if (l1 == len) | |||
return len; | |||
- if (sparse_seek) | |||
- do_lseek(f, sparse_seek, SEEK_CUR); | |||
+ if (sparse_seek) { | |||
+ if (sparse_past_write >= preallocated_len) { | |||
+ if (do_lseek(f, sparse_seek, SEEK_CUR) < 0) | |||
+ return -1; | |||
+ } else if (do_punch_hole(f, sparse_past_write, sparse_seek) < 0) { | |||
+ sparse_seek = 0; | |||
+ return -1; | |||
+ } | |||
+ } | |||
sparse_seek = l2; | |||
+ sparse_past_write = offset + len - l2; | |||
+ | |||
+ if (use_seek) { | |||
+ /* The in-place data already matches. */ | |||
+ if (do_lseek(f, len - (l1+l2), SEEK_CUR) < 0) | |||
+ return -1; | |||
+ return len; | |||
+ } | |||
while ((ret = write(f, buf + l1, len - (l1+l2))) <= 0) { | |||
if (ret < 0 && errno == EINTR) | |||
@@ -96,7 +116,6 @@ static int write_sparse(int f, char *buf, int len) | |||
return len; | |||
} | |||
- | |||
static char *wf_writeBuf; | |||
static size_t wf_writeBufSize; | |||
static size_t wf_writeBufCnt; | |||
@@ -118,12 +137,10 @@ int flush_write_file(int f) | |||
return ret; | |||
} | |||
- | |||
-/* | |||
- * write_file does not allow incomplete writes. It loops internally | |||
- * until len bytes are written or errno is set. | |||
- */ | |||
-int write_file(int f, char *buf, int len) | |||
+/* write_file does not allow incomplete writes. It loops internally | |||
+ * until len bytes are written or errno is set. Note that use_seek and | |||
+ * offset are only used in sparse processing (see write_sparse()). */ | |||
+int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) | |||
{ | |||
int ret = 0; | |||
@@ -131,7 +148,8 @@ int write_file(int f, char *buf, int len) | |||
int r1; | |||
if (sparse_files > 0) { | |||
int len1 = MIN(len, SPARSE_WRITE_SIZE); | |||
- r1 = write_sparse(f, buf, len1); | |||
+ r1 = write_sparse(f, use_seek, offset, buf, len1); | |||
+ offset += r1; | |||
} else { | |||
if (!wf_writeBuf) { | |||
wf_writeBufSize = WRITE_SIZE * 8; | |||
@@ -164,6 +182,30 @@ int write_file(int f, char *buf, int len) | |||
return ret; | |||
} | |||
+/* An in-place update found identical data at an identical location. We either | |||
+ * just seek past it, or (for an in-place sparse update), we give the data to | |||
+ * the sparse processor with the use_seek flag set. */ | |||
+int skip_matched(int fd, OFF_T offset, const char *buf, int len) | |||
+{ | |||
+ OFF_T pos; | |||
+ | |||
+ if (sparse_files > 0) { | |||
+ if (write_file(fd, 1, offset, buf, len) != len) | |||
+ return -1; | |||
+ return 0; | |||
+ } | |||
+ | |||
+ if (flush_write_file(fd) < 0) | |||
+ return -1; | |||
+ | |||
+ if ((pos = do_lseek(fd, len, SEEK_CUR)) != offset + len) { | |||
+ rsyserr(FERROR_XFER, errno, "lseek returned %s, not %s", | |||
+ big_num(pos), big_num(offset)); | |||
+ return -1; | |||
+ } | |||
+ | |||
+ return 0; | |||
+} | |||
/* This provides functionality somewhat similar to mmap() but using read(). | |||
* It gives sliding window access to a file. mmap() is not used because of | |||
@@ -271,7 +313,6 @@ char *map_ptr(struct map_struct *map, OFF_T offset, int32 len) | |||
return map->p + align_fudge; | |||
} | |||
- | |||
int unmap_file(struct map_struct *map) | |||
{ | |||
int ret; | |||
diff --git a/options.c b/options.c | |||
index 308443b..6ba13b7 100644 | |||
--- a/options.c | |||
+++ b/options.c | |||
@@ -714,7 +714,7 @@ void usage(enum logcode F) | |||
#ifdef SUPPORT_XATTRS | |||
rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); | |||
#endif | |||
- rprintf(F," -S, --sparse handle sparse files efficiently\n"); | |||
+ rprintf(F," -S, --sparse turn sequences of nulls into sparse blocks\n"); | |||
#ifdef SUPPORT_PREALLOCATION | |||
rprintf(F," --preallocate allocate dest files before writing them\n"); | |||
#else | |||
@@ -2237,14 +2237,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) | |||
bwlimit_writemax = 512; | |||
} | |||
- if (sparse_files && inplace) { | |||
- /* Note: we don't check for this below, because --append is | |||
- * OK with --sparse (as long as redos are handled right). */ | |||
- snprintf(err_buf, sizeof err_buf, | |||
- "--sparse cannot be used with --inplace\n"); | |||
- return 0; | |||
- } | |||
- | |||
if (append_mode) { | |||
if (whole_file > 0) { | |||
snprintf(err_buf, sizeof err_buf, | |||
diff --git a/receiver.c b/receiver.c | |||
index f9b97dd..bed5328 100644 | |||
--- a/receiver.c | |||
+++ b/receiver.c | |||
@@ -49,6 +49,7 @@ extern int sparse_files; | |||
extern int preallocate_files; | |||
extern int keep_partial; | |||
extern int checksum_seed; | |||
+extern int whole_file; | |||
extern int inplace; | |||
extern int allowed_lull; | |||
extern int delay_updates; | |||
@@ -61,6 +62,9 @@ extern char *basis_dir[MAX_BASIS_DIRS+1]; | |||
extern char sender_file_sum[MAX_DIGEST_LEN]; | |||
extern struct file_list *cur_flist, *first_flist, *dir_flist; | |||
extern filter_rule_list daemon_filter_list; | |||
+#ifdef SUPPORT_PREALLOCATION | |||
+extern OFF_T preallocated_len; | |||
+#endif | |||
static struct bitbag *delayed_bits = NULL; | |||
static int phase = 0, redoing = 0; | |||
@@ -241,22 +245,25 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
char *data; | |||
int32 i; | |||
char *map = NULL; | |||
-#ifdef SUPPORT_PREALLOCATION | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
- OFF_T preallocated_len = 0; | |||
-#endif | |||
+#ifdef SUPPORT_PREALLOCATION | |||
if (preallocate_files && fd != -1 && total_size > 0 && (!inplace || total_size > size_r)) { | |||
/* Try to preallocate enough space for file's eventual length. Can | |||
* reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ | |||
- if (do_fallocate(fd, 0, total_size) == 0) { | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
- preallocated_len = total_size; | |||
-#endif | |||
- } else | |||
+ if ((preallocated_len = do_fallocate(fd, 0, total_size)) < 0) | |||
rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); | |||
- } | |||
+ } else | |||
+#endif | |||
+ if (inplace) { | |||
+#ifdef HAVE_FTRUNCATE | |||
+ /* The most compatible way to create a sparse file is to start with no length. */ | |||
+ if (sparse_files > 0 && whole_file && fd >= 0 && do_ftruncate(fd, 0) == 0) | |||
+ preallocated_len = 0; | |||
+ else | |||
#endif | |||
+ preallocated_len = size_r; | |||
+ } else | |||
+ preallocated_len = 0; | |||
read_sum_head(f_in, &sum); | |||
@@ -318,7 +325,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
sum_update(data, i); | |||
- if (fd != -1 && write_file(fd,data,i) != i) | |||
+ if (fd != -1 && write_file(fd, 0, offset, data, i) != i) | |||
goto report_write_error; | |||
offset += i; | |||
continue; | |||
@@ -348,37 +355,33 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
if (updating_basis_or_equiv) { | |||
if (offset == offset2 && fd != -1) { | |||
- OFF_T pos; | |||
- if (flush_write_file(fd) < 0) | |||
+ if (skip_matched(fd, offset, map, len) < 0) | |||
goto report_write_error; | |||
offset += len; | |||
- if ((pos = do_lseek(fd, len, SEEK_CUR)) != offset) { | |||
- rsyserr(FERROR_XFER, errno, | |||
- "lseek of %s returned %s, not %s", | |||
- full_fname(fname), | |||
- big_num(pos), big_num(offset)); | |||
- exit_cleanup(RERR_FILEIO); | |||
- } | |||
continue; | |||
} | |||
} | |||
- if (fd != -1 && map && write_file(fd, map, len) != (int)len) | |||
+ if (fd != -1 && map && write_file(fd, 0, offset, map, len) != (int)len) | |||
goto report_write_error; | |||
offset += len; | |||
} | |||
- if (flush_write_file(fd) < 0) | |||
- goto report_write_error; | |||
+ if (fd != -1 && offset > 0) { | |||
+ if (sparse_files > 0) { | |||
+ if (sparse_end(fd, offset) != 0) | |||
+ goto report_write_error; | |||
+ } else if (flush_write_file(fd) < 0) { | |||
+ report_write_error: | |||
+ rsyserr(FERROR_XFER, errno, "write failed on %s", full_fname(fname)); | |||
+ exit_cleanup(RERR_FILEIO); | |||
+ } | |||
+ } | |||
#ifdef HAVE_FTRUNCATE | |||
/* inplace: New data could be shorter than old data. | |||
* preallocate_files: total_size could have been an overestimate. | |||
* Cut off any extra preallocated zeros from dest file. */ | |||
- if ((inplace | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
- || preallocated_len > offset | |||
-#endif | |||
- ) && fd != -1 && do_ftruncate(fd, offset) < 0) { | |||
+ if ((inplace || preallocated_len > offset) && fd != -1 && do_ftruncate(fd, offset) < 0) { | |||
rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", | |||
full_fname(fname)); | |||
} | |||
@@ -387,13 +390,6 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
if (INFO_GTE(PROGRESS, 1)) | |||
end_progress(total_size); | |||
- if (fd != -1 && offset > 0 && sparse_end(fd, offset) != 0) { | |||
- report_write_error: | |||
- rsyserr(FERROR_XFER, errno, "write failed on %s", | |||
- full_fname(fname)); | |||
- exit_cleanup(RERR_FILEIO); | |||
- } | |||
- | |||
checksum_len = sum_end(file_sum1); | |||
if (mapbuf) | |||
diff --git a/rsync.yo b/rsync.yo | |||
index bfe43b9..d1e6fdf 100644 | |||
--- a/rsync.yo | |||
+++ b/rsync.yo | |||
@@ -376,7 +376,7 @@ to the detailed description below for a complete description. verb( | |||
-J, --omit-link-times omit symlinks from --times | |||
--super receiver attempts super-user activities | |||
--fake-super store/recover privileged attrs using xattrs | |||
- -S, --sparse handle sparse files efficiently | |||
+ -S, --sparse turn sequences of nulls into sparse blocks | |||
--preallocate allocate dest files before writing | |||
-n, --dry-run perform a trial run with no changes made | |||
-W, --whole-file copy files whole (w/o delta-xfer algorithm) | |||
@@ -873,9 +873,7 @@ the same or longer than the size on the sender, the file is skipped. This | |||
does not interfere with the updating of a file's non-content attributes | |||
(e.g. permissions, ownership, etc.) when the file does not need to be | |||
transferred, nor does it affect the updating of any non-regular files. | |||
-Implies bf(--inplace), | |||
-but does not conflict with bf(--sparse) (since it is always extending a | |||
-file's length). | |||
+Implies bf(--inplace). | |||
The use of bf(--append) can be dangerous if you aren't 100% sure that the files | |||
that are longer have only grown by the appending of data onto the end. You | |||
@@ -1252,20 +1250,30 @@ This option is overridden by both bf(--super) and bf(--no-super). | |||
See also the "fake super" setting in the daemon's rsyncd.conf file. | |||
dit(bf(-S, --sparse)) Try to handle sparse files efficiently so they take | |||
-up less space on the destination. Conflicts with bf(--inplace) because it's | |||
-not possible to overwrite data in a sparse fashion. | |||
+up less space on the destination. If combined with bf(--inplace) the | |||
+file created might not end up with sparse blocks with some combinations | |||
+of kernel version and/or filesystem type. If bf(--whole-file) is in | |||
+effect (e.g. for a local copy) then it will always work because rsync | |||
+truncates the file prior to writing out the updated version. | |||
+ | |||
+Note that versions of rsync older than 3.1.3 will reject the combination of | |||
+bf(--sparse) and bf(--inplace). | |||
dit(bf(--preallocate)) This tells the receiver to allocate each destination | |||
-file to its eventual size before writing data to the file. Rsync will only use | |||
-the real filesystem-level preallocation support provided by Linux's | |||
+file to its eventual size before writing data to the file. Rsync will only | |||
+use the real filesystem-level preallocation support provided by Linux's | |||
bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow | |||
-glibc implementation that writes a zero byte into each block. | |||
+glibc implementation that writes a null byte into each block. | |||
Without this option, larger files may not be entirely contiguous on the | |||
filesystem, but with this option rsync will probably copy more slowly. If the | |||
destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS, | |||
etc.), this option may have no positive effect at all. | |||
+If combined with bf(--sparse), the file will only have sparse blocks (as | |||
+opposed to allocated sequences of null bytes) if the kernel version and | |||
+filesystem type support creating holes in the allocated data. | |||
+ | |||
dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't | |||
make any changes (and produces mostly the same output as a real run). It | |||
is most commonly used in combination with the bf(-v, --verbose) and/or | |||
diff --git a/syscall.c b/syscall.c | |||
index ecca2f1..fa53b63 100644 | |||
--- a/syscall.c | |||
+++ b/syscall.c | |||
@@ -38,6 +38,8 @@ extern int am_root; | |||
extern int am_sender; | |||
extern int read_only; | |||
extern int list_only; | |||
+extern int inplace; | |||
+extern int preallocate_files; | |||
extern int preserve_perms; | |||
extern int preserve_executability; | |||
@@ -423,27 +425,80 @@ int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)) | |||
#endif | |||
#ifdef SUPPORT_PREALLOCATION | |||
-int do_fallocate(int fd, OFF_T offset, OFF_T length) | |||
-{ | |||
#ifdef FALLOC_FL_KEEP_SIZE | |||
#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE | |||
#else | |||
#define DO_FALLOC_OPTIONS 0 | |||
#endif | |||
+ | |||
+OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length) | |||
+{ | |||
+ int opts = inplace || preallocate_files ? 0 : DO_FALLOC_OPTIONS; | |||
+ int ret; | |||
RETURN_ERROR_IF(dry_run, 0); | |||
RETURN_ERROR_IF_RO_OR_LO; | |||
+ if (length & 1) /* make the length not match the desired length */ | |||
+ length++; | |||
+ else | |||
+ length--; | |||
#if defined HAVE_FALLOCATE | |||
- return fallocate(fd, DO_FALLOC_OPTIONS, offset, length); | |||
+ ret = fallocate(fd, opts, offset, length); | |||
#elif defined HAVE_SYS_FALLOCATE | |||
- return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length); | |||
+ ret = syscall(SYS_fallocate, fd, opts, (loff_t)offset, (loff_t)length); | |||
#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE | |||
- return posix_fallocate(fd, offset, length); | |||
+ ret = posix_fallocate(fd, offset, length); | |||
#else | |||
#error Coding error in SUPPORT_PREALLOCATION logic. | |||
#endif | |||
+ if (ret < 0) | |||
+ return ret; | |||
+ if (opts == 0) { | |||
+ STRUCT_STAT st; | |||
+ if (do_fstat(fd, &st) < 0) | |||
+ return length; | |||
+ return st.st_blocks * 512; | |||
+ } | |||
+ return 0; | |||
} | |||
#endif | |||
+/* Punch a hole at pos for len bytes. The current file position must be at pos and will be | |||
+ * changed to be at pos + len. */ | |||
+int do_punch_hole(int fd, UNUSED(OFF_T pos), int len) | |||
+{ | |||
+#ifdef HAVE_FALLOCATE | |||
+# ifdef HAVE_FALLOC_FL_PUNCH_HOLE | |||
+ if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, pos, len) == 0) { | |||
+ if (do_lseek(fd, len, SEEK_CUR) != pos + len) | |||
+ return -1; | |||
+ return 0; | |||
+ } | |||
+# endif | |||
+# ifdef HAVE_FALLOC_FL_ZERO_RANGE | |||
+ if (fallocate(fd, FALLOC_FL_ZERO_RANGE, pos, len) == 0) { | |||
+ if (do_lseek(fd, len, SEEK_CUR) != pos + len) | |||
+ return -1; | |||
+ return 0; | |||
+ } | |||
+# endif | |||
+#endif | |||
+ { | |||
+ char zeros[4096]; | |||
+ memset(zeros, 0, sizeof zeros); | |||
+ while (len > 0) { | |||
+ int chunk = len > (int)sizeof zeros ? (int)sizeof zeros : len; | |||
+ int wrote = write(fd, zeros, chunk); | |||
+ if (wrote <= 0) { | |||
+ if (wrote < 0 && errno == EINTR) | |||
+ continue; | |||
+ return -1; | |||
+ } | |||
+ len -= wrote; | |||
+ } | |||
+ } | |||
+ return 0; | |||
+} | |||
+ | |||
int do_open_nofollow(const char *pathname, int flags) | |||
{ | |||
#ifndef O_NOFOLLOW | |||
diff --git a/t_stub.c b/t_stub.c | |||
index 26951a6..fc1ee3b 100644 | |||
--- a/t_stub.c | |||
+++ b/t_stub.c | |||
@@ -21,6 +21,7 @@ | |||
#include "rsync.h" | |||
+int inplace = 0; | |||
int modify_window = 0; | |||
int preallocate_files = 0; | |||
int protect_args = 0; | |||
diff --git a/tls.c b/tls.c | |||
index 45d1e10..d5a2896 100644 | |||
--- a/tls.c | |||
+++ b/tls.c | |||
@@ -51,6 +51,8 @@ int link_owner = 0; | |||
int nsec_times = 0; | |||
int preserve_perms = 0; | |||
int preserve_executability = 0; | |||
+int preallocate_files = 0; | |||
+int inplace = 0; | |||
#ifdef SUPPORT_XATTRS | |||
diff --git a/trimslash.c b/trimslash.c | |||
index 207eaf2..5db6f3e 100644 | |||
--- a/trimslash.c | |||
+++ b/trimslash.c | |||
@@ -28,6 +28,8 @@ int read_only = 1; | |||
int list_only = 0; | |||
int preserve_perms = 0; | |||
int preserve_executability = 0; | |||
+int preallocate_files = 0; | |||
+int inplace = 0; | |||
int | |||
main(int argc, char **argv) | |||
diff --git a/util.c b/util.c | |||
index ca38f3e..49c5b71 100644 | |||
--- a/util.c | |||
+++ b/util.c | |||
@@ -323,9 +323,7 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) | |||
int ifd; | |||
char buf[1024 * 8]; | |||
int len; /* Number of bytes read into `buf'. */ | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
- OFF_T preallocated_len = 0, offset = 0; | |||
-#endif | |||
+ OFF_T prealloc_len = 0, offset = 0; | |||
if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { | |||
int save_errno = errno; | |||
@@ -365,11 +363,8 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) | |||
if (do_fstat(ifd, &srcst) < 0) | |||
rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); | |||
else if (srcst.st_size > 0) { | |||
- if (do_fallocate(ofd, 0, srcst.st_size) == 0) { | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
- preallocated_len = srcst.st_size; | |||
-#endif | |||
- } else | |||
+ prealloc_len = do_fallocate(ofd, 0, srcst.st_size); | |||
+ if (prealloc_len < 0) | |||
rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); | |||
} | |||
} | |||
@@ -384,9 +379,7 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) | |||
errno = save_errno; | |||
return -1; | |||
} | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
offset += len; | |||
-#endif | |||
} | |||
if (len < 0) { | |||
@@ -403,15 +396,13 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) | |||
full_fname(source)); | |||
} | |||
-#ifdef PREALLOCATE_NEEDS_TRUNCATE | |||
/* Source file might have shrunk since we fstatted it. | |||
* Cut off any extra preallocated zeros from dest file. */ | |||
- if (offset < preallocated_len && do_ftruncate(ofd, offset) < 0) { | |||
+ if (offset < prealloc_len && do_ftruncate(ofd, offset) < 0) { | |||
/* If we fail to truncate, the dest file may be wrong, so we | |||
* must trigger the "partial transfer" error. */ | |||
rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); | |||
} | |||
-#endif | |||
if (close(ofd) < 0) { | |||
int save_errno = errno; |
@ -1,21 +0,0 @@ | |||
commit e02b89d0d35ab8acbd522983c08d2519d8bd12d4 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Sat Oct 29 14:33:44 2016 -0700 | |||
We need a LF after filelist-progress with a CR. | |||
Fixes bug 12367. | |||
diff --git a/flist.c b/flist.c | |||
index acb95f7..4a9f4e6 100644 | |||
--- a/flist.c | |||
+++ b/flist.c | |||
@@ -156,7 +156,9 @@ static void start_filelist_progress(char *kind) | |||
static void emit_filelist_progress(int count) | |||
{ | |||
+ output_needs_newline = 0; /* avoid a newline in the middle of this filelist-progress output */ | |||
rprintf(FCLIENT, " %d files...\r", count); | |||
+ output_needs_newline = 1; | |||
} | |||
static void maybe_emit_filelist_progress(int count) |
@ -1,140 +0,0 @@ | |||
commit ff66fd4bb6cc76488c6ea1e4b651a869847f6375 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Sat Oct 29 14:47:58 2016 -0700 | |||
More fixes for --progress quirks. | |||
This patch avoids inconsistent evaluation of options in the | |||
show_filelist_p() function by turning it into a var. We | |||
also avoid setting "output_needs_newline" if --quiet was | |||
specified. | |||
diff --git a/flist.c b/flist.c | |||
index 4a9f4e6..54ced36 100644 | |||
--- a/flist.c | |||
+++ b/flist.c | |||
@@ -37,6 +37,7 @@ extern int checksum_type; | |||
extern int module_id; | |||
extern int ignore_errors; | |||
extern int numeric_ids; | |||
+extern int quiet; | |||
extern int recurse; | |||
extern int use_qsort; | |||
extern int xfer_dirs; | |||
@@ -128,6 +129,7 @@ static char tmp_sum[MAX_DIGEST_LEN]; | |||
static char empty_sum[MAX_DIGEST_LEN]; | |||
static int flist_count_offset; /* for --delete --progress */ | |||
+static int show_filelist_progress; | |||
static void flist_sort_and_clean(struct file_list *flist, int strip_root); | |||
static void output_flist(struct file_list *flist); | |||
@@ -140,15 +142,14 @@ void init_flist(void) | |||
} | |||
parse_checksum_choice(); /* Sets checksum_type && xfersum_type */ | |||
checksum_len = csum_len_for_type(checksum_type); | |||
-} | |||
-static int show_filelist_p(void) | |||
-{ | |||
- return INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse; | |||
+ show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse; | |||
} | |||
static void start_filelist_progress(char *kind) | |||
{ | |||
+ if (quiet) | |||
+ return; | |||
rprintf(FCLIENT, "%s ... ", kind); | |||
output_needs_newline = 1; | |||
rflush(FINFO); | |||
@@ -156,25 +157,28 @@ static void start_filelist_progress(char *kind) | |||
static void emit_filelist_progress(int count) | |||
{ | |||
- output_needs_newline = 0; /* avoid a newline in the middle of this filelist-progress output */ | |||
+ if (quiet) | |||
+ return; | |||
+ if (output_needs_newline == 2) /* avoid a newline in the middle of this filelist-progress output */ | |||
+ output_needs_newline = 0; | |||
rprintf(FCLIENT, " %d files...\r", count); | |||
- output_needs_newline = 1; | |||
+ output_needs_newline = 2; | |||
} | |||
static void maybe_emit_filelist_progress(int count) | |||
{ | |||
- if (INFO_GTE(FLIST, 2) && show_filelist_p() && (count % 100) == 0) | |||
+ if (INFO_GTE(FLIST, 2) && show_filelist_progress && (count % 100) == 0) | |||
emit_filelist_progress(count); | |||
} | |||
static void finish_filelist_progress(const struct file_list *flist) | |||
{ | |||
+ output_needs_newline = 0; | |||
if (INFO_GTE(FLIST, 2)) { | |||
/* This overwrites the progress line */ | |||
rprintf(FINFO, "%d file%sto consider\n", | |||
flist->used, flist->used == 1 ? " " : "s "); | |||
} else { | |||
- output_needs_newline = 0; | |||
rprintf(FINFO, "done\n"); | |||
} | |||
} | |||
@@ -2089,7 +2093,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) | |||
int implied_dot_dir = 0; | |||
rprintf(FLOG, "building file list\n"); | |||
- if (show_filelist_p()) | |||
+ if (show_filelist_progress) | |||
start_filelist_progress("building file list"); | |||
else if (inc_recurse && INFO_GTE(FLIST, 1) && !am_server) | |||
rprintf(FCLIENT, "sending incremental file list\n"); | |||
@@ -2363,7 +2367,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) | |||
idev_destroy(); | |||
#endif | |||
- if (show_filelist_p()) | |||
+ if (show_filelist_progress) | |||
finish_filelist_progress(flist); | |||
gettimeofday(&end_tv, NULL); | |||
@@ -2445,7 +2449,7 @@ struct file_list *recv_file_list(int f, int dir_ndx) | |||
int64 start_read; | |||
if (!first_flist) { | |||
- if (show_filelist_p()) | |||
+ if (show_filelist_progress) | |||
start_filelist_progress("receiving file list"); | |||
else if (inc_recurse && INFO_GTE(FLIST, 1) && !am_server) | |||
rprintf(FCLIENT, "receiving incremental file list\n"); | |||
@@ -2541,7 +2545,7 @@ struct file_list *recv_file_list(int f, int dir_ndx) | |||
if (DEBUG_GTE(FLIST, 2)) | |||
rprintf(FINFO, "received %d names\n", flist->used); | |||
- if (show_filelist_p()) | |||
+ if (show_filelist_progress) | |||
finish_filelist_progress(flist); | |||
if (need_unsorted_flist) { | |||
diff --git a/progress.c b/progress.c | |||
index 3858fc4..d19fa25 100644 | |||
--- a/progress.c | |||
+++ b/progress.c | |||
@@ -25,6 +25,7 @@ | |||
extern int am_server; | |||
extern int flist_eof; | |||
+extern int quiet; | |||
extern int need_unsorted_flist; | |||
extern int output_needs_newline; | |||
extern struct stats stats; | |||
@@ -127,7 +128,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, | |||
pct = ofs == size ? 100 : (int) (100.0 * ofs / size); | |||
rprintf(FCLIENT, "\r%15s %3d%% %7.2f%s %s%s", | |||
human_num(ofs), pct, rate, units, rembuf, eol); | |||
- if (!is_last) { | |||
+ if (!is_last && !quiet) { | |||
output_needs_newline = 1; | |||
rflush(FCLIENT); | |||
} |
@ -1,100 +0,0 @@ | |||
commit 7b8a4ecd6ff9cdf4e5d3850ebf822f1e989255b3 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Tue Oct 24 15:40:37 2017 -0700 | |||
Handle archaic checksums properly. | |||
diff --git a/checksum.c b/checksum.c | |||
index 8b38833..9382694 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -27,7 +27,7 @@ extern int proper_seed_order; | |||
extern char *checksum_choice; | |||
#define CSUM_NONE 0 | |||
-#define CSUM_ARCHAIC 1 | |||
+#define CSUM_MD4_ARCHAIC 1 | |||
#define CSUM_MD4_BUSTED 2 | |||
#define CSUM_MD4_OLD 3 | |||
#define CSUM_MD4 4 | |||
@@ -60,7 +60,7 @@ int parse_csum_name(const char *name, int len) | |||
return CSUM_MD4_OLD; | |||
if (protocol_version >= 21) | |||
return CSUM_MD4_BUSTED; | |||
- return CSUM_ARCHAIC; | |||
+ return CSUM_MD4_ARCHAIC; | |||
} | |||
if (len == 3 && strncasecmp(name, "md4", 3) == 0) | |||
return CSUM_MD4; | |||
@@ -78,7 +78,7 @@ int csum_len_for_type(int cst) | |||
switch (cst) { | |||
case CSUM_NONE: | |||
return 1; | |||
- case CSUM_ARCHAIC: | |||
+ case CSUM_MD4_ARCHAIC: | |||
return 2; | |||
case CSUM_MD4: | |||
case CSUM_MD4_OLD: | |||
@@ -143,7 +143,8 @@ void get_checksum2(char *buf, int32 len, char *sum) | |||
} | |||
case CSUM_MD4: | |||
case CSUM_MD4_OLD: | |||
- case CSUM_MD4_BUSTED: { | |||
+ case CSUM_MD4_BUSTED: | |||
+ case CSUM_MD4_ARCHAIC: { | |||
int32 i; | |||
static char *buf1; | |||
static int32 len1; | |||
@@ -174,7 +175,7 @@ void get_checksum2(char *buf, int32 len, char *sum) | |||
* are multiples of 64. This is fixed by calling mdfour_update() | |||
* even when there are no more bytes. | |||
*/ | |||
- if (len - i > 0 || xfersum_type != CSUM_MD4_BUSTED) | |||
+ if (len - i > 0 || xfersum_type > CSUM_MD4_BUSTED) | |||
mdfour_update(&m, (uchar *)(buf1+i), len-i); | |||
mdfour_result(&m, (uchar *)sum); | |||
@@ -217,6 +218,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
case CSUM_MD4: | |||
case CSUM_MD4_OLD: | |||
case CSUM_MD4_BUSTED: | |||
+ case CSUM_MD4_ARCHAIC: | |||
mdfour_begin(&m); | |||
for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { | |||
@@ -229,7 +231,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) | |||
* are multiples of 64. This is fixed by calling mdfour_update() | |||
* even when there are no more bytes. */ | |||
remainder = (int32)(len - i); | |||
- if (remainder > 0 || checksum_type != CSUM_MD4_BUSTED) | |||
+ if (remainder > 0 || checksum_type > CSUM_MD4_BUSTED) | |||
mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); | |||
mdfour_result(&m, (uchar *)sum); | |||
@@ -265,6 +267,7 @@ void sum_init(int csum_type, int seed) | |||
break; | |||
case CSUM_MD4_OLD: | |||
case CSUM_MD4_BUSTED: | |||
+ case CSUM_MD4_ARCHAIC: | |||
mdfour_begin(&md); | |||
sumresidue = 0; | |||
SIVAL(s, 0, seed); | |||
@@ -321,6 +324,10 @@ void sum_update(const char *p, int32 len) | |||
} | |||
} | |||
+/* NOTE: all the callers of sum_end() pass in a pointer to a buffer that is | |||
+ * MAX_DIGEST_LEN in size, so even if the csum-len is shorter that that (i.e. | |||
+ * CSUM_MD4_ARCHAIC), we don't have to worry about limiting the data we write | |||
+ * into the "sum" buffer. */ | |||
int sum_end(char *sum) | |||
{ | |||
switch (cursum_type) { | |||
@@ -333,6 +340,7 @@ int sum_end(char *sum) | |||
mdfour_result(&md, (uchar *)sum); | |||
break; | |||
case CSUM_MD4_BUSTED: | |||
+ case CSUM_MD4_ARCHAIC: | |||
if (sumresidue) | |||
mdfour_update(&md, (uchar *)md.buffer, sumresidue); | |||
mdfour_result(&md, (uchar *)sum); |
@ -1,18 +0,0 @@ | |||
commit c252546ceeb0925eb8a4061315e3ff0a8c55b48b | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Tue Oct 24 20:42:41 2017 -0700 | |||
Don't forget to tweak sum_update(). | |||
diff --git a/checksum.c b/checksum.c | |||
index 9382694..c119f97 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -295,6 +295,7 @@ void sum_update(const char *p, int32 len) | |||
case CSUM_MD4: | |||
case CSUM_MD4_OLD: | |||
case CSUM_MD4_BUSTED: | |||
+ case CSUM_MD4_ARCHAIC: | |||
if (len + sumresidue < CSUM_CHUNK) { | |||
memcpy(md.buffer + sumresidue, p, len); | |||
sumresidue += len; |
@ -1,30 +0,0 @@ | |||
commit 9a480deec4d20277d8e20bc55515ef0640ca1e55 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Tue Oct 24 20:44:37 2017 -0700 | |||
Only allow a modern checksum method for passwords. | |||
diff --git a/authenticate.c b/authenticate.c | |||
index d60ee20..a106b0f 100644 | |||
--- a/authenticate.c | |||
+++ b/authenticate.c | |||
@@ -22,6 +22,7 @@ | |||
#include "itypes.h" | |||
extern int read_only; | |||
+extern int protocol_version; | |||
extern char *password_file; | |||
/*************************************************************************** | |||
@@ -237,6 +238,11 @@ char *auth_server(int f_in, int f_out, int module, const char *host, | |||
if (!users || !*users) | |||
return ""; | |||
+ if (protocol_version < 21) { /* Don't allow a weak checksum for the password. */ | |||
+ rprintf(FERROR, "ERROR: protocol version is too old!\n"); | |||
+ exit_cleanup(RERR_PROTOCOL); | |||
+ } | |||
+ | |||
gen_challenge(addr, challenge); | |||
io_printf(f_out, "%s%s\n", leader, challenge); |
@ -1,97 +0,0 @@ | |||
commit bc112b0e7feece62ce98708092306639a8a53cce | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Mon Oct 30 09:11:16 2017 -0700 | |||
Use full MD4 len for archaic protocol auth. | |||
diff --git a/authenticate.c b/authenticate.c | |||
index a106b0f..519429d 100644 | |||
--- a/authenticate.c | |||
+++ b/authenticate.c | |||
@@ -22,7 +22,6 @@ | |||
#include "itypes.h" | |||
extern int read_only; | |||
-extern int protocol_version; | |||
extern char *password_file; | |||
/*************************************************************************** | |||
@@ -75,6 +74,8 @@ static void gen_challenge(const char *addr, char *challenge) | |||
sum_init(-1, 0); | |||
sum_update(input, sizeof input); | |||
len = sum_end(digest); | |||
+ if (len == 2) /* The archaic checksum is 2 bytes, but sum_end() filled in the full MD4 checksum for us. */ | |||
+ len = MD4_DIGEST_LEN; | |||
base64_encode(digest, len, challenge, 0); | |||
} | |||
@@ -90,6 +91,8 @@ static void generate_hash(const char *in, const char *challenge, char *out) | |||
sum_update(in, strlen(in)); | |||
sum_update(challenge, strlen(challenge)); | |||
len = sum_end(buf); | |||
+ if (len == 2) /* The archaic checksum is 2 bytes, but sum_end() filled in the full MD4 checksum for us. */ | |||
+ len = MD4_DIGEST_LEN; | |||
base64_encode(buf, len, out, 0); | |||
} | |||
@@ -238,11 +241,6 @@ char *auth_server(int f_in, int f_out, int module, const char *host, | |||
if (!users || !*users) | |||
return ""; | |||
- if (protocol_version < 21) { /* Don't allow a weak checksum for the password. */ | |||
- rprintf(FERROR, "ERROR: protocol version is too old!\n"); | |||
- exit_cleanup(RERR_PROTOCOL); | |||
- } | |||
- | |||
gen_challenge(addr, challenge); | |||
io_printf(f_out, "%s%s\n", leader, challenge); | |||
diff --git a/checksum.c b/checksum.c | |||
index c119f97..741ad7d 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -86,6 +86,8 @@ int csum_len_for_type(int cst) | |||
return MD4_DIGEST_LEN; | |||
case CSUM_MD5: | |||
return MD5_DIGEST_LEN; | |||
+ default: /* paranoia to prevent missing case values */ | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
return 0; | |||
} | |||
@@ -181,6 +183,8 @@ void get_checksum2(char *buf, int32 len, char *sum) | |||
mdfour_result(&m, (uchar *)sum); | |||
break; | |||
} | |||
+ default: /* paranoia to prevent missing case values */ | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
} | |||
@@ -275,6 +279,8 @@ void sum_init(int csum_type, int seed) | |||
break; | |||
case CSUM_NONE: | |||
break; | |||
+ default: /* paranoia to prevent missing case values */ | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
} | |||
@@ -322,6 +328,8 @@ void sum_update(const char *p, int32 len) | |||
break; | |||
case CSUM_NONE: | |||
break; | |||
+ default: /* paranoia to prevent missing case values */ | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
} | |||
@@ -349,6 +357,8 @@ int sum_end(char *sum) | |||
case CSUM_NONE: | |||
*sum = '\0'; | |||
break; | |||
+ default: /* paranoia to prevent missing case values */ | |||
+ exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
return csum_len_for_type(cursum_type); |
@ -1,22 +0,0 @@ | |||
commit 47a63d90e71d3e19e0e96052bb8c6b9cb140ecc1 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Sun Nov 5 11:33:15 2017 -0800 | |||
Enforce trailing \0 when receiving xattr name values. | |||
Fixes bug 13112. | |||
diff --git a/xattrs.c b/xattrs.c | |||
index 68305d7..4867e6f 100644 | |||
--- a/xattrs.c | |||
+++ b/xattrs.c | |||
@@ -824,6 +824,10 @@ void receive_xattr(int f, struct file_struct *file) | |||
out_of_memory("receive_xattr"); | |||
name = ptr + dget_len + extra_len; | |||
read_buf(f, name, name_len); | |||
+ if (name_len < 1 || name[name_len-1] != '\0') { | |||
+ rprintf(FERROR, "Invalid xattr name received (missing trailing \\0).\n"); | |||
+ exit_cleanup(RERR_FILEIO); | |||
+ } | |||
if (dget_len == datum_len) | |||
read_buf(f, ptr, dget_len); | |||
else { |
@ -1,271 +0,0 @@ | |||
commit 416e719bea4f5466c8dd2b34cac0059b6ff84ff3 | |||
Author: Wayne Davison <wayned@samba.org> | |||
Date: Tue Nov 7 14:01:13 2017 -0800 | |||
More archaic-checksum improvements. This makes the len vars clearer | |||
and ensures that only the flist code gets the 2-byte digest len. | |||
diff --git a/authenticate.c b/authenticate.c | |||
index 519429d..d60ee20 100644 | |||
--- a/authenticate.c | |||
+++ b/authenticate.c | |||
@@ -74,8 +74,6 @@ static void gen_challenge(const char *addr, char *challenge) | |||
sum_init(-1, 0); | |||
sum_update(input, sizeof input); | |||
len = sum_end(digest); | |||
- if (len == 2) /* The archaic checksum is 2 bytes, but sum_end() filled in the full MD4 checksum for us. */ | |||
- len = MD4_DIGEST_LEN; | |||
base64_encode(digest, len, challenge, 0); | |||
} | |||
@@ -91,8 +89,6 @@ static void generate_hash(const char *in, const char *challenge, char *out) | |||
sum_update(in, strlen(in)); | |||
sum_update(challenge, strlen(challenge)); | |||
len = sum_end(buf); | |||
- if (len == 2) /* The archaic checksum is 2 bytes, but sum_end() filled in the full MD4 checksum for us. */ | |||
- len = MD4_DIGEST_LEN; | |||
base64_encode(buf, len, out, 0); | |||
} | |||
diff --git a/checksum.c b/checksum.c | |||
index 741ad7d..4c9351c 100644 | |||
--- a/checksum.c | |||
+++ b/checksum.c | |||
@@ -73,13 +73,15 @@ int parse_csum_name(const char *name, int len) | |||
exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
-int csum_len_for_type(int cst) | |||
+int csum_len_for_type(int cst, int flist_csum) | |||
{ | |||
switch (cst) { | |||
case CSUM_NONE: | |||
return 1; | |||
case CSUM_MD4_ARCHAIC: | |||
- return 2; | |||
+ /* The oldest checksum code is rather weird: the file-list code only sent | |||
+ * 2-byte checksums, but all other checksums were full MD4 length. */ | |||
+ return flist_csum ? 2 : MD4_DIGEST_LEN; | |||
case CSUM_MD4: | |||
case CSUM_MD4_OLD: | |||
case CSUM_MD4_BUSTED: | |||
@@ -361,5 +363,5 @@ int sum_end(char *sum) | |||
exit_cleanup(RERR_UNSUPPORTED); | |||
} | |||
- return csum_len_for_type(cursum_type); | |||
+ return csum_len_for_type(cursum_type, 0); | |||
} | |||
diff --git a/flist.c b/flist.c | |||
index 24b3506..bedba63 100644 | |||
--- a/flist.c | |||
+++ b/flist.c | |||
@@ -91,7 +91,7 @@ extern iconv_t ic_send, ic_recv; | |||
#define PTR_SIZE (sizeof (struct file_struct *)) | |||
int io_error; | |||
-int checksum_len; | |||
+int flist_csum_len; | |||
dev_t filesystem_dev; /* used to implement -x */ | |||
struct file_list *cur_flist, *first_flist, *dir_flist; | |||
@@ -141,7 +141,7 @@ void init_flist(void) | |||
(int)FILE_STRUCT_LEN, (int)EXTRA_LEN); | |||
} | |||
parse_checksum_choice(); /* Sets checksum_type && xfersum_type */ | |||
- checksum_len = csum_len_for_type(checksum_type); | |||
+ flist_csum_len = csum_len_for_type(checksum_type, 1); | |||
show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse; | |||
} | |||
@@ -638,7 +638,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, | |||
/* Prior to 28, we sent a useless set of nulls. */ | |||
sum = empty_sum; | |||
} | |||
- write_buf(f, sum, checksum_len); | |||
+ write_buf(f, sum, flist_csum_len); | |||
} | |||
#ifdef SUPPORT_HARD_LINKS | |||
@@ -1094,9 +1094,9 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x | |||
} | |||
if (first_hlink_ndx >= flist->ndx_start) { | |||
struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; | |||
- memcpy(bp, F_SUM(first), checksum_len); | |||
+ memcpy(bp, F_SUM(first), flist_csum_len); | |||
} else | |||
- read_buf(f, bp, checksum_len); | |||
+ read_buf(f, bp, flist_csum_len); | |||
} | |||
#ifdef SUPPORT_ACLS | |||
@@ -1384,7 +1384,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, | |||
} | |||
if (sender_keeps_checksum && S_ISREG(st.st_mode)) | |||
- memcpy(F_SUM(file), tmp_sum, checksum_len); | |||
+ memcpy(F_SUM(file), tmp_sum, flist_csum_len); | |||
if (unsort_ndx) | |||
F_NDX(file) = stats.num_dirs; | |||
diff --git a/generator.c b/generator.c | |||
index af2e290..3d65839 100644 | |||
--- a/generator.c | |||
+++ b/generator.c | |||
@@ -74,7 +74,7 @@ extern int protocol_version; | |||
extern int file_total; | |||
extern int fuzzy_basis; | |||
extern int always_checksum; | |||
-extern int checksum_len; | |||
+extern int flist_csum_len; | |||
extern char *partial_dir; | |||
extern int compare_dest; | |||
extern int copy_dest; | |||
@@ -583,7 +583,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) | |||
if (always_checksum > 0 && S_ISREG(st->st_mode)) { | |||
char sum[MAX_DIGEST_LEN]; | |||
file_checksum(fn, st, sum); | |||
- return memcmp(sum, F_SUM(file), checksum_len) == 0; | |||
+ return memcmp(sum, F_SUM(file), flist_csum_len) == 0; | |||
} | |||
if (size_only > 0) | |||
diff --git a/log.c b/log.c | |||
index b145ee1..7b23a2c 100644 | |||
--- a/log.c | |||
+++ b/log.c | |||
@@ -673,14 +673,15 @@ static void log_formatted(enum logcode code, const char *format, const char *op, | |||
n = NULL; | |||
if (S_ISREG(file->mode)) { | |||
if (always_checksum && canonical_checksum(checksum_type)) | |||
- n = sum_as_hex(checksum_type, F_SUM(file)); | |||
+ n = sum_as_hex(checksum_type, F_SUM(file), 1); | |||
else if (iflags & ITEM_TRANSFER && canonical_checksum(xfersum_type)) | |||
- n = sum_as_hex(xfersum_type, sender_file_sum); | |||
+ n = sum_as_hex(xfersum_type, sender_file_sum, 0); | |||
} | |||
if (!n) { | |||
- int checksum_len = csum_len_for_type(always_checksum ? checksum_type : xfersum_type); | |||
- memset(buf2, ' ', checksum_len*2); | |||
- buf2[checksum_len*2] = '\0'; | |||
+ int sum_len = csum_len_for_type(always_checksum ? checksum_type : xfersum_type, | |||
+ always_checksum); | |||
+ memset(buf2, ' ', sum_len*2); | |||
+ buf2[sum_len*2] = '\0'; | |||
n = buf2; | |||
} | |||
break; | |||
diff --git a/match.c b/match.c | |||
index ff10310..280038b 100644 | |||
--- a/match.c | |||
+++ b/match.c | |||
@@ -360,7 +360,7 @@ static void hash_search(int f,struct sum_struct *s, | |||
**/ | |||
void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) | |||
{ | |||
- int checksum_len; | |||
+ int sum_len; | |||
last_match = 0; | |||
false_alarms = 0; | |||
@@ -409,22 +409,22 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) | |||
matched(f, s, buf, len, -1); | |||
} | |||
- checksum_len = sum_end(sender_file_sum); | |||
+ sum_len = sum_end(sender_file_sum); | |||
/* If we had a read error, send a bad checksum. We use all bits | |||
* off as long as the checksum doesn't happen to be that, in | |||
* which case we turn the last 0 bit into a 1. */ | |||
if (buf && buf->status != 0) { | |||
int i; | |||
- for (i = 0; i < checksum_len && sender_file_sum[i] == 0; i++) {} | |||
- memset(sender_file_sum, 0, checksum_len); | |||
- if (i == checksum_len) | |||
+ for (i = 0; i < sum_len && sender_file_sum[i] == 0; i++) {} | |||
+ memset(sender_file_sum, 0, sum_len); | |||
+ if (i == sum_len) | |||
sender_file_sum[i-1]++; | |||
} | |||
if (DEBUG_GTE(DELTASUM, 2)) | |||
rprintf(FINFO,"sending file_sum\n"); | |||
- write_buf(f, sender_file_sum, checksum_len); | |||
+ write_buf(f, sender_file_sum, sum_len); | |||
if (DEBUG_GTE(DELTASUM, 2)) { | |||
rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n", | |||
diff --git a/receiver.c b/receiver.c | |||
index 3d9bc65..baae3a9 100644 | |||
--- a/receiver.c | |||
+++ b/receiver.c | |||
@@ -236,7 +236,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
static char file_sum1[MAX_DIGEST_LEN]; | |||
struct map_struct *mapbuf; | |||
struct sum_struct sum; | |||
- int checksum_len; | |||
+ int sum_len; | |||
int32 len; | |||
OFF_T offset = 0; | |||
OFF_T offset2; | |||
@@ -388,15 +388,15 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |||
if (INFO_GTE(PROGRESS, 1)) | |||
end_progress(total_size); | |||
- checksum_len = sum_end(file_sum1); | |||
+ sum_len = sum_end(file_sum1); | |||
if (mapbuf) | |||
unmap_file(mapbuf); | |||
- read_buf(f_in, sender_file_sum, checksum_len); | |||
+ read_buf(f_in, sender_file_sum, sum_len); | |||
if (DEBUG_GTE(DELTASUM, 2)) | |||
rprintf(FINFO,"got file_sum\n"); | |||
- if (fd != -1 && memcmp(file_sum1, sender_file_sum, checksum_len) != 0) | |||
+ if (fd != -1 && memcmp(file_sum1, sender_file_sum, sum_len) != 0) | |||
return 0; | |||
return 1; | |||
} | |||
diff --git a/t_stub.c b/t_stub.c | |||
index fc1ee3b..0c49d9c 100644 | |||
--- a/t_stub.c | |||
+++ b/t_stub.c | |||
@@ -98,7 +98,7 @@ filter_rule_list daemon_filter_list; | |||
return "tester"; | |||
} | |||
- int csum_len_for_type(int cst) | |||
+ int csum_len_for_type(int cst, int flg) | |||
{ | |||
- return cst ? 16 : 1; | |||
+ return cst || !flg ? 16 : 1; | |||
} | |||
diff --git a/util2.c b/util2.c | |||
index a892e51..b1adf81 100644 | |||
--- a/util2.c | |||
+++ b/util2.c | |||
@@ -77,18 +77,18 @@ void *_realloc_array(void *ptr, unsigned int size, size_t num) | |||
return realloc(ptr, size * num); | |||
} | |||
-const char *sum_as_hex(int csum_type, const char *sum) | |||
+const char *sum_as_hex(int csum_type, const char *sum, int flist_csum) | |||
{ | |||
static char buf[MAX_DIGEST_LEN*2+1]; | |||
int i, x1, x2; | |||
- int checksum_len = csum_len_for_type(csum_type); | |||
- char *c = buf + checksum_len*2; | |||
+ int sum_len = csum_len_for_type(csum_type, flist_csum); | |||
+ char *c = buf + sum_len*2; | |||
assert(c - buf < (int)sizeof buf); | |||
*c = '\0'; | |||
- for (i = checksum_len; --i >= 0; ) { | |||
+ for (i = sum_len; --i >= 0; ) { | |||
x1 = CVAL(sum, i); | |||
x2 = x1 >> 4; | |||
x1 &= 0xF; |
@ -1,36 +0,0 @@ | |||
commit 3e06d40029cfdce9d0f73d87cfd4edaf54be9c51 | |||
Author: Jeriko One <jeriko.one@gmx.us> | |||
Date: Thu Nov 2 23:44:19 2017 -0700 | |||
Check fname in recv_files sooner. | |||
diff --git a/receiver.c b/receiver.c | |||
index baae3a9..9fdafa1 100644 | |||
--- a/receiver.c | |||
+++ b/receiver.c | |||
@@ -574,6 +574,12 @@ int recv_files(int f_in, int f_out, char *local_name) | |||
file = dir_flist->files[cur_flist->parent_ndx]; | |||
fname = local_name ? local_name : f_name(file, fbuf); | |||
+ if (daemon_filter_list.head | |||
+ && check_filter(&daemon_filter_list, FLOG, fname, 0) < 0) { | |||
+ rprintf(FERROR, "attempt to hack rsync failed.\n"); | |||
+ exit_cleanup(RERR_PROTOCOL); | |||
+ } | |||
+ | |||
if (DEBUG_GTE(RECV, 1)) | |||
rprintf(FINFO, "recv_files(%s)\n", fname); | |||
@@ -645,12 +651,6 @@ int recv_files(int f_in, int f_out, char *local_name) | |||
cleanup_got_literal = 0; | |||
- if (daemon_filter_list.head | |||
- && check_filter(&daemon_filter_list, FLOG, fname, 0) < 0) { | |||
- rprintf(FERROR, "attempt to hack rsync failed.\n"); | |||
- exit_cleanup(RERR_PROTOCOL); | |||
- } | |||
- | |||
if (read_batch) { | |||
int wanted = redoing | |||
? we_want_redo(ndx) |
@ -1,19 +0,0 @@ | |||
commit 5509597decdbd7b91994210f700329d8a35e70a1 | |||
Author: Jeriko One <jeriko.one@gmx.us> | |||
Date: Thu Nov 16 17:26:03 2017 -0800 | |||
Check daemon filter against fnamecmp in recv_files(). | |||
diff --git a/receiver.c b/receiver.c | |||
index 9fdafa1..9c46242 100644 | |||
--- a/receiver.c | |||
+++ b/receiver.c | |||
@@ -722,7 +722,7 @@ int recv_files(int f_in, int f_out, char *local_name) | |||
break; | |||
} | |||
if (!fnamecmp || (daemon_filter_list.head | |||
- && check_filter(&daemon_filter_list, FLOG, fname, 0) < 0)) { | |||
+ && check_filter(&daemon_filter_list, FLOG, fnamecmp, 0) < 0)) { | |||
fnamecmp = fname; | |||
fnamecmp_type = FNAMECMP_FNAME; | |||
} |
@ -1,30 +0,0 @@ | |||
commit 70aeb5fddd1b2f8e143276f8d5a085db16c593b9 | |||
Author: Jeriko One <jeriko.one@gmx.us> | |||
Date: Thu Nov 16 17:05:42 2017 -0800 | |||
Sanitize xname in read_ndx_and_attrs. | |||
diff --git a/rsync.c b/rsync.c | |||
index b82e598..a0945ba 100644 | |||
--- a/rsync.c | |||
+++ b/rsync.c | |||
@@ -49,6 +49,7 @@ extern int flist_eof; | |||
extern int file_old_total; | |||
extern int keep_dirlinks; | |||
extern int make_backups; | |||
+extern int sanitize_paths; | |||
extern struct file_list *cur_flist, *first_flist, *dir_flist; | |||
extern struct chmod_mode_struct *daemon_chmod_modes; | |||
#ifdef ICONV_OPTION | |||
@@ -396,6 +397,11 @@ int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, uchar *type_ptr, | |||
if (iflags & ITEM_XNAME_FOLLOWS) { | |||
if ((len = read_vstring(f_in, buf, MAXPATHLEN)) < 0) | |||
exit_cleanup(RERR_PROTOCOL); | |||
+ | |||
+ if (sanitize_paths) { | |||
+ sanitize_path(buf, buf, "", 0, SP_DEFAULT); | |||
+ len = strlen(buf); | |||
+ } | |||
} else { | |||
*buf = '\0'; | |||
len = -1; |