[PATCH]: scp program improved
Miika Pekkarinen
miipekk at ihme.org
Sat Jul 13 19:16:37 EST 2002
Hi,
I have made a patch which improves scp utility. It adds two new features:
rate limit and resume support. With rate limit it's possible to limit
transfer speed. Resume allows to continue file transfer where it was last
interrupted. Also the progress meter was improved.
Here is my patch, please send comments about it and what I can do better
if there is something to fix.
--- openssh-3.4p1.orig/scp.c Sat Jul 13 11:38:42 2002
+++ openssh-3.4p1/scp.c Sat Jul 13 11:49:23 2002
@@ -14,6 +14,25 @@
* called by a name other than "ssh" or "Secure Shell".
*/
/*
+ * Few patches for the original version:
+ * + Added rate limit feature (-R)
+ * + Added resume feature (-a)
+ * + Progress meter improved
+ *
+ * Description:
+ * Rate limit adds a new switch -R. With rate limit it's possible to
+ * control how much traffic you would like to allow for the file
+ * transfer. Option format for this switch is -R <value>[kMG].
+ * Value is in bytes per second.
+ *
+ * Resume allows to continue last transfer if it was interrupted etc.
+ * But be careful when using resume feature because it will ALWAYS
+ * append to the remote file if any exists, i.e., resume doesn't make
+ * any timestamp checks currently.
+ *
+ * 2002 Miika Pekkarinen <miika at ihme.org>
+ */
+/*
* Copyright (c) 1999 Theo de Raadt. All rights reserved.
* Copyright (c) 1999 Aaron Campbell. All rights reserved.
*
@@ -119,6 +138,9 @@
/* This is set to non-zero to enable verbose mode. */
int verbose_mode = 0;
+/* This disables resume mode for default */
+int resume_mode = 0;
+
/* This is set to zero if the progressmeter is not desired. */
int showprogress = 1;
@@ -202,6 +224,7 @@
uid_t userid;
int errs, remin, remout;
int pflag, iamremote, iamrecursive, targetshouldbedirectory;
+off_t rate_limit = 0;
#define CMDNEEDS 64
char cmd[CMDNEEDS]; /* must hold "rcp -r -p -d\0" */
@@ -223,7 +246,10 @@
char *targ;
extern char *optarg;
extern int optind;
-
+ int factor, i; /* For rate limit */
+ const char factors[] = "kMMGGG";
+ char *c;
+
__progname = get_progname(argv[0]);
args.list = NULL;
@@ -233,7 +259,7 @@
addargs(&args, "-oClearAllForwardings yes");
fflag = tflag = 0;
- while ((ch = getopt(argc, argv, "dfprtvBCc:i:P:q1246S:o:F:")) != -1)
+ while ((ch = getopt(argc, argv, "adfprtvBCc:i:P:q1246S:o:F:R:")) != -1)
switch (ch) {
/* User-visible flags. */
case '1':
@@ -249,6 +275,18 @@
case 'F':
addargs(&args, "-%c%s", ch, optarg);
break;
+ case 'a': /* Resume mode */
+ resume_mode = 1;
+ break;
+ case 'R': /* Rate limit */
+ factor = 1;
+ for (i = 0; i < sizeof(factors) - 1; i++)
+ if ((c = strchr(optarg, factors[i])) != NULL)
+ factor *= 1000;
+ if (c)
+ (*c) = '\0';
+ rate_limit = atoi(optarg) * factor;
+ break;
case 'P':
addargs(&args, "-p%s", optarg);
break;
@@ -320,10 +358,13 @@
remin = remout = -1;
/* Command to be executed on remote system using "ssh". */
- (void) snprintf(cmd, sizeof cmd, "scp%s%s%s%s",
- verbose_mode ? " -v" : "",
- iamrecursive ? " -r" : "", pflag ? " -p" : "",
- targetshouldbedirectory ? " -d" : "");
+ (void) snprintf(cmd, sizeof cmd, "scp%s%s%s%s%s -R %llu",
+ verbose_mode ? " -v" : "",
+ iamrecursive ? " -r" : "", pflag ? " -p" : "",
+ targetshouldbedirectory ? " -d" : "",
+ resume_mode ? " -a" : "", rate_limit);
+ if (!rate_limit)
+ *(strstr(cmd, " -R")) = '\0';
(void) signal(SIGPIPE, lostconn);
@@ -482,14 +523,18 @@
char *argv[];
{
struct stat stb;
+ struct timeval wait, lastupdate, now;
static BUF buffer;
BUF *bp;
- off_t i, amt, result;
+ off_t i, amt, result, bcounter;
int fd, haderr, indx;
- char *last, *name, buf[2048];
+ char *last, *name, buf[2048], c;
int len;
+ /* For rate limit */
+ int buf_size, cur_mod;
+ long int sleep_time;
- for (indx = 0; indx < argc; ++indx) {
+ for (indx = 0; indx < argc; ++indx) {
name = argv[indx];
statbytes = 0;
len = strlen(name);
@@ -536,14 +581,19 @@
if (response() < 0)
goto next;
}
+ if (resume_mode)
+ c = 'R';
+ else
+ c = 'C';
+
#define FILEMODEMASK (S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)
#ifdef HAVE_LONG_LONG_INT
- snprintf(buf, sizeof buf, "C%04o %lld %s\n",
+ snprintf(buf, sizeof buf, "%c%04o %lld %s\n", c,
(u_int) (stb.st_mode & FILEMODEMASK),
(long long)stb.st_size, last);
#else
/* XXX: Handle integer overflow? */
- snprintf(buf, sizeof buf, "C%04o %lu %s\n",
+ snprintf(buf, sizeof buf, "%c%04o %lu %s\n", c,
(u_int) (stb.st_mode & FILEMODEMASK),
(u_long) stb.st_size, last);
#endif
@@ -558,12 +608,42 @@
next: (void) close(fd);
continue;
}
+
+ /* Calculate optimal buffer size for rate limit if enabled */
+ if (rate_limit) {
+ for (i = buf_size = cur_mod = bp->cnt; i >= 100; i--) {
+ result = rate_limit % i;
+ if (result < cur_mod) {
+ cur_mod = result;
+ buf_size = i;
+ }
+ }
+ bp->cnt = buf_size;
+ if (verbose_mode)
+ printf ("Buffer size: %u\n", buf_size);
+ }
+
+ i = 0;
+ if (resume_mode) {
+ if (bp->cnt < 20)
+ goto next;
+
+ /* Get the file size if resume mode used */
+ (void) atomicio(read, remin, buf, 20);
+ buf[20] = '\0';
+ statbytes = i = atoll(buf);
+ lseek(fd, i, SEEK_SET);
+ if (verbose_mode)
+ printf("Continuing at: %llu\n", i);
+ }
+
if (showprogress) {
totalbytes = stb.st_size;
progressmeter(-1);
}
+
/* Keep writing after an error so that we stay sync'd up. */
- for (haderr = i = 0; i < stb.st_size; i += bp->cnt) {
+ for (haderr = bcounter = 0; i < stb.st_size; i += bp->cnt) {
amt = bp->cnt;
if (i + amt > stb.st_size)
amt = stb.st_size - i;
@@ -572,6 +652,26 @@
if (result != amt)
haderr = result >= 0 ? EIO : errno;
}
+
+ if (rate_limit) {
+ if (bcounter >= rate_limit / 2) {
+ (void) gettimeofday(&now, (struct timezone *) 0);
+ timersub(&now, &lastupdate, &wait);
+
+ sleep_time = ((double)(bcounter / (rate_limit
+ * (double)(wait.tv_usec / 1000000.0)) - 0)
+ * (double)wait.tv_usec);
+ if (sleep_time < 0)
+ sleep_time = 0;
+
+ usleep(sleep_time);
+ (void) gettimeofday(&now, (struct timezone *) 0);
+ lastupdate = now;
+
+ bcounter = 0;
+ }
+ }
+
if (haderr)
(void) atomicio(write, remout, bp->buf, amt);
else {
@@ -579,8 +679,10 @@
if (result != amt)
haderr = result >= 0 ? EIO : errno;
statbytes += result;
+ bcounter += result;
}
}
+
if (showprogress)
progressmeter(1);
@@ -666,7 +768,8 @@
int setimes, targisdir, wrerrno = 0;
char ch, *cp, *np, *targ, *why, *vect[1], buf[2048];
struct timeval tv[2];
-
+ int resume = 0;
+
#define atime tv[0]
#define mtime tv[1]
#define SCREWUP(str) do { why = str; goto screwup; } while (0)
@@ -734,7 +837,7 @@
(void) atomicio(write, remout, "", 1);
continue;
}
- if (*cp != 'C' && *cp != 'D') {
+ if (*cp != 'C' && *cp != 'D' && *cp != 'R') {
/*
* Check for the case "rcp remote:foo\* local:bar".
* In this case, the line "No match." can be returned
@@ -748,6 +851,11 @@
}
SCREWUP("expected control record");
}
+
+ /* Set resume mode on if requested */
+ if (*cp == 'R')
+ resume = 1;
+
mode = 0;
for (++cp; cp < buf + 5; cp++) {
if (*cp < '0' || *cp > '7')
@@ -812,11 +920,31 @@
}
omode = mode;
mode |= S_IWRITE;
- if ((ofd = open(np, O_WRONLY|O_CREAT, mode)) < 0) {
+ i = statbytes = 0;
+ if (resume != 1) {
+ if ((ofd = open(np, O_WRONLY|O_CREAT, mode)) < 0) {
bad: run_err("%s: %s", np, strerror(errno));
- continue;
+ continue;
+ }
+ /* Send response */
+ (void) atomicio(write, remout, "", 1);
+ } else {
+ if ((ofd = open(np, O_WRONLY | O_APPEND | O_CREAT, mode)) < 0) {
+ run_err("%s: %s", np, strerror(errno));
+ continue;
+ }
+ /* Send response */
+ (void) atomicio(write, remout, "", 1);
+
+ /* Get the file size and send it to the remote
+ * process so it can continue sending data from
+ * the right position */
+ i = statbytes = lseek(ofd, 0, SEEK_END);
+
+ snprintf(buf, sizeof(buf), "%llu", i);
+ (void) atomicio(write, remout, buf, 20);
}
- (void) atomicio(write, remout, "", 1);
+
if ((bp = allocbuf(&buffer, ofd, 4096)) == NULL) {
(void) close(ofd);
continue;
@@ -828,8 +956,7 @@
totalbytes = size;
progressmeter(-1);
}
- statbytes = 0;
- for (count = i = 0; i < size; i += 4096) {
+ for (count = 0; i < size; i += 4096) {
amt = 4096;
if (i + amt > size)
amt = size - i;
@@ -957,8 +1084,8 @@
usage(void)
{
(void) fprintf(stderr,
- "usage: scp [-pqrvBC46] [-F config] [-S program] [-P port]\n"
- " [-c cipher] [-i identity] [-o option]\n"
+ "usage: scp [-apqrvBC46] [-F config] [-S program] [-P port]\n"
+ " [-c cipher] [-i identity] [-R rate] [-o option]\n"
" [[user@]host1:]file1 [...] [[user@]host2:]file2\n");
exit(1);
}
@@ -1098,110 +1225,191 @@
#endif
}
-void
-progressmeter(int flag)
-{
- static const char prefixes[] = " KMGTP";
- static struct timeval lastupdate;
- static off_t lastsize;
- struct timeval now, td, wait;
- off_t cursize, abbrevsize;
- double elapsed;
- int ratio, barlength, i, remaining;
- char buf[512];
-
- if (flag == -1) {
- (void) gettimeofday(&start, (struct timezone *) 0);
- lastupdate = start;
- lastsize = 0;
- }
- if (foregroundproc() == 0)
- return;
+/* Functions for progress meter */
+#define MAX_PREFIXLEN 30
+#define MAX_GAUGELEN 200
- (void) gettimeofday(&now, (struct timezone *) 0);
- cursize = statbytes;
- if (totalbytes != 0) {
- ratio = 100.0 * cursize / totalbytes;
- ratio = MAX(ratio, 0);
- ratio = MIN(ratio, 100);
- } else
- ratio = 100;
-
- snprintf(buf, sizeof(buf), "\r%-20.20s %3d%% ", curfile, ratio);
-
- barlength = getttywidth() - 51;
- if (barlength > 0) {
- i = barlength * ratio / 100;
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- "|%.*s%*s|", i,
- "*******************************************************"
- "*******************************************************"
- "*******************************************************"
- "*******************************************************"
- "*******************************************************"
- "*******************************************************"
- "*******************************************************",
- barlength - i, "");
- }
- i = 0;
- abbrevsize = cursize;
- while (abbrevsize >= 100000 && i < sizeof(prefixes)) {
- i++;
- abbrevsize >>= 10;
- }
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), " %5lu %c%c ",
- (unsigned long) abbrevsize, prefixes[i],
- prefixes[i] == ' ' ? ' ' : 'B');
-
- timersub(&now, &lastupdate, &wait);
- if (cursize > lastsize) {
- lastupdate = now;
- lastsize = cursize;
- if (wait.tv_sec >= STALLTIME) {
- start.tv_sec += wait.tv_sec;
- start.tv_usec += wait.tv_usec;
- }
- wait.tv_sec = 0;
- }
- timersub(&now, &start, &td);
- elapsed = td.tv_sec + (td.tv_usec / 1000000.0);
+void get_prefixed(off_t value, char *prefixed)
+{
+ static const char prefixes[] = " KMGT";
+ int i = 0;
+
+ for (i = 0; value >= 1024 && i < sizeof(prefixes); i++) {
+ value >>= 10;
+ }
+
+ if (i > 0)
+ snprintf(prefixed, MAX_PREFIXLEN, "%u %ciB",
+ (unsigned int)value, prefixes[i]);
+ else
+ snprintf(prefixed, MAX_PREFIXLEN, "%u B", (unsigned int)value);
+}
- if (flag != 1 &&
- (statbytes <= 0 || elapsed <= 0.0 || cursize > totalbytes)) {
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- " --:-- ETA");
- } else if (wait.tv_sec >= STALLTIME) {
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- " - stalled -");
- } else {
- if (flag != 1)
- remaining = (int)(totalbytes / (statbytes / elapsed) -
- elapsed);
- else
- remaining = elapsed;
+void get_prefixed_data(off_t value, char *prefixed)
+{
+ static const char prefixes[] = " kMGT";
+ double temp = value;
+ int i = 0;
+
+ for (i = 0; temp >= 1000 && i < sizeof(prefixes); i++) {
+ temp /= 1000.0;
+ }
+
+ if (i > 0)
+ snprintf(prefixed, MAX_PREFIXLEN, "%u %cB",
+ (unsigned int)temp, prefixes[i]);
+ else
+ snprintf(prefixed, MAX_PREFIXLEN, "%u B", (unsigned int)temp);
+}
- i = remaining / 3600;
- if (i)
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- "%2d:", i);
- else
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- " ");
- i = remaining % 3600;
- snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
- "%02d:%02d%s", i / 60, i % 60,
- (flag != 1) ? " ETA" : " ");
- }
- atomicio(write, fileno(stdout), buf, strlen(buf));
+void gauge(unsigned int width, float ratio, char *buf)
+{
+ const float step = width / 100.0;
+ char temp[MAX_GAUGELEN + 1] = "";
+ int i, n;
+
+ if (width >= MAX_GAUGELEN)
+ return ;
+
+ n = (float)step * ((float)ratio);
+ for (i = 0; (i < n) && (i < width); i++)
+ temp[i] = '=';
+
+ if (i < width)
+ temp[i] = '>';
+
+ for (i++; i < width; i++)
+ temp[i] = '-';
+
+ snprintf(buf, MAX_GAUGELEN, "[%s]", temp);
+}
- if (flag == -1) {
- mysignal(SIGALRM, updateprogressmeter);
- alarm(PROGRESSTIME);
- } else if (flag == 1) {
- alarm(0);
- atomicio(write, fileno(stdout), "\n", 1);
- statbytes = 0;
- }
+void progressmeter(int flag)
+{
+ static const char indicator[] = "|/-\\";
+ static struct timeval lastupdate;
+ static off_t lastsize;
+ struct timeval now, td, wait;
+ static off_t cursize, cur_speed = 1;
+ static double temp = 1;
+ static char speed[MAX_PREFIXLEN] = "0 B";
+ double elapsed;
+ static int ind_value = 0;
+ float ratio;
+ int hours, mins, secs, remaining;
+ char buf[512], eta[20], buf_gauge[MAX_GAUGELEN] = "", filename[512];
+ char pre_cursize[MAX_PREFIXLEN], pre_totalsize[MAX_PREFIXLEN];
+ int len, len2, len3, ttywidth = getttywidth();
+
+ snprintf(filename, 512, "%s", curfile);
+ if (flag == -1) {
+ (void) gettimeofday(&start, (struct timezone *) 0);
+ lastupdate = start;
+ lastsize = 0;
+ }
+ if (foregroundproc() == 0)
+ return;
+
+ (void) gettimeofday(&now, (struct timezone *) 0);
+ cursize = statbytes;
+ if (totalbytes != 0) {
+ ratio = 100.0 * cursize / totalbytes;
+ /* ratio = MAX(ratio, 0);
+ ratio = MIN(ratio, 100); */
+ } else
+ ratio = 100;
+
+ get_prefixed(cursize, pre_cursize);
+ get_prefixed(totalbytes, pre_totalsize);
+
+ timersub(&now, &lastupdate, &wait);
+ if (cursize > lastsize) {
+ lastupdate = now;
+ if (wait.tv_sec >= STALLTIME) {
+ start.tv_sec += wait.tv_sec;
+ start.tv_usec += wait.tv_usec;
+ }
+
+ temp = (double)(wait.tv_usec / 1000000.0);
+ if ((long int)temp == 0)
+ temp = 1;
+
+ cur_speed = (double)(cursize - lastsize) / temp;
+ if (flag != 1)
+ get_prefixed_data(cur_speed, speed);
+
+ wait.tv_sec = 0;
+ lastsize = cursize;
+ }
+ if ((long int)cur_speed == 0)
+ cur_speed = 1;
+ timersub(&now, &start, &td);
+ elapsed = td.tv_sec + (td.tv_usec / 1000000.0);
+
+ if (flag != 1 &&
+ (statbytes <= 0 || elapsed <= 0.0 || cursize > totalbytes)) {
+ snprintf(eta, sizeof(eta), "--.--.--");
+ } else if (wait.tv_sec >= STALLTIME) {
+ snprintf(eta, sizeof(eta), "stalled");
+ } else {
+ if (flag != 1)
+ remaining = (int)((totalbytes - statbytes) / cur_speed);
+ else
+ remaining = elapsed;
+
+ hours = remaining / 3600;
+ mins = (remaining / 60) - (hours * 60);
+ secs = remaining - (3600 * hours) - (60 * mins);
+
+ snprintf(eta, sizeof(eta), "%2.2u.%2.2u.%2.2u",
+ hours, mins, secs);
+ }
+
+ /* Determine best size for progress meter */
+ snprintf(buf, sizeof(buf), "'' %s of %s [] %c (%2.1f%%) %s/s ETA: %s ",
+ pre_cursize, pre_totalsize, indicator[ind_value], ratio, speed, eta);
+
+ len = strlen(buf);
+ len2 = strlen(filename);
+
+ if (ttywidth - len - len2 < 10) {
+ len3 = ttywidth - len - len2 - 10;
+ len2 = MAX(len2 + len3, 20);
+ if (len2 < 20) {
+ filename[18] = '.';
+ filename[19] = '.';
+ filename[20] = '\0';
+ } else if (len2 < sizeof(filename)) {
+ filename[len2 - 2] = '.';
+ filename[len2 - 1] = '.';
+ filename[len2] = '\0';
+ }
+ }
+ len2 = strlen(filename);
+
+ gauge(MAX(ttywidth - len - len2, 0), ratio, buf_gauge);
+ snprintf(buf, sizeof(buf), "\r'%s' %s of %s %s %c (%2.1f%%) %s/s ETA: %s",
+ filename, pre_cursize, pre_totalsize, buf_gauge, indicator[ind_value],
+ ratio, speed, eta);
+
+ if (ttywidth < sizeof(buf))
+ buf[ttywidth] = '\0';
+
+ if (ind_value == 3)
+ ind_value = 0;
+ else
+ ind_value++;
+
+ atomicio(write, fileno(stdout), buf, strlen(buf));
+
+ if (flag == -1) {
+ mysignal(SIGALRM, updateprogressmeter);
+ alarm(PROGRESSTIME);
+ } else if (flag == 1) {
+ alarm(0);
+ atomicio(write, fileno(stdout), "\n", 1);
+ statbytes = 0;
+ }
}
int
--- openssh-3.4p1.orig/scp.1 Sat Jul 13 11:38:42 2002
+++ openssh-3.4p1/scp.1 Sat Jul 13 11:21:33 2002
@@ -3,6 +3,7 @@
.\" scp.1
.\"
.\" Author: Tatu Ylonen <ylo at cs.hut.fi>
+.\" Patched: Miika Pekkarinen <miika at ihme.org>
.\"
.\" Copyright (c) 1995 Tatu Ylonen <ylo at cs.hut.fi>, Espoo, Finland
.\" All rights reserved
@@ -19,12 +20,13 @@
.Nd secure copy (remote file copy program)
.Sh SYNOPSIS
.Nm scp
-.Op Fl pqrvBC1246
+.Op Fl apqrvBC1246
.Op Fl F Ar ssh_config
.Op Fl S Ar program
.Op Fl P Ar port
.Op Fl c Ar cipher
.Op Fl i Ar identity_file
+.Op Fl R Ar rate_limit
.Op Fl o Ar ssh_option
.Sm off
.Oo
@@ -59,6 +61,12 @@
.Pp
The options are as follows:
.Bl -tag -width Ds
+.It Fl a
+Sets resume mode on. When copying a file which already exists on the
+remote machine scp will try to continue the data transfer where it was
+last interrupted. Please be careful when using this flag because scp
+will always append to the remote file if any exists. Both sides have to
+support this feature.
.It Fl c Ar cipher
Selects the cipher to use for encrypting the data transfer.
This option is directly passed to
@@ -68,6 +76,12 @@
authentication is read.
This option is directly passed to
.Xr ssh 1 .
+.It Fl R Ar rate_limit
+With rate limit it's possible to control how much traffic you would
+like to allow for the file transfer. For example to limit bandwidth
+usage to 20 kilobytes per second is specified using
+.Ic scp -R 20k .
+Source host must support rate limit.
.It Fl p
Preserves modification times, access times, and modes from the
original file.
--
Miika Pekkarinen
miika at ihme.org
More information about the openssh-unix-dev
mailing list