[PATCH 3/5] sftp-client: use new limits at openssh.com extension
Mike Frysinger
vapier at gentoo.org
Mon Nov 30 15:57:02 AEDT 2020
Now that the server will tell us its limits, we can use those to select
a good transfer size. In practice, the default increases from 32KiB to
64KiB when using newer OpenSSH servers.
---
sftp-client.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++---
sftp-client.h | 11 ++++++
sftp.c | 7 ++--
3 files changed, 105 insertions(+), 9 deletions(-)
diff --git a/sftp-client.c b/sftp-client.c
index d82e31aeeec4..ef3906477bd5 100644
--- a/sftp-client.c
+++ b/sftp-client.c
@@ -61,6 +61,12 @@
extern volatile sig_atomic_t interrupted;
extern int showprogress;
+/* Default size of buffer for up/download */
+#define DEFAULT_COPY_BUFLEN 32768
+
+/* Default number of concurrent outstanding requests */
+#define DEFAULT_NUM_REQUESTS 64
+
/* Minimum amount of data to read at a time */
#define MIN_READ_SIZE 512
@@ -87,6 +93,7 @@ struct sftp_conn {
#define SFTP_EXT_HARDLINK 0x00000008
#define SFTP_EXT_FSYNC 0x00000010
#define SFTP_EXT_LSETSTAT 0x00000020
+#define SFTP_EXT_LIMITS 0x00000040
u_int exts;
u_int64_t limit_kbps;
struct bwlimit bwlimit_in, bwlimit_out;
@@ -405,8 +412,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
ret->msg_id = 1;
ret->fd_in = fd_in;
ret->fd_out = fd_out;
- ret->transfer_buflen = transfer_buflen;
- ret->num_requests = num_requests;
+ ret->transfer_buflen =
+ transfer_buflen ? transfer_buflen : DEFAULT_COPY_BUFLEN;
+ ret->num_requests =
+ num_requests ? num_requests : DEFAULT_NUM_REQUESTS;
ret->exts = 0;
ret->limit_kbps = 0;
@@ -418,8 +427,6 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
send_msg(ret, msg);
- sshbuf_reset(msg);
-
get_msg_extended(ret, msg, 1);
/* Expecting a VERSION reply */
@@ -471,6 +478,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
strcmp((char *)value, "1") == 0) {
ret->exts |= SFTP_EXT_LSETSTAT;
known = 1;
+ } else if (strcmp(name, "limits at openssh.com") == 0 &&
+ strcmp((char *)value, "1") == 0) {
+ ret->exts |= SFTP_EXT_LIMITS;
+ known = 1;
}
if (known) {
debug2("Server supports extension \"%s\" revision %s",
@@ -484,6 +495,33 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
sshbuf_free(msg);
+ /* Query the server for its limits */
+ if (ret->exts & SFTP_EXT_LIMITS) {
+ struct sftp_limits limits;
+ if (do_limits(ret, &limits) != 0)
+ fatal_f("limits failed");
+
+ /* If the caller did not specify, find a good value */
+ if (transfer_buflen == 0) {
+ /* TODO: We should have sep read & write limits */
+ ret->transfer_buflen =
+ MINIMUM(limits.read_length, limits.write_length);
+ debug("Using smaller of server transfer limits "
+ "[read:%llu, write:%llu] -> %u",
+ (unsigned long long)limits.read_length,
+ (unsigned long long)limits.write_length,
+ ret->transfer_buflen);
+ }
+
+ /* Use the server limit to scale down our value only */
+ if (num_requests == 0 && limits.open_handles) {
+ ret->num_requests =
+ MINIMUM(DEFAULT_NUM_REQUESTS, limits.open_handles);
+ debug("Server handle limit %llu; using %u",
+ (unsigned long long)limits.open_handles, ret->num_requests);
+ }
+ }
+
/* Some filexfer v.0 servers don't support large packets */
if (ret->version == 0)
ret->transfer_buflen = MINIMUM(ret->transfer_buflen, 20480);
@@ -505,6 +543,56 @@ sftp_proto_version(struct sftp_conn *conn)
return conn->version;
}
+int
+do_limits(struct sftp_conn *conn, struct sftp_limits *limits)
+{
+ u_int id, msg_id;
+ u_char type;
+ struct sshbuf *msg;
+ int r;
+
+ if ((conn->exts & SFTP_EXT_LIMITS) == 0) {
+ error("Server does not support limits at openssh.com extension");
+ return -1;
+ }
+
+ if ((msg = sshbuf_new()) == NULL)
+ fatal_f("sshbuf_new failed");
+
+ id = conn->msg_id++;
+ if ((r = sshbuf_put_u8(msg, SSH2_FXP_EXTENDED)) != 0 ||
+ (r = sshbuf_put_u32(msg, id)) != 0 ||
+ (r = sshbuf_put_cstring(msg, "limits at openssh.com")) != 0)
+ fatal_fr(r, "compose");
+ send_msg(conn, msg);
+ debug3("Sent message limits at openssh.com I:%u", id);
+
+ get_msg(conn, msg);
+
+ if ((r = sshbuf_get_u8(msg, &type)) != 0 ||
+ (r = sshbuf_get_u32(msg, &msg_id)) != 0)
+ fatal_fr(r, "parse");
+
+ debug3("Received limits reply T:%u I:%u", type, msg_id);
+ if (id != msg_id)
+ fatal("ID mismatch (%u != %u)", msg_id, id);
+ if (type != SSH2_FXP_EXTENDED_REPLY) {
+ fatal("Expected SSH2_FXP_EXTENDED_REPLY(%u) packet, got %u",
+ SSH2_FXP_EXTENDED_REPLY, type);
+ }
+
+ memset(limits, 0, sizeof(*limits));
+ if ((r = sshbuf_get_u64(msg, &limits->packet_length)) != 0 ||
+ (r = sshbuf_get_u64(msg, &limits->read_length)) != 0 ||
+ (r = sshbuf_get_u64(msg, &limits->write_length)) != 0 ||
+ (r = sshbuf_get_u64(msg, &limits->open_handles)) != 0)
+ fatal_fr(r, "parse limits");
+
+ sshbuf_free(msg);
+
+ return 0;
+}
+
int
do_close(struct sftp_conn *conn, const u_char *handle, u_int handle_len)
{
diff --git a/sftp-client.h b/sftp-client.h
index 63a9b8b13bdc..51ba72a3ab22 100644
--- a/sftp-client.h
+++ b/sftp-client.h
@@ -53,6 +53,14 @@ struct sftp_statvfs {
u_int64_t f_namemax;
};
+/* Used for limits response on the wire from the server */
+struct sftp_limits {
+ u_int64_t packet_length;
+ u_int64_t read_length;
+ u_int64_t write_length;
+ u_int64_t open_handles;
+};
+
/*
* Initialise a SSH filexfer connection. Returns NULL on error or
* a pointer to a initialized sftp_conn struct on success.
@@ -61,6 +69,9 @@ struct sftp_conn *do_init(int, int, u_int, u_int, u_int64_t);
u_int sftp_proto_version(struct sftp_conn *);
+/* Query server limits */
+int do_limits(struct sftp_conn *, struct sftp_limits *);
+
/* Close file referred to by 'handle' */
int do_close(struct sftp_conn *, const u_char *, u_int);
diff --git a/sftp.c b/sftp.c
index b641be2bc4c7..b63f43b5af48 100644
--- a/sftp.c
+++ b/sftp.c
@@ -70,9 +70,6 @@ typedef void EditLine;
#include "sftp-common.h"
#include "sftp-client.h"
-#define DEFAULT_COPY_BUFLEN 32768 /* Size of buffer for up/download */
-#define DEFAULT_NUM_REQUESTS 64 /* # concurrent outstanding requests */
-
/* File to read commands from */
FILE* infile;
@@ -2386,8 +2383,8 @@ main(int argc, char **argv)
extern int optind;
extern char *optarg;
struct sftp_conn *conn;
- size_t copy_buffer_len = DEFAULT_COPY_BUFLEN;
- size_t num_requests = DEFAULT_NUM_REQUESTS;
+ size_t copy_buffer_len = 0;
+ size_t num_requests = 0;
long long limit_kbps = 0;
/* Ensure that fds 0, 1 and 2 are open or directed to /dev/null */
--
2.28.0
More information about the openssh-unix-dev
mailing list