From 819b929d3389f6007e1c469d9060e7876caeb97f Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:02:28 -0500 Subject: pkt-line: teach packet_read_line to chomp newlines The packets sent during ref negotiation are all terminated by newline; even though the code to chomp these newlines is short, we end up doing it in a lot of places. This patch teaches packet_read_line to auto-chomp the trailing newline; this lets us get rid of a lot of inline chomping code. As a result, some call-sites which are not reading line-oriented data (e.g., when reading chunks of packfiles alongside sideband) transition away from packet_read_line to the generic packet_read interface. This patch converts all of the existing callsites. Since the function signature of packet_read_line does not change (but its behavior does), there is a possibility of new callsites being introduced in later commits, silently introducing an incompatibility. However, since a later patch in this series will change the signature, such a commit would have to be merged directly into this commit, not to the tip of the series; we can therefore ignore the issue. This is an internal cleanup and should produce no change of behavior in the normal case. However, there is one corner case to note. Callers of packet_read_line have never been able to tell the difference between a flush packet ("0000") and an empty packet ("0004"), as both cause packet_read_line to return a length of 0. Readers treat them identically, even though Documentation/technical/protocol-common.txt says we must not; it also says that implementations should not send an empty pkt-line. By stripping out the newline before the result gets to the caller, we will now treat the newline-only packet ("0005\n") the same as an empty packet, which in turn gets treated like a flush packet. In practice this doesn't matter, as neither empty nor newline-only packets are part of git's protocols (at least not for the line-oriented bits, and readers who are not expecting line-oriented packets will be calling packet_read directly, anyway). But even if we do decide to care about the distinction later, it is orthogonal to this patch. The right place to tighten would be to stop treating empty packets as flush packets, and this change does not make doing so any harder. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 940ae35dc2..f73664f433 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -105,8 +105,6 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) int n = packet_read_line(0, line, sizeof(line)); if (!n) break; - if (line[n-1] == '\n') - n--; string_list_append(&sought, xmemdupz(line, n)); } } -- cgit v1.2.3 From 74543a0423c96130b3b07946c20b10735c3b5b15 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:02:57 -0500 Subject: pkt-line: provide a LARGE_PACKET_MAX static buffer Most of the callers of packet_read_line just read into a static 1000-byte buffer (callers which handle arbitrary binary data already use LARGE_PACKET_MAX). This works fine in practice, because: 1. The only variable-sized data in these lines is a ref name, and refs tend to be a lot shorter than 1000 characters. 2. When sending ref lines, git-core always limits itself to 1000 byte packets. However, the only limit given in the protocol specification in Documentation/technical/protocol-common.txt is LARGE_PACKET_MAX; the 1000 byte limit is mentioned only in pack-protocol.txt, and then only describing what we write, not as a specific limit for readers. This patch lets us bump the 1000-byte limit to LARGE_PACKET_MAX. Even though git-core will never write a packet where this makes a difference, there are two good reasons to do this: 1. Other git implementations may have followed protocol-common.txt and used a larger maximum size. We don't bump into it in practice because it would involve very long ref names. 2. We may want to increase the 1000-byte limit one day. Since packets are transferred before any capabilities, it's difficult to do this in a backwards-compatible way. But if we bump the size of buffer the readers can handle, eventually older versions of git will be obsolete enough that we can justify bumping the writers, as well. We don't have plans to do this anytime soon, but there is no reason not to start the clock ticking now. Just bumping all of the reading bufs to LARGE_PACKET_MAX would waste memory. Instead, since most readers just read into a temporary buffer anyway, let's provide a single static buffer that all callers can use. We can further wrap this detail away by having the packet_read_line wrapper just use the buffer transparently and return a pointer to the static storage. That covers most of the cases, and the remaining ones already read into their own LARGE_PACKET_MAX buffers. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/archive.c | 15 +++++++-------- builtin/fetch-pack.c | 7 +++---- builtin/receive-pack.c | 6 +++--- builtin/upload-archive.c | 7 ++----- connect.c | 4 ++-- daemon.c | 4 ++-- fetch-pack.c | 12 ++++++------ pkt-line.c | 9 +++++++-- pkt-line.h | 9 +++++++-- send-pack.c | 7 +++---- upload-pack.c | 12 +++++------- 11 files changed, 47 insertions(+), 45 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/archive.c b/builtin/archive.c index d381ac4147..49178f159e 100644 --- a/builtin/archive.c +++ b/builtin/archive.c @@ -27,8 +27,8 @@ static int run_remote_archiver(int argc, const char **argv, const char *remote, const char *exec, const char *name_hint) { - char buf[LARGE_PACKET_MAX]; - int fd[2], i, len, rv; + char *buf; + int fd[2], i, rv; struct transport *transport; struct remote *_remote; @@ -53,19 +53,18 @@ static int run_remote_archiver(int argc, const char **argv, packet_write(fd[1], "argument %s\n", argv[i]); packet_flush(fd[1]); - len = packet_read_line(fd[0], buf, sizeof(buf)); - if (!len) + buf = packet_read_line(fd[0], NULL); + if (!buf) die(_("git archive: expected ACK/NAK, got EOF")); if (strcmp(buf, "ACK")) { - if (len > 5 && !prefixcmp(buf, "NACK ")) + if (!prefixcmp(buf, "NACK ")) die(_("git archive: NACK %s"), buf + 5); - if (len > 4 && !prefixcmp(buf, "ERR ")) + if (!prefixcmp(buf, "ERR ")) die(_("remote error: %s"), buf + 4); die(_("git archive: protocol error")); } - len = packet_read_line(fd[0], buf, sizeof(buf)); - if (len) + if (packet_read_line(fd[0], NULL)) die(_("git archive: expected a flush")); /* Now, start reading from fd[0] and spit it out to stdout */ diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index f73664f433..c21cc2c778 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -100,12 +100,11 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) /* in stateless RPC mode we use pkt-line to read * from stdin, until we get a flush packet */ - static char line[1000]; for (;;) { - int n = packet_read_line(0, line, sizeof(line)); - if (!n) + char *line = packet_read_line(0, NULL); + if (!line) break; - string_list_append(&sought, xmemdupz(line, n)); + string_list_append(&sought, xstrdup(line)); } } else { diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 6679e636c7..ccebd74f16 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -754,14 +754,14 @@ static struct command *read_head_info(void) struct command *commands = NULL; struct command **p = &commands; for (;;) { - static char line[1000]; + char *line; unsigned char old_sha1[20], new_sha1[20]; struct command *cmd; char *refname; int len, reflen; - len = packet_read_line(0, line, sizeof(line)); - if (!len) + line = packet_read_line(0, &len); + if (!line) break; if (len < 83 || line[40] != ' ' || diff --git a/builtin/upload-archive.c b/builtin/upload-archive.c index d90f0aba44..af2da35e7d 100644 --- a/builtin/upload-archive.c +++ b/builtin/upload-archive.c @@ -21,8 +21,6 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix) { struct argv_array sent_argv = ARGV_ARRAY_INIT; const char *arg_cmd = "argument "; - char buf[4096]; - int len; if (argc != 2) usage(upload_archive_usage); @@ -33,9 +31,8 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix) /* put received options in sent_argv[] */ argv_array_push(&sent_argv, "git-upload-archive"); for (;;) { - /* This will die if not enough free space in buf */ - len = packet_read_line(0, buf, sizeof(buf)); - if (len == 0) + char *buf = packet_read_line(0, NULL); + if (!buf) break; /* got a flush */ if (sent_argv.argc > MAX_ARGS) die("Too many options (>%d)", MAX_ARGS - 1); diff --git a/connect.c b/connect.c index fe8eb01ae2..611ffb4419 100644 --- a/connect.c +++ b/connect.c @@ -72,11 +72,11 @@ struct ref **get_remote_heads(int in, struct ref **list, for (;;) { struct ref *ref; unsigned char old_sha1[20]; - static char buffer[1000]; char *name; int len, name_len; + char *buffer = packet_buffer; - len = packet_read(in, buffer, sizeof(buffer), + len = packet_read(in, packet_buffer, sizeof(packet_buffer), PACKET_READ_GENTLE_ON_EOF | PACKET_READ_CHOMP_NEWLINE); if (len < 0) diff --git a/daemon.c b/daemon.c index 4f5cd61558..3f70e79b8e 100644 --- a/daemon.c +++ b/daemon.c @@ -604,7 +604,7 @@ static void parse_host_arg(char *extra_args, int buflen) static int execute(void) { - static char line[1000]; + char *line = packet_buffer; int pktlen, len, i; char *addr = getenv("REMOTE_ADDR"), *port = getenv("REMOTE_PORT"); @@ -612,7 +612,7 @@ static int execute(void) loginfo("Connection from %s:%s", addr, port); alarm(init_timeout ? init_timeout : timeout); - pktlen = packet_read(0, line, sizeof(line), 0); + pktlen = packet_read(0, packet_buffer, sizeof(packet_buffer), 0); alarm(0); len = strlen(line); diff --git a/fetch-pack.c b/fetch-pack.c index f830db224b..66ff9add89 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -172,8 +172,8 @@ static void consume_shallow_list(struct fetch_pack_args *args, int fd) * shallow and unshallow commands every time there * is a block of have lines exchanged. */ - char line[1000]; - while (packet_read_line(fd, line, sizeof(line))) { + char *line; + while ((line = packet_read_line(fd, NULL))) { if (!prefixcmp(line, "shallow ")) continue; if (!prefixcmp(line, "unshallow ")) @@ -215,8 +215,8 @@ static int write_shallow_commits(struct strbuf *out, int use_pack_protocol) static enum ack_type get_ack(int fd, unsigned char *result_sha1) { - static char line[1000]; - int len = packet_read_line(fd, line, sizeof(line)); + int len; + char *line = packet_read_line(fd, &len); if (!len) die("git fetch-pack: expected ACK/NAK, got EOF"); @@ -346,11 +346,11 @@ static int find_common(struct fetch_pack_args *args, state_len = req_buf.len; if (args->depth > 0) { - char line[1024]; + char *line; unsigned char sha1[20]; send_request(args, fd[1], &req_buf); - while (packet_read_line(fd[0], line, sizeof(line))) { + while ((line = packet_read_line(fd[0], NULL))) { if (!prefixcmp(line, "shallow ")) { if (get_sha1_hex(line + 8, sha1)) die("invalid shallow line: %s", line); diff --git a/pkt-line.c b/pkt-line.c index dc11c407cd..55fb688899 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -1,6 +1,7 @@ #include "cache.h" #include "pkt-line.h" +char packet_buffer[LARGE_PACKET_MAX]; static const char *packet_trace_prefix = "git"; static const char trace_key[] = "GIT_TRACE_PACKET"; @@ -174,9 +175,13 @@ int packet_read(int fd, char *buffer, unsigned size, int options) return len; } -int packet_read_line(int fd, char *buffer, unsigned size) +char *packet_read_line(int fd, int *len_p) { - return packet_read(fd, buffer, size, PACKET_READ_CHOMP_NEWLINE); + int len = packet_read(fd, packet_buffer, sizeof(packet_buffer), + PACKET_READ_CHOMP_NEWLINE); + if (len_p) + *len_p = len; + return len ? packet_buffer : NULL; } int packet_get_line(struct strbuf *out, diff --git a/pkt-line.h b/pkt-line.h index 6927ea521b..fa93e32071 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -54,12 +54,17 @@ int packet_read(int fd, char *buffer, unsigned size, int options); /* * Convenience wrapper for packet_read that is not gentle, and sets the - * CHOMP_NEWLINE option. + * CHOMP_NEWLINE option. The return value is NULL for a flush packet, + * and otherwise points to a static buffer (that may be overwritten by + * subsequent calls). If the size parameter is not NULL, the length of the + * packet is written to it. */ -int packet_read_line(int fd, char *buffer, unsigned size); +char *packet_read_line(int fd, int *size); + #define DEFAULT_PACKET_MAX 1000 #define LARGE_PACKET_MAX 65520 +extern char packet_buffer[LARGE_PACKET_MAX]; int packet_get_line(struct strbuf *out, char **src_buf, size_t *src_len); diff --git a/send-pack.c b/send-pack.c index 8c230bf6c9..7d172ef37f 100644 --- a/send-pack.c +++ b/send-pack.c @@ -106,9 +106,8 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext static int receive_status(int in, struct ref *refs) { struct ref *hint; - char line[1000]; int ret = 0; - int len = packet_read_line(in, line, sizeof(line)); + char *line = packet_read_line(in, NULL); if (prefixcmp(line, "unpack ")) return error("did not receive remote status"); if (strcmp(line, "unpack ok")) { @@ -119,8 +118,8 @@ static int receive_status(int in, struct ref *refs) while (1) { char *refname; char *msg; - len = packet_read_line(in, line, sizeof(line)); - if (!len) + line = packet_read_line(in, NULL); + if (!line) break; if (prefixcmp(line, "ok ") && prefixcmp(line, "ng ")) { error("invalid ref status from remote: %s", line); diff --git a/upload-pack.c b/upload-pack.c index 6e6d166876..98ddb69581 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -408,7 +408,6 @@ static int ok_to_give_up(void) static int get_common_commits(void) { - static char line[1000]; unsigned char sha1[20]; char last_hex[41]; int got_common = 0; @@ -418,10 +417,10 @@ static int get_common_commits(void) save_commit_buffer = 0; for (;;) { - int len = packet_read_line(0, line, sizeof(line)); + char *line = packet_read_line(0, NULL); reset_timeout(); - if (!len) { + if (!line) { if (multi_ack == 2 && got_common && !got_other && ok_to_give_up()) { sent_ready = 1; @@ -567,8 +566,7 @@ error: static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; - static char line[1000]; - int len, depth = 0; + int depth = 0; int has_non_tip = 0; shallow_nr = 0; @@ -576,9 +574,9 @@ static void receive_needs(void) struct object *o; const char *features; unsigned char sha1_buf[20]; - len = packet_read_line(0, line, sizeof(line)); + char *line = packet_read_line(0, NULL); reset_timeout(); - if (!len) + if (!line) break; if (!prefixcmp(line, "shallow ")) { -- cgit v1.2.3 From 85edf4f58b5368e2f2acc4bce0d10e1ca9d6c879 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:06:45 -0500 Subject: teach get_remote_heads to read from a memory buffer Now that we can read packet data from memory as easily as a descriptor, get_remote_heads can take either one as a source. This will allow further refactoring in remote-curl. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 +- builtin/send-pack.c | 2 +- cache.h | 4 +++- connect.c | 6 +++--- remote-curl.c | 2 +- transport.c | 6 +++--- 6 files changed, 12 insertions(+), 10 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index c21cc2c778..03ed2caae3 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -125,7 +125,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.verbose ? CONNECT_VERBOSE : 0); } - get_remote_heads(fd[0], &ref, 0, NULL); + get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL); ref = fetch_pack(&args, fd, conn, ref, dest, &sought, pack_lockfile_ptr); diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 87785197cd..152c4ea092 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -207,7 +207,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) memset(&extra_have, 0, sizeof(extra_have)); - get_remote_heads(fd[0], &remote_refs, REF_NORMAL, &extra_have); + get_remote_heads(fd[0], NULL, 0, &remote_refs, REF_NORMAL, &extra_have); transport_verify_remote_names(nr_refspecs, refspecs); diff --git a/cache.h b/cache.h index e493563f4c..db646a2ff8 100644 --- a/cache.h +++ b/cache.h @@ -1049,7 +1049,9 @@ struct extra_have_objects { int nr, alloc; unsigned char (*array)[20]; }; -extern struct ref **get_remote_heads(int in, struct ref **list, unsigned int flags, struct extra_have_objects *); +extern struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, + struct ref **list, unsigned int flags, + struct extra_have_objects *); extern int server_supports(const char *feature); extern int parse_feature_request(const char *features, const char *feature); extern const char *server_feature_value(const char *feature, int *len_ret); diff --git a/connect.c b/connect.c index 3d999999e5..f57efd06c1 100644 --- a/connect.c +++ b/connect.c @@ -62,8 +62,8 @@ static void die_initial_contact(int got_at_least_one_head) /* * Read all the refs from the other end */ -struct ref **get_remote_heads(int in, struct ref **list, - unsigned int flags, +struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, + struct ref **list, unsigned int flags, struct extra_have_objects *extra_have) { int got_at_least_one_head = 0; @@ -76,7 +76,7 @@ struct ref **get_remote_heads(int in, struct ref **list, int len, name_len; char *buffer = packet_buffer; - len = packet_read(in, NULL, NULL, + len = packet_read(in, &src_buf, &src_len, packet_buffer, sizeof(packet_buffer), PACKET_READ_GENTLE_ON_EOF | PACKET_READ_CHOMP_NEWLINE); diff --git a/remote-curl.c b/remote-curl.c index c8379a53f0..24c86264d2 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -192,7 +192,7 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) if (start_async(&async)) die("cannot start thread to parse advertised refs"); - get_remote_heads(async.out, &list, + get_remote_heads(async.out, NULL, 0, &list, for_push ? REF_NORMAL : 0, NULL); close(async.out); if (finish_async(&async)) diff --git a/transport.c b/transport.c index 886ffd8b1e..62df466c1a 100644 --- a/transport.c +++ b/transport.c @@ -507,7 +507,7 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus struct ref *refs; connect_setup(transport, for_push, 0); - get_remote_heads(data->fd[0], &refs, + get_remote_heads(data->fd[0], NULL, 0, &refs, for_push ? REF_NORMAL : 0, &data->extra_have); data->got_remote_heads = 1; @@ -541,7 +541,7 @@ static int fetch_refs_via_pack(struct transport *transport, if (!data->got_remote_heads) { connect_setup(transport, 0, 0); - get_remote_heads(data->fd[0], &refs_tmp, 0, NULL); + get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, NULL); data->got_remote_heads = 1; } @@ -799,7 +799,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re struct ref *tmp_refs; connect_setup(transport, 1, 0); - get_remote_heads(data->fd[0], &tmp_refs, REF_NORMAL, NULL); + get_remote_heads(data->fd[0], NULL, 0, &tmp_refs, REF_NORMAL, NULL); data->got_remote_heads = 1; } -- cgit v1.2.3