From 2153d478b74cfef58ee49ee0305cccf5e8a77b4f Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:38 -0700 Subject: pkt-line: introduce packet_read_with_status The current pkt-line API encodes the status of a pkt-line read in the length of the read content. An error is indicated with '-1', a flush with '0' (which can be confusing since a return value of '0' can also indicate an empty pkt-line), and a positive integer for the length of the read content otherwise. This doesn't leave much room for allowing the addition of additional special packets in the future. To solve this introduce 'packet_read_with_status()' which reads a packet and returns the status of the read encoded as an 'enum packet_status' type. This allows for easily identifying between special and normal packets as well as errors. It also enables easily adding a new special packet in the future. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/pkt-line.c b/pkt-line.c index 2827ca7..db2fb29 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -280,28 +280,39 @@ static int packet_length(const char *linelen) return (val < 0) ? val : (val << 8) | hex2chr(linelen + 2); } -int packet_read(int fd, char **src_buf, size_t *src_len, - char *buffer, unsigned size, int options) +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options) { - int len, ret; + int len; char linelen[4]; - ret = get_packet_data(fd, src_buf, src_len, linelen, 4, options); - if (ret < 0) - return ret; + if (get_packet_data(fd, src_buffer, src_len, linelen, 4, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } + len = packet_length(linelen); - if (len < 0) + + if (len < 0) { die("protocol error: bad line length character: %.4s", linelen); - if (!len) { + } else if (!len) { packet_trace("0000", 4, 0); - return 0; + *pktlen = 0; + return PACKET_READ_FLUSH; + } else if (len < 4) { + die("protocol error: bad line length %d", len); } + len -= 4; - if (len >= size) + if ((unsigned)len >= size) die("protocol error: bad line length %d", len); - ret = get_packet_data(fd, src_buf, src_len, buffer, len, options); - if (ret < 0) - return ret; + + if (get_packet_data(fd, src_buffer, src_len, buffer, len, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } if ((options & PACKET_READ_CHOMP_NEWLINE) && len && buffer[len-1] == '\n') @@ -309,7 +320,19 @@ int packet_read(int fd, char **src_buf, size_t *src_len, buffer[len] = 0; packet_trace(buffer, len, 0); - return len; + *pktlen = len; + return PACKET_READ_NORMAL; +} + +int packet_read(int fd, char **src_buffer, size_t *src_len, + char *buffer, unsigned size, int options) +{ + int pktlen = -1; + + packet_read_with_status(fd, src_buffer, src_len, buffer, size, + &pktlen, options); + + return pktlen; } static char *packet_read_line_generic(int fd, diff --git a/pkt-line.h b/pkt-line.h index 3dad583..099b26b 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -66,6 +66,22 @@ int packet_read(int fd, char **src_buffer, size_t *src_len, char *buffer, unsigned size, int options); /* + * Read a packetized line into a buffer like the 'packet_read()' function but + * returns an 'enum packet_read_status' which indicates the status of the read. + * The number of bytes read will be assigined to *pktlen if the status of the + * read was 'PACKET_READ_NORMAL'. + */ +enum packet_read_status { + PACKET_READ_EOF, + PACKET_READ_NORMAL, + PACKET_READ_FLUSH, +}; +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options); + +/* * Convenience wrapper for packet_read that is not gentle, and sets the * CHOMP_NEWLINE option. The return value is NULL for a flush packet, * and otherwise points to a static buffer (that may be overwritten by -- cgit v0.10.2-6-g49f6 From 77dabc14c497d2498b814967bf21a3ed1618207f Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:39 -0700 Subject: pkt-line: allow peeking a packet line without consuming it Sometimes it is advantageous to be able to peek the next packet line without consuming it (e.g. to be able to determine the protocol version a server is speaking). In order to do that introduce 'struct packet_reader' which is an abstraction around the normal packet reading logic. This enables a caller to be able to peek a single line at a time using 'packet_reader_peek()' and having a caller consume a line by calling 'packet_reader_read()'. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/pkt-line.c b/pkt-line.c index db2fb29..1881dc8 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -400,3 +400,53 @@ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out) } return sb_out->len - orig_len; } + +/* Packet Reader Functions */ +void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options) +{ + memset(reader, 0, sizeof(*reader)); + + reader->fd = fd; + reader->src_buffer = src_buffer; + reader->src_len = src_len; + reader->buffer = packet_buffer; + reader->buffer_size = sizeof(packet_buffer); + reader->options = options; +} + +enum packet_read_status packet_reader_read(struct packet_reader *reader) +{ + if (reader->line_peeked) { + reader->line_peeked = 0; + return reader->status; + } + + reader->status = packet_read_with_status(reader->fd, + &reader->src_buffer, + &reader->src_len, + reader->buffer, + reader->buffer_size, + &reader->pktlen, + reader->options); + + if (reader->status == PACKET_READ_NORMAL) + reader->line = reader->buffer; + else + reader->line = NULL; + + return reader->status; +} + +enum packet_read_status packet_reader_peek(struct packet_reader *reader) +{ + /* Only allow peeking a single line */ + if (reader->line_peeked) + return reader->status; + + /* Peek a line by reading it and setting peeked flag */ + packet_reader_read(reader); + reader->line_peeked = 1; + return reader->status; +} diff --git a/pkt-line.h b/pkt-line.h index 099b26b..11b04f0 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -112,6 +112,64 @@ char *packet_read_line_buf(char **src_buf, size_t *src_len, int *size); */ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out); +struct packet_reader { + /* source file descriptor */ + int fd; + + /* source buffer and its size */ + char *src_buffer; + size_t src_len; + + /* buffer that pkt-lines are read into and its size */ + char *buffer; + unsigned buffer_size; + + /* options to be used during reads */ + int options; + + /* status of the last read */ + enum packet_read_status status; + + /* length of data read during the last read */ + int pktlen; + + /* the last line read */ + const char *line; + + /* indicates if a line has been peeked */ + int line_peeked; +}; + +/* + * Initialize a 'struct packet_reader' object which is an + * abstraction around the 'packet_read_with_status()' function. + */ +extern void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options); + +/* + * Perform a packet read and return the status of the read. + * The values of 'pktlen' and 'line' are updated based on the status of the + * read as follows: + * + * PACKET_READ_ERROR: 'pktlen' is set to '-1' and 'line' is set to NULL + * PACKET_READ_NORMAL: 'pktlen' is set to the number of bytes read + * 'line' is set to point at the read line + * PACKET_READ_FLUSH: 'pktlen' is set to '0' and 'line' is set to NULL + */ +extern enum packet_read_status packet_reader_read(struct packet_reader *reader); + +/* + * Peek the next packet line without consuming it and return the status. + * The next call to 'packet_reader_read()' will perform a read of the same line + * that was peeked, consuming the line. + * + * Peeking multiple times without calling 'packet_reader_read()' will return + * the same result. + */ +extern enum packet_read_status packet_reader_peek(struct packet_reader *reader); + #define DEFAULT_PACKET_MAX 1000 #define LARGE_PACKET_MAX 65520 #define LARGE_PACKET_DATA_MAX (LARGE_PACKET_MAX - 4) -- cgit v0.10.2-6-g49f6 From a4cfd41c7be81c10b925532979e81e4d07f6b376 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:40 -0700 Subject: pkt-line: add delim packet support One of the design goals of protocol-v2 is to improve the semantics of flush packets. Currently in protocol-v1, flush packets are used both to indicate a break in a list of packet lines as well as an indication that one side has finished speaking. This makes it particularly difficult to implement proxies as a proxy would need to completely understand git protocol instead of simply looking for a flush packet. To do this, introduce the special deliminator packet '0001'. A delim packet can then be used as a deliminator between lists of packet lines while flush packets can be reserved to indicate the end of a response. Documentation for how this packet will be used in protocol v2 will included in a future patch. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/pkt-line.c b/pkt-line.c index 1881dc8..7296731 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -91,6 +91,12 @@ void packet_flush(int fd) write_or_die(fd, "0000", 4); } +void packet_delim(int fd) +{ + packet_trace("0001", 4, 1); + write_or_die(fd, "0001", 4); +} + int packet_flush_gently(int fd) { packet_trace("0000", 4, 1); @@ -105,6 +111,12 @@ void packet_buf_flush(struct strbuf *buf) strbuf_add(buf, "0000", 4); } +void packet_buf_delim(struct strbuf *buf) +{ + packet_trace("0001", 4, 1); + strbuf_add(buf, "0001", 4); +} + static void set_packet_header(char *buf, const int size) { static char hexchar[] = "0123456789abcdef"; @@ -301,6 +313,10 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, packet_trace("0000", 4, 0); *pktlen = 0; return PACKET_READ_FLUSH; + } else if (len == 1) { + packet_trace("0001", 4, 0); + *pktlen = 0; + return PACKET_READ_DELIM; } else if (len < 4) { die("protocol error: bad line length %d", len); } diff --git a/pkt-line.h b/pkt-line.h index 11b04f0..9570bd7 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -20,8 +20,10 @@ * side can't, we stay with pure read/write interfaces. */ void packet_flush(int fd); +void packet_delim(int fd); void packet_write_fmt(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); void packet_buf_flush(struct strbuf *buf); +void packet_buf_delim(struct strbuf *buf); void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int packet_flush_gently(int fd); @@ -75,6 +77,7 @@ enum packet_read_status { PACKET_READ_EOF, PACKET_READ_NORMAL, PACKET_READ_FLUSH, + PACKET_READ_DELIM, }; enum packet_read_status packet_read_with_status(int fd, char **src_buffer, size_t *src_len, char *buffer, -- cgit v0.10.2-6-g49f6 From a3d6b53e92528d20fdaca504614f260c5b457b29 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:41 -0700 Subject: upload-pack: convert to a builtin In order to allow for code sharing with the server-side of fetch in protocol-v2 convert upload-pack to be a builtin. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index 2a81ae2..91a59a6 100644 --- a/Makefile +++ b/Makefile @@ -636,7 +636,6 @@ PROGRAM_OBJS += imap-send.o PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o PROGRAM_OBJS += show-index.o -PROGRAM_OBJS += upload-pack.o PROGRAM_OBJS += remote-testsvn.o # Binary suffix, set to .exe for Windows builds @@ -904,6 +903,7 @@ LIB_OBJS += tree-diff.o LIB_OBJS += tree.o LIB_OBJS += tree-walk.o LIB_OBJS += unpack-trees.o +LIB_OBJS += upload-pack.o LIB_OBJS += url.o LIB_OBJS += urlmatch.o LIB_OBJS += usage.o @@ -1021,6 +1021,7 @@ BUILTIN_OBJS += builtin/update-index.o BUILTIN_OBJS += builtin/update-ref.o BUILTIN_OBJS += builtin/update-server-info.o BUILTIN_OBJS += builtin/upload-archive.o +BUILTIN_OBJS += builtin/upload-pack.o BUILTIN_OBJS += builtin/var.o BUILTIN_OBJS += builtin/verify-commit.o BUILTIN_OBJS += builtin/verify-pack.o diff --git a/builtin.h b/builtin.h index 42378f3..f332a12 100644 --- a/builtin.h +++ b/builtin.h @@ -231,6 +231,7 @@ extern int cmd_update_ref(int argc, const char **argv, const char *prefix); extern int cmd_update_server_info(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix); +extern int cmd_upload_pack(int argc, const char **argv, const char *prefix); extern int cmd_var(int argc, const char **argv, const char *prefix); extern int cmd_verify_commit(int argc, const char **argv, const char *prefix); extern int cmd_verify_tag(int argc, const char **argv, const char *prefix); diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c new file mode 100644 index 0000000..2cb5cb3 --- /dev/null +++ b/builtin/upload-pack.c @@ -0,0 +1,67 @@ +#include "cache.h" +#include "builtin.h" +#include "exec_cmd.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "protocol.h" +#include "upload-pack.h" + +static const char * const upload_pack_usage[] = { + N_("git upload-pack [] "), + NULL +}; + +int cmd_upload_pack(int argc, const char **argv, const char *prefix) +{ + const char *dir; + int strict = 0; + struct upload_pack_options opts = { 0 }; + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-refs", &opts.advertise_refs, + N_("exit immediately after initial ref advertisement")), + OPT_BOOL(0, "strict", &strict, + N_("do not try /.git/ if is no Git directory")), + OPT_INTEGER(0, "timeout", &opts.timeout, + N_("interrupt transfer after seconds of inactivity")), + OPT_END() + }; + + packet_trace_identity("upload-pack"); + check_replace_refs = 0; + + argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + + if (argc != 1) + usage_with_options(upload_pack_usage, options); + + if (opts.timeout) + opts.daemon_mode = 1; + + setup_path(); + + dir = argv[0]; + + if (!enter_repo(dir, strict)) + die("'%s' does not appear to be a git repository", dir); + + switch (determine_protocol_version_server()) { + case protocol_v1: + /* + * v1 is just the original protocol with a version string, + * so just fall through after writing the version string. + */ + if (opts.advertise_refs || !opts.stateless_rpc) + packet_write_fmt(1, "version 1\n"); + + /* fallthrough */ + case protocol_v0: + upload_pack(&opts); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return 0; +} diff --git a/git.c b/git.c index c870b97..f71073d 100644 --- a/git.c +++ b/git.c @@ -478,6 +478,7 @@ static struct cmd_struct commands[] = { { "update-server-info", cmd_update_server_info, RUN_SETUP }, { "upload-archive", cmd_upload_archive }, { "upload-archive--writer", cmd_upload_archive_writer }, + { "upload-pack", cmd_upload_pack }, { "var", cmd_var, RUN_SETUP_GENTLY }, { "verify-commit", cmd_verify_commit, RUN_SETUP }, { "verify-pack", cmd_verify_pack }, diff --git a/upload-pack.c b/upload-pack.c index d5de181..2ad73a9 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -6,7 +6,6 @@ #include "tag.h" #include "object.h" #include "commit.h" -#include "exec_cmd.h" #include "diff.h" #include "revision.h" #include "list-objects.h" @@ -15,15 +14,10 @@ #include "sigchain.h" #include "version.h" #include "string-list.h" -#include "parse-options.h" #include "argv-array.h" #include "prio-queue.h" #include "protocol.h" - -static const char * const upload_pack_usage[] = { - N_("git upload-pack [] "), - NULL -}; +#include "upload-pack.h" /* Remember to update object flag allocation in object.h */ #define THEY_HAVE (1u << 11) @@ -61,7 +55,6 @@ static int keepalive = 5; * otherwise maximum packet size (up to 65520 bytes). */ static int use_sideband; -static int advertise_refs; static int stateless_rpc; static const char *pack_objects_hook; @@ -977,33 +970,6 @@ static int find_symref(const char *refname, const struct object_id *oid, return 0; } -static void upload_pack(void) -{ - struct string_list symref = STRING_LIST_INIT_DUP; - - head_ref_namespaced(find_symref, &symref); - - if (advertise_refs || !stateless_rpc) { - reset_timeout(); - head_ref_namespaced(send_ref, &symref); - for_each_namespaced_ref(send_ref, &symref); - advertise_shallow_grafts(1); - packet_flush(1); - } else { - head_ref_namespaced(check_ref, NULL); - for_each_namespaced_ref(check_ref, NULL); - } - string_list_clear(&symref, 1); - if (advertise_refs) - return; - - receive_needs(); - if (want_obj.nr) { - get_common_commits(); - create_pack_file(); - } -} - static int upload_pack_config(const char *var, const char *value, void *unused) { if (!strcmp("uploadpack.allowtipsha1inwant", var)) { @@ -1032,58 +998,35 @@ static int upload_pack_config(const char *var, const char *value, void *unused) return parse_hide_refs_config(var, value, "uploadpack"); } -int cmd_main(int argc, const char **argv) +void upload_pack(struct upload_pack_options *options) { - const char *dir; - int strict = 0; - struct option options[] = { - OPT_BOOL(0, "stateless-rpc", &stateless_rpc, - N_("quit after a single request/response exchange")), - OPT_BOOL(0, "advertise-refs", &advertise_refs, - N_("exit immediately after initial ref advertisement")), - OPT_BOOL(0, "strict", &strict, - N_("do not try /.git/ if is no Git directory")), - OPT_INTEGER(0, "timeout", &timeout, - N_("interrupt transfer after seconds of inactivity")), - OPT_END() - }; - - packet_trace_identity("upload-pack"); - check_replace_refs = 0; - - argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); - - if (argc != 1) - usage_with_options(upload_pack_usage, options); - - if (timeout) - daemon_mode = 1; - - setup_path(); - - dir = argv[0]; + struct string_list symref = STRING_LIST_INIT_DUP; - if (!enter_repo(dir, strict)) - die("'%s' does not appear to be a git repository", dir); + stateless_rpc = options->stateless_rpc; + timeout = options->timeout; + daemon_mode = options->daemon_mode; git_config(upload_pack_config, NULL); - switch (determine_protocol_version_server()) { - case protocol_v1: - /* - * v1 is just the original protocol with a version string, - * so just fall through after writing the version string. - */ - if (advertise_refs || !stateless_rpc) - packet_write_fmt(1, "version 1\n"); - - /* fallthrough */ - case protocol_v0: - upload_pack(); - break; - case protocol_unknown_version: - BUG("unknown protocol version"); + head_ref_namespaced(find_symref, &symref); + + if (options->advertise_refs || !stateless_rpc) { + reset_timeout(); + head_ref_namespaced(send_ref, &symref); + for_each_namespaced_ref(send_ref, &symref); + advertise_shallow_grafts(1); + packet_flush(1); + } else { + head_ref_namespaced(check_ref, NULL); + for_each_namespaced_ref(check_ref, NULL); } + string_list_clear(&symref, 1); + if (options->advertise_refs) + return; - return 0; + receive_needs(); + if (want_obj.nr) { + get_common_commits(); + create_pack_file(); + } } diff --git a/upload-pack.h b/upload-pack.h new file mode 100644 index 0000000..a71e4dc --- /dev/null +++ b/upload-pack.h @@ -0,0 +1,13 @@ +#ifndef UPLOAD_PACK_H +#define UPLOAD_PACK_H + +struct upload_pack_options { + int stateless_rpc; + int advertise_refs; + unsigned int timeout; + int daemon_mode; +}; + +void upload_pack(struct upload_pack_options *options); + +#endif /* UPLOAD_PACK_H */ -- cgit v0.10.2-6-g49f6 From ae2948f30c7de56e2805d26f78ea8205f6c98682 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:42 -0700 Subject: upload-pack: factor out processing lines Factor out the logic for processing shallow, deepen, deepen_since, and deepen_not lines into their own functions to simplify the 'receive_needs()' function in addition to making it easier to reuse some of this logic when implementing protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/upload-pack.c b/upload-pack.c index 2ad73a9..1e8a9e1 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -724,6 +724,75 @@ static void deepen_by_rev_list(int ac, const char **av, packet_flush(1); } +static int process_shallow(const char *line, struct object_array *shallows) +{ + const char *arg; + if (skip_prefix(line, "shallow ", &arg)) { + struct object_id oid; + struct object *object; + if (get_oid_hex(arg, &oid)) + die("invalid shallow line: %s", line); + object = parse_object(&oid); + if (!object) + return 1; + if (object->type != OBJ_COMMIT) + die("invalid shallow object %s", oid_to_hex(&oid)); + if (!(object->flags & CLIENT_SHALLOW)) { + object->flags |= CLIENT_SHALLOW; + add_object_array(object, NULL, shallows); + } + return 1; + } + + return 0; +} + +static int process_deepen(const char *line, int *depth) +{ + const char *arg; + if (skip_prefix(line, "deepen ", &arg)) { + char *end = NULL; + *depth = (int)strtol(arg, &end, 0); + if (!end || *end || *depth <= 0) + die("Invalid deepen: %s", line); + return 1; + } + + return 0; +} + +static int process_deepen_since(const char *line, timestamp_t *deepen_since, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-since ", &arg)) { + char *end = NULL; + *deepen_since = parse_timestamp(arg, &end, 0); + if (!end || *end || !deepen_since || + /* revisions.c's max_age -1 is special */ + *deepen_since == -1) + die("Invalid deepen-since: %s", line); + *deepen_rev_list = 1; + return 1; + } + return 0; +} + +static int process_deepen_not(const char *line, struct string_list *deepen_not, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-not ", &arg)) { + char *ref = NULL; + struct object_id oid; + if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) + die("git upload-pack: ambiguous deepen-not: %s", line); + string_list_append(deepen_not, ref); + free(ref); + *deepen_rev_list = 1; + return 1; + } + return 0; +} + static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; @@ -745,49 +814,15 @@ static void receive_needs(void) if (!line) break; - if (skip_prefix(line, "shallow ", &arg)) { - struct object_id oid; - struct object *object; - if (get_oid_hex(arg, &oid)) - die("invalid shallow line: %s", line); - object = parse_object(&oid); - if (!object) - continue; - if (object->type != OBJ_COMMIT) - die("invalid shallow object %s", oid_to_hex(&oid)); - if (!(object->flags & CLIENT_SHALLOW)) { - object->flags |= CLIENT_SHALLOW; - add_object_array(object, NULL, &shallows); - } + if (process_shallow(line, &shallows)) continue; - } - if (skip_prefix(line, "deepen ", &arg)) { - char *end = NULL; - depth = strtol(arg, &end, 0); - if (!end || *end || depth <= 0) - die("Invalid deepen: %s", line); + if (process_deepen(line, &depth)) continue; - } - if (skip_prefix(line, "deepen-since ", &arg)) { - char *end = NULL; - deepen_since = parse_timestamp(arg, &end, 0); - if (!end || *end || !deepen_since || - /* revisions.c's max_age -1 is special */ - deepen_since == -1) - die("Invalid deepen-since: %s", line); - deepen_rev_list = 1; + if (process_deepen_since(line, &deepen_since, &deepen_rev_list)) continue; - } - if (skip_prefix(line, "deepen-not ", &arg)) { - char *ref = NULL; - struct object_id oid; - if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) - die("git upload-pack: ambiguous deepen-not: %s", line); - string_list_append(&deepen_not, ref); - free(ref); - deepen_rev_list = 1; + if (process_deepen_not(line, &deepen_not, &deepen_rev_list)) continue; - } + if (!skip_prefix(line, "want ", &arg) || get_oid_hex(arg, &oid_buf)) die("git upload-pack: protocol error, " -- cgit v0.10.2-6-g49f6 From 635365eb2f67bc17482c5dbe076f20832cf92ae2 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:43 -0700 Subject: transport: use get_refs_via_connect to get refs Remove code duplication and use the existing 'get_refs_via_connect()' function to retrieve a remote's heads in 'fetch_refs_via_pack()' and 'git_transport_push()'. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport.c b/transport.c index fc80226..8e87790 100644 --- a/transport.c +++ b/transport.c @@ -230,12 +230,8 @@ static int fetch_refs_via_pack(struct transport *transport, args.cloning = transport->cloning; args.update_shallow = data->options.update_shallow; - if (!data->got_remote_heads) { - connect_setup(transport, 0); - get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, - NULL, &data->shallow); - data->got_remote_heads = 1; - } + if (!data->got_remote_heads) + refs_tmp = get_refs_via_connect(transport, 0); refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, @@ -541,14 +537,8 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re struct send_pack_args args; int ret; - if (!data->got_remote_heads) { - struct ref *tmp_refs; - connect_setup(transport, 1); - - get_remote_heads(data->fd[0], NULL, 0, &tmp_refs, REF_NORMAL, - NULL, &data->shallow); - data->got_remote_heads = 1; - } + if (!data->got_remote_heads) + get_refs_via_connect(transport, 1); memset(&args, 0, sizeof(args)); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); -- cgit v0.10.2-6-g49f6 From 7e3e479b90fd618fb8eb8222738f7cc53ab288fa Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:44 -0700 Subject: connect: convert get_remote_heads to use struct packet_reader In order to allow for better control flow when protocol_v2 is introduced convert 'get_remote_heads()' to use 'struct packet_reader' to read packet lines. This enables a client to be able to peek the first line of a server's response (without consuming it) in order to determine the protocol version its speaking and then passing control to the appropriate handler. This is needed because the initial response from a server speaking protocol_v0 includes the first ref, while subsequent protocol versions respond with a version line. We want to be able to read this first line without consuming the first ref sent in the protocol_v0 case so that the protocol version the server is speaking can be determined outside of 'get_remote_heads()' in a future patch. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/connect.c b/connect.c index c3a014c..c82c90b 100644 --- a/connect.c +++ b/connect.c @@ -48,6 +48,12 @@ int check_ref_type(const struct ref *ref, int flags) static void die_initial_contact(int unexpected) { + /* + * A hang-up after seeing some response from the other end + * means that it is unexpected, as we know the other end is + * willing to talk to us. A hang-up before seeing any + * response does not necessarily mean an ACL problem, though. + */ if (unexpected) die(_("The remote end hung up upon initial contact")); else @@ -56,6 +62,40 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } +static enum protocol_version discover_version(struct packet_reader *reader) +{ + enum protocol_version version = protocol_unknown_version; + + /* + * Peek the first line of the server's response to + * determine the protocol version the server is speaking. + */ + switch (packet_reader_peek(reader)) { + case PACKET_READ_EOF: + die_initial_contact(0); + case PACKET_READ_FLUSH: + case PACKET_READ_DELIM: + version = protocol_v0; + break; + case PACKET_READ_NORMAL: + version = determine_protocol_version_client(reader->line); + break; + } + + switch (version) { + case protocol_v1: + /* Read the peeked version line */ + packet_reader_read(reader); + break; + case protocol_v0: + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return version; +} + static void parse_one_symref_info(struct string_list *symref, const char *val, int len) { char *sym, *target; @@ -109,60 +149,21 @@ static void annotate_refs_with_symref_info(struct ref *ref) string_list_clear(&symref, 0); } -/* - * Read one line of a server's ref advertisement into packet_buffer. - */ -static int read_remote_ref(int in, char **src_buf, size_t *src_len, - int *responded) +static void process_capabilities(const char *line, int *len) { - int len = packet_read(in, src_buf, src_len, - packet_buffer, sizeof(packet_buffer), - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); - const char *arg; - if (len < 0) - die_initial_contact(*responded); - if (len > 4 && skip_prefix(packet_buffer, "ERR ", &arg)) - die("remote error: %s", arg); - - *responded = 1; - - return len; -} - -#define EXPECTING_PROTOCOL_VERSION 0 -#define EXPECTING_FIRST_REF 1 -#define EXPECTING_REF 2 -#define EXPECTING_SHALLOW 3 - -/* Returns 1 if packet_buffer is a protocol version pkt-line, 0 otherwise. */ -static int process_protocol_version(void) -{ - switch (determine_protocol_version_client(packet_buffer)) { - case protocol_v1: - return 1; - case protocol_v0: - return 0; - default: - die("server is speaking an unknown protocol"); - } -} - -static void process_capabilities(int *len) -{ - int nul_location = strlen(packet_buffer); + int nul_location = strlen(line); if (nul_location == *len) return; - server_capabilities = xstrdup(packet_buffer + nul_location + 1); + server_capabilities = xstrdup(line + nul_location + 1); *len = nul_location; } -static int process_dummy_ref(void) +static int process_dummy_ref(const char *line) { struct object_id oid; const char *name; - if (parse_oid_hex(packet_buffer, &oid, &name)) + if (parse_oid_hex(line, &oid, &name)) return 0; if (*name != ' ') return 0; @@ -171,20 +172,20 @@ static int process_dummy_ref(void) return !oidcmp(&null_oid, &oid) && !strcmp(name, "capabilities^{}"); } -static void check_no_capabilities(int len) +static void check_no_capabilities(const char *line, int len) { - if (strlen(packet_buffer) != len) + if (strlen(line) != len) warning("Ignoring capabilities after first line '%s'", - packet_buffer + strlen(packet_buffer)); + line + strlen(line)); } -static int process_ref(int len, struct ref ***list, unsigned int flags, - struct oid_array *extra_have) +static int process_ref(const char *line, int len, struct ref ***list, + unsigned int flags, struct oid_array *extra_have) { struct object_id old_oid; const char *name; - if (parse_oid_hex(packet_buffer, &old_oid, &name)) + if (parse_oid_hex(line, &old_oid, &name)) return 0; if (*name != ' ') return 0; @@ -200,16 +201,17 @@ static int process_ref(int len, struct ref ***list, unsigned int flags, **list = ref; *list = &ref->next; } - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } -static int process_shallow(int len, struct oid_array *shallow_points) +static int process_shallow(const char *line, int len, + struct oid_array *shallow_points) { const char *arg; struct object_id old_oid; - if (!skip_prefix(packet_buffer, "shallow ", &arg)) + if (!skip_prefix(line, "shallow ", &arg)) return 0; if (get_oid_hex(arg, &old_oid)) @@ -217,10 +219,17 @@ static int process_shallow(int len, struct oid_array *shallow_points) if (!shallow_points) die("repository on the other end cannot be shallow"); oid_array_append(shallow_points, &old_oid); - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } +enum get_remote_heads_state { + EXPECTING_FIRST_REF = 0, + EXPECTING_REF, + EXPECTING_SHALLOW, + EXPECTING_DONE, +}; + /* * Read all the refs from the other end */ @@ -230,47 +239,55 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, struct oid_array *shallow_points) { struct ref **orig_list = list; + int len = 0; + enum get_remote_heads_state state = EXPECTING_FIRST_REF; + struct packet_reader reader; + const char *arg; - /* - * A hang-up after seeing some response from the other end - * means that it is unexpected, as we know the other end is - * willing to talk to us. A hang-up before seeing any - * response does not necessarily mean an ACL problem, though. - */ - int responded = 0; - int len; - int state = EXPECTING_PROTOCOL_VERSION; + packet_reader_init(&reader, in, src_buf, src_len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + discover_version(&reader); *list = NULL; - while ((len = read_remote_ref(in, &src_buf, &src_len, &responded))) { + while (state != EXPECTING_DONE) { + switch (packet_reader_read(&reader)) { + case PACKET_READ_EOF: + die_initial_contact(1); + case PACKET_READ_NORMAL: + len = reader.pktlen; + if (len > 4 && skip_prefix(reader.line, "ERR ", &arg)) + die("remote error: %s", arg); + break; + case PACKET_READ_FLUSH: + state = EXPECTING_DONE; + break; + case PACKET_READ_DELIM: + die("invalid packet"); + } + switch (state) { - case EXPECTING_PROTOCOL_VERSION: - if (process_protocol_version()) { - state = EXPECTING_FIRST_REF; - break; - } - state = EXPECTING_FIRST_REF; - /* fallthrough */ case EXPECTING_FIRST_REF: - process_capabilities(&len); - if (process_dummy_ref()) { + process_capabilities(reader.line, &len); + if (process_dummy_ref(reader.line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(len, &list, flags, extra_have)) + if (process_ref(reader.line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(len, shallow_points)) + if (process_shallow(reader.line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", packet_buffer); - default: - die("unexpected state %d", state); + die("protocol error: unexpected '%s'", reader.line); + case EXPECTING_DONE: + break; } } -- cgit v0.10.2-6-g49f6 From ad6ac1244fd175d08bcee62060a9a0b7975930fb Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:45 -0700 Subject: connect: discover protocol version outside of get_remote_heads In order to prepare for the addition of protocol_v2 push the protocol version discovery outside of 'get_remote_heads()'. This will allow for keeping the logic for processing the reference advertisement for protocol_v1 and protocol_v0 separate from the logic for protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 366b9d1..85d4faf 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -4,6 +4,7 @@ #include "remote.h" #include "connect.h" #include "sha1-array.h" +#include "protocol.h" static const char fetch_pack_usage[] = "git fetch-pack [--all] [--stdin] [--quiet | -q] [--keep | -k] [--thin] " @@ -52,6 +53,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct packet_reader reader; packet_trace_identity("fetch-pack"); @@ -193,7 +195,19 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!conn) return args.diag_url ? 0 : 1; } - get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow); + + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &ref, 0, NULL, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, &shallow, pack_lockfile_ptr); diff --git a/builtin/send-pack.c b/builtin/send-pack.c index fc4f0bb..83cb125 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -14,6 +14,7 @@ #include "sha1-array.h" #include "gpg-interface.h" #include "gettext.h" +#include "protocol.h" static const char * const send_pack_usage[] = { N_("git send-pack [--all | --mirror] [--dry-run] [--force] " @@ -154,6 +155,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) int progress = -1; int from_stdin = 0; struct push_cas_option cas = {0}; + struct packet_reader reader; struct option options[] = { OPT__VERBOSITY(&verbose), @@ -256,8 +258,19 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.verbose ? CONNECT_VERBOSE : 0); } - get_remote_heads(fd[0], NULL, 0, &remote_refs, REF_NORMAL, - &extra_have, &shallow); + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &remote_refs, REF_NORMAL, + &extra_have, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } transport_verify_remote_names(nr_refspecs, refspecs); diff --git a/connect.c b/connect.c index c82c90b..0b111e6 100644 --- a/connect.c +++ b/connect.c @@ -62,7 +62,7 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } -static enum protocol_version discover_version(struct packet_reader *reader) +enum protocol_version discover_version(struct packet_reader *reader) { enum protocol_version version = protocol_unknown_version; @@ -233,7 +233,7 @@ enum get_remote_heads_state { /* * Read all the refs from the other end */ -struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points) @@ -241,24 +241,17 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, struct ref **orig_list = list; int len = 0; enum get_remote_heads_state state = EXPECTING_FIRST_REF; - struct packet_reader reader; const char *arg; - packet_reader_init(&reader, in, src_buf, src_len, - PACKET_READ_CHOMP_NEWLINE | - PACKET_READ_GENTLE_ON_EOF); - - discover_version(&reader); - *list = NULL; while (state != EXPECTING_DONE) { - switch (packet_reader_read(&reader)) { + switch (packet_reader_read(reader)) { case PACKET_READ_EOF: die_initial_contact(1); case PACKET_READ_NORMAL: - len = reader.pktlen; - if (len > 4 && skip_prefix(reader.line, "ERR ", &arg)) + len = reader->pktlen; + if (len > 4 && skip_prefix(reader->line, "ERR ", &arg)) die("remote error: %s", arg); break; case PACKET_READ_FLUSH: @@ -270,22 +263,22 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, switch (state) { case EXPECTING_FIRST_REF: - process_capabilities(reader.line, &len); - if (process_dummy_ref(reader.line)) { + process_capabilities(reader->line, &len); + if (process_dummy_ref(reader->line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(reader.line, len, &list, flags, extra_have)) + if (process_ref(reader->line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(reader.line, len, shallow_points)) + if (process_shallow(reader->line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", reader.line); + die("protocol error: unexpected '%s'", reader->line); case EXPECTING_DONE: break; } diff --git a/connect.h b/connect.h index 01f14cd..cdb8979 100644 --- a/connect.h +++ b/connect.h @@ -13,4 +13,7 @@ extern int parse_feature_request(const char *features, const char *feature); extern const char *server_feature_value(const char *feature, int *len_ret); extern int url_is_local_not_ssh(const char *url); +struct packet_reader; +extern enum protocol_version discover_version(struct packet_reader *reader); + #endif diff --git a/remote-curl.c b/remote-curl.c index 0053b09..9f6d076 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,6 +1,7 @@ #include "cache.h" #include "config.h" #include "remote.h" +#include "connect.h" #include "strbuf.h" #include "walker.h" #include "http.h" @@ -13,6 +14,7 @@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" +#include "protocol.h" static struct remote *remote; /* always ends with a trailing slash */ @@ -176,8 +178,22 @@ static struct discovery *last_discovery; static struct ref *parse_git_refs(struct discovery *heads, int for_push) { struct ref *list = NULL; - get_remote_heads(-1, heads->buf, heads->len, &list, - for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + struct packet_reader reader; + + packet_reader_init(&reader, -1, heads->buf, heads->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, + NULL, &heads->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + return list; } diff --git a/remote.h b/remote.h index 1f6611b..2016461 100644 --- a/remote.h +++ b/remote.h @@ -150,10 +150,11 @@ int check_ref_type(const struct ref *ref, int flags); void free_refs(struct ref *ref); struct oid_array; -extern struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct packet_reader; +extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, - struct oid_array *shallow); + struct oid_array *shallow_points); int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); diff --git a/transport.c b/transport.c index 8e87790..63c3dba 100644 --- a/transport.c +++ b/transport.c @@ -18,6 +18,7 @@ #include "sha1-array.h" #include "sigchain.h" #include "transport-internal.h" +#include "protocol.h" static void set_upstreams(struct transport *transport, struct ref *refs, int pretend) @@ -190,13 +191,26 @@ static int connect_setup(struct transport *transport, int for_push) static struct ref *get_refs_via_connect(struct transport *transport, int for_push) { struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; + struct packet_reader reader; connect_setup(transport, for_push); - get_remote_heads(data->fd[0], NULL, 0, &refs, - for_push ? REF_NORMAL : 0, - &data->extra_have, - &data->shallow); + + packet_reader_init(&reader, data->fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &refs, + for_push ? REF_NORMAL : 0, + &data->extra_have, + &data->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } data->got_remote_heads = 1; return refs; -- cgit v0.10.2-6-g49f6 From 432e95651031025e421ae2f521f7c0d9f60c527c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:46 -0700 Subject: transport: store protocol version Once protocol_v2 is introduced requesting a fetch or a push will need to be handled differently depending on the protocol version. Store the protocol version the server is speaking in 'struct git_transport_data' and use it to determine what to do in the case of a fetch or a push. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport.c b/transport.c index 63c3dba..2378dcb 100644 --- a/transport.c +++ b/transport.c @@ -118,6 +118,7 @@ struct git_transport_data { struct child_process *conn; int fd[2]; unsigned got_remote_heads : 1; + enum protocol_version version; struct oid_array extra_have; struct oid_array shallow; }; @@ -200,7 +201,8 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus PACKET_READ_CHOMP_NEWLINE | PACKET_READ_GENTLE_ON_EOF); - switch (discover_version(&reader)) { + data->version = discover_version(&reader); + switch (data->version) { case protocol_v1: case protocol_v0: get_remote_heads(&reader, &refs, @@ -221,7 +223,7 @@ static int fetch_refs_via_pack(struct transport *transport, { int ret = 0; struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; char *dest = xstrdup(transport->url); struct fetch_pack_args args; struct ref *refs_tmp = NULL; @@ -247,10 +249,18 @@ static int fetch_refs_via_pack(struct transport *transport, if (!data->got_remote_heads) refs_tmp = get_refs_via_connect(transport, 0); - refs = fetch_pack(&args, data->fd, data->conn, - refs_tmp ? refs_tmp : transport->remote_refs, - dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); + switch (data->version) { + case protocol_v1: + case protocol_v0: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + close(data->fd[0]); close(data->fd[1]); if (finish_connect(data->conn)) @@ -549,7 +559,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re { struct git_transport_data *data = transport->data; struct send_pack_args args; - int ret; + int ret = 0; if (!data->got_remote_heads) get_refs_via_connect(transport, 1); @@ -574,8 +584,15 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re else args.push_cert = SEND_PACK_PUSH_CERT_NEVER; - ret = send_pack(&args, data->fd, data->conn, remote_refs, - &data->extra_have); + switch (data->version) { + case protocol_v1: + case protocol_v0: + ret = send_pack(&args, data->fd, data->conn, remote_refs, + &data->extra_have); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } close(data->fd[1]); close(data->fd[0]); -- cgit v0.10.2-6-g49f6 From 8f6982b4e16c60bba713a3b6592b2ff5c7476974 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:47 -0700 Subject: protocol: introduce enum protocol_version value protocol_v2 Introduce protocol_v2, a new value for 'enum protocol_version'. Subsequent patches will fill in the implementation of protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 85d4faf..b2374dd 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -201,6 +201,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); case protocol_v1: case protocol_v0: get_remote_heads(&reader, &ref, 0, NULL, &shallow); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index b7ce7c7..3656e94 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1963,6 +1963,12 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) unpack_limit = receive_unpack_limit; switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * push support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 83cb125..b5427f7 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -263,6 +263,9 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &remote_refs, REF_NORMAL, diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 2cb5cb3..8d53e97 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -47,6 +47,13 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) die("'%s' does not appear to be a git repository", dir); switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * fetch support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + upload_pack(&opts); + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/connect.c b/connect.c index 0b111e6..4b89b98 100644 --- a/connect.c +++ b/connect.c @@ -83,6 +83,9 @@ enum protocol_version discover_version(struct packet_reader *reader) } switch (version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: /* Read the peeked version line */ packet_reader_read(reader); diff --git a/protocol.c b/protocol.c index 43012b7..5e63678 100644 --- a/protocol.c +++ b/protocol.c @@ -8,6 +8,8 @@ static enum protocol_version parse_protocol_version(const char *value) return protocol_v0; else if (!strcmp(value, "1")) return protocol_v1; + else if (!strcmp(value, "2")) + return protocol_v2; else return protocol_unknown_version; } diff --git a/protocol.h b/protocol.h index 1b2bc94..2ad35e4 100644 --- a/protocol.h +++ b/protocol.h @@ -5,6 +5,7 @@ enum protocol_version { protocol_unknown_version = -1, protocol_v0 = 0, protocol_v1 = 1, + protocol_v2 = 2, }; /* diff --git a/remote-curl.c b/remote-curl.c index 9f6d076..dae8a4a 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -185,6 +185,9 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, diff --git a/transport.c b/transport.c index 2378dcb..83d9dd1 100644 --- a/transport.c +++ b/transport.c @@ -203,6 +203,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &refs, @@ -250,6 +253,9 @@ static int fetch_refs_via_pack(struct transport *transport, refs_tmp = get_refs_via_connect(transport, 0); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, @@ -585,6 +591,9 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re args.push_cert = SEND_PACK_PUSH_CERT_NEVER; switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: ret = send_pack(&args, data->fd, data->conn, remote_refs, -- cgit v0.10.2-6-g49f6 From 74e70029615ead3e7203fed715abf0dc31bcb211 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:48 -0700 Subject: test-pkt-line: introduce a packet-line test helper Introduce a packet-line test helper which can either pack or unpack an input stream into packet-lines and writes out the result to stdout. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index 91a59a6..c26fb93 100644 --- a/Makefile +++ b/Makefile @@ -666,6 +666,7 @@ TEST_PROGRAMS_NEED_X += test-mktemp TEST_PROGRAMS_NEED_X += test-online-cpus TEST_PROGRAMS_NEED_X += test-parse-options TEST_PROGRAMS_NEED_X += test-path-utils +TEST_PROGRAMS_NEED_X += test-pkt-line TEST_PROGRAMS_NEED_X += test-prio-queue TEST_PROGRAMS_NEED_X += test-read-cache TEST_PROGRAMS_NEED_X += test-write-cache diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c new file mode 100644 index 0000000..0f19e53 --- /dev/null +++ b/t/helper/test-pkt-line.c @@ -0,0 +1,64 @@ +#include "pkt-line.h" + +static void pack_line(const char *line) +{ + if (!strcmp(line, "0000") || !strcmp(line, "0000\n")) + packet_flush(1); + else if (!strcmp(line, "0001") || !strcmp(line, "0001\n")) + packet_delim(1); + else + packet_write_fmt(1, "%s", line); +} + +static void pack(int argc, const char **argv) +{ + if (argc) { /* read from argv */ + int i; + for (i = 0; i < argc; i++) + pack_line(argv[i]); + } else { /* read from stdin */ + char line[LARGE_PACKET_MAX]; + while (fgets(line, sizeof(line), stdin)) { + pack_line(line); + } + } +} + +static void unpack(void) +{ + struct packet_reader reader; + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_CHOMP_NEWLINE); + + while (packet_reader_read(&reader) != PACKET_READ_EOF) { + switch (reader.status) { + case PACKET_READ_EOF: + break; + case PACKET_READ_NORMAL: + printf("%s\n", reader.line); + break; + case PACKET_READ_FLUSH: + printf("0000\n"); + break; + case PACKET_READ_DELIM: + printf("0001\n"); + break; + } + } +} + +int cmd_main(int argc, const char **argv) +{ + if (argc < 2) + die("too few arguments"); + + if (!strcmp(argv[1], "pack")) + pack(argc - 2, argv + 2); + else if (!strcmp(argv[1], "unpack")) + unpack(); + else + die("invalid argument '%s'", argv[1]); + + return 0; +} -- cgit v0.10.2-6-g49f6 From ed10cb952d315ae47d2f7f6f29333a9ee227bf6b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:19 -0700 Subject: serve: introduce git-serve Introduce git-serve, the base server for protocol version 2. Protocol version 2 is intended to be a replacement for Git's current wire protocol. The intention is that it will be a simpler, less wasteful protocol which can evolve over time. Protocol version 2 improves upon version 1 by eliminating the initial ref advertisement. In its place a server will export a list of capabilities and commands which it supports in a capability advertisement. A client can then request that a particular command be executed by providing a number of capabilities and command specific parameters. At the completion of a command, a client can request that another command be executed or can terminate the connection by sending a flush packet. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/.gitignore b/.gitignore index 833ef3b..2d0450c 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ /git-rm /git-send-email /git-send-pack +/git-serve /git-sh-i18n /git-sh-i18n--envsubst /git-sh-setup diff --git a/Documentation/Makefile b/Documentation/Makefile index 4ae9ba5..b105775 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -77,6 +77,7 @@ TECH_DOCS += technical/pack-heuristics TECH_DOCS += technical/pack-protocol TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common +TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt new file mode 100644 index 0000000..270b28f --- /dev/null +++ b/Documentation/technical/protocol-v2.txt @@ -0,0 +1,170 @@ + Git Wire Protocol, Version 2 +============================== + +This document presents a specification for a version 2 of Git's wire +protocol. Protocol v2 will improve upon v1 in the following ways: + + * Instead of multiple service names, multiple commands will be + supported by a single service + * Easily extendable as capabilities are moved into their own section + of the protocol, no longer being hidden behind a NUL byte and + limited by the size of a pkt-line + * Separate out other information hidden behind NUL bytes (e.g. agent + string as a capability and symrefs can be requested using 'ls-refs') + * Reference advertisement will be omitted unless explicitly requested + * ls-refs command to explicitly request some refs + * Designed with http and stateless-rpc in mind. With clear flush + semantics the http remote helper can simply act as a proxy + +In protocol v2 communication is command oriented. When first contacting a +server a list of capabilities will advertised. Some of these capabilities +will be commands which a client can request be executed. Once a command +has completed, a client can reuse the connection and request that other +commands be executed. + + Packet-Line Framing +--------------------- + +All communication is done using packet-line framing, just as in v1. See +`Documentation/technical/pack-protocol.txt` and +`Documentation/technical/protocol-common.txt` for more information. + +In protocol v2 these special packets will have the following semantics: + + * '0000' Flush Packet (flush-pkt) - indicates the end of a message + * '0001' Delimiter Packet (delim-pkt) - separates sections of a message + + Initial Client Request +------------------------ + +In general a client can request to speak protocol v2 by sending +`version=2` through the respective side-channel for the transport being +used which inevitably sets `GIT_PROTOCOL`. More information can be +found in `pack-protocol.txt` and `http-protocol.txt`. In all cases the +response from the server is the capability advertisement. + + Git Transport +~~~~~~~~~~~~~~~ + +When using the git:// transport, you can request to use protocol v2 by +sending "version=2" as an extra parameter: + + 003egit-upload-pack /project.git\0host=myserver.com\0\0version=2\0 + + SSH and File Transport +~~~~~~~~~~~~~~~~~~~~~~~~ + +When using either the ssh:// or file:// transport, the GIT_PROTOCOL +environment variable must be set explicitly to include "version=2". + + HTTP Transport +~~~~~~~~~~~~~~~~ + +When using the http:// or https:// transport a client makes a "smart" +info/refs request as described in `http-protocol.txt` and requests that +v2 be used by supplying "version=2" in the `Git-Protocol` header. + + C: Git-Protocol: version=2 + C: + C: GET $GIT_URL/info/refs?service=git-upload-pack HTTP/1.0 + +A v2 server would reply: + + S: 200 OK + S: + S: ... + S: + S: 000eversion 2\n + S: + +Subsequent requests are then made directly to the service +`$GIT_URL/git-upload-pack`. (This works the same for git-receive-pack). + + Capability Advertisement +-------------------------- + +A server which decides to communicate (based on a request from a client) +using protocol version 2, notifies the client by sending a version string +in its initial response followed by an advertisement of its capabilities. +Each capability is a key with an optional value. Clients must ignore all +unknown keys. Semantics of unknown values are left to the definition of +each key. Some capabilities will describe commands which can be requested +to be executed by the client. + + capability-advertisement = protocol-version + capability-list + flush-pkt + + protocol-version = PKT-LINE("version 2" LF) + capability-list = *capability + capability = PKT-LINE(key[=value] LF) + + key = 1*(ALPHA | DIGIT | "-_") + value = 1*(ALPHA | DIGIT | " -_.,?\/{}[]()<>!@#$%^&*+=:;") + + Command Request +----------------- + +After receiving the capability advertisement, a client can then issue a +request to select the command it wants with any particular capabilities +or arguments. There is then an optional section where the client can +provide any command specific parameters or queries. Only a single +command can be requested at a time. + + request = empty-request | command-request + empty-request = flush-pkt + command-request = command + capability-list + [command-args] + flush-pkt + command = PKT-LINE("command=" key LF) + command-args = delim-pkt + *command-specific-arg + + command-specific-args are packet line framed arguments defined by + each individual command. + +The server will then check to ensure that the client's request is +comprised of a valid command as well as valid capabilities which were +advertised. If the request is valid the server will then execute the +command. A server MUST wait till it has received the client's entire +request before issuing a response. The format of the response is +determined by the command being executed, but in all cases a flush-pkt +indicates the end of the response. + +When a command has finished, and the client has received the entire +response from the server, a client can either request that another +command be executed or can terminate the connection. A client may +optionally send an empty request consisting of just a flush-pkt to +indicate that no more requests will be made. + + Capabilities +-------------- + +There are two different types of capabilities: normal capabilities, +which can be used to to convey information or alter the behavior of a +request, and commands, which are the core actions that a client wants to +perform (fetch, push, etc). + +Protocol version 2 is stateless by default. This means that all commands +must only last a single round and be stateless from the perspective of the +server side, unless the client has requested a capability indicating that +state should be maintained by the server. Clients MUST NOT require state +management on the server side in order to function correctly. This +permits simple round-robin load-balancing on the server side, without +needing to worry about state management. + + agent +~~~~~~~ + +The server can advertise the `agent` capability with a value `X` (in the +form `agent=X`) to notify the client that the server is running version +`X`. The client may optionally send its own agent string by including +the `agent` capability with a value `Y` (in the form `agent=Y`) in its +request to the server (but it MUST NOT do so if the server did not +advertise the agent capability). The `X` and `Y` strings may contain any +printable ASCII characters except space (i.e., the byte range 32 < x < +127), and are typically of the form "package/version" (e.g., +"git/1.8.3.1"). The agent strings are purely informative for statistics +and debugging purposes, and MUST NOT be used to programmatically assume +the presence or absence of particular features. diff --git a/Makefile b/Makefile index c26fb93..fee1d5c 100644 --- a/Makefile +++ b/Makefile @@ -876,6 +876,7 @@ LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o LIB_OBJS += sequencer.o +LIB_OBJS += serve.o LIB_OBJS += server-info.o LIB_OBJS += setup.o LIB_OBJS += sha1-array.o @@ -1009,6 +1010,7 @@ BUILTIN_OBJS += builtin/rev-parse.o BUILTIN_OBJS += builtin/revert.o BUILTIN_OBJS += builtin/rm.o BUILTIN_OBJS += builtin/send-pack.o +BUILTIN_OBJS += builtin/serve.o BUILTIN_OBJS += builtin/shortlog.o BUILTIN_OBJS += builtin/show-branch.o BUILTIN_OBJS += builtin/show-ref.o diff --git a/builtin.h b/builtin.h index f332a12..3f3fdfc 100644 --- a/builtin.h +++ b/builtin.h @@ -215,6 +215,7 @@ extern int cmd_rev_parse(int argc, const char **argv, const char *prefix); extern int cmd_revert(int argc, const char **argv, const char *prefix); extern int cmd_rm(int argc, const char **argv, const char *prefix); extern int cmd_send_pack(int argc, const char **argv, const char *prefix); +extern int cmd_serve(int argc, const char **argv, const char *prefix); extern int cmd_shortlog(int argc, const char **argv, const char *prefix); extern int cmd_show(int argc, const char **argv, const char *prefix); extern int cmd_show_branch(int argc, const char **argv, const char *prefix); diff --git a/builtin/serve.c b/builtin/serve.c new file mode 100644 index 0000000..d3fd240 --- /dev/null +++ b/builtin/serve.c @@ -0,0 +1,30 @@ +#include "cache.h" +#include "builtin.h" +#include "parse-options.h" +#include "serve.h" + +static char const * const serve_usage[] = { + N_("git serve []"), + NULL +}; + +int cmd_serve(int argc, const char **argv, const char *prefix) +{ + struct serve_options opts = SERVE_OPTIONS_INIT; + + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-capabilities", &opts.advertise_capabilities, + N_("exit immediately after advertising capabilities")), + OPT_END() + }; + + /* ignore all unknown cmdline switches for now */ + argc = parse_options(argc, argv, prefix, options, serve_usage, + PARSE_OPT_KEEP_DASHDASH | + PARSE_OPT_KEEP_UNKNOWN); + serve(&opts); + + return 0; +} diff --git a/git.c b/git.c index f71073d..f85d682 100644 --- a/git.c +++ b/git.c @@ -461,6 +461,7 @@ static struct cmd_struct commands[] = { { "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE }, { "rm", cmd_rm, RUN_SETUP }, { "send-pack", cmd_send_pack, RUN_SETUP }, + { "serve", cmd_serve, RUN_SETUP }, { "shortlog", cmd_shortlog, RUN_SETUP_GENTLY | USE_PAGER }, { "show", cmd_show, RUN_SETUP }, { "show-branch", cmd_show_branch, RUN_SETUP }, diff --git a/serve.c b/serve.c new file mode 100644 index 0000000..7ddcba9 --- /dev/null +++ b/serve.c @@ -0,0 +1,247 @@ +#include "cache.h" +#include "repository.h" +#include "config.h" +#include "pkt-line.h" +#include "version.h" +#include "argv-array.h" +#include "serve.h" + +static int agent_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, git_user_agent_sanitized()); + return 1; +} + +struct protocol_capability { + /* + * The name of the capability. The server uses this name when + * advertising this capability, and the client uses this name to + * specify this capability. + */ + const char *name; + + /* + * Function queried to see if a capability should be advertised. + * Optionally a value can be specified by adding it to 'value'. + * If a value is added to 'value', the server will advertise this + * capability as "=" instead of "". + */ + int (*advertise)(struct repository *r, struct strbuf *value); + + /* + * Function called when a client requests the capability as a command. + * The function will be provided the capabilities requested via 'keys' + * as well as a struct packet_reader 'request' which the command should + * use to read the command specific part of the request. Every command + * MUST read until a flush packet is seen before sending a response. + * + * This field should be NULL for capabilities which are not commands. + */ + int (*command)(struct repository *r, + struct argv_array *keys, + struct packet_reader *request); +}; + +static struct protocol_capability capabilities[] = { + { "agent", agent_advertise, NULL }, +}; + +static void advertise_capabilities(void) +{ + struct strbuf capability = STRBUF_INIT; + struct strbuf value = STRBUF_INIT; + int i; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + + if (c->advertise(the_repository, &value)) { + strbuf_addstr(&capability, c->name); + + if (value.len) { + strbuf_addch(&capability, '='); + strbuf_addbuf(&capability, &value); + } + + strbuf_addch(&capability, '\n'); + packet_write(1, capability.buf, capability.len); + } + + strbuf_reset(&capability); + strbuf_reset(&value); + } + + packet_flush(1); + strbuf_release(&capability); + strbuf_release(&value); +} + +static struct protocol_capability *get_capability(const char *key) +{ + int i; + + if (!key) + return NULL; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + const char *out; + if (skip_prefix(key, c->name, &out) && (!*out || *out == '=')) + return c; + } + + return NULL; +} + +static int is_valid_capability(const char *key) +{ + const struct protocol_capability *c = get_capability(key); + + return c && c->advertise(the_repository, NULL); +} + +static int is_command(const char *key, struct protocol_capability **command) +{ + const char *out; + + if (skip_prefix(key, "command=", &out)) { + struct protocol_capability *cmd = get_capability(out); + + if (*command) + die("command '%s' requested after already requesting command '%s'", + out, (*command)->name); + if (!cmd || !cmd->advertise(the_repository, NULL) || !cmd->command) + die("invalid command '%s'", out); + + *command = cmd; + return 1; + } + + return 0; +} + +int has_capability(const struct argv_array *keys, const char *capability, + const char **value) +{ + int i; + for (i = 0; i < keys->argc; i++) { + const char *out; + if (skip_prefix(keys->argv[i], capability, &out) && + (!*out || *out == '=')) { + if (value) { + if (*out == '=') + out++; + *value = out; + } + return 1; + } + } + + return 0; +} + +enum request_state { + PROCESS_REQUEST_KEYS, + PROCESS_REQUEST_DONE, +}; + +static int process_request(void) +{ + enum request_state state = PROCESS_REQUEST_KEYS; + struct packet_reader reader; + struct argv_array keys = ARGV_ARRAY_INIT; + struct protocol_capability *command = NULL; + + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + /* + * Check to see if the client closed their end before sending another + * request. If so we can terminate the connection. + */ + if (packet_reader_peek(&reader) == PACKET_READ_EOF) + return 1; + reader.options = PACKET_READ_CHOMP_NEWLINE; + + while (state != PROCESS_REQUEST_DONE) { + switch (packet_reader_peek(&reader)) { + case PACKET_READ_EOF: + BUG("Should have already died when seeing EOF"); + case PACKET_READ_NORMAL: + /* collect request; a sequence of keys and values */ + if (is_command(reader.line, &command) || + is_valid_capability(reader.line)) + argv_array_push(&keys, reader.line); + else + die("unknown capability '%s'", reader.line); + + /* Consume the peeked line */ + packet_reader_read(&reader); + break; + case PACKET_READ_FLUSH: + /* + * If no command and no keys were given then the client + * wanted to terminate the connection. + */ + if (!keys.argc) + return 1; + + /* + * The flush packet isn't consume here like it is in + * the other parts of this switch statement. This is + * so that the command can read the flush packet and + * see the end of the request in the same way it would + * if command specific arguments were provided after a + * delim packet. + */ + state = PROCESS_REQUEST_DONE; + break; + case PACKET_READ_DELIM: + /* Consume the peeked line */ + packet_reader_read(&reader); + + state = PROCESS_REQUEST_DONE; + break; + } + } + + if (!command) + die("no command requested"); + + command->command(the_repository, &keys, &reader); + + argv_array_clear(&keys); + return 0; +} + +/* Main serve loop for protocol version 2 */ +void serve(struct serve_options *options) +{ + if (options->advertise_capabilities || !options->stateless_rpc) { + /* serve by default supports v2 */ + packet_write_fmt(1, "version 2\n"); + + advertise_capabilities(); + /* + * If only the list of capabilities was requested exit + * immediately after advertising capabilities + */ + if (options->advertise_capabilities) + return; + } + + /* + * If stateless-rpc was requested then exit after + * a single request/response exchange + */ + if (options->stateless_rpc) { + process_request(); + } else { + for (;;) + if (process_request()) + break; + } +} diff --git a/serve.h b/serve.h new file mode 100644 index 0000000..fe65ba9 --- /dev/null +++ b/serve.h @@ -0,0 +1,15 @@ +#ifndef SERVE_H +#define SERVE_H + +struct argv_array; +extern int has_capability(const struct argv_array *keys, const char *capability, + const char **value); + +struct serve_options { + unsigned advertise_capabilities; + unsigned stateless_rpc; +}; +#define SERVE_OPTIONS_INIT { 0 } +extern void serve(struct serve_options *options); + +#endif /* SERVE_H */ diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh new file mode 100755 index 0000000..affbad0 --- /dev/null +++ b/t/t5701-git-serve.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +test_description='test git-serve and server commands' + +. ./test-lib.sh + +test_expect_success 'test capability advertisement' ' + cat >expect <<-EOF && + version 2 + agent=git/$(git version | cut -d" " -f3) + 0000 + EOF + + git serve --advertise-capabilities >out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'stateless-rpc flag does not list capabilities' ' + # Empty request + test-pkt-line pack >in <<-EOF && + 0000 + EOF + git serve --stateless-rpc >out out && + test_must_be_empty out +' + +test_expect_success 'request invalid capability' ' + test-pkt-line pack >in <<-EOF && + foobar + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err in <<-EOF && + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err in <<-EOF && + command=foo + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err Date: Thu, 15 Mar 2018 10:31:20 -0700 Subject: ls-refs: introduce ls-refs server command Introduce the ls-refs server command. In protocol v2, the ls-refs command is used to request the ref advertisement from the server. Since it is a command which can be requested (as opposed to mandatory in v1), a client can sent a number of parameters in its request to limit the ref advertisement based on provided ref-prefixes. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 270b28f..bbb8b14 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -168,3 +168,34 @@ printable ASCII characters except space (i.e., the byte range 32 < x < "git/1.8.3.1"). The agent strings are purely informative for statistics and debugging purposes, and MUST NOT be used to programmatically assume the presence or absence of particular features. + + ls-refs +~~~~~~~~~ + +`ls-refs` is the command used to request a reference advertisement in v2. +Unlike the current reference advertisement, ls-refs takes in arguments +which can be used to limit the refs sent from the server. + +Additional features not supported in the base command will be advertised +as the value of the command in the capability advertisement in the form +of a space separated list of features: "= " + +ls-refs takes in the following arguments: + + symrefs + In addition to the object pointed by it, show the underlying ref + pointed by it when showing a symbolic ref. + peel + Show peeled tags. + ref-prefix + When specified, only references having a prefix matching one of + the provided prefixes are displayed. + +The output of ls-refs is as follows: + + output = *ref + flush-pkt + ref = PKT-LINE(obj-id SP refname *(SP ref-attribute) LF) + ref-attribute = (symref | peeled) + symref = "symref-target:" symref-target + peeled = "peeled:" obj-id diff --git a/Makefile b/Makefile index fee1d5c..fb2175b 100644 --- a/Makefile +++ b/Makefile @@ -820,6 +820,7 @@ LIB_OBJS += list-objects-filter-options.o LIB_OBJS += ll-merge.o LIB_OBJS += lockfile.o LIB_OBJS += log-tree.o +LIB_OBJS += ls-refs.o LIB_OBJS += mailinfo.o LIB_OBJS += mailmap.o LIB_OBJS += match-trees.o diff --git a/ls-refs.c b/ls-refs.c new file mode 100644 index 0000000..a06f12e --- /dev/null +++ b/ls-refs.c @@ -0,0 +1,96 @@ +#include "cache.h" +#include "repository.h" +#include "refs.h" +#include "remote.h" +#include "argv-array.h" +#include "ls-refs.h" +#include "pkt-line.h" + +/* + * Check if one of the prefixes is a prefix of the ref. + * If no prefixes were provided, all refs match. + */ +static int ref_match(const struct argv_array *prefixes, const char *refname) +{ + int i; + + if (!prefixes->argc) + return 1; /* no restriction */ + + for (i = 0; i < prefixes->argc; i++) { + const char *prefix = prefixes->argv[i]; + + if (starts_with(refname, prefix)) + return 1; + } + + return 0; +} + +struct ls_refs_data { + unsigned peel; + unsigned symrefs; + struct argv_array prefixes; +}; + +static int send_ref(const char *refname, const struct object_id *oid, + int flag, void *cb_data) +{ + struct ls_refs_data *data = cb_data; + const char *refname_nons = strip_namespace(refname); + struct strbuf refline = STRBUF_INIT; + + if (!ref_match(&data->prefixes, refname)) + return 0; + + strbuf_addf(&refline, "%s %s", oid_to_hex(oid), refname_nons); + if (data->symrefs && flag & REF_ISSYMREF) { + struct object_id unused; + const char *symref_target = resolve_ref_unsafe(refname, 0, + &unused, + &flag); + + if (!symref_target) + die("'%s' is a symref but it is not?", refname); + + strbuf_addf(&refline, " symref-target:%s", symref_target); + } + + if (data->peel) { + struct object_id peeled; + if (!peel_ref(refname, &peeled)) + strbuf_addf(&refline, " peeled:%s", oid_to_hex(&peeled)); + } + + strbuf_addch(&refline, '\n'); + packet_write(1, refline.buf, refline.len); + + strbuf_release(&refline); + return 0; +} + +int ls_refs(struct repository *r, struct argv_array *keys, + struct packet_reader *request) +{ + struct ls_refs_data data; + + memset(&data, 0, sizeof(data)); + + while (packet_reader_read(request) != PACKET_READ_FLUSH) { + const char *arg = request->line; + const char *out; + + if (!strcmp("peel", arg)) + data.peel = 1; + else if (!strcmp("symrefs", arg)) + data.symrefs = 1; + else if (skip_prefix(arg, "ref-prefix ", &out)) + argv_array_push(&data.prefixes, out); + } + + head_ref_namespaced(send_ref, &data); + for_each_namespaced_ref(send_ref, &data); + packet_flush(1); + argv_array_clear(&data.prefixes); + return 0; +} diff --git a/ls-refs.h b/ls-refs.h new file mode 100644 index 0000000..b62877e --- /dev/null +++ b/ls-refs.h @@ -0,0 +1,10 @@ +#ifndef LS_REFS_H +#define LS_REFS_H + +struct repository; +struct argv_array; +struct packet_reader; +extern int ls_refs(struct repository *r, struct argv_array *keys, + struct packet_reader *request); + +#endif /* LS_REFS_H */ diff --git a/serve.c b/serve.c index 7ddcba9..1cb5fb9 100644 --- a/serve.c +++ b/serve.c @@ -4,8 +4,15 @@ #include "pkt-line.h" #include "version.h" #include "argv-array.h" +#include "ls-refs.h" #include "serve.h" +static int always_advertise(struct repository *r, + struct strbuf *value) +{ + return 1; +} + static int agent_advertise(struct repository *r, struct strbuf *value) { @@ -46,6 +53,7 @@ struct protocol_capability { static struct protocol_capability capabilities[] = { { "agent", agent_advertise, NULL }, + { "ls-refs", always_advertise, ls_refs }, }; static void advertise_capabilities(void) diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index affbad0..d03e08e 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -8,6 +8,7 @@ test_expect_success 'test capability advertisement' ' cat >expect <<-EOF && version 2 agent=git/$(git version | cut -d" " -f3) + ls-refs 0000 EOF @@ -57,4 +58,118 @@ test_expect_success 'request invalid command' ' test_i18ngrep "invalid command" err ' +# Test the basics of ls-refs +# +test_expect_success 'setup some refs and tags' ' + test_commit one && + git branch dev master && + test_commit two && + git symbolic-ref refs/heads/release refs/heads/master && + git tag -a -m "annotated tag" annotated-tag +' + +test_expect_success 'basics of ls-refs' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse HEAD) HEAD + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release + $(git rev-parse refs/tags/annotated-tag) refs/tags/annotated-tag + $(git rev-parse refs/tags/one) refs/tags/one + $(git rev-parse refs/tags/two) refs/tags/two + 0000 + EOF + + git serve --stateless-rpc out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'basic ref-prefixes' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + ref-prefix refs/heads/master + ref-prefix refs/tags/one + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/tags/one) refs/tags/one + 0000 + EOF + + git serve --stateless-rpc out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'refs/heads prefix' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + ref-prefix refs/heads/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release + 0000 + EOF + + git serve --stateless-rpc out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'peel parameter' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + peel + ref-prefix refs/tags/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/tags/annotated-tag) refs/tags/annotated-tag peeled:$(git rev-parse refs/tags/annotated-tag^{}) + $(git rev-parse refs/tags/one) refs/tags/one + $(git rev-parse refs/tags/two) refs/tags/two + 0000 + EOF + + git serve --stateless-rpc out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'symrefs parameter' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + symrefs + ref-prefix refs/heads/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release symref-target:refs/heads/master + 0000 + EOF + + git serve --stateless-rpc out && + test-pkt-line unpack actual && + test_cmp actual expect +' + test_done -- cgit v0.10.2-6-g49f6 From e52449b672210f4b49e116ca34dcd46657287f61 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:21 -0700 Subject: connect: request remote refs using v2 Teach the client to be able to request a remote's refs using protocol v2. This is done by having a client issue a 'ls-refs' request to a v2 server. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 8d53e97..a757df8 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -5,6 +5,7 @@ #include "parse-options.h" #include "protocol.h" #include "upload-pack.h" +#include "serve.h" static const char * const upload_pack_usage[] = { N_("git upload-pack [] "), @@ -16,6 +17,7 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) const char *dir; int strict = 0; struct upload_pack_options opts = { 0 }; + struct serve_options serve_opts = SERVE_OPTIONS_INIT; struct option options[] = { OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, N_("quit after a single request/response exchange")), @@ -48,11 +50,9 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) switch (determine_protocol_version_server()) { case protocol_v2: - /* - * fetch support for protocol v2 has not been implemented yet, - * so ignore the request to use v2 and fallback to using v0. - */ - upload_pack(&opts); + serve_opts.advertise_capabilities = opts.advertise_refs; + serve_opts.stateless_rpc = opts.stateless_rpc; + serve(&serve_opts); break; case protocol_v1: /* diff --git a/connect.c b/connect.c index 4b89b98..e42d779 100644 --- a/connect.c +++ b/connect.c @@ -12,9 +12,11 @@ #include "sha1-array.h" #include "transport.h" #include "strbuf.h" +#include "version.h" #include "protocol.h" -static char *server_capabilities; +static char *server_capabilities_v1; +static struct argv_array server_capabilities_v2 = ARGV_ARRAY_INIT; static const char *parse_feature_value(const char *, const char *, int *); static int check_ref(const char *name, unsigned int flags) @@ -62,6 +64,33 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } +/* Checks if the server supports the capability 'c' */ +int server_supports_v2(const char *c, int die_on_error) +{ + int i; + + for (i = 0; i < server_capabilities_v2.argc; i++) { + const char *out; + if (skip_prefix(server_capabilities_v2.argv[i], c, &out) && + (!*out || *out == '=')) + return 1; + } + + if (die_on_error) + die("server doesn't support '%s'", c); + + return 0; +} + +static void process_capabilities_v2(struct packet_reader *reader) +{ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) + argv_array_push(&server_capabilities_v2, reader->line); + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after capabilities"); +} + enum protocol_version discover_version(struct packet_reader *reader) { enum protocol_version version = protocol_unknown_version; @@ -84,7 +113,7 @@ enum protocol_version discover_version(struct packet_reader *reader) switch (version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + process_capabilities_v2(reader); break; case protocol_v1: /* Read the peeked version line */ @@ -128,7 +157,7 @@ reject: static void annotate_refs_with_symref_info(struct ref *ref) { struct string_list symref = STRING_LIST_INIT_DUP; - const char *feature_list = server_capabilities; + const char *feature_list = server_capabilities_v1; while (feature_list) { int len; @@ -157,7 +186,7 @@ static void process_capabilities(const char *line, int *len) int nul_location = strlen(line); if (nul_location == *len) return; - server_capabilities = xstrdup(line + nul_location + 1); + server_capabilities_v1 = xstrdup(line + nul_location + 1); *len = nul_location; } @@ -292,6 +321,105 @@ struct ref **get_remote_heads(struct packet_reader *reader, return list; } +/* Returns 1 when a valid ref has been added to `list`, 0 otherwise */ +static int process_ref_v2(const char *line, struct ref ***list) +{ + int ret = 1; + int i = 0; + struct object_id old_oid; + struct ref *ref; + struct string_list line_sections = STRING_LIST_INIT_DUP; + const char *end; + + /* + * Ref lines have a number of fields which are space deliminated. The + * first field is the OID of the ref. The second field is the ref + * name. Subsequent fields (symref-target and peeled) are optional and + * don't have a particular order. + */ + if (string_list_split(&line_sections, line, ' ', -1) < 2) { + ret = 0; + goto out; + } + + if (parse_oid_hex(line_sections.items[i++].string, &old_oid, &end) || + *end) { + ret = 0; + goto out; + } + + ref = alloc_ref(line_sections.items[i++].string); + + oidcpy(&ref->old_oid, &old_oid); + **list = ref; + *list = &ref->next; + + for (; i < line_sections.nr; i++) { + const char *arg = line_sections.items[i].string; + if (skip_prefix(arg, "symref-target:", &arg)) + ref->symref = xstrdup(arg); + + if (skip_prefix(arg, "peeled:", &arg)) { + struct object_id peeled_oid; + char *peeled_name; + struct ref *peeled; + if (parse_oid_hex(arg, &peeled_oid, &end) || *end) { + ret = 0; + goto out; + } + + peeled_name = xstrfmt("%s^{}", ref->name); + peeled = alloc_ref(peeled_name); + + oidcpy(&peeled->old_oid, &peeled_oid); + **list = peeled; + *list = &peeled->next; + + free(peeled_name); + } + } + +out: + string_list_clear(&line_sections, 0); + return ret; +} + +struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes) +{ + int i; + *list = NULL; + + if (server_supports_v2("ls-refs", 1)) + packet_write_fmt(fd_out, "command=ls-refs\n"); + + if (server_supports_v2("agent", 0)) + packet_write_fmt(fd_out, "agent=%s", git_user_agent_sanitized()); + + packet_delim(fd_out); + /* When pushing we don't want to request the peeled tags */ + if (!for_push) + packet_write_fmt(fd_out, "peel\n"); + packet_write_fmt(fd_out, "symrefs\n"); + for (i = 0; ref_prefixes && i < ref_prefixes->argc; i++) { + packet_write_fmt(fd_out, "ref-prefix %s\n", + ref_prefixes->argv[i]); + } + packet_flush(fd_out); + + /* Process response from server */ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + if (!process_ref_v2(reader->line, &list)) + die("invalid ls-refs response: %s", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after ref listing"); + + return list; +} + static const char *parse_feature_value(const char *feature_list, const char *feature, int *lenp) { int len; @@ -336,7 +464,7 @@ int parse_feature_request(const char *feature_list, const char *feature) const char *server_feature_value(const char *feature, int *len) { - return parse_feature_value(server_capabilities, feature, len); + return parse_feature_value(server_capabilities_v1, feature, len); } int server_supports(const char *feature) diff --git a/connect.h b/connect.h index cdb8979..8898d44 100644 --- a/connect.h +++ b/connect.h @@ -16,4 +16,6 @@ extern int url_is_local_not_ssh(const char *url); struct packet_reader; extern enum protocol_version discover_version(struct packet_reader *reader); +extern int server_supports_v2(const char *c, int die_on_error); + #endif diff --git a/remote.h b/remote.h index 2016461..368ba22 100644 --- a/remote.h +++ b/remote.h @@ -151,11 +151,17 @@ void free_refs(struct ref *ref); struct oid_array; struct packet_reader; +struct argv_array; extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points); +/* Used for protocol v2 in order to retrieve refs from a remote */ +extern struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes); + int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh new file mode 100755 index 0000000..dc5f813 --- /dev/null +++ b/t/t5702-protocol-v2.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +test_description='test git wire-protocol version 2' + +TEST_NO_CREATE_REPO=1 + +. ./test-lib.sh + +# Test protocol v2 with 'git://' transport +# +. "$TEST_DIRECTORY"/lib-git-daemon.sh +start_git_daemon --export-all --enable=receive-pack +daemon_parent=$GIT_DAEMON_DOCUMENT_ROOT_PATH/parent + +test_expect_success 'create repo to be served by git-daemon' ' + git init "$daemon_parent" && + test_commit -C "$daemon_parent" one +' + +test_expect_success 'list refs with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "$GIT_DAEMON_URL/parent" >actual && + + # Client requested to use protocol v2 + grep "git> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "$GIT_DAEMON_URL/parent" >expect && + test_cmp actual expect +' + +stop_git_daemon + +# Test protocol v2 with 'file://' transport +# +test_expect_success 'create repo to be served by file:// transport' ' + git init file_parent && + test_commit -C file_parent one +' + +test_expect_success 'list refs with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "file://$(pwd)/file_parent" >actual && + + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "file://$(pwd)/file_parent" >expect && + test_cmp actual expect +' + +test_done diff --git a/transport.c b/transport.c index 83d9dd1..ffc6b26 100644 --- a/transport.c +++ b/transport.c @@ -204,7 +204,7 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + get_remote_refs(data->fd[1], &reader, &refs, for_push, NULL); break; case protocol_v1: case protocol_v0: -- cgit v0.10.2-6-g49f6 From 834cf34b26035abc58f7c2b0f2afd5527e93c30d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:22 -0700 Subject: transport: convert get_refs_list to take a list of ref prefixes Convert the 'struct transport' virtual function 'get_refs_list()' to optionally take an argv_array of ref prefixes. When communicating with a server using protocol v2 these ref prefixes can be sent when requesting a listing of their refs allowing the server to filter the refs it sends based on the sent prefixes. This list will be ignored when not using protocol v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport-helper.c b/transport-helper.c index 5080150..8774ab3 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -1026,7 +1026,8 @@ static int has_attribute(const char *attrs, const char *attr) { } } -static struct ref *get_refs_list(struct transport *transport, int for_push) +static struct ref *get_refs_list(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes) { struct helper_data *data = transport->data; struct child_process *helper; @@ -1039,7 +1040,7 @@ static struct ref *get_refs_list(struct transport *transport, int for_push) if (process_connect(transport, for_push)) { do_take_over(transport); - return transport->vtable->get_refs_list(transport, for_push); + return transport->vtable->get_refs_list(transport, for_push, ref_prefixes); } if (data->push && for_push) diff --git a/transport-internal.h b/transport-internal.h index 3c1a29d..1cde625 100644 --- a/transport-internal.h +++ b/transport-internal.h @@ -3,6 +3,7 @@ struct ref; struct transport; +struct argv_array; struct transport_vtable { /** @@ -17,11 +18,19 @@ struct transport_vtable { * the transport to try to share connections, for_push is a * hint as to whether the ultimate operation is a push or a fetch. * + * If communicating using protocol v2 a list of prefixes can be + * provided to be sent to the server to enable it to limit the ref + * advertisement. Since ref filtering is done on the server's end, and + * only when using protocol v2, this list will be ignored when not + * using protocol v2 meaning this function can return refs which don't + * match the provided ref_prefixes. + * * If the transport is able to determine the remote hash for * the ref without a huge amount of effort, it should store it * in the ref's old_sha1 field; otherwise it should be all 0. **/ - struct ref *(*get_refs_list)(struct transport *transport, int for_push); + struct ref *(*get_refs_list)(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes); /** * Fetch the objects for the given refs. Note that this gets diff --git a/transport.c b/transport.c index ffc6b26..2e68010 100644 --- a/transport.c +++ b/transport.c @@ -72,7 +72,9 @@ struct bundle_transport_data { struct bundle_header header; }; -static struct ref *get_refs_from_bundle(struct transport *transport, int for_push) +static struct ref *get_refs_from_bundle(struct transport *transport, + int for_push, + const struct argv_array *ref_prefixes) { struct bundle_transport_data *data = transport->data; struct ref *result = NULL; @@ -189,7 +191,8 @@ static int connect_setup(struct transport *transport, int for_push) return 0; } -static struct ref *get_refs_via_connect(struct transport *transport, int for_push) +static struct ref *get_refs_via_connect(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes) { struct git_transport_data *data = transport->data; struct ref *refs = NULL; @@ -204,7 +207,8 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { case protocol_v2: - get_remote_refs(data->fd[1], &reader, &refs, for_push, NULL); + get_remote_refs(data->fd[1], &reader, &refs, for_push, + ref_prefixes); break; case protocol_v1: case protocol_v0: @@ -250,7 +254,7 @@ static int fetch_refs_via_pack(struct transport *transport, args.update_shallow = data->options.update_shallow; if (!data->got_remote_heads) - refs_tmp = get_refs_via_connect(transport, 0); + refs_tmp = get_refs_via_connect(transport, 0, NULL); switch (data->version) { case protocol_v2: @@ -568,7 +572,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re int ret = 0; if (!data->got_remote_heads) - get_refs_via_connect(transport, 1); + get_refs_via_connect(transport, 1, NULL); memset(&args, 0, sizeof(args)); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); @@ -1028,7 +1032,7 @@ int transport_push(struct transport *transport, if (check_push_refs(local_refs, refspec_nr, refspec) < 0) return -1; - remote_refs = transport->vtable->get_refs_list(transport, 1); + remote_refs = transport->vtable->get_refs_list(transport, 1, NULL); if (flags & TRANSPORT_PUSH_ALL) match_flags |= MATCH_REFS_ALL; @@ -1137,7 +1141,7 @@ int transport_push(struct transport *transport, const struct ref *transport_get_remote_refs(struct transport *transport) { if (!transport->got_remote_refs) { - transport->remote_refs = transport->vtable->get_refs_list(transport, 0); + transport->remote_refs = transport->vtable->get_refs_list(transport, 0, NULL); transport->got_remote_refs = 1; } -- cgit v0.10.2-6-g49f6 From 1af8ae1cfa44a3f3c3f1efb7f9ebe68bc0ce7578 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:23 -0700 Subject: transport: convert transport_get_remote_refs to take a list of ref prefixes Teach transport_get_remote_refs() to accept a list of ref prefixes, which will be sent to the server for use in filtering when using protocol v2. (This list will be ignored when not using protocol v2.) Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/clone.c b/builtin/clone.c index 2da71db..4db3079 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1104,7 +1104,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (transport->smart_options && !deepen) transport->smart_options->check_self_contained_and_connected = 1; - refs = transport_get_remote_refs(transport); + refs = transport_get_remote_refs(transport, NULL); if (refs) { mapped_refs = wanted_peer_refs(refs, refspec); diff --git a/builtin/fetch.c b/builtin/fetch.c index 7bbcd26..850382f 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -250,7 +250,7 @@ static void find_non_local_tags(struct transport *transport, struct string_list_item *item = NULL; for_each_ref(add_existing, &existing_refs); - for (ref = transport_get_remote_refs(transport); ref; ref = ref->next) { + for (ref = transport_get_remote_refs(transport, NULL); ref; ref = ref->next) { if (!starts_with(ref->name, "refs/tags/")) continue; @@ -336,7 +336,7 @@ static struct ref *get_ref_map(struct transport *transport, /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs = transport_get_remote_refs(transport); + const struct ref *remote_refs = transport_get_remote_refs(transport, NULL); if (refspec_count) { struct refspec *fetch_refspec; diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index c4be98a..c6e9847 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -96,7 +96,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); - ref = transport_get_remote_refs(transport); + ref = transport_get_remote_refs(transport, NULL); if (transport_disconnect(transport)) return 1; diff --git a/builtin/remote.c b/builtin/remote.c index d95bf90..d0b6ff6 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -862,7 +862,7 @@ static int get_remote_ref_states(const char *name, if (query) { transport = transport_get(states->remote, states->remote->url_nr > 0 ? states->remote->url[0] : NULL); - remote_refs = transport_get_remote_refs(transport); + remote_refs = transport_get_remote_refs(transport, NULL); transport_disconnect(transport); states->queried = 1; diff --git a/transport.c b/transport.c index 2e68010..3f13051 100644 --- a/transport.c +++ b/transport.c @@ -1138,10 +1138,13 @@ int transport_push(struct transport *transport, return 1; } -const struct ref *transport_get_remote_refs(struct transport *transport) +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes) { if (!transport->got_remote_refs) { - transport->remote_refs = transport->vtable->get_refs_list(transport, 0, NULL); + transport->remote_refs = + transport->vtable->get_refs_list(transport, 0, + ref_prefixes); transport->got_remote_refs = 1; } diff --git a/transport.h b/transport.h index 731c78b..83992a42 100644 --- a/transport.h +++ b/transport.h @@ -178,7 +178,17 @@ int transport_push(struct transport *connection, int refspec_nr, const char **refspec, int flags, unsigned int * reject_reasons); -const struct ref *transport_get_remote_refs(struct transport *transport); +/* + * Retrieve refs from a remote. + * + * Optionally a list of ref prefixes can be provided which can be sent to the + * server (when communicating using protocol v2) to enable it to limit the ref + * advertisement. Since ref filtering is done on the server's end (and only + * when using protocol v2), this can return refs which don't match the provided + * ref_prefixes. + */ +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes); int transport_fetch_refs(struct transport *transport, struct ref *refs); void transport_unlock_pack(struct transport *transport); -- cgit v0.10.2-6-g49f6 From b4be74105febef5c6235e1e2f1e468e76166cad8 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:24 -0700 Subject: ls-remote: pass ref prefixes when requesting a remote's refs Construct an argv_array of ref prefixes based on the patterns supplied via the command line and pass them to 'transport_get_remote_refs()' to be used when communicating protocol v2 so that the server can limit the ref advertisement based on those prefixes. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index c6e9847..4276bf9 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -2,6 +2,7 @@ #include "cache.h" #include "transport.h" #include "remote.h" +#include "refs.h" static const char * const ls_remote_usage[] = { N_("git ls-remote [--heads] [--tags] [--refs] [--upload-pack=]\n" @@ -43,6 +44,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) int show_symref_target = 0; const char *uploadpack = NULL; const char **pattern = NULL; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; struct remote *remote; struct transport *transport; @@ -74,8 +76,17 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (argc > 1) { int i; pattern = xcalloc(argc, sizeof(const char *)); - for (i = 1; i < argc; i++) + for (i = 1; i < argc; i++) { + const char *glob; pattern[i - 1] = xstrfmt("*/%s", argv[i]); + + glob = strchr(argv[i], '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - argv[i]), argv[i]); + else + expand_ref_prefix(&ref_prefixes, argv[i]); + } } remote = remote_get(dest); @@ -96,7 +107,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); - ref = transport_get_remote_refs(transport, NULL); + ref = transport_get_remote_refs(transport, &ref_prefixes); if (transport_disconnect(transport)) return 1; diff --git a/refs.c b/refs.c index 20ba82b..cefbad2 100644 --- a/refs.c +++ b/refs.c @@ -13,6 +13,7 @@ #include "tag.h" #include "submodule.h" #include "worktree.h" +#include "argv-array.h" /* * List of all available backends @@ -502,6 +503,19 @@ int refname_match(const char *abbrev_name, const char *full_name) } /* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix) +{ + const char **p; + int len = strlen(prefix); + + for (p = ref_rev_parse_rules; *p; p++) + argv_array_pushf(prefixes, *p, len, prefix); +} + +/* * *string and *len will only be substituted, and *string returned (for * later free()ing) if the string passed in is a magic short-hand form * to name a branch. diff --git a/refs.h b/refs.h index 01be5ae..93b6dce 100644 --- a/refs.h +++ b/refs.h @@ -139,6 +139,13 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, */ int refname_match(const char *abbrev_name, const char *full_name); +/* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +struct argv_array; +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix); + int expand_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_log(const char *str, int len, struct object_id *oid, char **ref); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index dc5f813..562610f 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -32,6 +32,19 @@ test_expect_success 'list refs with git:// using protocol v2' ' test_cmp actual expect ' +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "$GIT_DAEMON_URL/parent" master >actual && + + cat >expect <<-EOF && + $(git -C "$daemon_parent" rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + stop_git_daemon # Test protocol v2 with 'file://' transport @@ -54,4 +67,17 @@ test_expect_success 'list refs with file:// using protocol v2' ' test_cmp actual expect ' +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "file://$(pwd)/file_parent" master >actual && + + cat >expect <<-EOF && + $(git -C file_parent rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + test_done -- cgit v0.10.2-6-g49f6 From 230d7dd3917b80d94bf8d522b220d587634f1885 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:25 -0700 Subject: fetch: pass ref prefixes when fetching Construct a list of ref prefixes to be passed to 'transport_get_remote_refs()' from the refspec to be used during the fetch. This list of ref prefixes will be used to allow the server to filter the ref advertisement when communicating using protocol v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index 850382f..8258bbf 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -332,11 +332,28 @@ static struct ref *get_ref_map(struct transport *transport, struct ref *rm; struct ref *ref_map = NULL; struct ref **tail = &ref_map; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs = transport_get_remote_refs(transport, NULL); + const struct ref *remote_refs; + + for (i = 0; i < refspec_count; i++) { + if (!refspecs[i].exact_sha1) { + const char *glob = strchr(refspecs[i].src, '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - refspecs[i].src), + refspecs[i].src); + else + expand_ref_prefix(&ref_prefixes, refspecs[i].src); + } + } + + remote_refs = transport_get_remote_refs(transport, &ref_prefixes); + + argv_array_clear(&ref_prefixes); if (refspec_count) { struct refspec *fetch_refspec; -- cgit v0.10.2-6-g49f6 From 5b872fff187037829fdd3e494acdf4299ff6e80b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:26 -0700 Subject: push: pass ref prefixes when pushing Construct a list of ref prefixes to be passed to 'get_refs_list()' from the refspec to be used during the push. This list of ref prefixes will be used to allow the server to filter the ref advertisement when communicating using protocol v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport.c b/transport.c index 3f13051..57bdbb5 100644 --- a/transport.c +++ b/transport.c @@ -1028,11 +1028,38 @@ int transport_push(struct transport *transport, int porcelain = flags & TRANSPORT_PUSH_PORCELAIN; int pretend = flags & TRANSPORT_PUSH_DRY_RUN; int push_ret, ret, err; + struct refspec *tmp_rs; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; + int i; if (check_push_refs(local_refs, refspec_nr, refspec) < 0) return -1; - remote_refs = transport->vtable->get_refs_list(transport, 1, NULL); + tmp_rs = parse_push_refspec(refspec_nr, refspec); + for (i = 0; i < refspec_nr; i++) { + const char *prefix = NULL; + + if (tmp_rs[i].dst) + prefix = tmp_rs[i].dst; + else if (tmp_rs[i].src && !tmp_rs[i].exact_sha1) + prefix = tmp_rs[i].src; + + if (prefix) { + const char *glob = strchr(prefix, '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - prefix), + prefix); + else + expand_ref_prefix(&ref_prefixes, prefix); + } + } + + remote_refs = transport->vtable->get_refs_list(transport, 1, + &ref_prefixes); + + argv_array_clear(&ref_prefixes); + free_refspec(refspec_nr, tmp_rs); if (flags & TRANSPORT_PUSH_ALL) match_flags |= MATCH_REFS_ALL; -- cgit v0.10.2-6-g49f6 From 3145ea957d2ca6ba7afb087c9ed680e08d705690 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:27 -0700 Subject: upload-pack: introduce fetch server command Introduce the 'fetch' server command. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index bbb8b14..307dc14 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -199,3 +199,128 @@ The output of ls-refs is as follows: ref-attribute = (symref | peeled) symref = "symref-target:" symref-target peeled = "peeled:" obj-id + + fetch +~~~~~~~ + +`fetch` is the command used to fetch a packfile in v2. It can be looked +at as a modified version of the v1 fetch where the ref-advertisement is +stripped out (since the `ls-refs` command fills that role) and the +message format is tweaked to eliminate redundancies and permit easy +addition of future extensions. + +Additional features not supported in the base command will be advertised +as the value of the command in the capability advertisement in the form +of a space separated list of features: "= " + +A `fetch` request can take the following arguments: + + want + Indicates to the server an object which the client wants to + retrieve. Wants can be anything and are not limited to + advertised objects. + + have + Indicates to the server an object which the client has locally. + This allows the server to make a packfile which only contains + the objects that the client needs. Multiple 'have' lines can be + supplied. + + done + Indicates to the server that negotiation should terminate (or + not even begin if performing a clone) and that the server should + use the information supplied in the request to construct the + packfile. + + thin-pack + Request that a thin pack be sent, which is a pack with deltas + which reference base objects not contained within the pack (but + are known to exist at the receiving end). This can reduce the + network traffic significantly, but it requires the receiving end + to know how to "thicken" these packs by adding the missing bases + to the pack. + + no-progress + Request that progress information that would normally be sent on + side-band channel 2, during the packfile transfer, should not be + sent. However, the side-band channel 3 is still used for error + responses. + + include-tag + Request that annotated tags should be sent if the objects they + point to are being sent. + + ofs-delta + Indicate that the client understands PACKv2 with delta referring + to its base by position in pack rather than by an oid. That is, + they can read OBJ_OFS_DELTA (ake type 6) in a packfile. + +The response of `fetch` is broken into a number of sections separated by +delimiter packets (0001), with each section beginning with its section +header. + + output = *section + section = (acknowledgments | packfile) + (flush-pkt | delim-pkt) + + acknowledgments = PKT-LINE("acknowledgments" LF) + (nak | *ack) + (ready) + ready = PKT-LINE("ready" LF) + nak = PKT-LINE("NAK" LF) + ack = PKT-LINE("ACK" SP obj-id LF) + + packfile = PKT-LINE("packfile" LF) + *PKT-LINE(%x01-03 *%x00-ff) + + acknowledgments section + * If the client determines that it is finished with negotiations + by sending a "done" line, the acknowledgments sections MUST be + omitted from the server's response. + + * Always begins with the section header "acknowledgments" + + * The server will respond with "NAK" if none of the object ids sent + as have lines were common. + + * The server will respond with "ACK obj-id" for all of the + object ids sent as have lines which are common. + + * A response cannot have both "ACK" lines as well as a "NAK" + line. + + * The server will respond with a "ready" line indicating that + the server has found an acceptable common base and is ready to + make and send a packfile (which will be found in the packfile + section of the same response) + + * If the server has found a suitable cut point and has decided + to send a "ready" line, then the server can decide to (as an + optimization) omit any "ACK" lines it would have sent during + its response. This is because the server will have already + determined the objects it plans to send to the client and no + further negotiation is needed. + + packfile section + * This section is only included if the client has sent 'want' + lines in its request and either requested that no more + negotiation be done by sending 'done' or if the server has + decided it has found a sufficient cut point to produce a + packfile. + + * Always begins with the section header "packfile" + + * The transmission of the packfile begins immediately after the + section header + + * The data transfer of the packfile is always multiplexed, using + the same semantics of the 'side-band-64k' capability from + protocol version 1. This means that each packet, during the + packfile data stream, is made up of a leading 4-byte pkt-line + length (typical of the pkt-line format), followed by a 1-byte + stream code, followed by the actual data. + + The stream code can be one of: + 1 - pack data + 2 - progress messages + 3 - fatal error message just before stream aborts diff --git a/serve.c b/serve.c index 1cb5fb9..3692dda 100644 --- a/serve.c +++ b/serve.c @@ -6,6 +6,7 @@ #include "argv-array.h" #include "ls-refs.h" #include "serve.h" +#include "upload-pack.h" static int always_advertise(struct repository *r, struct strbuf *value) @@ -54,6 +55,7 @@ struct protocol_capability { static struct protocol_capability capabilities[] = { { "agent", agent_advertise, NULL }, { "ls-refs", always_advertise, ls_refs }, + { "fetch", always_advertise, upload_pack_v2 }, }; static void advertise_capabilities(void) diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index d03e08e..888a700 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -9,6 +9,7 @@ test_expect_success 'test capability advertisement' ' version 2 agent=git/$(git version | cut -d" " -f3) ls-refs + fetch 0000 EOF diff --git a/upload-pack.c b/upload-pack.c index 1e8a9e1..198e9f0 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -18,6 +18,7 @@ #include "prio-queue.h" #include "protocol.h" #include "upload-pack.h" +#include "serve.h" /* Remember to update object flag allocation in object.h */ #define THEY_HAVE (1u << 11) @@ -1065,3 +1066,268 @@ void upload_pack(struct upload_pack_options *options) create_pack_file(); } } + +struct upload_pack_data { + struct object_array wants; + struct oid_array haves; + + unsigned stateless_rpc : 1; + + unsigned use_thin_pack : 1; + unsigned use_ofs_delta : 1; + unsigned no_progress : 1; + unsigned use_include_tag : 1; + unsigned done : 1; +}; + +static void upload_pack_data_init(struct upload_pack_data *data) +{ + struct object_array wants = OBJECT_ARRAY_INIT; + struct oid_array haves = OID_ARRAY_INIT; + + memset(data, 0, sizeof(*data)); + data->wants = wants; + data->haves = haves; +} + +static void upload_pack_data_clear(struct upload_pack_data *data) +{ + object_array_clear(&data->wants); + oid_array_clear(&data->haves); +} + +static int parse_want(const char *line) +{ + const char *arg; + if (skip_prefix(line, "want ", &arg)) { + struct object_id oid; + struct object *o; + + if (get_oid_hex(arg, &oid)) + die("git upload-pack: protocol error, " + "expected to get oid, not '%s'", line); + + o = parse_object(&oid); + if (!o) { + packet_write_fmt(1, + "ERR upload-pack: not our ref %s", + oid_to_hex(&oid)); + die("git upload-pack: not our ref %s", + oid_to_hex(&oid)); + } + + if (!(o->flags & WANTED)) { + o->flags |= WANTED; + add_object_array(o, NULL, &want_obj); + } + + return 1; + } + + return 0; +} + +static int parse_have(const char *line, struct oid_array *haves) +{ + const char *arg; + if (skip_prefix(line, "have ", &arg)) { + struct object_id oid; + + if (get_oid_hex(arg, &oid)) + die("git upload-pack: expected SHA1 object, got '%s'", arg); + oid_array_append(haves, &oid); + return 1; + } + + return 0; +} + +static void process_args(struct packet_reader *request, + struct upload_pack_data *data) +{ + while (packet_reader_read(request) != PACKET_READ_FLUSH) { + const char *arg = request->line; + + /* process want */ + if (parse_want(arg)) + continue; + /* process have line */ + if (parse_have(arg, &data->haves)) + continue; + + /* process args like thin-pack */ + if (!strcmp(arg, "thin-pack")) { + use_thin_pack = 1; + continue; + } + if (!strcmp(arg, "ofs-delta")) { + use_ofs_delta = 1; + continue; + } + if (!strcmp(arg, "no-progress")) { + no_progress = 1; + continue; + } + if (!strcmp(arg, "include-tag")) { + use_include_tag = 1; + continue; + } + if (!strcmp(arg, "done")) { + data->done = 1; + continue; + } + + /* ignore unknown lines maybe? */ + die("unexpect line: '%s'", arg); + } +} + +static int process_haves(struct oid_array *haves, struct oid_array *common) +{ + int i; + + /* Process haves */ + for (i = 0; i < haves->nr; i++) { + const struct object_id *oid = &haves->oid[i]; + struct object *o; + int we_knew_they_have = 0; + + if (!has_object_file(oid)) + continue; + + oid_array_append(common, oid); + + o = parse_object(oid); + if (!o) + die("oops (%s)", oid_to_hex(oid)); + if (o->type == OBJ_COMMIT) { + struct commit_list *parents; + struct commit *commit = (struct commit *)o; + if (o->flags & THEY_HAVE) + we_knew_they_have = 1; + else + o->flags |= THEY_HAVE; + if (!oldest_have || (commit->date < oldest_have)) + oldest_have = commit->date; + for (parents = commit->parents; + parents; + parents = parents->next) + parents->item->object.flags |= THEY_HAVE; + } + if (!we_knew_they_have) + add_object_array(o, NULL, &have_obj); + } + + return 0; +} + +static int send_acks(struct oid_array *acks, struct strbuf *response) +{ + int i; + + packet_buf_write(response, "acknowledgments\n"); + + /* Send Acks */ + if (!acks->nr) + packet_buf_write(response, "NAK\n"); + + for (i = 0; i < acks->nr; i++) { + packet_buf_write(response, "ACK %s\n", + oid_to_hex(&acks->oid[i])); + } + + if (ok_to_give_up()) { + /* Send Ready */ + packet_buf_write(response, "ready\n"); + return 1; + } + + return 0; +} + +static int process_haves_and_send_acks(struct upload_pack_data *data) +{ + struct oid_array common = OID_ARRAY_INIT; + struct strbuf response = STRBUF_INIT; + int ret = 0; + + process_haves(&data->haves, &common); + if (data->done) { + ret = 1; + } else if (send_acks(&common, &response)) { + packet_buf_delim(&response); + ret = 1; + } else { + /* Add Flush */ + packet_buf_flush(&response); + ret = 0; + } + + /* Send response */ + write_or_die(1, response.buf, response.len); + strbuf_release(&response); + + oid_array_clear(&data->haves); + oid_array_clear(&common); + return ret; +} + +enum fetch_state { + FETCH_PROCESS_ARGS = 0, + FETCH_SEND_ACKS, + FETCH_SEND_PACK, + FETCH_DONE, +}; + +int upload_pack_v2(struct repository *r, struct argv_array *keys, + struct packet_reader *request) +{ + enum fetch_state state = FETCH_PROCESS_ARGS; + struct upload_pack_data data; + + upload_pack_data_init(&data); + use_sideband = LARGE_PACKET_MAX; + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_PROCESS_ARGS: + process_args(request, &data); + + if (!want_obj.nr) { + /* + * Request didn't contain any 'want' lines, + * guess they didn't want anything. + */ + state = FETCH_DONE; + } else if (data.haves.nr) { + /* + * Request had 'have' lines, so lets ACK them. + */ + state = FETCH_SEND_ACKS; + } else { + /* + * Request had 'want's but no 'have's so we can + * immedietly go to construct and send a pack. + */ + state = FETCH_SEND_PACK; + } + break; + case FETCH_SEND_ACKS: + if (process_haves_and_send_acks(&data)) + state = FETCH_SEND_PACK; + else + state = FETCH_DONE; + break; + case FETCH_SEND_PACK: + packet_write_fmt(1, "packfile\n"); + create_pack_file(); + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + upload_pack_data_clear(&data); + return 0; +} diff --git a/upload-pack.h b/upload-pack.h index a71e4dc..5bbdb7e 100644 --- a/upload-pack.h +++ b/upload-pack.h @@ -10,4 +10,10 @@ struct upload_pack_options { void upload_pack(struct upload_pack_options *options); +struct repository; +struct argv_array; +struct packet_reader; +extern int upload_pack_v2(struct repository *r, struct argv_array *keys, + struct packet_reader *request); + #endif /* UPLOAD_PACK_H */ -- cgit v0.10.2-6-g49f6 From 685fbd32916f3e94bc89aa14e8fdce835b06f801 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:28 -0700 Subject: fetch-pack: perform a fetch using v2 When communicating with a v2 server, perform a fetch by requesting the 'fetch' command. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 307dc14..4f7f251 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -255,12 +255,43 @@ A `fetch` request can take the following arguments: to its base by position in pack rather than by an oid. That is, they can read OBJ_OFS_DELTA (ake type 6) in a packfile. + shallow + A client must notify the server of all commits for which it only + has shallow copies (meaning that it doesn't have the parents of + a commit) by supplying a 'shallow ' line for each such + object so that the server is aware of the limitations of the + client's history. This is so that the server is aware that the + client may not have all objects reachable from such commits. + + deepen + Requests that the fetch/clone should be shallow having a commit + depth of relative to the remote side. + + deepen-relative + Requests that the semantics of the "deepen" command be changed + to indicate that the depth requested is relative to the client's + current shallow boundary, instead of relative to the requested + commits. + + deepen-since + Requests that the shallow clone/fetch should be cut at a + specific time, instead of depth. Internally it's equivalent to + doing "git rev-list --max-age=". Cannot be used with + "deepen". + + deepen-not + Requests that the shallow clone/fetch should be cut at a + specific revision specified by '', instead of a depth. + Internally it's equivalent of doing "git rev-list --not ". + Cannot be used with "deepen", but can be used with + "deepen-since". + The response of `fetch` is broken into a number of sections separated by delimiter packets (0001), with each section beginning with its section header. output = *section - section = (acknowledgments | packfile) + section = (acknowledgments | shallow-info | packfile) (flush-pkt | delim-pkt) acknowledgments = PKT-LINE("acknowledgments" LF) @@ -270,6 +301,11 @@ header. nak = PKT-LINE("NAK" LF) ack = PKT-LINE("ACK" SP obj-id LF) + shallow-info = PKT-LINE("shallow-info" LF) + *PKT-LINE((shallow | unshallow) LF) + shallow = "shallow" SP obj-id + unshallow = "unshallow" SP obj-id + packfile = PKT-LINE("packfile" LF) *PKT-LINE(%x01-03 *%x00-ff) @@ -301,6 +337,35 @@ header. determined the objects it plans to send to the client and no further negotiation is needed. + shallow-info section + If the client has requested a shallow fetch/clone, a shallow + client requests a fetch or the server is shallow then the + server's response may include a shallow-info section. The + shallow-info section will be included if (due to one of the + above conditions) the server needs to inform the client of any + shallow boundaries or adjustments to the clients already + existing shallow boundaries. + + * Always begins with the section header "shallow-info" + + * If a positive depth is requested, the server will compute the + set of commits which are no deeper than the desired depth. + + * The server sends a "shallow obj-id" line for each commit whose + parents will not be sent in the following packfile. + + * The server sends an "unshallow obj-id" line for each commit + which the client has indicated is shallow, but is no longer + shallow as a result of the fetch (due to its parents being + sent in the following packfile). + + * The server MUST NOT send any "unshallow" lines for anything + which the client has not indicated was shallow as a part of + its request. + + * This section is only included if a packfile section is also + included in the response. + packfile section * This section is only included if the client has sent 'want' lines in its request and either requested that no more diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b2374dd..f9d7d0b 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -212,7 +212,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, - &shallow, pack_lockfile_ptr); + &shallow, pack_lockfile_ptr, protocol_v0); if (pack_lockfile) { printf("lock %s\n", pack_lockfile); fflush(stdout); diff --git a/fetch-pack.c b/fetch-pack.c index 9f6b07a..dffcfd6 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -303,9 +303,9 @@ static void insert_one_alternate_object(struct object *obj) #define PIPESAFE_FLUSH 32 #define LARGE_FLUSH 16384 -static int next_flush(struct fetch_pack_args *args, int count) +static int next_flush(int stateless_rpc, int count) { - if (args->stateless_rpc) { + if (stateless_rpc) { if (count < LARGE_FLUSH) count <<= 1; else @@ -461,7 +461,7 @@ static int find_common(struct fetch_pack_args *args, send_request(args, fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); flushes++; - flush_at = next_flush(args, count); + flush_at = next_flush(args->stateless_rpc, count); /* * We keep one window "ahead" of the other side, and @@ -1008,6 +1008,259 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, return ref; } +static void add_wants(const struct ref *wants, struct strbuf *req_buf) +{ + for ( ; wants ; wants = wants->next) { + const struct object_id *remote = &wants->old_oid; + const char *remote_hex; + struct object *o; + + /* + * If that object is complete (i.e. it is an ancestor of a + * local ref), we tell them we have it but do not have to + * tell them about its ancestors, which they already know + * about. + * + * We use lookup_object here because we are only + * interested in the case we *know* the object is + * reachable and we have already scanned it. + */ + if (((o = lookup_object(remote->hash)) != NULL) && + (o->flags & COMPLETE)) { + continue; + } + + remote_hex = oid_to_hex(remote); + packet_buf_write(req_buf, "want %s\n", remote_hex); + } +} + +static void add_common(struct strbuf *req_buf, struct oidset *common) +{ + struct oidset_iter iter; + const struct object_id *oid; + oidset_iter_init(common, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + } +} + +static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain) +{ + int ret = 0; + int haves_added = 0; + const struct object_id *oid; + + while ((oid = get_rev())) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + if (++haves_added >= *haves_to_send) + break; + } + + *in_vain += haves_added; + if (!haves_added || *in_vain >= MAX_IN_VAIN) { + /* Send Done */ + packet_buf_write(req_buf, "done\n"); + ret = 1; + } + + /* Increase haves to send on next round */ + *haves_to_send = next_flush(1, *haves_to_send); + + return ret; +} + +static int send_fetch_request(int fd_out, const struct fetch_pack_args *args, + const struct ref *wants, struct oidset *common, + int *haves_to_send, int *in_vain) +{ + int ret = 0; + struct strbuf req_buf = STRBUF_INIT; + + if (server_supports_v2("fetch", 1)) + packet_buf_write(&req_buf, "command=fetch"); + if (server_supports_v2("agent", 0)) + packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized()); + + packet_buf_delim(&req_buf); + if (args->use_thin_pack) + packet_buf_write(&req_buf, "thin-pack"); + if (args->no_progress) + packet_buf_write(&req_buf, "no-progress"); + if (args->include_tag) + packet_buf_write(&req_buf, "include-tag"); + if (prefer_ofs_delta) + packet_buf_write(&req_buf, "ofs-delta"); + + /* add wants */ + add_wants(wants, &req_buf); + + /* Add all of the common commits we've found in previous rounds */ + add_common(&req_buf, common); + + /* Add initial haves */ + ret = add_haves(&req_buf, haves_to_send, in_vain); + + /* Send request */ + packet_buf_flush(&req_buf); + write_or_die(fd_out, req_buf.buf, req_buf.len); + + strbuf_release(&req_buf); + return ret; +} + +/* + * Processes a section header in a server's response and checks if it matches + * `section`. If the value of `peek` is 1, the header line will be peeked (and + * not consumed); if 0, the line will be consumed and the function will die if + * the section header doesn't match what was expected. + */ +static int process_section_header(struct packet_reader *reader, + const char *section, int peek) +{ + int ret; + + if (packet_reader_peek(reader) != PACKET_READ_NORMAL) + die("error reading packet"); + + ret = !strcmp(reader->line, section); + + if (!peek) { + if (!ret) + die("expected '%s', received '%s'", + section, reader->line); + packet_reader_read(reader); + } + + return ret; +} + +static int process_acks(struct packet_reader *reader, struct oidset *common) +{ + /* received */ + int received_ready = 0; + int received_ack = 0; + + process_section_header(reader, "acknowledgments", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + + if (!strcmp(reader->line, "NAK")) + continue; + + if (skip_prefix(reader->line, "ACK ", &arg)) { + struct object_id oid; + if (!get_oid_hex(arg, &oid)) { + struct commit *commit; + oidset_insert(common, &oid); + commit = lookup_commit(&oid); + mark_common(commit, 0, 1); + } + continue; + } + + if (!strcmp(reader->line, "ready")) { + clear_prio_queue(&rev_list); + received_ready = 1; + continue; + } + + die("unexpected acknowledgment line: '%s'", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die("error processing acks: %d", reader->status); + + /* return 0 if no common, 1 if there are common, or 2 if ready */ + return received_ready ? 2 : (received_ack ? 1 : 0); +} + +enum fetch_state { + FETCH_CHECK_LOCAL = 0, + FETCH_SEND_REQUEST, + FETCH_PROCESS_ACKS, + FETCH_GET_PACK, + FETCH_DONE, +}; + +static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, + int fd[2], + const struct ref *orig_ref, + struct ref **sought, int nr_sought, + char **pack_lockfile) +{ + struct ref *ref = copy_ref_list(orig_ref); + enum fetch_state state = FETCH_CHECK_LOCAL; + struct oidset common = OIDSET_INIT; + struct packet_reader reader; + int in_vain = 0; + int haves_to_send = INITIAL_FLUSH; + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE); + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_CHECK_LOCAL: + sort_ref_list(&ref, ref_compare_name); + QSORT(sought, nr_sought, cmp_ref_by_name); + + /* v2 supports these by default */ + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; + use_sideband = 2; + + if (marked) + for_each_ref(clear_marks, NULL); + marked = 1; + + for_each_ref(rev_list_insert_ref_oid, NULL); + for_each_cached_alternate(insert_one_alternate_object); + + /* Filter 'ref' by 'sought' and those that aren't local */ + if (everything_local(args, &ref, sought, nr_sought)) + state = FETCH_DONE; + else + state = FETCH_SEND_REQUEST; + break; + case FETCH_SEND_REQUEST: + if (send_fetch_request(fd[1], args, ref, &common, + &haves_to_send, &in_vain)) + state = FETCH_GET_PACK; + else + state = FETCH_PROCESS_ACKS; + break; + case FETCH_PROCESS_ACKS: + /* Process ACKs/NAKs */ + switch (process_acks(&reader, &common)) { + case 2: + state = FETCH_GET_PACK; + break; + case 1: + in_vain = 0; + /* fallthrough */ + default: + state = FETCH_SEND_REQUEST; + break; + } + break; + case FETCH_GET_PACK: + /* get the pack */ + process_section_header(&reader, "packfile", 0); + if (get_pack(args, fd, pack_lockfile)) + die(_("git fetch-pack: fetch failed.")); + + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + oidset_clear(&common); + return ref; +} + static void fetch_pack_config(void) { git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit); @@ -1153,7 +1406,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const char *dest, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile) + char **pack_lockfile, + enum protocol_version version) { struct ref *ref_cpy; struct shallow_info si; @@ -1167,8 +1421,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args, die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); + if (version == protocol_v2) + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, + pack_lockfile); + else + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); reprepare_packed_git(); update_shallow(args, sought, nr_sought, &si); clear_shallow_info(&si); diff --git a/fetch-pack.h b/fetch-pack.h index b6aeb43..7afca73 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -3,6 +3,7 @@ #include "string-list.h" #include "run-command.h" +#include "protocol.h" struct oid_array; @@ -43,7 +44,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile); + char **pack_lockfile, + enum protocol_version version); /* * Print an appropriate error message for each sought ref that wasn't diff --git a/serve.c b/serve.c index 3692dda..a5a7b2f 100644 --- a/serve.c +++ b/serve.c @@ -55,7 +55,7 @@ struct protocol_capability { static struct protocol_capability capabilities[] = { { "agent", agent_advertise, NULL }, { "ls-refs", always_advertise, ls_refs }, - { "fetch", always_advertise, upload_pack_v2 }, + { "fetch", upload_pack_advertise, upload_pack_v2 }, }; static void advertise_capabilities(void) diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 888a700..72d7bc5 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -9,7 +9,7 @@ test_expect_success 'test capability advertisement' ' version 2 agent=git/$(git version | cut -d" " -f3) ls-refs - fetch + fetch=shallow 0000 EOF diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 562610f..4365ac2 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -45,6 +45,56 @@ test_expect_success 'ref advertisment is filtered with ls-remote using protocol test_cmp actual expect ' +test_expect_success 'clone with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "$GIT_DAEMON_URL/parent" daemon_child && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "clone> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C "$daemon_parent" two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + fetch && + + git -C daemon_child log -1 --format=%s origin/master >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'pull with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + pull && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + stop_git_daemon # Test protocol v2 with 'file://' transport @@ -80,4 +130,51 @@ test_expect_success 'ref advertisment is filtered with ls-remote using protocol test_cmp actual expect ' +test_expect_success 'clone with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "file://$(pwd)/file_parent" file_child && + + git -C file_child log -1 --format=%s >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'ref advertisment is filtered during fetch using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent three && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin master && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + ! grep "refs/tags/one" log && + ! grep "refs/tags/two" log && + ! grep "refs/tags/three" log +' + test_done diff --git a/transport.c b/transport.c index 57bdbb5..9152a45 100644 --- a/transport.c +++ b/transport.c @@ -258,14 +258,17 @@ static int fetch_refs_via_pack(struct transport *transport, switch (data->version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); + &transport->pack_lockfile, data->version); break; case protocol_unknown_version: BUG("unknown protocol version"); diff --git a/upload-pack.c b/upload-pack.c index 198e9f0..4c9428c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -710,7 +710,6 @@ static void deepen(int depth, int deepen_relative, } send_unshallow(shallows); - packet_flush(1); } static void deepen_by_rev_list(int ac, const char **av, @@ -722,7 +721,53 @@ static void deepen_by_rev_list(int ac, const char **av, send_shallow(result); free_commit_list(result); send_unshallow(shallows); - packet_flush(1); +} + +/* Returns 1 if a shallow list is sent or 0 otherwise */ +static int send_shallow_list(int depth, int deepen_rev_list, + timestamp_t deepen_since, + struct string_list *deepen_not, + struct object_array *shallows) +{ + int ret = 0; + + if (depth > 0 && deepen_rev_list) + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); + if (depth > 0) { + deepen(depth, deepen_relative, shallows); + ret = 1; + } else if (deepen_rev_list) { + struct argv_array av = ARGV_ARRAY_INIT; + int i; + + argv_array_push(&av, "rev-list"); + if (deepen_since) + argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); + if (deepen_not->nr) { + argv_array_push(&av, "--not"); + for (i = 0; i < deepen_not->nr; i++) { + struct string_list_item *s = deepen_not->items + i; + argv_array_push(&av, s->string); + } + argv_array_push(&av, "--not"); + } + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); + } + deepen_by_rev_list(av.argc, av.argv, shallows); + argv_array_clear(&av); + ret = 1; + } else { + if (shallows->nr > 0) { + int i; + for (i = 0; i < shallows->nr; i++) + register_shallow(&shallows->objects[i].item->oid); + } + } + + shallow_nr += shallows->nr; + return ret; } static int process_shallow(const char *line, struct object_array *shallows) @@ -884,40 +929,10 @@ static void receive_needs(void) if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; - if (depth > 0 && deepen_rev_list) - die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); - if (depth > 0) - deepen(depth, deepen_relative, &shallows); - else if (deepen_rev_list) { - struct argv_array av = ARGV_ARRAY_INIT; - int i; - argv_array_push(&av, "rev-list"); - if (deepen_since) - argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); - if (deepen_not.nr) { - argv_array_push(&av, "--not"); - for (i = 0; i < deepen_not.nr; i++) { - struct string_list_item *s = deepen_not.items + i; - argv_array_push(&av, s->string); - } - argv_array_push(&av, "--not"); - } - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - argv_array_push(&av, oid_to_hex(&o->oid)); - } - deepen_by_rev_list(av.argc, av.argv, &shallows); - argv_array_clear(&av); - } - else - if (shallows.nr > 0) { - int i; - for (i = 0; i < shallows.nr; i++) - register_shallow(&shallows.objects[i].item->oid); - } - - shallow_nr += shallows.nr; + if (send_shallow_list(depth, deepen_rev_list, deepen_since, + &deepen_not, &shallows)) + packet_flush(1); object_array_clear(&shallows); } @@ -1071,6 +1086,13 @@ struct upload_pack_data { struct object_array wants; struct oid_array haves; + struct object_array shallows; + struct string_list deepen_not; + int depth; + timestamp_t deepen_since; + int deepen_rev_list; + int deepen_relative; + unsigned stateless_rpc : 1; unsigned use_thin_pack : 1; @@ -1084,16 +1106,22 @@ static void upload_pack_data_init(struct upload_pack_data *data) { struct object_array wants = OBJECT_ARRAY_INIT; struct oid_array haves = OID_ARRAY_INIT; + struct object_array shallows = OBJECT_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; memset(data, 0, sizeof(*data)); data->wants = wants; data->haves = haves; + data->shallows = shallows; + data->deepen_not = deepen_not; } static void upload_pack_data_clear(struct upload_pack_data *data) { object_array_clear(&data->wants); oid_array_clear(&data->haves); + object_array_clear(&data->shallows); + string_list_clear(&data->deepen_not, 0); } static int parse_want(const char *line) @@ -1177,6 +1205,22 @@ static void process_args(struct packet_reader *request, continue; } + /* Shallow related arguments */ + if (process_shallow(arg, &data->shallows)) + continue; + if (process_deepen(arg, &data->depth)) + continue; + if (process_deepen_since(arg, &data->deepen_since, + &data->deepen_rev_list)) + continue; + if (process_deepen_not(arg, &data->deepen_not, + &data->deepen_rev_list)) + continue; + if (!strcmp(arg, "deepen-relative")) { + data->deepen_relative = 1; + continue; + } + /* ignore unknown lines maybe? */ die("unexpect line: '%s'", arg); } @@ -1272,6 +1316,23 @@ static int process_haves_and_send_acks(struct upload_pack_data *data) return ret; } +static void send_shallow_info(struct upload_pack_data *data) +{ + /* No shallow info needs to be sent */ + if (!data->depth && !data->deepen_rev_list && !data->shallows.nr && + !is_repository_shallow()) + return; + + packet_write_fmt(1, "shallow-info\n"); + + if (!send_shallow_list(data->depth, data->deepen_rev_list, + data->deepen_since, &data->deepen_not, + &data->shallows) && is_repository_shallow()) + deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows); + + packet_delim(1); +} + enum fetch_state { FETCH_PROCESS_ARGS = 0, FETCH_SEND_ACKS, @@ -1319,6 +1380,8 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys, state = FETCH_DONE; break; case FETCH_SEND_PACK: + send_shallow_info(&data); + packet_write_fmt(1, "packfile\n"); create_pack_file(); state = FETCH_DONE; @@ -1331,3 +1394,11 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys, upload_pack_data_clear(&data); return 0; } + +int upload_pack_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, "shallow"); + return 1; +} diff --git a/upload-pack.h b/upload-pack.h index 5bbdb7e..cab2178 100644 --- a/upload-pack.h +++ b/upload-pack.h @@ -16,4 +16,8 @@ struct packet_reader; extern int upload_pack_v2(struct repository *r, struct argv_array *keys, struct packet_reader *request); +struct strbuf; +extern int upload_pack_advertise(struct repository *r, + struct strbuf *value); + #endif /* UPLOAD_PACK_H */ -- cgit v0.10.2-6-g49f6 From f7e205010542dc9b712473d260058e43ca2b26f7 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:29 -0700 Subject: fetch-pack: support shallow requests Enable shallow clones and deepen requests using protocol version 2 if the server 'fetch' command supports the 'shallow' feature. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 4f7f251..136179d 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -255,6 +255,10 @@ A `fetch` request can take the following arguments: to its base by position in pack rather than by an oid. That is, they can read OBJ_OFS_DELTA (ake type 6) in a packfile. +If the 'shallow' feature is advertised the following arguments can be +included in the clients request as well as the potential addition of the +'shallow-info' section in the server's response as explained below. + shallow A client must notify the server of all commits for which it only has shallow copies (meaning that it doesn't have the parents of @@ -338,13 +342,13 @@ header. further negotiation is needed. shallow-info section - If the client has requested a shallow fetch/clone, a shallow - client requests a fetch or the server is shallow then the - server's response may include a shallow-info section. The - shallow-info section will be included if (due to one of the - above conditions) the server needs to inform the client of any - shallow boundaries or adjustments to the clients already - existing shallow boundaries. + * If the client has requested a shallow fetch/clone, a shallow + client requests a fetch or the server is shallow then the + server's response may include a shallow-info section. The + shallow-info section will be included if (due to one of the + above conditions) the server needs to inform the client of any + shallow boundaries or adjustments to the clients already + existing shallow boundaries. * Always begins with the section header "shallow-info" diff --git a/connect.c b/connect.c index e42d779..5bb9d34 100644 --- a/connect.c +++ b/connect.c @@ -82,6 +82,28 @@ int server_supports_v2(const char *c, int die_on_error) return 0; } +int server_supports_feature(const char *c, const char *feature, + int die_on_error) +{ + int i; + + for (i = 0; i < server_capabilities_v2.argc; i++) { + const char *out; + if (skip_prefix(server_capabilities_v2.argv[i], c, &out) && + (!*out || *(out++) == '=')) { + if (parse_feature_request(out, feature)) + return 1; + else + break; + } + } + + if (die_on_error) + die("server doesn't support feature '%s'", feature); + + return 0; +} + static void process_capabilities_v2(struct packet_reader *reader) { while (packet_reader_read(reader) == PACKET_READ_NORMAL) diff --git a/connect.h b/connect.h index 8898d44..0e69c67 100644 --- a/connect.h +++ b/connect.h @@ -17,5 +17,7 @@ struct packet_reader; extern enum protocol_version discover_version(struct packet_reader *reader); extern int server_supports_v2(const char *c, int die_on_error); +extern int server_supports_feature(const char *c, const char *feature, + int die_on_error); #endif diff --git a/fetch-pack.c b/fetch-pack.c index dffcfd6..837e1fd 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1008,6 +1008,26 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, return ref; } +static void add_shallow_requests(struct strbuf *req_buf, + const struct fetch_pack_args *args) +{ + if (is_repository_shallow()) + write_shallow_commits(req_buf, 1, NULL); + if (args->depth > 0) + packet_buf_write(req_buf, "deepen %d", args->depth); + if (args->deepen_since) { + timestamp_t max_age = approxidate(args->deepen_since); + packet_buf_write(req_buf, "deepen-since %"PRItime, max_age); + } + if (args->deepen_not) { + int i; + for (i = 0; i < args->deepen_not->nr; i++) { + struct string_list_item *s = args->deepen_not->items + i; + packet_buf_write(req_buf, "deepen-not %s", s->string); + } + } +} + static void add_wants(const struct ref *wants, struct strbuf *req_buf) { for ( ; wants ; wants = wants->next) { @@ -1093,6 +1113,12 @@ static int send_fetch_request(int fd_out, const struct fetch_pack_args *args, if (prefer_ofs_delta) packet_buf_write(&req_buf, "ofs-delta"); + /* Add shallow-info and deepen request */ + if (server_supports_feature("fetch", "shallow", 0)) + add_shallow_requests(&req_buf, args); + else if (is_repository_shallow() || args->deepen) + die(_("Server does not support shallow requests")); + /* add wants */ add_wants(wants, &req_buf); @@ -1122,7 +1148,7 @@ static int process_section_header(struct packet_reader *reader, int ret; if (packet_reader_peek(reader) != PACKET_READ_NORMAL) - die("error reading packet"); + die("error reading section header '%s'", section); ret = !strcmp(reader->line, section); @@ -1177,6 +1203,43 @@ static int process_acks(struct packet_reader *reader, struct oidset *common) return received_ready ? 2 : (received_ack ? 1 : 0); } +static void receive_shallow_info(struct fetch_pack_args *args, + struct packet_reader *reader) +{ + process_section_header(reader, "shallow-info", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + struct object_id oid; + + if (skip_prefix(reader->line, "shallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid shallow line: %s"), reader->line); + register_shallow(&oid); + continue; + } + if (skip_prefix(reader->line, "unshallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid unshallow line: %s"), reader->line); + if (!lookup_object(oid.hash)) + die(_("object not found: %s"), reader->line); + /* make sure that it is parsed as shallow */ + if (!parse_object(&oid)) + die(_("error in object: %s"), reader->line); + if (unregister_shallow(&oid)) + die(_("no shallow found: %s"), reader->line); + continue; + } + die(_("expected shallow/unshallow, got %s"), reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die("error processing shallow info: %d", reader->status); + + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); + args->deepen = 1; +} + enum fetch_state { FETCH_CHECK_LOCAL = 0, FETCH_SEND_REQUEST, @@ -1209,6 +1272,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, /* v2 supports these by default */ allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; use_sideband = 2; + if (args->depth > 0 || args->deepen_since || args->deepen_not) + args->deepen = 1; if (marked) for_each_ref(clear_marks, NULL); @@ -1245,6 +1310,10 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, } break; case FETCH_GET_PACK: + /* Check for shallow-info section */ + if (process_section_header(&reader, "shallow-info", 1)) + receive_shallow_info(args, &reader); + /* get the pack */ process_section_header(&reader, "packfile", 0); if (get_pack(args, fd, pack_lockfile)) -- cgit v0.10.2-6-g49f6 From 40fc51e39f178e81cbd7feadda51fca604f02ea8 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:30 -0700 Subject: connect: refactor git_connect to only get the protocol version once Instead of having each builtin transport asking for which protocol version the user has configured in 'protocol.version' by calling `get_protocol_version_config()` multiple times, factor this logic out so there is just a single call at the beginning of `git_connect()`. This will be helpful in the next patch where we can have centralized logic which determines if we need to request a different protocol version than what the user has configured. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/connect.c b/connect.c index 5bb9d34..a57a060 100644 --- a/connect.c +++ b/connect.c @@ -1035,6 +1035,7 @@ static enum ssh_variant determine_ssh_variant(const char *ssh_command, */ static struct child_process *git_connect_git(int fd[2], char *hostandport, const char *path, const char *prog, + enum protocol_version version, int flags) { struct child_process *conn; @@ -1073,10 +1074,10 @@ static struct child_process *git_connect_git(int fd[2], char *hostandport, target_host, 0); /* If using a new version put that stuff here after a second null byte */ - if (get_protocol_version_config() > 0) { + if (version > 0) { strbuf_addch(&request, '\0'); strbuf_addf(&request, "version=%d%c", - get_protocol_version_config(), '\0'); + version, '\0'); } packet_write(fd[1], request.buf, request.len); @@ -1092,14 +1093,14 @@ static struct child_process *git_connect_git(int fd[2], char *hostandport, */ static void push_ssh_options(struct argv_array *args, struct argv_array *env, enum ssh_variant variant, const char *port, - int flags) + enum protocol_version version, int flags) { if (variant == VARIANT_SSH && - get_protocol_version_config() > 0) { + version > 0) { argv_array_push(args, "-o"); argv_array_push(args, "SendEnv=" GIT_PROTOCOL_ENVIRONMENT); argv_array_pushf(env, GIT_PROTOCOL_ENVIRONMENT "=version=%d", - get_protocol_version_config()); + version); } if (flags & CONNECT_IPV4) { @@ -1152,7 +1153,8 @@ static void push_ssh_options(struct argv_array *args, struct argv_array *env, /* Prepare a child_process for use by Git's SSH-tunneled transport. */ static void fill_ssh_args(struct child_process *conn, const char *ssh_host, - const char *port, int flags) + const char *port, enum protocol_version version, + int flags) { const char *ssh; enum ssh_variant variant; @@ -1186,14 +1188,14 @@ static void fill_ssh_args(struct child_process *conn, const char *ssh_host, argv_array_push(&detect.args, ssh); argv_array_push(&detect.args, "-G"); push_ssh_options(&detect.args, &detect.env_array, - VARIANT_SSH, port, flags); + VARIANT_SSH, port, version, flags); argv_array_push(&detect.args, ssh_host); variant = run_command(&detect) ? VARIANT_SIMPLE : VARIANT_SSH; } argv_array_push(&conn->args, ssh); - push_ssh_options(&conn->args, &conn->env_array, variant, port, flags); + push_ssh_options(&conn->args, &conn->env_array, variant, port, version, flags); argv_array_push(&conn->args, ssh_host); } @@ -1214,6 +1216,7 @@ struct child_process *git_connect(int fd[2], const char *url, char *hostandport, *path; struct child_process *conn; enum protocol protocol; + enum protocol_version version = get_protocol_version_config(); /* Without this we cannot rely on waitpid() to tell * what happened to our children. @@ -1228,7 +1231,7 @@ struct child_process *git_connect(int fd[2], const char *url, printf("Diag: path=%s\n", path ? path : "NULL"); conn = NULL; } else if (protocol == PROTO_GIT) { - conn = git_connect_git(fd, hostandport, path, prog, flags); + conn = git_connect_git(fd, hostandport, path, prog, version, flags); } else { struct strbuf cmd = STRBUF_INIT; const char *const *var; @@ -1271,12 +1274,12 @@ struct child_process *git_connect(int fd[2], const char *url, strbuf_release(&cmd); return NULL; } - fill_ssh_args(conn, ssh_host, port, flags); + fill_ssh_args(conn, ssh_host, port, version, flags); } else { transport_check_allowed("file"); - if (get_protocol_version_config() > 0) { + if (version > 0) { argv_array_pushf(&conn->env_array, GIT_PROTOCOL_ENVIRONMENT "=version=%d", - get_protocol_version_config()); + version); } } argv_array_push(&conn->args, cmd.buf); -- cgit v0.10.2-6-g49f6 From 1aa8dded3afff28d8f4c24a97b237a0d9e633173 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:31 -0700 Subject: connect: don't request v2 when pushing In order to be able to ship protocol v2 with only supporting fetch, we need clients to not issue a request to use protocol v2 when pushing (since the client currently doesn't know how to push using protocol v2). This allows a client to have protocol v2 configured in `protocol.version` and take advantage of using v2 for fetch and falling back to using v0 when pushing while v2 for push is being designed. We could run into issues if we didn't fall back to protocol v2 when pushing right now. This is because currently a server will ignore a request to use v2 when contacting the 'receive-pack' endpoint and fall back to using v0, but when push v2 is rolled out to servers, the 'receive-pack' endpoint will start responding using v2. So we don't want to get into a state where a client is requesting to push with v2 before they actually know how to push using v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/connect.c b/connect.c index a57a060..5497116 100644 --- a/connect.c +++ b/connect.c @@ -1218,6 +1218,14 @@ struct child_process *git_connect(int fd[2], const char *url, enum protocol protocol; enum protocol_version version = get_protocol_version_config(); + /* + * NEEDSWORK: If we are trying to use protocol v2 and we are planning + * to perform a push, then fallback to v0 since the client doesn't know + * how to push yet using v2. + */ + if (version == protocol_v2 && !strcmp("git-receive-pack", prog)) + version = protocol_v0; + /* Without this we cannot rely on waitpid() to tell * what happened to our children. */ diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 4365ac2..e3a7c09 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -95,6 +95,30 @@ test_expect_success 'pull with git:// using protocol v2' ' grep "fetch< version 2" log ' +test_expect_success 'push with git:// and a config of v2 does not request v2' ' + test_when_finished "rm -f log" && + + # Till v2 for push is designed, make sure that if a client has + # protocol.version configured to use v2, that the client instead falls + # back and uses v0. + + test_commit -C daemon_child three && + + # Push to another branch, as the target repository has the + # master branch checked out and we cannot push into it. + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + push origin HEAD:client_branch && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s client_branch >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + ! grep "push> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + ! grep "push< version 2" log +' + stop_git_daemon # Test protocol v2 with 'file://' transport -- cgit v0.10.2-6-g49f6 From b1c2edfc18028169c5e64a8320b0877cdcbf3397 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:32 -0700 Subject: transport-helper: remove name parameter Commit 266f1fdfa (transport-helper: be quiet on read errors from helpers, 2013-06-21) removed a call to 'die()' which printed the name of the remote helper passed in to the 'recvline_fh()' function using the 'name' parameter. Once the call to 'die()' was removed the parameter was no longer necessary but wasn't removed. Clean up 'recvline_fh()' parameter list by removing the 'name' parameter. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport-helper.c b/transport-helper.c index 8774ab3..9677ead 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -49,7 +49,7 @@ static void sendline(struct helper_data *helper, struct strbuf *buffer) die_errno("Full write to remote helper failed"); } -static int recvline_fh(FILE *helper, struct strbuf *buffer, const char *name) +static int recvline_fh(FILE *helper, struct strbuf *buffer) { strbuf_reset(buffer); if (debug) @@ -67,7 +67,7 @@ static int recvline_fh(FILE *helper, struct strbuf *buffer, const char *name) static int recvline(struct helper_data *helper, struct strbuf *buffer) { - return recvline_fh(helper->out, buffer, helper->name); + return recvline_fh(helper->out, buffer); } static void write_constant(int fd, const char *str) @@ -586,7 +586,7 @@ static int process_connect_service(struct transport *transport, goto exit; sendline(data, &cmdbuf); - if (recvline_fh(input, &cmdbuf, name)) + if (recvline_fh(input, &cmdbuf)) exit(128); if (!strcmp(cmdbuf.buf, "")) { -- cgit v0.10.2-6-g49f6 From 176e85c1b4bd055633e01dd7bb717191d369e8a1 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:33 -0700 Subject: transport-helper: refactor process_connect_service A future patch will need to take advantage of the logic which runs and processes the response of the connect command on a remote helper so factor out this logic from 'process_connect_service()' and place it into a helper function 'run_connect()'. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/transport-helper.c b/transport-helper.c index 9677ead..830f21f 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -545,14 +545,13 @@ static int fetch_with_import(struct transport *transport, return 0; } -static int process_connect_service(struct transport *transport, - const char *name, const char *exec) +static int run_connect(struct transport *transport, struct strbuf *cmdbuf) { struct helper_data *data = transport->data; - struct strbuf cmdbuf = STRBUF_INIT; - struct child_process *helper; - int r, duped, ret = 0; + int ret = 0; + int duped; FILE *input; + struct child_process *helper; helper = get_helper(transport); @@ -568,44 +567,54 @@ static int process_connect_service(struct transport *transport, input = xfdopen(duped, "r"); setvbuf(input, NULL, _IONBF, 0); + sendline(data, cmdbuf); + if (recvline_fh(input, cmdbuf)) + exit(128); + + if (!strcmp(cmdbuf->buf, "")) { + data->no_disconnect_req = 1; + if (debug) + fprintf(stderr, "Debug: Smart transport connection " + "ready.\n"); + ret = 1; + } else if (!strcmp(cmdbuf->buf, "fallback")) { + if (debug) + fprintf(stderr, "Debug: Falling back to dumb " + "transport.\n"); + } else { + die("Unknown response to connect: %s", + cmdbuf->buf); + } + + fclose(input); + return ret; +} + +static int process_connect_service(struct transport *transport, + const char *name, const char *exec) +{ + struct helper_data *data = transport->data; + struct strbuf cmdbuf = STRBUF_INIT; + int ret = 0; + /* * Handle --upload-pack and friends. This is fire and forget... * just warn if it fails. */ if (strcmp(name, exec)) { - r = set_helper_option(transport, "servpath", exec); + int r = set_helper_option(transport, "servpath", exec); if (r > 0) warning("Setting remote service path not supported by protocol."); else if (r < 0) warning("Invalid remote service path."); } - if (data->connect) + if (data->connect) { strbuf_addf(&cmdbuf, "connect %s\n", name); - else - goto exit; - - sendline(data, &cmdbuf); - if (recvline_fh(input, &cmdbuf)) - exit(128); - - if (!strcmp(cmdbuf.buf, "")) { - data->no_disconnect_req = 1; - if (debug) - fprintf(stderr, "Debug: Smart transport connection " - "ready.\n"); - ret = 1; - } else if (!strcmp(cmdbuf.buf, "fallback")) { - if (debug) - fprintf(stderr, "Debug: Falling back to dumb " - "transport.\n"); - } else - die("Unknown response to connect: %s", - cmdbuf.buf); + ret = run_connect(transport, &cmdbuf); + } -exit: strbuf_release(&cmdbuf); - fclose(input); return ret; } -- cgit v0.10.2-6-g49f6 From edc9caf7e2ecd3d5327e78a3e539eda61a0e4d81 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:34 -0700 Subject: transport-helper: introduce stateless-connect Introduce the transport-helper capability 'stateless-connect'. This capability indicates that the transport-helper can be requested to run the 'stateless-connect' command which should attempt to make a stateless connection with a remote end. Once established, the connection can be used by the git client to communicate with the remote end natively in a stateless-rpc manner as supported by protocol v2. This means that the client must send everything the server needs in a single request as the client must not assume any state-storing on the part of the server or transport. If a stateless connection cannot be established then the remote-helper will respond in the same manner as the 'connect' command indicating that the client should fallback to using the dumb remote-helper commands. A future patch will implement the 'stateless-connect' capability in our http remote-helper (remote-curl) so that protocol v2 can be used using the http transport. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt index 4a584f3..cd9b34d 100644 --- a/Documentation/gitremote-helpers.txt +++ b/Documentation/gitremote-helpers.txt @@ -102,6 +102,14 @@ Capabilities for Pushing + Supported commands: 'connect'. +'stateless-connect':: + Experimental; for internal use only. + Can attempt to connect to a remote server for communication + using git's wire-protocol version 2. See the documentation + for the stateless-connect command for more information. ++ +Supported commands: 'stateless-connect'. + 'push':: Can discover remote refs and push local commits and the history leading up to them to new or existing remote refs. @@ -136,6 +144,14 @@ Capabilities for Fetching + Supported commands: 'connect'. +'stateless-connect':: + Experimental; for internal use only. + Can attempt to connect to a remote server for communication + using git's wire-protocol version 2. See the documentation + for the stateless-connect command for more information. ++ +Supported commands: 'stateless-connect'. + 'fetch':: Can discover remote refs and transfer objects reachable from them to the local object store. @@ -375,6 +391,22 @@ Supported if the helper has the "export" capability. + Supported if the helper has the "connect" capability. +'stateless-connect' :: + Experimental; for internal use only. + Connects to the given remote service for communication using + git's wire-protocol version 2. Valid replies to this command + are empty line (connection established), 'fallback' (no smart + transport support, fall back to dumb transports) and just + exiting with error message printed (can't connect, don't bother + trying to fall back). After line feed terminating the positive + (empty) response, the output of the service starts. Messages + (both request and response) must consist of zero or more + PKT-LINEs, terminating in a flush packet. The client must not + expect the server to store any state in between request-response + pairs. After the connection ends, the remote helper exits. ++ +Supported if the helper has the "stateless-connect" capability. + If a fatal error occurs, the program writes the error message to stderr and exits. The caller should expect that a suitable error message has been printed if the child closes the connection without diff --git a/transport-helper.c b/transport-helper.c index 830f21f..aecbc4a 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -12,6 +12,7 @@ #include "argv-array.h" #include "refs.h" #include "transport-internal.h" +#include "protocol.h" static int debug; @@ -26,6 +27,7 @@ struct helper_data { option : 1, push : 1, connect : 1, + stateless_connect : 1, signed_tags : 1, check_connectivity : 1, no_disconnect_req : 1, @@ -188,6 +190,8 @@ static struct child_process *get_helper(struct transport *transport) refspecs[refspec_nr++] = xstrdup(arg); } else if (!strcmp(capname, "connect")) { data->connect = 1; + } else if (!strcmp(capname, "stateless-connect")) { + data->stateless_connect = 1; } else if (!strcmp(capname, "signed-tags")) { data->signed_tags = 1; } else if (skip_prefix(capname, "export-marks ", &arg)) { @@ -612,6 +616,13 @@ static int process_connect_service(struct transport *transport, if (data->connect) { strbuf_addf(&cmdbuf, "connect %s\n", name); ret = run_connect(transport, &cmdbuf); + } else if (data->stateless_connect && + (get_protocol_version_config() == protocol_v2) && + !strcmp("git-upload-pack", name)) { + strbuf_addf(&cmdbuf, "stateless-connect %s\n", name); + ret = run_connect(transport, &cmdbuf); + if (ret) + transport->stateless_rpc = 1; } strbuf_release(&cmdbuf); diff --git a/transport.c b/transport.c index 9152a45..342db49 100644 --- a/transport.c +++ b/transport.c @@ -252,6 +252,7 @@ static int fetch_refs_via_pack(struct transport *transport, data->options.check_self_contained_and_connected; args.cloning = transport->cloning; args.update_shallow = data->options.update_shallow; + args.stateless_rpc = transport->stateless_rpc; if (!data->got_remote_heads) refs_tmp = get_refs_via_connect(transport, 0, NULL); diff --git a/transport.h b/transport.h index 83992a42..0e602d4 100644 --- a/transport.h +++ b/transport.h @@ -56,6 +56,12 @@ struct transport { unsigned cloning : 1; /* + * Indicates that the transport is connected via a half-duplex + * connection and should operate in stateless-rpc mode. + */ + unsigned stateless_rpc : 1; + + /* * These strings will be passed to the {pre, post}-receive hook, * on the remote side, if both sides support the push options capability. */ -- cgit v0.10.2-6-g49f6 From f1f4d8acf400912a9e2a7b97080afced56719c29 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:35 -0700 Subject: pkt-line: add packet_buf_write_len function Add the 'packet_buf_write_len()' function which allows for writing an arbitrary length buffer into a 'struct strbuf' and formatting it in packet-line format. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/pkt-line.c b/pkt-line.c index 7296731..555eb2a 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -215,6 +215,22 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) va_end(args); } +void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len) +{ + size_t orig_len, n; + + orig_len = buf->len; + strbuf_addstr(buf, "0000"); + strbuf_add(buf, data, len); + n = buf->len - orig_len; + + if (n > LARGE_PACKET_MAX) + die("protocol error: impossibly long line"); + + set_packet_header(&buf->buf[orig_len], n); + packet_trace(data, len, 1); +} + int write_packetized_from_fd(int fd_in, int fd_out) { static char buf[LARGE_PACKET_DATA_MAX]; diff --git a/pkt-line.h b/pkt-line.h index 9570bd7..5b28d43 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -26,6 +26,7 @@ void packet_buf_flush(struct strbuf *buf); void packet_buf_delim(struct strbuf *buf); void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int write_packetized_from_fd(int fd_in, int fd_out); -- cgit v0.10.2-6-g49f6 From f08a5d42ea0567af12e703948b6859e30129ec1c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:36 -0700 Subject: remote-curl: create copy of the service name Make a copy of the service name being requested instead of relying on the buffer pointed to by the passed in 'const char *' to remain unchanged. Currently, all service names are string constants, but a subsequent patch will introduce service names from external sources. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/remote-curl.c b/remote-curl.c index dae8a4a..4086aa7 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -165,7 +165,7 @@ static int set_option(const char *name, const char *value) } struct discovery { - const char *service; + char *service; char *buf_alloc; char *buf; size_t len; @@ -257,6 +257,7 @@ static void free_discovery(struct discovery *d) free(d->shallow.oid); free(d->buf_alloc); free_refs(d->refs); + free(d->service); free(d); } } @@ -343,7 +344,7 @@ static struct discovery *discover_refs(const char *service, int for_push) warning(_("redirecting to %s"), url.buf); last= xcalloc(1, sizeof(*last_discovery)); - last->service = service; + last->service = xstrdup(service); last->buf_alloc = strbuf_detach(&buffer, &last->len); last->buf = last->buf_alloc; -- cgit v0.10.2-6-g49f6 From 49e85e95006e6a8e544372e529834319fcb0987c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:37 -0700 Subject: remote-curl: store the protocol version the server responded with Store the protocol version the server responded with when performing discovery. This will be used in a future patch to either change the 'Git-Protocol' header sent in subsequent requests or to determine if a client needs to fallback to using a different protocol version. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/remote-curl.c b/remote-curl.c index 4086aa7..c540358 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -171,6 +171,7 @@ struct discovery { size_t len; struct ref *refs; struct oid_array shallow; + enum protocol_version version; unsigned proto_git : 1; }; static struct discovery *last_discovery; @@ -184,7 +185,8 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) PACKET_READ_CHOMP_NEWLINE | PACKET_READ_GENTLE_ON_EOF); - switch (discover_version(&reader)) { + heads->version = discover_version(&reader); + switch (heads->version) { case protocol_v2: die("support for protocol v2 not implemented yet"); break; -- cgit v0.10.2-6-g49f6 From 8ff14ed4127decbee3116aba59d7f8f897c4fe3b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:38 -0700 Subject: http: allow providing extra headers for http requests Add a way for callers to request that extra headers be included when making http requests. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/http.c b/http.c index 32d3326..a4c16a9 100644 --- a/http.c +++ b/http.c @@ -1723,6 +1723,14 @@ static int http_request(const char *url, headers = curl_slist_append(headers, buf.buf); + /* Add additional headers here */ + if (options && options->extra_headers) { + const struct string_list_item *item; + for_each_string_list_item(item, options->extra_headers) { + headers = curl_slist_append(headers, item->string); + } + } + curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip"); diff --git a/http.h b/http.h index f7bd3b2..4df4a25 100644 --- a/http.h +++ b/http.h @@ -172,6 +172,13 @@ struct http_get_options { * for details. */ struct strbuf *base_url; + + /* + * If not NULL, contains additional HTTP headers to be sent with the + * request. The strings in the list must not be freed until after the + * request has completed. + */ + struct string_list *extra_headers; }; /* Return values for http_get_*() */ -- cgit v0.10.2-6-g49f6 From 884e586f9ef46406263720523377298a90b41065 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:39 -0700 Subject: http: don't always add Git-Protocol header Instead of always sending the Git-Protocol header with the configured version with every http request, explicitly send it when discovering refs and then only send it on subsequent http requests if the server understood the version requested. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/http.c b/http.c index a4c16a9..531c59d 100644 --- a/http.c +++ b/http.c @@ -904,21 +904,6 @@ static void set_from_env(const char **var, const char *envname) *var = val; } -static void protocol_http_header(void) -{ - if (get_protocol_version_config() > 0) { - struct strbuf protocol_header = STRBUF_INIT; - - strbuf_addf(&protocol_header, GIT_PROTOCOL_HEADER ": version=%d", - get_protocol_version_config()); - - - extra_http_headers = curl_slist_append(extra_http_headers, - protocol_header.buf); - strbuf_release(&protocol_header); - } -} - void http_init(struct remote *remote, const char *url, int proactive_auth) { char *low_speed_limit; @@ -949,8 +934,6 @@ void http_init(struct remote *remote, const char *url, int proactive_auth) if (remote) var_override(&http_proxy_authmethod, remote->http_proxy_authmethod); - protocol_http_header(); - pragma_header = curl_slist_append(http_copy_default_headers(), "Pragma: no-cache"); no_pragma_header = curl_slist_append(http_copy_default_headers(), diff --git a/remote-curl.c b/remote-curl.c index c540358..b4e9db8 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -291,6 +291,19 @@ static int show_http_message(struct strbuf *type, struct strbuf *charset, return 0; } +static int get_protocol_http_header(enum protocol_version version, + struct strbuf *header) +{ + if (version > 0) { + strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d", + version); + + return 1; + } + + return 0; +} + static struct discovery *discover_refs(const char *service, int for_push) { struct strbuf exp = STRBUF_INIT; @@ -299,6 +312,8 @@ static struct discovery *discover_refs(const char *service, int for_push) struct strbuf buffer = STRBUF_INIT; struct strbuf refs_url = STRBUF_INIT; struct strbuf effective_url = STRBUF_INIT; + struct strbuf protocol_header = STRBUF_INIT; + struct string_list extra_headers = STRING_LIST_INIT_DUP; struct discovery *last = last_discovery; int http_ret, maybe_smart = 0; struct http_get_options http_options; @@ -318,11 +333,16 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_addf(&refs_url, "service=%s", service); } + /* Add the extra Git-Protocol header */ + if (get_protocol_http_header(get_protocol_version_config(), &protocol_header)) + string_list_append(&extra_headers, protocol_header.buf); + memset(&http_options, 0, sizeof(http_options)); http_options.content_type = &type; http_options.charset = &charset; http_options.effective_url = &effective_url; http_options.base_url = &url; + http_options.extra_headers = &extra_headers; http_options.initial_request = 1; http_options.no_cache = 1; http_options.keep_error = 1; @@ -389,6 +409,8 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_release(&charset); strbuf_release(&effective_url); strbuf_release(&buffer); + strbuf_release(&protocol_header); + string_list_clear(&extra_headers, 0); last_discovery = last; return last; } @@ -425,6 +447,7 @@ struct rpc_state { char *service_url; char *hdr_content_type; char *hdr_accept; + char *protocol_header; char *buf; size_t alloc; size_t len; @@ -611,6 +634,10 @@ static int post_rpc(struct rpc_state *rpc) headers = curl_slist_append(headers, needs_100_continue ? "Expect: 100-continue" : "Expect:"); + /* Add the extra Git-Protocol header */ + if (rpc->protocol_header) + headers = curl_slist_append(headers, rpc->protocol_header); + retry: slot = get_active_slot(); @@ -751,6 +778,11 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) strbuf_addf(&buf, "Accept: application/x-%s-result", svc); rpc->hdr_accept = strbuf_detach(&buf, NULL); + if (get_protocol_http_header(heads->version, &buf)) + rpc->protocol_header = strbuf_detach(&buf, NULL); + else + rpc->protocol_header = NULL; + while (!err) { int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); if (!n) @@ -778,6 +810,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) free(rpc->service_url); free(rpc->hdr_content_type); free(rpc->hdr_accept); + free(rpc->protocol_header); free(rpc->buf); strbuf_release(&buf); return err; -- cgit v0.10.2-6-g49f6 From 237ffedd46098a5b8a303cfbac7ecb0ff3a8a7a7 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:40 -0700 Subject: http: eliminate "# service" line when using protocol v2 When an http info/refs request is made, requesting that protocol v2 be used, don't send a "# service" line since this line is not part of the v2 spec. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/http-backend.c b/http-backend.c index f3dc218..5d241e9 100644 --- a/http-backend.c +++ b/http-backend.c @@ -10,6 +10,7 @@ #include "url.h" #include "argv-array.h" #include "packfile.h" +#include "protocol.h" static const char content_type[] = "Content-Type"; static const char content_length[] = "Content-Length"; @@ -466,8 +467,11 @@ static void get_info_refs(struct strbuf *hdr, char *arg) hdr_str(hdr, content_type, buf.buf); end_headers(hdr); - packet_write_fmt(1, "# service=git-%s\n", svc->name); - packet_flush(1); + + if (determine_protocol_version_server() != protocol_v2) { + packet_write_fmt(1, "# service=git-%s\n", svc->name); + packet_flush(1); + } argv[0] = svc->name; run_service(argv, 0); diff --git a/remote-curl.c b/remote-curl.c index b4e9db8..66a53f7 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -396,6 +396,9 @@ static struct discovery *discover_refs(const char *service, int for_push) ; last->proto_git = 1; + } else if (maybe_smart && + last->len > 5 && starts_with(last->buf + 4, "version 2")) { + last->proto_git = 1; } if (last->proto_git) -- cgit v0.10.2-6-g49f6 From 0f1dc53f45d307c8342bf547d79695b614cf9c26 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:41 -0700 Subject: remote-curl: implement stateless-connect command Teach remote-curl the 'stateless-connect' command which is used to establish a stateless connection with servers which support protocol version 2. This allows remote-curl to act as a proxy, allowing the git client to communicate natively with a remote end, simply using remote-curl as a pass through to convert requests to http. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/remote-curl.c b/remote-curl.c index 66a53f7..87f5b77 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -188,7 +188,12 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) heads->version = discover_version(&reader); switch (heads->version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + /* + * Do nothing. This isn't a list of refs but rather a + * capability advertisement. Client would have run + * 'stateless-connect' so we'll dump this capability listing + * and let them request the refs themselves. + */ break; case protocol_v1: case protocol_v0: @@ -1085,6 +1090,202 @@ static void parse_push(struct strbuf *buf) free(specs); } +/* + * Used to represent the state of a connection to an HTTP server when + * communicating using git's wire-protocol version 2. + */ +struct proxy_state { + char *service_name; + char *service_url; + struct curl_slist *headers; + struct strbuf request_buffer; + int in; + int out; + struct packet_reader reader; + size_t pos; + int seen_flush; +}; + +static void proxy_state_init(struct proxy_state *p, const char *service_name, + enum protocol_version version) +{ + struct strbuf buf = STRBUF_INIT; + + memset(p, 0, sizeof(*p)); + p->service_name = xstrdup(service_name); + + p->in = 0; + p->out = 1; + strbuf_init(&p->request_buffer, 0); + + strbuf_addf(&buf, "%s%s", url.buf, p->service_name); + p->service_url = strbuf_detach(&buf, NULL); + + p->headers = http_copy_default_headers(); + + strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked"); + + /* Add the Git-Protocol header */ + if (get_protocol_http_header(version, &buf)) + p->headers = curl_slist_append(p->headers, buf.buf); + + packet_reader_init(&p->reader, p->in, NULL, 0, + PACKET_READ_GENTLE_ON_EOF); + + strbuf_release(&buf); +} + +static void proxy_state_clear(struct proxy_state *p) +{ + free(p->service_name); + free(p->service_url); + curl_slist_free_all(p->headers); + strbuf_release(&p->request_buffer); +} + +/* + * CURLOPT_READFUNCTION callback function. + * Attempts to copy over a single packet-line at a time into the + * curl provided buffer. + */ +static size_t proxy_in(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t max; + struct proxy_state *p = userdata; + size_t avail = p->request_buffer.len - p->pos; + + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + max = nmemb; + + if (!avail) { + if (p->seen_flush) { + p->seen_flush = 0; + return 0; + } + + strbuf_reset(&p->request_buffer); + switch (packet_reader_read(&p->reader)) { + case PACKET_READ_EOF: + die("unexpected EOF when reading from parent process"); + case PACKET_READ_NORMAL: + packet_buf_write_len(&p->request_buffer, p->reader.line, + p->reader.pktlen); + break; + case PACKET_READ_DELIM: + packet_buf_delim(&p->request_buffer); + break; + case PACKET_READ_FLUSH: + packet_buf_flush(&p->request_buffer); + p->seen_flush = 1; + break; + } + p->pos = 0; + avail = p->request_buffer.len; + } + + if (max < avail) + avail = max; + memcpy(buffer, p->request_buffer.buf + p->pos, avail); + p->pos += avail; + return avail; +} + +static size_t proxy_out(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t size; + struct proxy_state *p = userdata; + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + size = nmemb; + + write_or_die(p->out, buffer, size); + return size; +} + +/* Issues a request to the HTTP server configured in `p` */ +static int proxy_request(struct proxy_state *p) +{ + struct active_request_slot *slot; + + slot = get_active_slot(); + + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(slot->curl, CURLOPT_POST, 1); + curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers); + + /* Setup function to read request from client */ + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in); + curl_easy_setopt(slot->curl, CURLOPT_READDATA, p); + + /* Setup function to write server response to client */ + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p); + + if (run_slot(slot, NULL) != HTTP_OK) + return -1; + + return 0; +} + +static int stateless_connect(const char *service_name) +{ + struct discovery *discover; + struct proxy_state p; + + /* + * Run the info/refs request and see if the server supports protocol + * v2. If and only if the server supports v2 can we successfully + * establish a stateless connection, otherwise we need to tell the + * client to fallback to using other transport helper functions to + * complete their request. + */ + discover = discover_refs(service_name, 0); + if (discover->version != protocol_v2) { + printf("fallback\n"); + fflush(stdout); + return -1; + } else { + /* Stateless Connection established */ + printf("\n"); + fflush(stdout); + } + + proxy_state_init(&p, service_name, discover->version); + + /* + * Dump the capability listing that we got from the server earlier + * during the info/refs request. + */ + write_or_die(p.out, discover->buf, discover->len); + + /* Peek the next packet line. Until we see EOF keep sending POSTs */ + while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) { + if (proxy_request(&p)) { + /* We would have an err here */ + break; + } + } + + proxy_state_clear(&p); + return 0; +} + int cmd_main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; @@ -1153,12 +1354,16 @@ int cmd_main(int argc, const char **argv) fflush(stdout); } else if (!strcmp(buf.buf, "capabilities")) { + printf("stateless-connect\n"); printf("fetch\n"); printf("option\n"); printf("push\n"); printf("check-connectivity\n"); printf("\n"); fflush(stdout); + } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) { + if (!stateless_connect(arg)) + break; } else { error("remote-curl: unknown command '%s' from git", buf.buf); return 1; diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index e3a7c09..124063c 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -201,4 +201,49 @@ test_expect_success 'ref advertisment is filtered during fetch using protocol v2 ! grep "refs/tags/three" log ' +# Test protocol v2 with 'http://' transport +# +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'create repo to be served by http:// transport' ' + git init "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config http.receivepack true && + test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" one +' + +test_expect_success 'clone with http:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git -c protocol.version=2 \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + git -C http_child log -1 --format=%s >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "Git-Protocol: version=2" log && + # Server responded using protocol v2 + grep "git< version 2" log +' + +test_expect_success 'fetch with http:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C http_child -c protocol.version=2 \ + fetch && + + git -C http_child log -1 --format=%s origin/master >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "git< version 2" log +' + +stop_httpd + test_done -- cgit v0.10.2-6-g49f6 From a4d78ce26bf4816d868be21c7355d0e0940460a7 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:42 -0700 Subject: remote-curl: don't request v2 when pushing In order to be able to ship protocol v2 with only supporting fetch, we need clients to not issue a request to use protocol v2 when pushing (since the client currently doesn't know how to push using protocol v2). This allows a client to have protocol v2 configured in `protocol.version` and take advantage of using v2 for fetch and falling back to using v0 when pushing while v2 for push is being designed. We could run into issues if we didn't fall back to protocol v2 when pushing right now. This is because currently a server will ignore a request to use v2 when contacting the 'receive-pack' endpoint and fall back to using v0, but when push v2 is rolled out to servers, the 'receive-pack' endpoint will start responding using v2. So we don't want to get into a state where a client is requesting to push with v2 before they actually know how to push using v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano diff --git a/remote-curl.c b/remote-curl.c index 87f5b77..595447b 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -322,6 +322,7 @@ static struct discovery *discover_refs(const char *service, int for_push) struct discovery *last = last_discovery; int http_ret, maybe_smart = 0; struct http_get_options http_options; + enum protocol_version version = get_protocol_version_config(); if (last && !strcmp(service, last->service)) return last; @@ -338,8 +339,16 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_addf(&refs_url, "service=%s", service); } + /* + * NEEDSWORK: If we are trying to use protocol v2 and we are planning + * to perform a push, then fallback to v0 since the client doesn't know + * how to push yet using v2. + */ + if (version == protocol_v2 && !strcmp("git-receive-pack", service)) + version = protocol_v0; + /* Add the extra Git-Protocol header */ - if (get_protocol_http_header(get_protocol_version_config(), &protocol_header)) + if (get_protocol_http_header(version, &protocol_header)) string_list_append(&extra_headers, protocol_header.buf); memset(&http_options, 0, sizeof(http_options)); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 124063c..56f7c3c 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -244,6 +244,30 @@ test_expect_success 'fetch with http:// using protocol v2' ' grep "git< version 2" log ' +test_expect_success 'push with http:// and a config of v2 does not request v2' ' + test_when_finished "rm -f log" && + # Till v2 for push is designed, make sure that if a client has + # protocol.version configured to use v2, that the client instead falls + # back and uses v0. + + test_commit -C http_child three && + + # Push to another branch, as the target repository has the + # master branch checked out and we cannot push into it. + GIT_TRACE_PACKET="$(pwd)/log" git -C http_child -c protocol.version=2 \ + push origin HEAD:client_branch && + + git -C http_child log -1 --format=%s >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s client_branch >expect && + test_cmp expect actual && + + # Client didnt request to use protocol v2 + ! grep "Git-Protocol: version=2" log && + # Server didnt respond using protocol v2 + ! grep "git< version 2" log +' + + stop_httpd test_done -- cgit v0.10.2-6-g49f6