diff options
Diffstat (limited to 'remote-curl.c')
| -rw-r--r-- | remote-curl.c | 454 |
1 files changed, 399 insertions, 55 deletions
diff --git a/remote-curl.c b/remote-curl.c index 954c41b..fb28309 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,9 +1,11 @@ #include "cache.h" +#include "config.h" #include "remote.h" +#include "connect.h" #include "strbuf.h" #include "walker.h" #include "http.h" -#include "exec_cmd.h" +#include "exec-cmd.h" #include "run-command.h" #include "pkt-line.h" #include "string-list.h" @@ -12,6 +14,8 @@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" +#include "protocol.h" +#include "quote.h" static struct remote *remote; /* always ends with a trailing slash */ @@ -20,6 +24,10 @@ static struct strbuf url = STRBUF_INIT; struct options { int verbosity; unsigned long depth; + char *deepen_since; + struct string_list deepen_not; + struct string_list push_options; + char *filter; unsigned progress : 1, check_self_contained_and_connected : 1, cloning : 1, @@ -28,7 +36,10 @@ struct options { dry_run : 1, thin : 1, /* One of the SEND_PACK_PUSH_CERT_* constants. */ - push_cert : 2; + push_cert : 2, + deepen_relative : 1, + from_promisor : 1, + no_dependents : 1; }; static struct options options; static struct string_list cas_options = STRING_LIST_INIT_DUP; @@ -60,6 +71,23 @@ static int set_option(const char *name, const char *value) options.depth = v; return 0; } + else if (!strcmp(name, "deepen-since")) { + options.deepen_since = xstrdup(value); + return 0; + } + else if (!strcmp(name, "deepen-not")) { + string_list_append(&options.deepen_not, value); + return 0; + } + else if (!strcmp(name, "deepen-relative")) { + if (!strcmp(value, "true")) + options.deepen_relative = 1; + else if (!strcmp(value, "false")) + options.deepen_relative = 0; + else + return -1; + return 0; + } else if (!strcmp(name, "followtags")) { if (!strcmp(value, "true")) options.followtags = 1; @@ -119,6 +147,17 @@ static int set_option(const char *name, const char *value) else return -1; return 0; + } else if (!strcmp(name, "push-option")) { + if (*value != '"') + string_list_append(&options.push_options, value); + else { + struct strbuf unquoted = STRBUF_INIT; + if (unquote_c_style(&unquoted, value, NULL) < 0) + die("invalid quoting in push-option value"); + string_list_append_nodup(&options.push_options, + strbuf_detach(&unquoted, NULL)); + } + return 0; #if LIBCURL_VERSION_NUM >= 0x070a08 } else if (!strcmp(name, "family")) { @@ -132,18 +171,28 @@ static int set_option(const char *name, const char *value) return -1; return 0; #endif /* LIBCURL_VERSION_NUM >= 0x070a08 */ + } else if (!strcmp(name, "from-promisor")) { + options.from_promisor = 1; + return 0; + } else if (!strcmp(name, "no-dependents")) { + options.no_dependents = 1; + return 0; + } else if (!strcmp(name, "filter")) { + options.filter = xstrdup(value);; + return 0; } else { return 1 /* unsupported */; } } struct discovery { - const char *service; + char *service; char *buf_alloc; char *buf; size_t len; struct ref *refs; - struct sha1_array shallow; + struct oid_array shallow; + enum protocol_version version; unsigned proto_git : 1; }; static struct discovery *last_discovery; @@ -151,8 +200,31 @@ static struct discovery *last_discovery; static struct ref *parse_git_refs(struct discovery *heads, int for_push) { struct ref *list = NULL; - get_remote_heads(-1, heads->buf, heads->len, &list, - for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + struct packet_reader reader; + + packet_reader_init(&reader, -1, heads->buf, heads->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + heads->version = discover_version(&reader); + switch (heads->version) { + case protocol_v2: + /* + * Do nothing. This isn't a list of refs but rather a + * capability advertisement. Client would have run + * 'stateless-connect' so we'll dump this capability listing + * and let them request the refs themselves. + */ + break; + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, + NULL, &heads->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + return list; } @@ -210,9 +282,10 @@ static void free_discovery(struct discovery *d) if (d) { if (d == last_discovery) last_discovery = NULL; - free(d->shallow.sha1); + free(d->shallow.oid); free(d->buf_alloc); free_refs(d->refs); + free(d->service); free(d); } } @@ -244,6 +317,19 @@ static int show_http_message(struct strbuf *type, struct strbuf *charset, return 0; } +static int get_protocol_http_header(enum protocol_version version, + struct strbuf *header) +{ + if (version > 0) { + strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d", + version); + + return 1; + } + + return 0; +} + static struct discovery *discover_refs(const char *service, int for_push) { struct strbuf exp = STRBUF_INIT; @@ -252,9 +338,12 @@ static struct discovery *discover_refs(const char *service, int for_push) struct strbuf buffer = STRBUF_INIT; struct strbuf refs_url = STRBUF_INIT; struct strbuf effective_url = STRBUF_INIT; + struct strbuf protocol_header = STRBUF_INIT; + struct string_list extra_headers = STRING_LIST_INIT_DUP; struct discovery *last = last_discovery; int http_ret, maybe_smart = 0; - struct http_get_options options; + struct http_get_options http_options; + enum protocol_version version = get_protocol_version_config(); if (last && !strcmp(service, last->service)) return last; @@ -271,15 +360,29 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_addf(&refs_url, "service=%s", service); } - memset(&options, 0, sizeof(options)); - options.content_type = &type; - options.charset = &charset; - options.effective_url = &effective_url; - options.base_url = &url; - options.no_cache = 1; - options.keep_error = 1; - - http_ret = http_get_strbuf(refs_url.buf, &buffer, &options); + /* + * NEEDSWORK: If we are trying to use protocol v2 and we are planning + * to perform a push, then fallback to v0 since the client doesn't know + * how to push yet using v2. + */ + if (version == protocol_v2 && !strcmp("git-receive-pack", service)) + version = protocol_v0; + + /* Add the extra Git-Protocol header */ + if (get_protocol_http_header(version, &protocol_header)) + string_list_append(&extra_headers, protocol_header.buf); + + memset(&http_options, 0, sizeof(http_options)); + http_options.content_type = &type; + http_options.charset = &charset; + http_options.effective_url = &effective_url; + http_options.base_url = &url; + http_options.extra_headers = &extra_headers; + http_options.initial_request = 1; + http_options.no_cache = 1; + http_options.keep_error = 1; + + http_ret = http_get_strbuf(refs_url.buf, &buffer, &http_options); switch (http_ret) { case HTTP_OK: break; @@ -294,8 +397,11 @@ static struct discovery *discover_refs(const char *service, int for_push) die("unable to access '%s': %s", url.buf, curl_errorstr); } + if (options.verbosity && !starts_with(refs_url.buf, url.buf)) + warning(_("redirecting to %s"), url.buf); + last= xcalloc(1, sizeof(*last_discovery)); - last->service = service; + last->service = xstrdup(service); last->buf_alloc = strbuf_detach(&buffer, &last->len); last->buf = last->buf_alloc; @@ -310,6 +416,8 @@ static struct discovery *discover_refs(const char *service, int for_push) * pkt-line matches our request. */ line = packet_read_line_buf(&last->buf, &last->len, NULL); + if (!line) + die("invalid server response; expected service, got flush packet"); strbuf_reset(&exp); strbuf_addf(&exp, "# service=%s", service); @@ -325,6 +433,9 @@ static struct discovery *discover_refs(const char *service, int for_push) ; last->proto_git = 1; + } else if (maybe_smart && + last->len > 5 && starts_with(last->buf + 4, "version 2")) { + last->proto_git = 1; } if (last->proto_git) @@ -338,6 +449,8 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_release(&charset); strbuf_release(&effective_url); strbuf_release(&buffer); + strbuf_release(&protocol_header); + string_list_clear(&extra_headers, 0); last_discovery = last; return last; } @@ -374,12 +487,14 @@ struct rpc_state { char *service_url; char *hdr_content_type; char *hdr_accept; + char *protocol_header; char *buf; size_t alloc; size_t len; size_t pos; int in; int out; + int any_written; struct strbuf result; unsigned gzip_request : 1; unsigned initial_buffer : 1; @@ -436,6 +551,8 @@ static size_t rpc_in(char *ptr, size_t eltsize, { size_t size = eltsize * nmemb; struct rpc_state *rpc = buffer_; + if (size) + rpc->any_written = 1; write_or_die(rpc->in, ptr, size); return size; } @@ -500,6 +617,12 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results) return err; } +static curl_off_t xcurl_off_t(ssize_t len) { + if (len > maximum_signed_value_of_type(curl_off_t)) + die("cannot handle pushes this big"); + return (curl_off_t) len; +} + static int post_rpc(struct rpc_state *rpc) { struct active_request_slot *slot; @@ -551,13 +674,17 @@ static int post_rpc(struct rpc_state *rpc) headers = curl_slist_append(headers, needs_100_continue ? "Expect: 100-continue" : "Expect:"); + /* Add the extra Git-Protocol header */ + if (rpc->protocol_header) + headers = curl_slist_append(headers, rpc->protocol_header); + retry: slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); curl_easy_setopt(slot->curl, CURLOPT_POST, 1); curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url); - curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip"); + curl_easy_setopt(slot->curl, CURLOPT_ENCODING, ""); if (large_request) { /* The request body is large and the size cannot be predicted. @@ -583,7 +710,7 @@ retry: * and we just need to send it. */ curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body); - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size)); } else if (use_gzip && 1024 < rpc->len) { /* The client backend isn't giving us compressed data so @@ -614,7 +741,7 @@ retry: headers = curl_slist_append(headers, "Content-Encoding: gzip"); curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body); - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size)); if (options.verbosity > 1) { fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n", @@ -627,7 +754,7 @@ retry: * more normal Content-Length approach. */ curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf); - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(rpc->len)); if (options.verbosity > 1) { fprintf(stderr, "POST %s (%lu bytes)\n", rpc->service_name, (unsigned long)rpc->len); @@ -639,6 +766,8 @@ retry: curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in); curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc); + + rpc->any_written = 0; err = run_slot(slot, NULL); if (err == HTTP_REAUTH && !large_request) { credential_fill(&http_auth); @@ -647,6 +776,9 @@ retry: if (err != HTTP_OK) err = -1; + if (!rpc->any_written) + err = -1; + curl_slist_free_all(headers); free(gzip_body); return err; @@ -686,6 +818,11 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) strbuf_addf(&buf, "Accept: application/x-%s-result", svc); rpc->hdr_accept = strbuf_detach(&buf, NULL); + if (get_protocol_http_header(heads->version, &buf)) + rpc->protocol_header = strbuf_detach(&buf, NULL); + else + rpc->protocol_header = NULL; + while (!err) { int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); if (!n) @@ -713,6 +850,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) free(rpc->service_url); free(rpc->hdr_content_type); free(rpc->hdr_accept); + free(rpc->protocol_header); free(rpc->buf); strbuf_release(&buf); return err; @@ -725,15 +863,12 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) int ret, i; ALLOC_ARRAY(targets, nr_heads); - if (options.depth) - die("dumb http transport does not support --depth"); + if (options.depth || options.deepen_since) + die("dumb http transport does not support shallow capabilities"); for (i = 0; i < nr_heads; i++) targets[i] = xstrdup(oid_to_hex(&to_fetch[i]->old_oid)); walker = get_http_walker(url.buf); - walker->get_all = 1; - walker->get_tree = 1; - walker->get_history = 1; walker->get_verbosely = options.verbosity >= 3; walker->get_recover = 0; ret = walker_fetch(walker, nr_heads, targets, NULL, NULL); @@ -751,38 +886,41 @@ static int fetch_git(struct discovery *heads, { struct rpc_state rpc; struct strbuf preamble = STRBUF_INIT; - char *depth_arg = NULL; - int argc = 0, i, err; - const char *argv[17]; - - argv[argc++] = "fetch-pack"; - argv[argc++] = "--stateless-rpc"; - argv[argc++] = "--stdin"; - argv[argc++] = "--lock-pack"; + int i, err; + struct argv_array args = ARGV_ARRAY_INIT; + + argv_array_pushl(&args, "fetch-pack", "--stateless-rpc", + "--stdin", "--lock-pack", NULL); if (options.followtags) - argv[argc++] = "--include-tag"; + argv_array_push(&args, "--include-tag"); if (options.thin) - argv[argc++] = "--thin"; - if (options.verbosity >= 3) { - argv[argc++] = "-v"; - argv[argc++] = "-v"; - } + argv_array_push(&args, "--thin"); + if (options.verbosity >= 3) + argv_array_pushl(&args, "-v", "-v", NULL); if (options.check_self_contained_and_connected) - argv[argc++] = "--check-self-contained-and-connected"; + argv_array_push(&args, "--check-self-contained-and-connected"); if (options.cloning) - argv[argc++] = "--cloning"; + argv_array_push(&args, "--cloning"); if (options.update_shallow) - argv[argc++] = "--update-shallow"; + argv_array_push(&args, "--update-shallow"); if (!options.progress) - argv[argc++] = "--no-progress"; - if (options.depth) { - struct strbuf buf = STRBUF_INIT; - strbuf_addf(&buf, "--depth=%lu", options.depth); - depth_arg = strbuf_detach(&buf, NULL); - argv[argc++] = depth_arg; - } - argv[argc++] = url.buf; - argv[argc++] = NULL; + argv_array_push(&args, "--no-progress"); + if (options.depth) + argv_array_pushf(&args, "--depth=%lu", options.depth); + if (options.deepen_since) + argv_array_pushf(&args, "--shallow-since=%s", options.deepen_since); + for (i = 0; i < options.deepen_not.nr; i++) + argv_array_pushf(&args, "--shallow-exclude=%s", + options.deepen_not.items[i].string); + if (options.deepen_relative && options.depth) + argv_array_push(&args, "--deepen-relative"); + if (options.from_promisor) + argv_array_push(&args, "--from-promisor"); + if (options.no_dependents) + argv_array_push(&args, "--no-dependents"); + if (options.filter) + argv_array_pushf(&args, "--filter=%s", options.filter); + argv_array_push(&args, url.buf); for (i = 0; i < nr_heads; i++) { struct ref *ref = to_fetch[i]; @@ -795,7 +933,7 @@ static int fetch_git(struct discovery *heads, memset(&rpc, 0, sizeof(rpc)); rpc.service_name = "git-upload-pack", - rpc.argv = argv; + rpc.argv = args.argv; rpc.stdin_preamble = &preamble; rpc.gzip_request = 1; @@ -804,7 +942,7 @@ static int fetch_git(struct discovery *heads, write_or_die(1, rpc.result.buf, rpc.result.len); strbuf_release(&rpc.result); strbuf_release(&preamble); - free(depth_arg); + argv_array_clear(&args); return err; } @@ -914,6 +1052,9 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs) argv_array_push(&args, "--quiet"); else if (options.verbosity > 1) argv_array_push(&args, "--verbose"); + for (i = 0; i < options.push_options.nr; i++) + argv_array_pushf(&args, "--push-option=%s", + options.push_options.items[i].string); argv_array_push(&args, options.progress ? "--progress" : "--no-progress"); for_each_string_list_item(cas_option, &cas_options) argv_array_push(&args, cas_option->string); @@ -984,6 +1125,203 @@ static void parse_push(struct strbuf *buf) free(specs); } +/* + * Used to represent the state of a connection to an HTTP server when + * communicating using git's wire-protocol version 2. + */ +struct proxy_state { + char *service_name; + char *service_url; + struct curl_slist *headers; + struct strbuf request_buffer; + int in; + int out; + struct packet_reader reader; + size_t pos; + int seen_flush; +}; + +static void proxy_state_init(struct proxy_state *p, const char *service_name, + enum protocol_version version) +{ + struct strbuf buf = STRBUF_INIT; + + memset(p, 0, sizeof(*p)); + p->service_name = xstrdup(service_name); + + p->in = 0; + p->out = 1; + strbuf_init(&p->request_buffer, 0); + + strbuf_addf(&buf, "%s%s", url.buf, p->service_name); + p->service_url = strbuf_detach(&buf, NULL); + + p->headers = http_copy_default_headers(); + + strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked"); + + /* Add the Git-Protocol header */ + if (get_protocol_http_header(version, &buf)) + p->headers = curl_slist_append(p->headers, buf.buf); + + packet_reader_init(&p->reader, p->in, NULL, 0, + PACKET_READ_GENTLE_ON_EOF); + + strbuf_release(&buf); +} + +static void proxy_state_clear(struct proxy_state *p) +{ + free(p->service_name); + free(p->service_url); + curl_slist_free_all(p->headers); + strbuf_release(&p->request_buffer); +} + +/* + * CURLOPT_READFUNCTION callback function. + * Attempts to copy over a single packet-line at a time into the + * curl provided buffer. + */ +static size_t proxy_in(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t max; + struct proxy_state *p = userdata; + size_t avail = p->request_buffer.len - p->pos; + + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + max = nmemb; + + if (!avail) { + if (p->seen_flush) { + p->seen_flush = 0; + return 0; + } + + strbuf_reset(&p->request_buffer); + switch (packet_reader_read(&p->reader)) { + case PACKET_READ_EOF: + die("unexpected EOF when reading from parent process"); + case PACKET_READ_NORMAL: + packet_buf_write_len(&p->request_buffer, p->reader.line, + p->reader.pktlen); + break; + case PACKET_READ_DELIM: + packet_buf_delim(&p->request_buffer); + break; + case PACKET_READ_FLUSH: + packet_buf_flush(&p->request_buffer); + p->seen_flush = 1; + break; + } + p->pos = 0; + avail = p->request_buffer.len; + } + + if (max < avail) + avail = max; + memcpy(buffer, p->request_buffer.buf + p->pos, avail); + p->pos += avail; + return avail; +} + +static size_t proxy_out(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t size; + struct proxy_state *p = userdata; + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + size = nmemb; + + write_or_die(p->out, buffer, size); + return size; +} + +/* Issues a request to the HTTP server configured in `p` */ +static int proxy_request(struct proxy_state *p) +{ + struct active_request_slot *slot; + + slot = get_active_slot(); + + curl_easy_setopt(slot->curl, CURLOPT_ENCODING, ""); + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(slot->curl, CURLOPT_POST, 1); + curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers); + + /* Setup function to read request from client */ + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in); + curl_easy_setopt(slot->curl, CURLOPT_READDATA, p); + + /* Setup function to write server response to client */ + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p); + + if (run_slot(slot, NULL) != HTTP_OK) + return -1; + + return 0; +} + +static int stateless_connect(const char *service_name) +{ + struct discovery *discover; + struct proxy_state p; + + /* + * Run the info/refs request and see if the server supports protocol + * v2. If and only if the server supports v2 can we successfully + * establish a stateless connection, otherwise we need to tell the + * client to fallback to using other transport helper functions to + * complete their request. + */ + discover = discover_refs(service_name, 0); + if (discover->version != protocol_v2) { + printf("fallback\n"); + fflush(stdout); + return -1; + } else { + /* Stateless Connection established */ + printf("\n"); + fflush(stdout); + } + + proxy_state_init(&p, service_name, discover->version); + + /* + * Dump the capability listing that we got from the server earlier + * during the info/refs request. + */ + write_or_die(p.out, discover->buf, discover->len); + + /* Peek the next packet line. Until we see EOF keep sending POSTs */ + while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) { + if (proxy_request(&p)) { + /* We would have an err here */ + break; + } + } + + proxy_state_clear(&p); + return 0; +} + int cmd_main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; @@ -998,6 +1336,8 @@ int cmd_main(int argc, const char **argv) options.verbosity = 1; options.progress = !!isatty(2); options.thin = 1; + string_list_init(&options.deepen_not, 1); + string_list_init(&options.push_options, 1); remote = remote_get(argv[1]); @@ -1050,12 +1390,16 @@ int cmd_main(int argc, const char **argv) fflush(stdout); } else if (!strcmp(buf.buf, "capabilities")) { + printf("stateless-connect\n"); printf("fetch\n"); printf("option\n"); printf("push\n"); printf("check-connectivity\n"); printf("\n"); fflush(stdout); + } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) { + if (!stateless_connect(arg)) + break; } else { error("remote-curl: unknown command '%s' from git", buf.buf); return 1; |