From 8102570374a87c032bff2d7114c67390f55f9a1b Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Thu, 8 Apr 2021 18:09:59 -0700 Subject: fetch-pack: refactor process_acks() A subsequent commit will need part, but not all, of the functionality in process_acks(), so move some of its functionality to its sole caller do_fetch_pack_v2(). As a side effect, the resulting code is also shorter. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index 2318ebe..9f3901c 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1351,35 +1351,11 @@ static int process_section_header(struct packet_reader *reader, return ret; } -enum common_found { - /* - * No commit was found to be possessed by both the client and the - * server, and "ready" was not received. - */ - NO_COMMON_FOUND, - - /* - * At least one commit was found to be possessed by both the client and - * the server, and "ready" was not received. - */ - COMMON_FOUND, - - /* - * "ready" was received, indicating that the server is ready to send - * the packfile without any further negotiation. - */ - READY -}; - -static enum common_found process_acks(struct fetch_negotiator *negotiator, - struct packet_reader *reader, - struct oidset *common) +static int process_ack(struct fetch_negotiator *negotiator, + struct packet_reader *reader, + struct object_id *common_oid, + int *received_ready) { - /* received */ - int received_ready = 0; - int received_ack = 0; - - process_section_header(reader, "acknowledgments", 0); while (packet_reader_read(reader) == PACKET_READ_NORMAL) { const char *arg; @@ -1387,20 +1363,17 @@ static enum common_found process_acks(struct fetch_negotiator *negotiator, continue; if (skip_prefix(reader->line, "ACK ", &arg)) { - struct object_id oid; - received_ack = 1; - if (!get_oid_hex(arg, &oid)) { + if (!get_oid_hex(arg, common_oid)) { struct commit *commit; - oidset_insert(common, &oid); - commit = lookup_commit(the_repository, &oid); + commit = lookup_commit(the_repository, common_oid); if (negotiator) negotiator->ack(negotiator, commit); } - continue; + return 1; } if (!strcmp(reader->line, "ready")) { - received_ready = 1; + *received_ready = 1; continue; } @@ -1418,13 +1391,12 @@ static enum common_found process_acks(struct fetch_negotiator *negotiator, * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH * otherwise. */ - if (received_ready && reader->status != PACKET_READ_DELIM) + if (*received_ready && reader->status != PACKET_READ_DELIM) die(_("expected packfile to be sent after 'ready'")); - if (!received_ready && reader->status != PACKET_READ_FLUSH) + if (!*received_ready && reader->status != PACKET_READ_FLUSH) die(_("expected no other sections to be sent after no 'ready'")); - return received_ready ? READY : - (received_ack ? COMMON_FOUND : NO_COMMON_FOUND); + return 0; } static void receive_shallow_info(struct fetch_pack_args *args, @@ -1573,6 +1545,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, struct fetch_negotiator negotiator_alloc; struct fetch_negotiator *negotiator; int seen_ack = 0; + struct object_id common_oid; + int received_ready = 0; struct string_list packfile_uris = STRING_LIST_INIT_DUP; int i; struct strvec index_pack_args = STRVEC_INIT; @@ -1631,22 +1605,22 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, break; case FETCH_PROCESS_ACKS: /* Process ACKs/NAKs */ - switch (process_acks(negotiator, &reader, &common)) { - case READY: + process_section_header(&reader, "acknowledgments", 0); + while (process_ack(negotiator, &reader, &common_oid, + &received_ready)) { + in_vain = 0; + seen_ack = 1; + oidset_insert(&common, &common_oid); + } + if (received_ready) { /* * Don't check for response delimiter; get_pack() will * read the rest of this response. */ state = FETCH_GET_PACK; - break; - case COMMON_FOUND: - in_vain = 0; - seen_ack = 1; - /* fallthrough */ - case NO_COMMON_FOUND: + } else { do_check_stateless_delimiter(args, &reader); state = FETCH_SEND_REQUEST; - break; } break; case FETCH_GET_PACK: -- cgit v0.10.2-6-g49f6 From 57c3451b2e36d003aa7fc398c644ce71ab668b5e Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Thu, 8 Apr 2021 18:10:00 -0700 Subject: fetch-pack: refactor add_haves() A subsequent commit will need part, but not all, of the functionality in add_haves(), so move some of its functionality to its sole caller send_fetch_request(). Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index 9f3901c..128ad47 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1195,11 +1195,9 @@ static void add_common(struct strbuf *req_buf, struct oidset *common) } static int add_haves(struct fetch_negotiator *negotiator, - int seen_ack, struct strbuf *req_buf, - int *haves_to_send, int *in_vain) + int *haves_to_send) { - int ret = 0; int haves_added = 0; const struct object_id *oid; @@ -1209,17 +1207,10 @@ static int add_haves(struct fetch_negotiator *negotiator, break; } - *in_vain += haves_added; - if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) { - /* Send Done */ - packet_buf_write(req_buf, "done\n"); - ret = 1; - } - /* Increase haves to send on next round */ *haves_to_send = next_flush(1, *haves_to_send); - return ret; + return haves_added; } static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, @@ -1228,7 +1219,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, int *haves_to_send, int *in_vain, int sideband_all, int seen_ack) { - int ret = 0; + int haves_added; + int done_sent = 0; const char *hash_name; struct strbuf req_buf = STRBUF_INIT; @@ -1312,9 +1304,13 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, /* Add all of the common commits we've found in previous rounds */ add_common(&req_buf, common); - /* Add initial haves */ - ret = add_haves(negotiator, seen_ack, &req_buf, - haves_to_send, in_vain); + haves_added = add_haves(negotiator, &req_buf, haves_to_send); + *in_vain += haves_added; + if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) { + /* Send Done */ + packet_buf_write(&req_buf, "done\n"); + done_sent = 1; + } /* Send request */ packet_buf_flush(&req_buf); @@ -1322,7 +1318,7 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, die_errno(_("unable to write request to remote")); strbuf_release(&req_buf); - return ret; + return done_sent; } /* -- cgit v0.10.2-6-g49f6 From 6871d0cec62dc12d0c5f7390eee8a80614919578 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Thu, 8 Apr 2021 18:10:01 -0700 Subject: fetch-pack: refactor command and capability write A subsequent commit will need this functionality independent of the rest of send_fetch_request(), so put this into its own function. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index 128ad47..512fe54 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1213,29 +1213,23 @@ static int add_haves(struct fetch_negotiator *negotiator, return haves_added; } -static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, - struct fetch_pack_args *args, - const struct ref *wants, struct oidset *common, - int *haves_to_send, int *in_vain, - int sideband_all, int seen_ack) +static void write_fetch_command_and_capabilities(struct strbuf *req_buf, + const struct string_list *server_options) { - int haves_added; - int done_sent = 0; const char *hash_name; - struct strbuf req_buf = STRBUF_INIT; if (server_supports_v2("fetch", 1)) - packet_buf_write(&req_buf, "command=fetch"); + packet_buf_write(req_buf, "command=fetch"); if (server_supports_v2("agent", 0)) - packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized()); + packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized()); if (advertise_sid && server_supports_v2("session-id", 0)) - packet_buf_write(&req_buf, "session-id=%s", trace2_session_id()); - if (args->server_options && args->server_options->nr && + packet_buf_write(req_buf, "session-id=%s", trace2_session_id()); + if (server_options && server_options->nr && server_supports_v2("server-option", 1)) { int i; - for (i = 0; i < args->server_options->nr; i++) - packet_buf_write(&req_buf, "server-option=%s", - args->server_options->items[i].string); + for (i = 0; i < server_options->nr; i++) + packet_buf_write(req_buf, "server-option=%s", + server_options->items[i].string); } if (server_feature_v2("object-format", &hash_name)) { @@ -1243,13 +1237,26 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, if (hash_algo_by_ptr(the_hash_algo) != hash_algo) die(_("mismatched algorithms: client %s; server %s"), the_hash_algo->name, hash_name); - packet_buf_write(&req_buf, "object-format=%s", the_hash_algo->name); + packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name); } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1) { die(_("the server does not support algorithm '%s'"), the_hash_algo->name); } + packet_buf_delim(req_buf); +} + +static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, + struct fetch_pack_args *args, + const struct ref *wants, struct oidset *common, + int *haves_to_send, int *in_vain, + int sideband_all, int seen_ack) +{ + int haves_added; + int done_sent = 0; + struct strbuf req_buf = STRBUF_INIT; + + write_fetch_command_and_capabilities(&req_buf, args->server_options); - packet_buf_delim(&req_buf); if (args->use_thin_pack) packet_buf_write(&req_buf, "thin-pack"); if (args->no_progress) -- cgit v0.10.2-6-g49f6 From 9c1e657a8fd26fa3ed8d13fb8c796cef8db8b124 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Tue, 4 May 2021 14:16:01 -0700 Subject: fetch: teach independent negotiation (no packfile) Currently, the packfile negotiation step within a Git fetch cannot be done independent of sending the packfile, even though there is at least one application wherein this is useful. Therefore, make it possible for this negotiation step to be done independently. A subsequent commit will use this for one such application - push negotiation. This feature is for protocol v2 only. (An implementation for protocol v0 would require a separate implementation in the fetch, transport, and transport helper code.) In the protocol, the main hindrance towards independent negotiation is that the server can unilaterally decide to send the packfile. This is solved by a "wait-for-done" argument: the server will then wait for the client to say "done". In practice, the client will never say it; instead it will cease requests once it is satisfied. In the client, the main change lies in the transport and transport helper code. fetch_refs_via_pack() performs everything needed - protocol version and capability checks, and the negotiation itself. There are 2 code paths that do not go through fetch_refs_via_pack() that needed to be individually excluded: the bundle transport (excluded through requiring smart_options, which the bundle transport doesn't support) and transport helpers that do not support takeover. If or when we support independent negotiation for protocol v0, we will need to modify these 2 code paths to support it. But for now, report failure if independent negotiation is requested in these cases. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index a7c806a..0b371d8 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -346,6 +346,14 @@ explained below. client should download from all given URIs. Currently, the protocols supported are "http" and "https". +If the 'wait-for-done' feature is advertised, the following argument +can be included in the client's request. + + wait-for-done + Indicates to the server that it should never send "ready", but + should wait for the client to say "done" before sending the + packfile. + The response of `fetch` is broken into a number of sections separated by delimiter packets (0001), with each section beginning with its section header. Most sections are sent only when the packfile is sent. diff --git a/builtin/fetch.c b/builtin/fetch.c index 0b90de8..a732295 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -82,6 +82,7 @@ static struct string_list server_options = STRING_LIST_INIT_DUP; static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; static int fetch_write_commit_graph = -1; static int stdin_refspecs = 0; +static int negotiate_only; static int git_fetch_config(const char *k, const char *v, void *cb) { @@ -202,6 +203,8 @@ static struct option builtin_fetch_options[] = { TRANSPORT_FAMILY_IPV6), OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), N_("report that we have only objects reachable from this object")), + OPT_BOOL(0, "negotiate-only", &negotiate_only, + N_("do not fetch a packfile; instead, print ancestors of negotiation tips")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), OPT_BOOL(0, "auto-maintenance", &enable_auto_gc, N_("run 'maintenance --auto' after fetching")), @@ -1986,7 +1989,29 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) } } - if (remote) { + if (negotiate_only) { + struct oidset acked_commits = OIDSET_INIT; + struct oidset_iter iter; + const struct object_id *oid; + + if (!remote) + die(_("must supply remote when using --negotiate-only")); + gtransport = prepare_transport(remote, 1); + if (gtransport->smart_options) { + gtransport->smart_options->acked_commits = &acked_commits; + } else { + warning(_("Protocol does not support --negotiate-only, exiting.")); + return 1; + } + if (server_options.nr) + gtransport->server_options = &server_options; + result = transport_fetch_refs(gtransport, NULL); + + oidset_iter_init(&acked_commits, &iter); + while ((oid = oidset_iter_next(&iter))) + printf("%s\n", oid_to_hex(oid)); + oidset_clear(&acked_commits); + } else if (remote) { if (filter_options.choice || has_promisor_remote()) fetch_one_setup_partial(remote); result = fetch_one(remote, argc, argv, prune_tags_ok, stdin_refspecs); diff --git a/fetch-pack.c b/fetch-pack.c index 512fe54..c135635 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -23,6 +23,8 @@ #include "fetch-negotiator.h" #include "fsck.h" #include "shallow.h" +#include "commit-reach.h" +#include "commit-graph.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -45,6 +47,8 @@ static struct string_list uri_protocols = STRING_LIST_INIT_DUP; /* Remember to update object flag allocation in object.h */ #define COMPLETE (1U << 0) #define ALTERNATE (1U << 1) +#define COMMON (1U << 6) +#define REACH_SCRATCH (1U << 7) /* * After sending this many "have"s if we do not get any new ACK , we @@ -1523,10 +1527,10 @@ enum fetch_state { FETCH_DONE, }; -static void do_check_stateless_delimiter(const struct fetch_pack_args *args, +static void do_check_stateless_delimiter(int stateless_rpc, struct packet_reader *reader) { - check_stateless_delimiter(args->stateless_rpc, reader, + check_stateless_delimiter(stateless_rpc, reader, _("git fetch-pack: expected response end packet")); } @@ -1622,7 +1626,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, */ state = FETCH_GET_PACK; } else { - do_check_stateless_delimiter(args, &reader); + do_check_stateless_delimiter(args->stateless_rpc, &reader); state = FETCH_SEND_REQUEST; } break; @@ -1645,7 +1649,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, packfile_uris.nr ? &index_pack_args : NULL, sought, nr_sought, &fsck_options.gitmodules_found)) die(_("git fetch-pack: fetch failed.")); - do_check_stateless_delimiter(args, &reader); + do_check_stateless_delimiter(args->stateless_rpc, &reader); state = FETCH_DONE; break; @@ -1962,6 +1966,105 @@ cleanup: return ref_cpy; } +static int add_to_object_array(const struct object_id *oid, void *data) +{ + struct object_array *a = data; + + add_object_array(lookup_object(the_repository, oid), "", a); + return 0; +} + +static void clear_common_flag(struct oidset *s) +{ + struct oidset_iter iter; + const struct object_id *oid; + oidset_iter_init(s, &iter); + + while ((oid = oidset_iter_next(&iter))) { + struct object *obj = lookup_object(the_repository, oid); + obj->flags &= ~COMMON; + } +} + +void negotiate_using_fetch(const struct oid_array *negotiation_tips, + const struct string_list *server_options, + int stateless_rpc, + int fd[], + struct oidset *acked_commits) +{ + struct fetch_negotiator negotiator; + struct packet_reader reader; + struct object_array nt_object_array = OBJECT_ARRAY_INIT; + struct strbuf req_buf = STRBUF_INIT; + int haves_to_send = INITIAL_FLUSH; + int in_vain = 0; + int seen_ack = 0; + int last_iteration = 0; + timestamp_t min_generation = GENERATION_NUMBER_INFINITY; + + fetch_negotiator_init(the_repository, &negotiator); + mark_tips(&negotiator, negotiation_tips); + + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_DIE_ON_ERR_PACKET); + + oid_array_for_each((struct oid_array *) negotiation_tips, + add_to_object_array, + &nt_object_array); + + while (!last_iteration) { + int haves_added; + struct object_id common_oid; + int received_ready = 0; + + strbuf_reset(&req_buf); + write_fetch_command_and_capabilities(&req_buf, server_options); + + packet_buf_write(&req_buf, "wait-for-done"); + + haves_added = add_haves(&negotiator, &req_buf, &haves_to_send); + in_vain += haves_added; + if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN)) + last_iteration = 1; + + /* Send request */ + packet_buf_flush(&req_buf); + if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0) + die_errno(_("unable to write request to remote")); + + /* Process ACKs/NAKs */ + process_section_header(&reader, "acknowledgments", 0); + while (process_ack(&negotiator, &reader, &common_oid, + &received_ready)) { + struct commit *commit = lookup_commit(the_repository, + &common_oid); + if (commit) { + timestamp_t generation; + + parse_commit_or_die(commit); + commit->object.flags |= COMMON; + generation = commit_graph_generation(commit); + if (generation < min_generation) + min_generation = generation; + } + in_vain = 0; + seen_ack = 1; + oidset_insert(acked_commits, &common_oid); + } + if (received_ready) + die(_("unexpected 'ready' from remote")); + else + do_check_stateless_delimiter(stateless_rpc, &reader); + if (can_all_from_reach_with_flag(&nt_object_array, COMMON, + REACH_SCRATCH, 0, + min_generation)) + last_iteration = 1; + } + clear_common_flag(acked_commits); + strbuf_release(&req_buf); +} + int report_unmatched_refs(struct ref **sought, int nr_sought) { int i, ret = 0; diff --git a/fetch-pack.h b/fetch-pack.h index f114d72..7f94a2a 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -5,6 +5,7 @@ #include "run-command.h" #include "protocol.h" #include "list-objects-filter-options.h" +#include "oidset.h" struct oid_array; @@ -82,6 +83,19 @@ struct ref *fetch_pack(struct fetch_pack_args *args, enum protocol_version version); /* + * Execute the --negotiate-only mode of "git fetch", adding all known common + * commits to acked_commits. + * + * In the capability advertisement that has happened prior to invoking this + * function, the "wait-for-done" capability must be present. + */ +void negotiate_using_fetch(const struct oid_array *negotiation_tips, + const struct string_list *server_options, + int stateless_rpc, + int fd[], + struct oidset *acked_commits); + +/* * Print an appropriate error message for each sought ref that wasn't * matched. Return 0 if all sought refs were matched, otherwise 1. */ diff --git a/object.h b/object.h index 59daadc..4806fa8 100644 --- a/object.h +++ b/object.h @@ -60,7 +60,7 @@ struct object_array { /* * object flag allocation: * revision.h: 0---------10 15 23------26 - * fetch-pack.c: 01 + * fetch-pack.c: 01 67 * negotiator/default.c: 2--5 * walker.c: 0-2 * upload-pack.c: 4 11-----14 16-----19 diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 509f379..f03bb04 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -16,7 +16,7 @@ test_expect_success 'test capability advertisement' ' version 2 agent=git/$(git version | cut -d" " -f3) ls-refs=unborn - fetch=shallow + fetch=shallow wait-for-done server-option object-format=$(test_oid algo) 0000 diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 2e1243c..66af411 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -585,6 +585,49 @@ test_expect_success 'deepen-relative' ' test_cmp expected actual ' +setup_negotiate_only () { + SERVER="$1" + URI="$2" + + rm -rf "$SERVER" client + + git init "$SERVER" + test_commit -C "$SERVER" one + test_commit -C "$SERVER" two + + git clone "$URI" client + test_commit -C client three +} + +test_expect_success 'file:// --negotiate-only' ' + SERVER="server" && + URI="file://$(pwd)/server" && + + setup_negotiate_only "$SERVER" "$URI" && + + git -c protocol.version=2 -C client fetch \ + --no-tags \ + --negotiate-only \ + --negotiation-tip=$(git -C client rev-parse HEAD) \ + origin >out && + COMMON=$(git -C "$SERVER" rev-parse two) && + grep "$COMMON" out +' + +test_expect_success 'file:// --negotiate-only with protocol v0' ' + SERVER="server" && + URI="file://$(pwd)/server" && + + setup_negotiate_only "$SERVER" "$URI" && + + test_must_fail git -c protocol.version=0 -C client fetch \ + --no-tags \ + --negotiate-only \ + --negotiation-tip=$(git -C client rev-parse HEAD) \ + origin 2>err && + test_i18ngrep "negotiate-only requires protocol v2" err +' + # Test protocol v2 with 'http://' transport # . "$TEST_DIRECTORY"/lib-httpd.sh @@ -1035,6 +1078,52 @@ test_expect_success 'packfile-uri with transfer.fsckobjects fails when .gitmodul test_i18ngrep "disallowed submodule name" err ' +test_expect_success 'http:// --negotiate-only' ' + SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && + URI="$HTTPD_URL/smart/server" && + + setup_negotiate_only "$SERVER" "$URI" && + + git -c protocol.version=2 -C client fetch \ + --no-tags \ + --negotiate-only \ + --negotiation-tip=$(git -C client rev-parse HEAD) \ + origin >out && + COMMON=$(git -C "$SERVER" rev-parse two) && + grep "$COMMON" out +' + +test_expect_success 'http:// --negotiate-only without wait-for-done support' ' + SERVER="server" && + URI="$HTTPD_URL/one_time_perl/server" && + + setup_negotiate_only "$SERVER" "$URI" && + + echo "s/ wait-for-done/ xxxx-xxx-xxxx/" \ + >"$HTTPD_ROOT_PATH/one-time-perl" && + + test_must_fail git -c protocol.version=2 -C client fetch \ + --no-tags \ + --negotiate-only \ + --negotiation-tip=$(git -C client rev-parse HEAD) \ + origin 2>err && + test_i18ngrep "server does not support wait-for-done" err +' + +test_expect_success 'http:// --negotiate-only with protocol v0' ' + SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && + URI="$HTTPD_URL/smart/server" && + + setup_negotiate_only "$SERVER" "$URI" && + + test_must_fail git -c protocol.version=0 -C client fetch \ + --no-tags \ + --negotiate-only \ + --negotiation-tip=$(git -C client rev-parse HEAD) \ + origin 2>err && + test_i18ngrep "negotiate-only requires protocol v2" err +' + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled. diff --git a/transport-helper.c b/transport-helper.c index 4cd7636..4be035e 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -684,6 +684,16 @@ static int fetch(struct transport *transport, return transport->vtable->fetch(transport, nr_heads, to_fetch); } + /* + * If we reach here, then the server, the client, and/or the transport + * helper does not support protocol v2. --negotiate-only requires + * protocol v2. + */ + if (data->transport_options.acked_commits) { + warning(_("--negotiate-only requires protocol v2")); + return -1; + } + if (!data->get_refs_list_called) get_refs_list_using_list(transport, 0); diff --git a/transport.c b/transport.c index ef66e73..80caeaa 100644 --- a/transport.c +++ b/transport.c @@ -392,16 +392,29 @@ static int fetch_refs_via_pack(struct transport *transport, else if (data->version <= protocol_v1) die_if_server_options(transport); + if (data->options.acked_commits) { + if (data->version < protocol_v2) { + warning(_("--negotiate-only requires protocol v2")); + ret = -1; + } else if (!server_supports_feature("fetch", "wait-for-done", 0)) { + warning(_("server does not support wait-for-done")); + ret = -1; + } else { + negotiate_using_fetch(data->options.negotiation_tips, + transport->server_options, + transport->stateless_rpc, + data->fd, + data->options.acked_commits); + ret = 0; + } + goto cleanup; + } + refs = fetch_pack(&args, data->fd, refs_tmp ? refs_tmp : transport->remote_refs, to_fetch, nr_heads, &data->shallow, &transport->pack_lockfiles, data->version); - close(data->fd[0]); - close(data->fd[1]); - if (finish_connect(data->conn)) - ret = -1; - data->conn = NULL; data->got_remote_heads = 0; data->options.self_contained_and_connected = args.self_contained_and_connected; @@ -412,6 +425,13 @@ static int fetch_refs_via_pack(struct transport *transport, if (report_unmatched_refs(to_fetch, nr_heads)) ret = -1; +cleanup: + close(data->fd[0]); + close(data->fd[1]); + if (finish_connect(data->conn)) + ret = -1; + data->conn = NULL; + free_refs(refs_tmp); free_refs(refs); return ret; diff --git a/transport.h b/transport.h index 4d5db0a..1cbab11 100644 --- a/transport.h +++ b/transport.h @@ -47,6 +47,12 @@ struct git_transport_options { * transport_set_option(). */ struct oid_array *negotiation_tips; + + /* + * If allocated, whenever transport_fetch_refs() is called, add known + * common commits to this oidset instead of fetching any packfiles. + */ + struct oidset *acked_commits; }; enum transport_family { diff --git a/upload-pack.c b/upload-pack.c index e19583a..b432ef0 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -103,6 +103,7 @@ struct upload_pack_data { unsigned use_ofs_delta : 1; unsigned no_progress : 1; unsigned use_include_tag : 1; + unsigned wait_for_done : 1; unsigned allow_filter : 1; unsigned allow_filter_fallback : 1; unsigned long tree_filter_max_depth; @@ -1496,6 +1497,10 @@ static void process_args(struct packet_reader *request, data->done = 1; continue; } + if (!strcmp(arg, "wait-for-done")) { + data->wait_for_done = 1; + continue; + } /* Shallow related arguments */ if (process_shallow(arg, &data->shallows)) @@ -1578,7 +1583,7 @@ static int send_acks(struct upload_pack_data *data, struct oid_array *acks) oid_to_hex(&acks->oid[i])); } - if (ok_to_give_up(data)) { + if (!data->wait_for_done && ok_to_give_up(data)) { /* Send Ready */ packet_writer_write(&data->writer, "ready\n"); return 1; @@ -1668,10 +1673,13 @@ int upload_pack_v2(struct repository *r, struct strvec *keys, case FETCH_PROCESS_ARGS: process_args(request, &data); - if (!data.want_obj.nr) { + if (!data.want_obj.nr && !data.wait_for_done) { /* - * Request didn't contain any 'want' lines, - * guess they didn't want anything. + * Request didn't contain any 'want' lines (and + * the request does not contain + * "wait-for-done", in which it is reasonable + * to just send 'have's without 'want's); guess + * they didn't want anything. */ state = FETCH_DONE; } else if (data.haves.nr) { @@ -1723,7 +1731,7 @@ int upload_pack_advertise(struct repository *r, int allow_sideband_all_value; char *str = NULL; - strbuf_addstr(value, "shallow"); + strbuf_addstr(value, "shallow wait-for-done"); if (!repo_config_get_bool(the_repository, "uploadpack.allowfilter", -- cgit v0.10.2-6-g49f6 From 477673d6f39b4829baa98ad88d6b65b35b79fd0f Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Tue, 4 May 2021 14:16:02 -0700 Subject: send-pack: support push negotiation Teach Git the push.negotiate config variable. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/Documentation/config/push.txt b/Documentation/config/push.txt index 21b256e..f2667b2 100644 --- a/Documentation/config/push.txt +++ b/Documentation/config/push.txt @@ -120,3 +120,10 @@ push.useForceIfIncludes:: `--force-if-includes` as an option to linkgit:git-push[1] in the command line. Adding `--no-force-if-includes` at the time of push overrides this configuration setting. + +push.negotiate:: + If set to "true", attempt to reduce the size of the packfile + sent by rounds of negotiation in which the client and the + server attempt to find commits in common. If "false", Git will + rely solely on the server's ref advertisement to find commits + in common. diff --git a/send-pack.c b/send-pack.c index 5f215b1..9cb9f71 100644 --- a/send-pack.c +++ b/send-pack.c @@ -56,7 +56,9 @@ static void feed_object(const struct object_id *oid, FILE *fh, int negative) /* * Make a pack stream and spit it out into file descriptor fd */ -static int pack_objects(int fd, struct ref *refs, struct oid_array *extra, struct send_pack_args *args) +static int pack_objects(int fd, struct ref *refs, struct oid_array *advertised, + struct oid_array *negotiated, + struct send_pack_args *args) { /* * The child becomes pack-objects --revs; we feed @@ -94,8 +96,10 @@ static int pack_objects(int fd, struct ref *refs, struct oid_array *extra, struc * parameters by writing to the pipe. */ po_in = xfdopen(po.in, "w"); - for (i = 0; i < extra->nr; i++) - feed_object(&extra->oid[i], po_in, 1); + for (i = 0; i < advertised->nr; i++) + feed_object(&advertised->oid[i], po_in, 1); + for (i = 0; i < negotiated->nr; i++) + feed_object(&negotiated->oid[i], po_in, 1); while (refs) { if (!is_null_oid(&refs->old_oid)) @@ -409,11 +413,55 @@ static void reject_invalid_nonce(const char *nonce, int len) } } +static void get_commons_through_negotiation(const char *url, + const struct ref *remote_refs, + struct oid_array *commons) +{ + struct child_process child = CHILD_PROCESS_INIT; + const struct ref *ref; + int len = the_hash_algo->hexsz + 1; /* hash + NL */ + + child.git_cmd = 1; + child.no_stdin = 1; + child.out = -1; + strvec_pushl(&child.args, "fetch", "--negotiate-only", NULL); + for (ref = remote_refs; ref; ref = ref->next) + strvec_pushf(&child.args, "--negotiation-tip=%s", oid_to_hex(&ref->new_oid)); + strvec_push(&child.args, url); + + if (start_command(&child)) + die(_("send-pack: unable to fork off fetch subprocess")); + + do { + char hex_hash[GIT_MAX_HEXSZ + 1]; + int read_len = read_in_full(child.out, hex_hash, len); + struct object_id oid; + const char *end; + + if (!read_len) + break; + if (read_len != len) + die("invalid length read %d", read_len); + if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n') + die("invalid hash"); + oid_array_append(commons, &oid); + } while (1); + + if (finish_command(&child)) { + /* + * The information that push negotiation provides is useful but + * not mandatory. + */ + warning(_("push negotiation failed; proceeding anyway with push")); + } +} + int send_pack(struct send_pack_args *args, int fd[], struct child_process *conn, struct ref *remote_refs, struct oid_array *extra_have) { + struct oid_array commons = OID_ARRAY_INIT; int in = fd[0]; int out = fd[1]; struct strbuf req_buf = STRBUF_INIT; @@ -426,6 +474,7 @@ int send_pack(struct send_pack_args *args, int quiet_supported = 0; int agent_supported = 0; int advertise_sid = 0; + int push_negotiate = 0; int use_atomic = 0; int atomic_supported = 0; int use_push_options = 0; @@ -437,6 +486,10 @@ int send_pack(struct send_pack_args *args, const char *push_cert_nonce = NULL; struct packet_reader reader; + git_config_get_bool("push.negotiate", &push_negotiate); + if (push_negotiate) + get_commons_through_negotiation(args->url, remote_refs, &commons); + git_config_get_bool("transfer.advertisesid", &advertise_sid); /* Does the other end support the reporting? */ @@ -625,7 +678,7 @@ int send_pack(struct send_pack_args *args, PACKET_READ_DIE_ON_ERR_PACKET); if (need_pack_data && cmds_sent) { - if (pack_objects(out, remote_refs, extra_have, args) < 0) { + if (pack_objects(out, remote_refs, extra_have, &commons, args) < 0) { if (args->stateless_rpc) close(out); if (git_connection_is_socket(conn)) diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index f11742e..0916f76 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -191,6 +191,41 @@ test_expect_success 'fetch with pushInsteadOf (should not rewrite)' ' ) ' +grep_wrote () { + object_count=$1 + file_name=$2 + grep 'write_pack_file/wrote.*"value":"'$1'"' $2 +} + +test_expect_success 'push with negotiation' ' + # Without negotiation + mk_empty testrepo && + git push testrepo $the_first_commit:refs/remotes/origin/first_commit && + git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit && + echo now pushing without negotiation && + GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 push testrepo refs/heads/main:refs/remotes/origin/main && + grep_wrote 5 event && # 2 commits, 2 trees, 1 blob + + # Same commands, but with negotiation + rm event && + mk_empty testrepo && + git push testrepo $the_first_commit:refs/remotes/origin/first_commit && + git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit && + GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main && + grep_wrote 2 event # 1 commit, 1 tree +' + +test_expect_success 'push with negotiation proceeds anyway even if negotiation fails' ' + rm event && + mk_empty testrepo && + git push testrepo $the_first_commit:refs/remotes/origin/first_commit && + git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit && + GIT_TEST_PROTOCOL_VERSION=0 GIT_TRACE2_EVENT="$(pwd)/event" \ + git -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main 2>err && + grep_wrote 5 event && # 2 commits, 2 trees, 1 blob + test_i18ngrep "push negotiation failed" err +' + test_expect_success 'push without wildcard' ' mk_empty testrepo && -- cgit v0.10.2-6-g49f6