From cbe566a0710b39691ca8f6cb6a764f6c553cd97b Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:31 -0700 Subject: negotiator/noop: add noop fetch negotiator Add a noop fetch negotiator. This is introduced to allow partial clones to skip the unneeded negotiation step when fetching missing objects using a "git fetch" subprocess. (The implementation of spawning a "git fetch" subprocess will be done in a subsequent patch.) But this can also be useful for end users, e.g. as a blunt fix for object corruption. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/Documentation/config/fetch.txt b/Documentation/config/fetch.txt index b203940..6af6f5e 100644 --- a/Documentation/config/fetch.txt +++ b/Documentation/config/fetch.txt @@ -60,7 +60,10 @@ fetch.negotiationAlgorithm:: sent when negotiating the contents of the packfile to be sent by the server. Set to "skipping" to use an algorithm that skips commits in an effort to converge faster, but may result in a larger-than-necessary - packfile; The default is "default" which instructs Git to use the default algorithm + packfile; or set to "noop" to not send any information at all, which + will almost certainly result in a larger-than-necessary packfile, but + will skip the negotiation step. + The default is "default" which instructs Git to use the default algorithm that never skips commits (unless the server has acknowledged it or one of its descendants). If `feature.experimental` is enabled, then this setting defaults to "skipping". diff --git a/Makefile b/Makefile index 65f8cfb..acdd2e3 100644 --- a/Makefile +++ b/Makefile @@ -916,6 +916,7 @@ LIB_OBJS += mergesort.o LIB_OBJS += midx.o LIB_OBJS += name-hash.o LIB_OBJS += negotiator/default.o +LIB_OBJS += negotiator/noop.o LIB_OBJS += negotiator/skipping.o LIB_OBJS += notes-cache.o LIB_OBJS += notes-merge.o diff --git a/fetch-negotiator.c b/fetch-negotiator.c index 0a1357d..57ed578 100644 --- a/fetch-negotiator.c +++ b/fetch-negotiator.c @@ -2,6 +2,7 @@ #include "fetch-negotiator.h" #include "negotiator/default.h" #include "negotiator/skipping.h" +#include "negotiator/noop.h" #include "repository.h" void fetch_negotiator_init(struct repository *r, @@ -13,6 +14,10 @@ void fetch_negotiator_init(struct repository *r, skipping_negotiator_init(negotiator); return; + case FETCH_NEGOTIATION_NOOP: + noop_negotiator_init(negotiator); + return; + case FETCH_NEGOTIATION_DEFAULT: default: default_negotiator_init(negotiator); diff --git a/negotiator/noop.c b/negotiator/noop.c new file mode 100644 index 0000000..60569b8 --- /dev/null +++ b/negotiator/noop.c @@ -0,0 +1,44 @@ +#include "cache.h" +#include "noop.h" +#include "../commit.h" +#include "../fetch-negotiator.h" + +static void known_common(struct fetch_negotiator *n, struct commit *c) +{ + /* do nothing */ +} + +static void add_tip(struct fetch_negotiator *n, struct commit *c) +{ + /* do nothing */ +} + +static const struct object_id *next(struct fetch_negotiator *n) +{ + return NULL; +} + +static int ack(struct fetch_negotiator *n, struct commit *c) +{ + /* + * This negotiator does not emit any commits, so there is no commit to + * be acknowledged. If there is any ack, there is a bug. + */ + BUG("ack with noop negotiator, which does not emit any commits"); + return 0; +} + +static void release(struct fetch_negotiator *n) +{ + /* nothing to release */ +} + +void noop_negotiator_init(struct fetch_negotiator *negotiator) +{ + negotiator->known_common = known_common; + negotiator->add_tip = add_tip; + negotiator->next = next; + negotiator->ack = ack; + negotiator->release = release; + negotiator->data = NULL; +} diff --git a/negotiator/noop.h b/negotiator/noop.h new file mode 100644 index 0000000..2b4ec5d --- /dev/null +++ b/negotiator/noop.h @@ -0,0 +1,8 @@ +#ifndef NEGOTIATOR_NOOP_H +#define NEGOTIATOR_NOOP_H + +struct fetch_negotiator; + +void noop_negotiator_init(struct fetch_negotiator *negotiator); + +#endif diff --git a/repo-settings.c b/repo-settings.c index 0918408..aa61a35 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -39,6 +39,8 @@ void prepare_repo_settings(struct repository *r) if (!repo_config_get_string(r, "fetch.negotiationalgorithm", &strval)) { if (!strcasecmp(strval, "skipping")) r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_SKIPPING; + else if (!strcasecmp(strval, "noop")) + r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_NOOP; else r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_DEFAULT; } diff --git a/repository.h b/repository.h index 3c1f7d5..628c834 100644 --- a/repository.h +++ b/repository.h @@ -23,6 +23,7 @@ enum fetch_negotiation_setting { FETCH_NEGOTIATION_NONE = 0, FETCH_NEGOTIATION_DEFAULT = 1, FETCH_NEGOTIATION_SKIPPING = 2, + FETCH_NEGOTIATION_NOOP = 3, }; struct repo_settings { diff --git a/t/t5554-noop-fetch-negotiator.sh b/t/t5554-noop-fetch-negotiator.sh new file mode 100755 index 0000000..2ac7b58 --- /dev/null +++ b/t/t5554-noop-fetch-negotiator.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='test noop fetch negotiator' +. ./test-lib.sh + +test_expect_success 'noop negotiator does not emit any "have"' ' + rm -f trace && + + test_create_repo server && + test_commit -C server to_fetch && + + test_create_repo client && + test_commit -C client we_have && + + test_config -C client fetch.negotiationalgorithm noop && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" && + + ! grep "fetch> have" trace && + grep "fetch> done" trace +' + +test_done -- cgit v0.10.2-6-g49f6 From 2b713c272cd8f9dc70ad746e8ba01a93d8baa023 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:32 -0700 Subject: fetch: allow refspecs specified through stdin In a subsequent patch, partial clones will be taught to fetch missing objects using a "git fetch" subprocess. Because the number of objects fetched may be too numerous to fit on the command line, teach "fetch" to accept refspecs passed through stdin. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/Documentation/git-fetch.txt b/Documentation/git-fetch.txt index 45b6d8e..9067c20 100644 --- a/Documentation/git-fetch.txt +++ b/Documentation/git-fetch.txt @@ -48,6 +48,10 @@ include::fetch-options.txt[] include::pull-fetch-param.txt[] +--stdin:: + Read refspecs, one per line, from stdin in addition to those provided + as arguments. The "tag " format is not supported. + include::urls-remotes.txt[] diff --git a/builtin/fetch.c b/builtin/fetch.c index 2eb8d6a..9fca92b 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -80,6 +80,7 @@ static struct list_objects_filter_options filter_options; static struct string_list server_options = STRING_LIST_INIT_DUP; static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; static int fetch_write_commit_graph = -1; +static int stdin_refspecs = 0; static int git_fetch_config(const char *k, const char *v, void *cb) { @@ -205,6 +206,8 @@ static struct option builtin_fetch_options[] = { N_("check for forced-updates on all updated branches")), OPT_BOOL(0, "write-commit-graph", &fetch_write_commit_graph, N_("write the commit-graph after fetching")), + OPT_BOOL(0, "stdin", &stdin_refspecs, + N_("accept refspecs from stdin")), OPT_END() }; @@ -1680,7 +1683,8 @@ static inline void fetch_one_setup_partial(struct remote *remote) return; } -static int fetch_one(struct remote *remote, int argc, const char **argv, int prune_tags_ok) +static int fetch_one(struct remote *remote, int argc, const char **argv, + int prune_tags_ok, int use_stdin_refspecs) { struct refspec rs = REFSPEC_INIT_FETCH; int i; @@ -1737,6 +1741,13 @@ static int fetch_one(struct remote *remote, int argc, const char **argv, int pru } } + if (use_stdin_refspecs) { + struct strbuf line = STRBUF_INIT; + while (strbuf_getline_lf(&line, stdin) != EOF) + refspec_append(&rs, line.buf); + strbuf_release(&line); + } + if (server_options.nr) gtransport->server_options = &server_options; @@ -1837,7 +1848,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) if (remote) { if (filter_options.choice || has_promisor_remote()) fetch_one_setup_partial(remote); - result = fetch_one(remote, argc, argv, prune_tags_ok); + result = fetch_one(remote, argc, argv, prune_tags_ok, stdin_refspecs); } else { int max_children = max_jobs; @@ -1845,6 +1856,10 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) die(_("--filter can only be used with the remote " "configured in extensions.partialclone")); + if (stdin_refspecs) + die(_("--stdin can only be used when fetching " + "from one remote")); + if (max_children < 0) max_children = fetch_parallel_config; -- cgit v0.10.2-6-g49f6 From e5b942136ec1af98551b259997872e9849b2766c Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:33 -0700 Subject: fetch: avoid reading submodule config until needed In "fetch", there are two parameters submodule_fetch_jobs_config and recurse_submodules that can be set in a variety of ways: through .gitmodules, through .git/config, and through the command line. Currently "fetch" handles this by first reading .gitmodules, then reading .git/config (allowing it to overwrite existing values), then reading the command line (allowing it to overwrite existing values). Notice that we can avoid reading .gitmodules if .git/config and/or the command line already provides us with what we need. In addition, if recurse_submodules is found to be "no", we do not need the value of submodule_fetch_jobs_config. Avoiding reading .gitmodules is especially important when we use "git fetch" to perform lazy fetches in a partial clone because the .gitmodules file itself might need to be lazy fetched (and otherwise causing an infinite loop). In light of all this, avoid reading .gitmodules until necessary. When reading it, we may only need one of the two parameters it provides, so teach fetch_config_from_gitmodules() to support NULL arguments. With this patch, users (including Git itself when invoking "git fetch" to lazy-fetch) will be able to guarantee avoiding reading .gitmodules by passing --recurse-submodules=no. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index 9fca92b..ecd041e 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1782,12 +1782,18 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) free(anon); } - fetch_config_from_gitmodules(&submodule_fetch_jobs_config, - &recurse_submodules); git_config(git_fetch_config, NULL); argc = parse_options(argc, argv, prefix, builtin_fetch_options, builtin_fetch_usage, 0); + if (recurse_submodules != RECURSE_SUBMODULES_OFF) { + int *sfjc = submodule_fetch_jobs_config == -1 + ? &submodule_fetch_jobs_config : NULL; + int *rs = recurse_submodules == RECURSE_SUBMODULES_DEFAULT + ? &recurse_submodules : NULL; + + fetch_config_from_gitmodules(sfjc, rs); + } if (deepen_relative) { if (deepen_relative < 0) diff --git a/submodule-config.c b/submodule-config.c index e175dfb..c569e22 100644 --- a/submodule-config.c +++ b/submodule-config.c @@ -777,10 +777,14 @@ static int gitmodules_fetch_config(const char *var, const char *value, void *cb) { struct fetch_config *config = cb; if (!strcmp(var, "submodule.fetchjobs")) { - *(config->max_children) = parse_submodule_fetchjobs(var, value); + if (config->max_children) + *(config->max_children) = + parse_submodule_fetchjobs(var, value); return 0; } else if (!strcmp(var, "fetch.recursesubmodules")) { - *(config->recurse_submodules) = parse_fetch_recurse_submodules_arg(var, value); + if (config->recurse_submodules) + *(config->recurse_submodules) = + parse_fetch_recurse_submodules_arg(var, value); return 0; } -- cgit v0.10.2-6-g49f6 From abcb7eeb31d0267ada469646be993ad25e127d0d Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:34 -0700 Subject: fetch: only populate existing_refs if needed In "fetch", get_ref_map() iterates over all refs to populate "existing_refs" in order to populate peer_ref->old_oid in the returned refmap, even if the refmap has no peer_ref set - which is the case when only literal hashes (i.e. no refs by name) are fetched. Iterating over refs causes the targets of those refs to be checked for existence. Avoiding this is especially important when we use "git fetch" to perform lazy fetches in a partial clone because a target of such a ref may need to be itself lazy-fetched (and otherwise causing an infinite loop). Therefore, avoid populating "existing_refs" until necessary. With this patch, because Git lazy-fetches objects by literal hashes (to be done in a subsequent commit), it will then be able to guarantee avoiding reading targets of refs. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index ecd041e..320ba94 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -445,6 +445,7 @@ static struct ref *get_ref_map(struct remote *remote, struct ref *orefs = NULL, **oref_tail = &orefs; struct hashmap existing_refs; + int existing_refs_populated = 0; if (rs->nr) { struct refspec *fetch_refspec; @@ -538,15 +539,18 @@ static struct ref *get_ref_map(struct remote *remote, ref_map = ref_remove_duplicates(ref_map); - refname_hash_init(&existing_refs); - for_each_ref(add_one_refname, &existing_refs); - for (rm = ref_map; rm; rm = rm->next) { if (rm->peer_ref) { const char *refname = rm->peer_ref->name; struct refname_hash_entry *peer_item; unsigned int hash = strhash(refname); + if (!existing_refs_populated) { + refname_hash_init(&existing_refs); + for_each_ref(add_one_refname, &existing_refs); + existing_refs_populated = 1; + } + peer_item = hashmap_get_entry_from_hash(&existing_refs, hash, refname, struct refname_hash_entry, ent); @@ -556,7 +560,8 @@ static struct ref *get_ref_map(struct remote *remote, } } } - hashmap_free_entries(&existing_refs, struct refname_hash_entry, ent); + if (existing_refs_populated) + hashmap_free_entries(&existing_refs, struct refname_hash_entry, ent); return ref_map; } -- cgit v0.10.2-6-g49f6 From 5c3b801dab9d29c63c6c929405f808f064c11b77 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:35 -0700 Subject: fetch-pack: do not lazy-fetch during ref iteration In order to determine negotiation tips, "fetch-pack" iterates over all refs and dereferences all annotated tags found. This causes the existence of targets of refs and annotated tags to be checked. Avoiding this is especially important when we use "git fetch" (which invokes "fetch-pack") to perform lazy fetches in a partial clone because a target of such a ref or annotated tag may need to be itself lazy-fetched (and otherwise causing an infinite loop). Therefore, teach "fetch-pack" not to lazy fetch whenever iterating over refs. This is done by using the raw form of ref iteration and by dereferencing tags ourselves. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index 7f20eca..71dcc32 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -108,24 +108,48 @@ static void for_each_cached_alternate(struct fetch_negotiator *negotiator, cb(negotiator, cache.items[i]); } +static struct commit *deref_without_lazy_fetch(const struct object_id *oid, + int mark_tags_complete) +{ + enum object_type type; + struct object_info info = { .typep = &type }; + + while (1) { + if (oid_object_info_extended(the_repository, oid, &info, + OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK)) + return NULL; + if (type == OBJ_TAG) { + struct tag *tag = (struct tag *) + parse_object(the_repository, oid); + + if (!tag->tagged) + return NULL; + if (mark_tags_complete) + tag->object.flags |= COMPLETE; + oid = &tag->tagged->oid; + } else { + break; + } + } + if (type == OBJ_COMMIT) + return (struct commit *) parse_object(the_repository, oid); + return NULL; +} + static int rev_list_insert_ref(struct fetch_negotiator *negotiator, - const char *refname, const struct object_id *oid) { - struct object *o = deref_tag(the_repository, - parse_object(the_repository, oid), - refname, 0); - - if (o && o->type == OBJ_COMMIT) - negotiator->add_tip(negotiator, (struct commit *)o); + struct commit *c = deref_without_lazy_fetch(oid, 0); + if (c) + negotiator->add_tip(negotiator, c); return 0; } static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid, int flag, void *cb_data) { - return rev_list_insert_ref(cb_data, refname, oid); + return rev_list_insert_ref(cb_data, oid); } enum ack_type { @@ -201,7 +225,7 @@ static void send_request(struct fetch_pack_args *args, static void insert_one_alternate_object(struct fetch_negotiator *negotiator, struct object *obj) { - rev_list_insert_ref(negotiator, NULL, &obj->oid); + rev_list_insert_ref(negotiator, &obj->oid); } #define INITIAL_FLUSH 16 @@ -230,13 +254,12 @@ static void mark_tips(struct fetch_negotiator *negotiator, int i; if (!negotiation_tips) { - for_each_ref(rev_list_insert_ref_oid, negotiator); + for_each_rawref(rev_list_insert_ref_oid, negotiator); return; } for (i = 0; i < negotiation_tips->nr; i++) - rev_list_insert_ref(negotiator, NULL, - &negotiation_tips->oid[i]); + rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]); return; } @@ -503,21 +526,11 @@ static struct commit_list *complete; static int mark_complete(const struct object_id *oid) { - struct object *o = parse_object(the_repository, oid); - - while (o && o->type == OBJ_TAG) { - struct tag *t = (struct tag *) o; - if (!t->tagged) - break; /* broken repository */ - o->flags |= COMPLETE; - o = parse_object(the_repository, &t->tagged->oid); - } - if (o && o->type == OBJ_COMMIT) { - struct commit *commit = (struct commit *)o; - if (!(commit->object.flags & COMPLETE)) { - commit->object.flags |= COMPLETE; - commit_list_insert(commit, &complete); - } + struct commit *commit = deref_without_lazy_fetch(oid, 1); + + if (commit && !(commit->object.flags & COMPLETE)) { + commit->object.flags |= COMPLETE; + commit_list_insert(commit, &complete); } return 0; } @@ -702,7 +715,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, */ trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL); if (!args->deepen) { - for_each_ref(mark_complete_oid, NULL); + for_each_rawref(mark_complete_oid, NULL); for_each_cached_alternate(NULL, mark_alternate_complete); commit_list_sort_by_date(&complete); if (cutoff) @@ -716,16 +729,12 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, */ trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL); for (ref = *refs; ref; ref = ref->next) { - struct object *o = deref_tag(the_repository, - lookup_object(the_repository, - &ref->old_oid), - NULL, 0); + struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0); - if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) + if (!c || !(c->object.flags & COMPLETE)) continue; - negotiator->known_common(negotiator, - (struct commit *)o); + negotiator->known_common(negotiator, c); } trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL); diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 8827c2e..b1a7bd1 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -417,6 +417,26 @@ test_expect_success 'fetch lazy-fetches only to resolve deltas, protocol v2' ' grep "want $(cat hash)" trace ' +test_expect_success 'fetch does not lazy-fetch missing targets of its refs' ' + rm -rf server client trace && + + test_create_repo server && + test_config -C server uploadpack.allowfilter 1 && + test_config -C server uploadpack.allowanysha1inwant 1 && + test_commit -C server foo && + + git clone --filter=blob:none "file://$(pwd)/server" client && + # Make all refs point to nothing by deleting all objects. + rm client/.git/objects/pack/* && + + test_commit -C server bar && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --no-tags --recurse-submodules=no \ + origin refs/tags/bar && + FOO_HASH=$(git -C server rev-parse foo) && + ! grep "want $FOO_HASH" trace +' + # The following two tests must be in this order. It is important that # the srv.bare repository did not have tags during clone, but has tags # in the fetch. -- cgit v0.10.2-6-g49f6 From 7ca3c0ac37bfb803f11db14b7223f6d8f0cd142e Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:36 -0700 Subject: promisor-remote: lazy-fetch objects in subprocess Teach Git to lazy-fetch missing objects in a subprocess instead of doing it in-process. This allows any fatal errors that occur during the fetch to be isolated and converted into an error return value, instead of causing the current command being run to terminate. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/partial-clone.txt b/Documentation/technical/partial-clone.txt index b9e17e7..0780d30 100644 --- a/Documentation/technical/partial-clone.txt +++ b/Documentation/technical/partial-clone.txt @@ -171,20 +171,13 @@ additional flag. Fetching Missing Objects ------------------------ -- Fetching of objects is done using the existing transport mechanism using - transport_fetch_refs(), setting a new transport option - TRANS_OPT_NO_DEPENDENTS to indicate that only the objects themselves are - desired, not any object that they refer to. -+ -Because some transports invoke fetch_pack() in the same process, fetch_pack() -has been updated to not use any object flags when the corresponding argument -(no_dependents) is set. +- Fetching of objects is done by invoking a "git fetch" subprocess. - The local repository sends a request with the hashes of all requested - objects as "want" lines, and does not perform any packfile negotiation. + objects, and does not perform any packfile negotiation. It then receives a packfile. -- Because we are reusing the existing fetch-pack mechanism, fetching +- Because we are reusing the existing fetch mechanism, fetching currently fetches all objects referred to by the requested objects, even though they are not necessary. diff --git a/promisor-remote.c b/promisor-remote.c index baaea12..6530e26 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -3,6 +3,7 @@ #include "promisor-remote.h" #include "config.h" #include "transport.h" +#include "strvec.h" static char *repository_format_partial_clone; static const char *core_partial_clone_filter_default; @@ -12,39 +13,34 @@ void set_repository_format_partial_clone(char *partial_clone) repository_format_partial_clone = xstrdup_or_null(partial_clone); } -static int fetch_refs(const char *remote_name, struct ref *ref) -{ - struct remote *remote; - struct transport *transport; - int res; - - remote = remote_get(remote_name); - if (!remote->url[0]) - die(_("Remote with no URL")); - transport = transport_get(remote, remote->url[0]); - - transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1"); - res = transport_fetch_refs(transport, ref); - - return res; -} - static int fetch_objects(const char *remote_name, const struct object_id *oids, int oid_nr) { - struct ref *ref = NULL; + struct child_process child = CHILD_PROCESS_INIT; int i; + FILE *child_in; + + child.git_cmd = 1; + child.in = -1; + strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop", + "fetch", remote_name, "--no-tags", + "--no-write-fetch-head", "--recurse-submodules=no", + "--filter=blob:none", "--stdin", NULL); + if (start_command(&child)) + die(_("promisor-remote: unable to fork off fetch subprocess")); + child_in = xfdopen(child.in, "w"); for (i = 0; i < oid_nr; i++) { - struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i])); - oidcpy(&new_ref->old_oid, &oids[i]); - new_ref->exact_oid = 1; - new_ref->next = ref; - ref = new_ref; + if (fputs(oid_to_hex(&oids[i]), child_in) < 0) + die_errno(_("promisor-remote: could not write to fetch subprocess")); + if (fputc('\n', child_in) < 0) + die_errno(_("promisor-remote: could not write to fetch subprocess")); } - return fetch_refs(remote_name, ref); + + if (fclose(child_in) < 0) + die_errno(_("promisor-remote: could not close stdin to fetch subprocess")); + return finish_command(&child) ? -1 : 0; } static struct promisor_remote *promisors; diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index a5ebdf9..d681e90 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -214,7 +214,7 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled' rm -rf repo/.git/objects/* && rm -f trace && GIT_TRACE_PACKET="$(pwd)/trace" git -C repo cat-file -p "$HASH" && - grep "git< fetch=.*ref-in-want" trace + grep "fetch< fetch=.*ref-in-want" trace ' test_expect_success 'fetching of missing objects from another promisor remote' ' diff --git a/t/t4067-diff-partial-clone.sh b/t/t4067-diff-partial-clone.sh index ef8e0e9..804f2a8 100755 --- a/t/t4067-diff-partial-clone.sh +++ b/t/t4067-diff-partial-clone.sh @@ -20,7 +20,7 @@ test_expect_success 'git show batches blobs' ' # Ensure that there is exactly 1 negotiation by checking that there is # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -44,7 +44,7 @@ test_expect_success 'diff batches blobs' ' # Ensure that there is exactly 1 negotiation by checking that there is # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -127,7 +127,7 @@ test_expect_success 'diff with rename detection batches blobs' ' # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --raw -M HEAD^ HEAD >out && grep ":100644 100644.*R[0-9][0-9][0-9].*b.*c" out && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -175,7 +175,7 @@ test_expect_success 'diff --break-rewrites fetches only if necessary, and batche # by checking that there is only 1 "done" line sent. ("done" marks the # end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --break-rewrites --raw -M HEAD^ HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 3d6a933..392201c 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -528,7 +528,7 @@ test_expect_success 'prefetch objects' ' TWO=$(git -C server rev-parse three_branch^) && git -C client fetch --filter=blob:none origin "$TWO" && GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch && - grep "git> done" trace >donelines && + grep "fetch> done" trace >donelines && test_line_count = 1 donelines ' diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index eb9a093..15fb64c 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -704,7 +704,7 @@ test_expect_success 'batch missing blob request during checkout' ' # Ensure that there is only one negotiation by checking that there is # only "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' -- cgit v0.10.2-6-g49f6 From 9dfa8dbeee18a126aabcdd36a06e5d6b5eb6a58a Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:37 -0700 Subject: fetch-pack: remove no_dependents code Now that Git has switched to using a subprocess to lazy-fetch missing objects, remove the no_dependents code as it is no longer used. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index bbb5c96..58b7c1f 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -153,10 +153,6 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.from_promisor = 1; continue; } - if (!strcmp("--no-dependents", arg)) { - args.no_dependents = 1; - continue; - } if (skip_prefix(arg, ("--" CL_ARG__FILTER "="), &arg)) { parse_list_objects_filter(&args.filter_options, arg); continue; diff --git a/fetch-pack.c b/fetch-pack.c index 71dcc32..6c576a9 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -285,10 +285,8 @@ static int find_common(struct fetch_negotiator *negotiator, PACKET_READ_CHOMP_NEWLINE | PACKET_READ_DIE_ON_ERR_PACKET); - if (!args->no_dependents) { - mark_tips(negotiator, args->negotiation_tips); - for_each_cached_alternate(negotiator, insert_one_alternate_object); - } + mark_tips(negotiator, args->negotiation_tips); + for_each_cached_alternate(negotiator, insert_one_alternate_object); fetching = 0; for ( ; refs ; refs = refs->next) { @@ -305,12 +303,8 @@ static int find_common(struct fetch_negotiator *negotiator, * We use lookup_object here because we are only * interested in the case we *know* the object is * reachable and we have already scanned it. - * - * Do this only if args->no_dependents is false (if it is true, - * we cannot trust the object flags). */ - if (!args->no_dependents && - ((o = lookup_object(the_repository, remote)) != NULL) && + if (((o = lookup_object(the_repository, remote)) != NULL) && (o->flags & COMPLETE)) { continue; } @@ -410,8 +404,6 @@ static int find_common(struct fetch_negotiator *negotiator, trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository); flushes = 0; retval = -1; - if (args->no_dependents) - goto done; while ((oid = negotiator->next(negotiator))) { packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); print_verbose(args, "have %s", oid_to_hex(oid)); @@ -666,9 +658,7 @@ struct loose_object_iter { /* * Mark recent commits available locally and reachable from a local ref as - * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as - * COMMON_REF (otherwise, we are not planning to participate in negotiation, and - * thus do not need COMMON_REF marks). + * COMPLETE. * * The cutoff time for recency is determined by this heuristic: it is the * earliest commit time of the objects in refs that are commits and that we know @@ -969,12 +959,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, struct fetch_negotiator negotiator_alloc; struct fetch_negotiator *negotiator; - if (args->no_dependents) { - negotiator = NULL; - } else { - negotiator = &negotiator_alloc; - fetch_negotiator_init(r, negotiator); - } + negotiator = &negotiator_alloc; + fetch_negotiator_init(r, negotiator); sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); @@ -1062,15 +1048,11 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (!server_supports_hash(the_hash_algo->name, NULL)) die(_("Server does not support this repository's object format")); - if (!args->no_dependents) { - mark_complete_and_common_ref(negotiator, args, &ref); - filter_refs(args, &ref, sought, nr_sought); - if (everything_local(args, &ref)) { - packet_flush(fd[1]); - goto all_done; - } - } else { - filter_refs(args, &ref, sought, nr_sought); + mark_complete_and_common_ref(negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) { + packet_flush(fd[1]); + goto all_done; } if (find_common(negotiator, args, fd, &oid, ref) < 0) if (!args->keep_pack) @@ -1119,7 +1101,7 @@ static void add_shallow_requests(struct strbuf *req_buf, packet_buf_write(req_buf, "deepen-relative\n"); } -static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf) +static void add_wants(const struct ref *wants, struct strbuf *req_buf) { int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0); @@ -1136,12 +1118,8 @@ static void add_wants(int no_dependents, const struct ref *wants, struct strbuf * We use lookup_object here because we are only * interested in the case we *know* the object is * reachable and we have already scanned it. - * - * Do this only if args->no_dependents is false (if it is true, - * we cannot trust the object flags). */ - if (!no_dependents && - ((o = lookup_object(the_repository, remote)) != NULL) && + if (((o = lookup_object(the_repository, remote)) != NULL) && (o->flags & COMPLETE)) { continue; } @@ -1275,19 +1253,14 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, } /* add wants */ - add_wants(args->no_dependents, wants, &req_buf); + add_wants(wants, &req_buf); - if (args->no_dependents) { - packet_buf_write(&req_buf, "done"); - ret = 1; - } else { - /* Add all of the common commits we've found in previous rounds */ - add_common(&req_buf, common); + /* Add all of the common commits we've found in previous rounds */ + add_common(&req_buf, common); - /* Add initial haves */ - ret = add_haves(negotiator, seen_ack, &req_buf, - haves_to_send, in_vain); - } + /* Add initial haves */ + ret = add_haves(negotiator, seen_ack, &req_buf, + haves_to_send, in_vain); /* Send request */ packet_buf_flush(&req_buf); @@ -1547,12 +1520,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, struct string_list packfile_uris = STRING_LIST_INIT_DUP; int i; - if (args->no_dependents) { - negotiator = NULL; - } else { - negotiator = &negotiator_alloc; - fetch_negotiator_init(r, negotiator); - } + negotiator = &negotiator_alloc; + fetch_negotiator_init(r, negotiator); packet_reader_init(&reader, fd[0], NULL, 0, PACKET_READ_CHOMP_NEWLINE | @@ -1576,21 +1545,16 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, args->deepen = 1; /* Filter 'ref' by 'sought' and those that aren't local */ - if (!args->no_dependents) { - mark_complete_and_common_ref(negotiator, args, &ref); - filter_refs(args, &ref, sought, nr_sought); - if (everything_local(args, &ref)) - state = FETCH_DONE; - else - state = FETCH_SEND_REQUEST; - - mark_tips(negotiator, args->negotiation_tips); - for_each_cached_alternate(negotiator, - insert_one_alternate_object); - } else { - filter_refs(args, &ref, sought, nr_sought); + mark_complete_and_common_ref(negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) + state = FETCH_DONE; + else state = FETCH_SEND_REQUEST; - } + + mark_tips(negotiator, args->negotiation_tips); + for_each_cached_alternate(negotiator, + insert_one_alternate_object); break; case FETCH_SEND_REQUEST: if (!negotiation_started) { @@ -1911,20 +1875,6 @@ struct ref *fetch_pack(struct fetch_pack_args *args, if (nr_sought) nr_sought = remove_duplicates_in_refs(sought, nr_sought); - if (args->no_dependents && !args->filter_options.choice) { - /* - * The protocol does not support requesting that only the - * wanted objects be sent, so approximate this by setting a - * "blob:none" filter if no filter is already set. This works - * for all object types: note that wanted blobs will still be - * sent because they are directly specified as a "want". - * - * NEEDSWORK: Add an option in the protocol to request that - * only the wanted objects be sent, and implement it. - */ - parse_list_objects_filter(&args->filter_options, "blob:none"); - } - if (version != protocol_v2 && !ref) { packet_flush(fd[1]); die(_("no matching remote head")); diff --git a/fetch-pack.h b/fetch-pack.h index 85d1e39..bbe2938 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -43,20 +43,6 @@ struct fetch_pack_args { unsigned from_promisor:1; /* - * Attempt to fetch only the wanted objects, and not any objects - * referred to by them. Due to protocol limitations, extraneous - * objects may still be included. (When fetching non-blob - * objects, only blobs are excluded; when fetching a blob, the - * blob itself will still be sent. The client does not need to - * know whether a wanted object is a blob or not.) - * - * If 1, fetch_pack() will also not modify any object flags. - * This allows fetch_pack() to safely be called by any function, - * regardless of which object flags it uses (if any). - */ - unsigned no_dependents:1; - - /* * Because fetch_pack() overwrites the shallow file upon a * successful deepening non-clone fetch, if this struct * specifies such a fetch, fetch_pack() needs to perform a diff --git a/remote-curl.c b/remote-curl.c index 62b3a45..ce470f2 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -40,7 +40,6 @@ struct options { push_cert : 2, deepen_relative : 1, from_promisor : 1, - no_dependents : 1, atomic : 1, object_format : 1; const struct git_hash_algo *hash_algo; @@ -190,9 +189,6 @@ static int set_option(const char *name, const char *value) } else if (!strcmp(name, "from-promisor")) { options.from_promisor = 1; return 0; - } else if (!strcmp(name, "no-dependents")) { - options.no_dependents = 1; - return 0; } else if (!strcmp(name, "filter")) { options.filter = xstrdup(value); return 0; @@ -1175,8 +1171,6 @@ static int fetch_git(struct discovery *heads, strvec_push(&args, "--deepen-relative"); if (options.from_promisor) strvec_push(&args, "--from-promisor"); - if (options.no_dependents) - strvec_push(&args, "--no-dependents"); if (options.filter) strvec_pushf(&args, "--filter=%s", options.filter); strvec_push(&args, url.buf); diff --git a/transport.c b/transport.c index 2d4fd85..7efe323 100644 --- a/transport.c +++ b/transport.c @@ -232,9 +232,6 @@ static int set_git_option(struct git_transport_options *opts, } else if (!strcmp(name, TRANS_OPT_FROM_PROMISOR)) { opts->from_promisor = !!value; return 0; - } else if (!strcmp(name, TRANS_OPT_NO_DEPENDENTS)) { - opts->no_dependents = !!value; - return 0; } else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) { list_objects_filter_die_if_populated(&opts->filter_options); parse_list_objects_filter(&opts->filter_options, value); @@ -359,7 +356,6 @@ static int fetch_refs_via_pack(struct transport *transport, args.cloning = transport->cloning; args.update_shallow = data->options.update_shallow; args.from_promisor = data->options.from_promisor; - args.no_dependents = data->options.no_dependents; args.filter_options = data->options.filter_options; args.stateless_rpc = transport->stateless_rpc; args.server_options = transport->server_options; diff --git a/transport.h b/transport.h index 1be4013..fe5a0a6 100644 --- a/transport.h +++ b/transport.h @@ -16,7 +16,6 @@ struct git_transport_options { unsigned update_shallow : 1; unsigned deepen_relative : 1; unsigned from_promisor : 1; - unsigned no_dependents : 1; /* * If this transport supports connect or stateless-connect, @@ -201,12 +200,6 @@ void transport_check_allowed(const char *type); /* Indicate that these objects are being fetched by a promisor */ #define TRANS_OPT_FROM_PROMISOR "from-promisor" -/* - * Indicate that only the objects wanted need to be fetched, not their - * dependents - */ -#define TRANS_OPT_NO_DEPENDENTS "no-dependents" - /* Filter objects for partial clone and fetch */ #define TRANS_OPT_LIST_OBJECTS_FILTER "filter" -- cgit v0.10.2-6-g49f6 From db3c293ecded67128b74a03f01e65c6799ff1116 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Wed, 2 Sep 2020 14:05:39 -0700 Subject: fetch: no FETCH_HEAD display if --no-write-fetch-head 887952b8c6 ("fetch: optionally allow disabling FETCH_HEAD update", 2020-08-18) introduced the ability to disable writing to FETCH_HEAD during fetch, but did not suppress the " -> FETCH_HEAD" message when this ability is used. This message is misleading in this case, because FETCH_HEAD is not written. Also, because "fetch" is used to lazy-fetch missing objects in a partial clone, this significantly clutters up the output in that case since the objects to be fetched are potentially numerous. Therefore, suppress this message when --no-write-fetch-head is passed (but not when --dry-run is set). Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index 320ba94..c6c4689 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1023,11 +1023,17 @@ static int store_updated_refs(const char *raw_url, const char *remote_name, rc |= update_local_ref(ref, what, rm, ¬e, summary_width); free(ref); - } else + } else if (write_fetch_head || dry_run) { + /* + * Display fetches written to FETCH_HEAD (or + * would be written to FETCH_HEAD, if --dry-run + * is set). + */ format_display(¬e, '*', *kind ? kind : "branch", NULL, *what ? what : "HEAD", "FETCH_HEAD", summary_width); + } if (note.len) { if (verbosity >= 0 && !shown_url) { fprintf(stderr, _("From %.*s\n"), diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index d681e90..584a039 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -183,7 +183,7 @@ test_expect_success 'missing CLI object, but promised, passes fsck' ' ' test_expect_success 'fetching of missing objects' ' - rm -rf repo && + rm -rf repo err && test_create_repo server && test_commit -C server foo && git -C server repack -a -d --write-bitmap-index && @@ -194,7 +194,10 @@ test_expect_success 'fetching of missing objects' ' git -C repo config core.repositoryformatversion 1 && git -C repo config extensions.partialclone "origin" && - git -C repo cat-file -p "$HASH" && + git -C repo cat-file -p "$HASH" 2>err && + + # Ensure that no spurious FETCH_HEAD messages are written + ! grep FETCH_HEAD err && # Ensure that the .promisor file is written, and check that its # associated packfile contains the object diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 2a1abe9..759aec9 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -543,16 +543,18 @@ test_expect_success 'fetch into the current branch with --update-head-ok' ' ' -test_expect_success 'fetch --dry-run does not touch FETCH_HEAD' ' - rm -f .git/FETCH_HEAD && - git fetch --dry-run . && - ! test -f .git/FETCH_HEAD +test_expect_success 'fetch --dry-run does not touch FETCH_HEAD, but still prints what would be written' ' + rm -f .git/FETCH_HEAD err && + git fetch --dry-run . 2>err && + ! test -f .git/FETCH_HEAD && + grep FETCH_HEAD err ' -test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD' ' - rm -f .git/FETCH_HEAD && - git fetch --no-write-fetch-head . && - ! test -f .git/FETCH_HEAD +test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD, and does not print what would be written' ' + rm -f .git/FETCH_HEAD err && + git fetch --no-write-fetch-head . 2>err && + ! test -f .git/FETCH_HEAD && + ! grep FETCH_HEAD err ' test_expect_success '--write-fetch-head gets defeated by --dry-run' ' -- cgit v0.10.2-6-g49f6