From c59c7c879ef60b2b0d0b050982712c53ec4d0680 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:25 +0200 Subject: t0410: remove pipes after git commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's not run a git command, especially one with "verify" in its name, upstream of a pipe, because the pipe will hide the git command's exit code. While at it, let's also avoid a useless `cat` command piping into `sed`. Helped-by: SZEDER Gábor Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index 5bd892f..3559313 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -166,8 +166,9 @@ test_expect_success 'fetching of missing objects' ' # associated packfile contains the object ls repo/.git/objects/pack/pack-*.promisor >promisorlist && test_line_count = 1 promisorlist && - IDX=$(cat promisorlist | sed "s/promisor$/idx/") && - git verify-pack --verbose "$IDX" | grep "$HASH" + IDX=$(sed "s/promisor$/idx/" promisorlist) && + git verify-pack --verbose "$IDX" >out && + grep "$HASH" out ' test_expect_success 'fetching of missing objects works with ref-in-want enabled' ' @@ -514,8 +515,9 @@ test_expect_success 'fetching of missing objects from an HTTP server' ' # associated packfile contains the object ls repo/.git/objects/pack/pack-*.promisor >promisorlist && test_line_count = 1 promisorlist && - IDX=$(cat promisorlist | sed "s/promisor$/idx/") && - git verify-pack --verbose "$IDX" | grep "$HASH" + IDX=$(sed "s/promisor$/idx/" promisorlist) && + git verify-pack --verbose "$IDX" >out && + grep "$HASH" out ' test_done -- cgit v0.10.2-6-g49f6 From 2e860675b6572cf476e99888134a5b307fd7eb62 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:26 +0200 Subject: fetch-object: make functions return an error code The callers of the fetch_object() and fetch_objects() might be interested in knowing if these functions succeeded or not. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/fetch-object.c b/fetch-object.c index 4266548..eac4d44 100644 --- a/fetch-object.c +++ b/fetch-object.c @@ -5,11 +5,12 @@ #include "transport.h" #include "fetch-object.h" -static void fetch_refs(const char *remote_name, struct ref *ref) +static int fetch_refs(const char *remote_name, struct ref *ref) { struct remote *remote; struct transport *transport; int original_fetch_if_missing = fetch_if_missing; + int res; fetch_if_missing = 0; remote = remote_get(remote_name); @@ -19,12 +20,14 @@ static void fetch_refs(const char *remote_name, struct ref *ref) transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1"); - transport_fetch_refs(transport, ref); + res = transport_fetch_refs(transport, ref); fetch_if_missing = original_fetch_if_missing; + + return res; } -void fetch_objects(const char *remote_name, const struct object_id *oids, - int oid_nr) +int fetch_objects(const char *remote_name, const struct object_id *oids, + int oid_nr) { struct ref *ref = NULL; int i; @@ -36,5 +39,5 @@ void fetch_objects(const char *remote_name, const struct object_id *oids, new_ref->next = ref; ref = new_ref; } - fetch_refs(remote_name, ref); + return fetch_refs(remote_name, ref); } diff --git a/fetch-object.h b/fetch-object.h index d6444ca..7bcc7ca 100644 --- a/fetch-object.h +++ b/fetch-object.h @@ -3,7 +3,7 @@ struct object_id; -void fetch_objects(const char *remote_name, const struct object_id *oids, - int oid_nr); +int fetch_objects(const char *remote_name, const struct object_id *oids, + int oid_nr); #endif diff --git a/sha1-file.c b/sha1-file.c index 888b602..819d32c 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -1381,8 +1381,8 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, !already_retried && r == the_repository && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { /* - * TODO Investigate having fetch_object() return - * TODO error/success and stopping the music here. + * TODO Investigate checking fetch_object() return + * TODO value and stopping on error here. * TODO Pass a repository struct through fetch_object, * such that arbitrary repositories work. */ -- cgit v0.10.2-6-g49f6 From 48de315817281e49a5e9000d40550b5257b437c6 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:27 +0200 Subject: Add initial support for many promisor remotes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The promisor-remote.{c,h} files will contain functions to manage many promisor remotes. We expect that there will not be a lot of promisor remotes, so it is ok to use a simple linked list to manage them. Helped-by: Jeff King Helped-by: SZEDER Gábor Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index f58bf14..049bc8c 100644 --- a/Makefile +++ b/Makefile @@ -944,6 +944,7 @@ LIB_OBJS += preload-index.o LIB_OBJS += pretty.o LIB_OBJS += prio-queue.o LIB_OBJS += progress.o +LIB_OBJS += promisor-remote.o LIB_OBJS += prompt.o LIB_OBJS += protocol.o LIB_OBJS += quote.o diff --git a/promisor-remote.c b/promisor-remote.c new file mode 100644 index 0000000..c249b80 --- /dev/null +++ b/promisor-remote.c @@ -0,0 +1,92 @@ +#include "cache.h" +#include "promisor-remote.h" +#include "config.h" + +static struct promisor_remote *promisors; +static struct promisor_remote **promisors_tail = &promisors; + +static struct promisor_remote *promisor_remote_new(const char *remote_name) +{ + struct promisor_remote *r; + + if (*remote_name == '/') { + warning(_("promisor remote name cannot begin with '/': %s"), + remote_name); + return NULL; + } + + FLEX_ALLOC_STR(r, name, remote_name); + + *promisors_tail = r; + promisors_tail = &r->next; + + return r; +} + +static struct promisor_remote *promisor_remote_lookup(const char *remote_name, + struct promisor_remote **previous) +{ + struct promisor_remote *r, *p; + + for (p = NULL, r = promisors; r; p = r, r = r->next) + if (!strcmp(r->name, remote_name)) { + if (previous) + *previous = p; + return r; + } + + return NULL; +} + +static int promisor_remote_config(const char *var, const char *value, void *data) +{ + const char *name; + int namelen; + const char *subkey; + + if (parse_config_key(var, "remote", &name, &namelen, &subkey) < 0) + return 0; + + if (!strcmp(subkey, "promisor")) { + char *remote_name; + + if (!git_config_bool(var, value)) + return 0; + + remote_name = xmemdupz(name, namelen); + + if (!promisor_remote_lookup(remote_name, NULL)) + promisor_remote_new(remote_name); + + free(remote_name); + return 0; + } + + return 0; +} + +static void promisor_remote_init(void) +{ + static int initialized; + + if (initialized) + return; + initialized = 1; + + git_config(promisor_remote_config, NULL); +} + +struct promisor_remote *promisor_remote_find(const char *remote_name) +{ + promisor_remote_init(); + + if (!remote_name) + return promisors; + + return promisor_remote_lookup(remote_name, NULL); +} + +int has_promisor_remote(void) +{ + return !!promisor_remote_find(NULL); +} diff --git a/promisor-remote.h b/promisor-remote.h new file mode 100644 index 0000000..01dcdf4 --- /dev/null +++ b/promisor-remote.h @@ -0,0 +1,16 @@ +#ifndef PROMISOR_REMOTE_H +#define PROMISOR_REMOTE_H + +/* + * Promisor remote linked list + * Its information come from remote.XXX config entries. + */ +struct promisor_remote { + struct promisor_remote *next; + const char name[FLEX_ARRAY]; +}; + +extern struct promisor_remote *promisor_remote_find(const char *remote_name); +extern int has_promisor_remote(void); + +#endif /* PROMISOR_REMOTE_H */ -- cgit v0.10.2-6-g49f6 From 9e27beaa2344dc6dd422d7711a666c082785118f Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:28 +0200 Subject: promisor-remote: implement promisor_remote_get_direct() This is implemented for now by calling fetch_objects(). It fetches from all the promisor remotes. Helped-by: Ramsay Jones Helped-by: Derrick Stolee Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/promisor-remote.c b/promisor-remote.c index c249b80..b79a84c 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -1,6 +1,8 @@ #include "cache.h" +#include "object-store.h" #include "promisor-remote.h" #include "config.h" +#include "fetch-object.h" static struct promisor_remote *promisors; static struct promisor_remote **promisors_tail = &promisors; @@ -90,3 +92,68 @@ int has_promisor_remote(void) { return !!promisor_remote_find(NULL); } + +static int remove_fetched_oids(struct repository *repo, + struct object_id **oids, + int oid_nr, int to_free) +{ + int i, remaining_nr = 0; + int *remaining = xcalloc(oid_nr, sizeof(*remaining)); + struct object_id *old_oids = *oids; + struct object_id *new_oids; + + for (i = 0; i < oid_nr; i++) + if (oid_object_info_extended(repo, &old_oids[i], NULL, + OBJECT_INFO_SKIP_FETCH_OBJECT)) { + remaining[i] = 1; + remaining_nr++; + } + + if (remaining_nr) { + int j = 0; + new_oids = xcalloc(remaining_nr, sizeof(*new_oids)); + for (i = 0; i < oid_nr; i++) + if (remaining[i]) + oidcpy(&new_oids[j++], &old_oids[i]); + *oids = new_oids; + if (to_free) + free(old_oids); + } + + free(remaining); + + return remaining_nr; +} + +int promisor_remote_get_direct(struct repository *repo, + const struct object_id *oids, + int oid_nr) +{ + struct promisor_remote *r; + struct object_id *remaining_oids = (struct object_id *)oids; + int remaining_nr = oid_nr; + int to_free = 0; + int res = -1; + + promisor_remote_init(); + + for (r = promisors; r; r = r->next) { + if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) { + if (remaining_nr == 1) + continue; + remaining_nr = remove_fetched_oids(repo, &remaining_oids, + remaining_nr, to_free); + if (remaining_nr) { + to_free = 1; + continue; + } + } + res = 0; + break; + } + + if (to_free) + free(remaining_oids); + + return res; +} diff --git a/promisor-remote.h b/promisor-remote.h index 01dcdf4..ed4ecea 100644 --- a/promisor-remote.h +++ b/promisor-remote.h @@ -1,6 +1,8 @@ #ifndef PROMISOR_REMOTE_H #define PROMISOR_REMOTE_H +struct object_id; + /* * Promisor remote linked list * Its information come from remote.XXX config entries. @@ -12,5 +14,8 @@ struct promisor_remote { extern struct promisor_remote *promisor_remote_find(const char *remote_name); extern int has_promisor_remote(void); +extern int promisor_remote_get_direct(struct repository *repo, + const struct object_id *oids, + int oid_nr); #endif /* PROMISOR_REMOTE_H */ -- cgit v0.10.2-6-g49f6 From 9cfebc1f3b2b60290b6321b95e5038b6e5b758ab Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:29 +0200 Subject: promisor-remote: add promisor_remote_reinit() We will need to reinitialize the promisor remote configuration as we will make some changes to it in a later commit. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/promisor-remote.c b/promisor-remote.c index b79a84c..763d98a 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -67,10 +67,10 @@ static int promisor_remote_config(const char *var, const char *value, void *data return 0; } +static int initialized; + static void promisor_remote_init(void) { - static int initialized; - if (initialized) return; initialized = 1; @@ -78,6 +78,24 @@ static void promisor_remote_init(void) git_config(promisor_remote_config, NULL); } +static void promisor_remote_clear(void) +{ + while (promisors) { + struct promisor_remote *r = promisors; + promisors = promisors->next; + free(r); + } + + promisors_tail = &promisors; +} + +void promisor_remote_reinit(void) +{ + initialized = 0; + promisor_remote_clear(); + promisor_remote_init(); +} + struct promisor_remote *promisor_remote_find(const char *remote_name) { promisor_remote_init(); diff --git a/promisor-remote.h b/promisor-remote.h index ed4ecea..dddd404 100644 --- a/promisor-remote.h +++ b/promisor-remote.h @@ -12,6 +12,7 @@ struct promisor_remote { const char name[FLEX_ARRAY]; }; +extern void promisor_remote_reinit(void); extern struct promisor_remote *promisor_remote_find(const char *remote_name); extern int has_promisor_remote(void); extern int promisor_remote_get_direct(struct repository *repo, -- cgit v0.10.2-6-g49f6 From faf2abf496bb8e5a5fbf3818f3e78077b2f3e143 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:30 +0200 Subject: promisor-remote: use repository_format_partial_clone A remote specified using the extensions.partialClone config option should be considered a promisor remote too. For simplicity and to make things predictable, this promisor remote should be either always the last one we try to get objects from, or the first one. So it should always be either at the end of the promisor remote list, or at its start. We decided to make it the last one we try, because it is likely that someone using many promisor remotes is doing so because the other promisor remotes are better for some reason (maybe they are closer or faster for some kind of objects) than the origin, and the origin is likely to be the remote specified by extensions.partialClone. This justification is not very strong, but one choice had to be made, and anyway the long term plan should be to make the order somehow fully configurable. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/promisor-remote.c b/promisor-remote.c index 763d98a..6a8856f 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -40,6 +40,18 @@ static struct promisor_remote *promisor_remote_lookup(const char *remote_name, return NULL; } +static void promisor_remote_move_to_tail(struct promisor_remote *r, + struct promisor_remote *previous) +{ + if (previous) + previous->next = r->next; + else + promisors = r->next ? r->next : r; + r->next = NULL; + *promisors_tail = r; + promisors_tail = &r->next; +} + static int promisor_remote_config(const char *var, const char *value, void *data) { const char *name; @@ -76,6 +88,17 @@ static void promisor_remote_init(void) initialized = 1; git_config(promisor_remote_config, NULL); + + if (repository_format_partial_clone) { + struct promisor_remote *o, *previous; + + o = promisor_remote_lookup(repository_format_partial_clone, + &previous); + if (o) + promisor_remote_move_to_tail(o, previous); + else + promisor_remote_new(repository_format_partial_clone); + } } static void promisor_remote_clear(void) -- cgit v0.10.2-6-g49f6 From b14ed5adaf87c5943433fd6b1d2cbe8c060f9264 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:31 +0200 Subject: Use promisor_remote_get_direct() and has_promisor_remote() Instead of using the repository_format_partial_clone global and fetch_objects() directly, let's use has_promisor_remote() and promisor_remote_get_direct(). This way all the configured promisor remotes will be taken into account, not only the one specified by extensions.partialClone. Also when cloning or fetching using a partial clone filter, remote.origin.promisor will be set to "true" instead of setting extensions.partialClone to "origin". This makes it possible to use many promisor remote just by fetching from them. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 0f09238..85ae10b 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -15,6 +15,7 @@ #include "sha1-array.h" #include "packfile.h" #include "object-store.h" +#include "promisor-remote.h" struct batch_options { int enabled; @@ -523,8 +524,8 @@ static int batch_objects(struct batch_options *opt) if (opt->all_objects) { struct object_cb_data cb; - if (repository_format_partial_clone) - warning("This repository has extensions.partialClone set. Some objects may not be loaded."); + if (has_promisor_remote()) + warning("This repository uses promisor remotes. Some objects may not be loaded."); cb.opt = opt; cb.expand = &data; diff --git a/builtin/fetch.c b/builtin/fetch.c index 4ba63d5..f74bd78 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -23,6 +23,7 @@ #include "packfile.h" #include "list-objects-filter-options.h" #include "commit-reach.h" +#include "promisor-remote.h" static const char * const builtin_fetch_usage[] = { N_("git fetch [] [ [...]]"), @@ -1460,7 +1461,7 @@ static inline void fetch_one_setup_partial(struct remote *remote) * If no prior partial clone/fetch and the current fetch DID NOT * request a partial-fetch, do a normal fetch. */ - if (!repository_format_partial_clone && !filter_options.choice) + if (!has_promisor_remote() && !filter_options.choice) return; /* @@ -1468,7 +1469,7 @@ static inline void fetch_one_setup_partial(struct remote *remote) * on this repo and remember the given filter-spec as the default * for subsequent fetches to this remote. */ - if (!repository_format_partial_clone && filter_options.choice) { + if (!has_promisor_remote() && filter_options.choice) { partial_clone_register(remote->name, &filter_options); return; } @@ -1477,7 +1478,7 @@ static inline void fetch_one_setup_partial(struct remote *remote) * We are currently limited to only ONE promisor remote and only * allow partial-fetches from the promisor remote. */ - if (strcmp(remote->name, repository_format_partial_clone)) { + if (!promisor_remote_find(remote->name)) { if (filter_options.choice) die(_("--filter can only be used with the remote " "configured in extensions.partialClone")); @@ -1611,7 +1612,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) if (depth || deepen_since || deepen_not.nr) deepen = 1; - if (filter_options.choice && !repository_format_partial_clone) + if (filter_options.choice && !has_promisor_remote()) die("--filter can only be used when extensions.partialClone is set"); if (all) { @@ -1645,7 +1646,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) } if (remote) { - if (filter_options.choice || repository_format_partial_clone) + if (filter_options.choice || has_promisor_remote()) fetch_one_setup_partial(remote); result = fetch_one(remote, argc, argv, prune_tags_ok); } else { diff --git a/builtin/gc.c b/builtin/gc.c index 8943bcc..824a883 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -27,6 +27,7 @@ #include "pack-objects.h" #include "blob.h" #include "tree.h" +#include "promisor-remote.h" #define FAILED_RUN "failed to run %s" @@ -661,7 +662,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix) argv_array_push(&prune, prune_expire); if (quiet) argv_array_push(&prune, "--no-progress"); - if (repository_format_partial_clone) + if (has_promisor_remote()) argv_array_push(&prune, "--exclude-promisor-objects"); if (run_command_v_opt(prune.argv, RUN_GIT_CMD)) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 0d55f73..a23454d 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -14,7 +14,7 @@ #include "thread-utils.h" #include "packfile.h" #include "object-store.h" -#include "fetch-object.h" +#include "promisor-remote.h" static const char index_pack_usage[] = "git index-pack [-v] [-o ] [--keep | --keep=] [--verify] [--strict] ( | --stdin [--fix-thin] [])"; @@ -1352,7 +1352,7 @@ static void fix_unresolved_deltas(struct hashfile *f) sorted_by_pos[i] = &ref_deltas[i]; QSORT(sorted_by_pos, nr_ref_deltas, delta_pos_compare); - if (repository_format_partial_clone) { + if (has_promisor_remote()) { /* * Prefetch the delta bases. */ @@ -1366,8 +1366,8 @@ static void fix_unresolved_deltas(struct hashfile *f) oid_array_append(&to_fetch, &d->oid); } if (to_fetch.nr) - fetch_objects(repository_format_partial_clone, - to_fetch.oid, to_fetch.nr); + promisor_remote_get_direct(the_repository, + to_fetch.oid, to_fetch.nr); oid_array_clear(&to_fetch); } diff --git a/builtin/repack.c b/builtin/repack.c index caca113..df9a32c 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -11,6 +11,7 @@ #include "midx.h" #include "packfile.h" #include "object-store.h" +#include "promisor-remote.h" static int delta_base_offset = 1; static int pack_kept_objects = -1; @@ -369,7 +370,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) argv_array_push(&cmd.args, "--all"); argv_array_push(&cmd.args, "--reflog"); argv_array_push(&cmd.args, "--indexed-objects"); - if (repository_format_partial_clone) + if (has_promisor_remote()) argv_array_push(&cmd.args, "--exclude-promisor-objects"); if (write_bitmaps) argv_array_push(&cmd.args, "--write-bitmap-index"); diff --git a/cache-tree.c b/cache-tree.c index b13bfaf..64c285a 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -5,6 +5,7 @@ #include "cache-tree.h" #include "object-store.h" #include "replace-object.h" +#include "promisor-remote.h" #ifndef DEBUG #define DEBUG 0 @@ -357,7 +358,7 @@ static int update_one(struct cache_tree *it, } ce_missing_ok = mode == S_IFGITLINK || missing_ok || - (repository_format_partial_clone && + (has_promisor_remote() && ce_skip_worktree(ce)); if (is_null_oid(oid) || (!ce_missing_ok && !has_object_file(oid))) { diff --git a/connected.c b/connected.c index 1ab481f..b0e4968 100644 --- a/connected.c +++ b/connected.c @@ -5,6 +5,7 @@ #include "connected.h" #include "transport.h" #include "packfile.h" +#include "promisor-remote.h" /* * If we feed all the commits we want to verify to this command @@ -73,7 +74,7 @@ int check_connected(oid_iterate_fn fn, void *cb_data, argv_array_push(&rev_list.args,"rev-list"); argv_array_push(&rev_list.args, "--objects"); argv_array_push(&rev_list.args, "--stdin"); - if (repository_format_partial_clone) + if (has_promisor_remote()) argv_array_push(&rev_list.args, "--exclude-promisor-objects"); if (!opt->is_deepening_fetch) { argv_array_push(&rev_list.args, "--not"); diff --git a/diff.c b/diff.c index 1ee04e3..249cc6e 100644 --- a/diff.c +++ b/diff.c @@ -25,7 +25,7 @@ #include "packfile.h" #include "parse-options.h" #include "help.h" -#include "fetch-object.h" +#include "promisor-remote.h" #ifdef NO_FAST_WORKING_DIRECTORY #define FAST_WORKING_DIRECTORY 0 @@ -6514,8 +6514,7 @@ static void add_if_missing(struct repository *r, void diffcore_std(struct diff_options *options) { - if (options->repo == the_repository && - repository_format_partial_clone) { + if (options->repo == the_repository && has_promisor_remote()) { /* * Prefetch the diff pairs that are about to be flushed. */ @@ -6532,8 +6531,8 @@ void diffcore_std(struct diff_options *options) /* * NEEDSWORK: Consider deduplicating the OIDs sent. */ - fetch_objects(repository_format_partial_clone, - to_fetch.oid, to_fetch.nr); + promisor_remote_get_direct(options->repo, + to_fetch.oid, to_fetch.nr); oid_array_clear(&to_fetch); } diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 1cb20c6..b0de7d3 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -6,6 +6,7 @@ #include "list-objects.h" #include "list-objects-filter.h" #include "list-objects-filter-options.h" +#include "promisor-remote.h" /* * Parse value of the argument to the "filter" keyword. @@ -146,30 +147,31 @@ void partial_clone_register( const char *remote, const struct list_objects_filter_options *filter_options) { - /* - * Record the name of the partial clone remote in the - * config and in the global variable -- the latter is - * used throughout to indicate that partial clone is - * enabled and to expect missing objects. - */ - if (repository_format_partial_clone && - *repository_format_partial_clone && - strcmp(remote, repository_format_partial_clone)) - die(_("cannot change partial clone promisor remote")); + char *cfg_name; - git_config_set("core.repositoryformatversion", "1"); - git_config_set("extensions.partialclone", remote); + /* Check if it is already registered */ + if (!promisor_remote_find(remote)) { + git_config_set("core.repositoryformatversion", "1"); - repository_format_partial_clone = xstrdup(remote); + /* Add promisor config for the remote */ + cfg_name = xstrfmt("remote.%s.promisor", remote); + git_config_set(cfg_name, "true"); + free(cfg_name); + } /* * Record the initial filter-spec in the config as * the default for subsequent fetches from this remote. + * + * TODO: record it into remote..partialclonefilter */ core_partial_clone_filter_default = xstrdup(filter_options->filter_spec); git_config_set("core.partialclonefilter", core_partial_clone_filter_default); + + /* Make sure the config info are reset */ + promisor_remote_reinit(); } void partial_clone_get_default_filter_spec( diff --git a/packfile.c b/packfile.c index d786ec7..50aaf93 100644 --- a/packfile.c +++ b/packfile.c @@ -16,6 +16,7 @@ #include "tree.h" #include "object-store.h" #include "midx.h" +#include "promisor-remote.h" char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, @@ -2119,7 +2120,7 @@ int is_promisor_object(const struct object_id *oid) static int promisor_objects_prepared; if (!promisor_objects_prepared) { - if (repository_format_partial_clone) { + if (has_promisor_remote()) { for_each_packed_object(add_promisor_object, &promisor_objects, FOR_EACH_OBJECT_PROMISOR_ONLY); diff --git a/sha1-file.c b/sha1-file.c index 819d32c..fe250c4 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -30,8 +30,8 @@ #include "mergesort.h" #include "quote.h" #include "packfile.h" -#include "fetch-object.h" #include "object-store.h" +#include "promisor-remote.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -1377,16 +1377,17 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, } /* Check if it is a missing object */ - if (fetch_if_missing && repository_format_partial_clone && + if (fetch_if_missing && has_promisor_remote() && !already_retried && r == the_repository && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { /* - * TODO Investigate checking fetch_object() return - * TODO value and stopping on error here. - * TODO Pass a repository struct through fetch_object, - * such that arbitrary repositories work. + * TODO Investigate checking promisor_remote_get_direct() + * TODO return value and stopping on error here. + * TODO Pass a repository struct through + * promisor_remote_get_direct(), such that arbitrary + * repositories work. */ - fetch_objects(repository_format_partial_clone, real, 1); + promisor_remote_get_direct(r, real, 1); already_retried = 1; continue; } diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 37d7680..534d03a 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -654,7 +654,7 @@ partial_clone () { git -C client fsck && # Ensure that unneeded blobs are not inadvertently fetched. - test_config -C client extensions.partialclone "not a remote" && + test_config -C client remote.origin.promisor "false" && test_must_fail git -C client cat-file -e "$HASH1" && # But this blob was fetched, because clone performs an initial checkout diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index b91ef54..8f9a62a 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -42,7 +42,7 @@ test_expect_success 'do partial clone 1' ' test_cmp expect_1.oids observed.oids && test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" && - test "$(git -C pc1 config --local extensions.partialclone)" = "origin" && + test "$(git -C pc1 config --local remote.origin.promisor)" = "true" && test "$(git -C pc1 config --local core.partialclonefilter)" = "blob:none" ' diff --git a/unpack-trees.c b/unpack-trees.c index 5018990..aebd865 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -16,7 +16,7 @@ #include "submodule-config.h" #include "fsmonitor.h" #include "object-store.h" -#include "fetch-object.h" +#include "promisor-remote.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -400,7 +400,7 @@ static int check_updates(struct unpack_trees_options *o) load_gitmodules_file(index, &state); enable_delayed_checkout(&state); - if (repository_format_partial_clone && o->update && !o->dry_run) { + if (has_promisor_remote() && o->update && !o->dry_run) { /* * Prefetch the objects that are to be checked out in the loop * below. @@ -419,8 +419,8 @@ static int check_updates(struct unpack_trees_options *o) oid_array_append(&to_fetch, &ce->oid); } if (to_fetch.nr) - fetch_objects(repository_format_partial_clone, - to_fetch.oid, to_fetch.nr); + promisor_remote_get_direct(the_repository, + to_fetch.oid, to_fetch.nr); oid_array_clear(&to_fetch); } for (i = 0; i < index->cache_nr; i++) { -- cgit v0.10.2-6-g49f6 From fa3d1b63e866d6b893934ab69da10b4516150cdc Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:32 +0200 Subject: promisor-remote: parse remote.*.partialclonefilter This makes it possible to specify a different partial clone filter for each promisor remote. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index f74bd78..13d8133 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1491,7 +1491,7 @@ static inline void fetch_one_setup_partial(struct remote *remote) * the config. */ if (!filter_options.choice) - partial_clone_get_default_filter_spec(&filter_options); + partial_clone_get_default_filter_spec(&filter_options, remote->name); return; } diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index b0de7d3..28c571f 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -30,6 +30,9 @@ static int gently_parse_list_objects_filter( { const char *v0; + if (!arg) + return 0; + if (filter_options->choice) { if (errbuf) { strbuf_addstr( @@ -148,6 +151,7 @@ void partial_clone_register( const struct list_objects_filter_options *filter_options) { char *cfg_name; + char *filter_name; /* Check if it is already registered */ if (!promisor_remote_find(remote)) { @@ -162,27 +166,26 @@ void partial_clone_register( /* * Record the initial filter-spec in the config as * the default for subsequent fetches from this remote. - * - * TODO: record it into remote..partialclonefilter */ - core_partial_clone_filter_default = - xstrdup(filter_options->filter_spec); - git_config_set("core.partialclonefilter", - core_partial_clone_filter_default); + filter_name = xstrfmt("remote.%s.partialclonefilter", remote); + git_config_set(filter_name, filter_options->filter_spec); + free(filter_name); /* Make sure the config info are reset */ promisor_remote_reinit(); } void partial_clone_get_default_filter_spec( - struct list_objects_filter_options *filter_options) + struct list_objects_filter_options *filter_options, + const char *remote) { + struct promisor_remote *promisor = promisor_remote_find(remote); + /* * Parse default value, but silently ignore it if it is invalid. */ - if (!core_partial_clone_filter_default) - return; - gently_parse_list_objects_filter(filter_options, - core_partial_clone_filter_default, - NULL); + if (promisor) + gently_parse_list_objects_filter(filter_options, + promisor->partial_clone_filter, + NULL); } diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index c54f000..8deaa28 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -87,6 +87,7 @@ void partial_clone_register( const char *remote, const struct list_objects_filter_options *filter_options); void partial_clone_get_default_filter_spec( - struct list_objects_filter_options *filter_options); + struct list_objects_filter_options *filter_options, + const char *remote); #endif /* LIST_OBJECTS_FILTER_OPTIONS_H */ diff --git a/promisor-remote.c b/promisor-remote.c index 6a8856f..826890f 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -75,6 +75,21 @@ static int promisor_remote_config(const char *var, const char *value, void *data free(remote_name); return 0; } + if (!strcmp(subkey, "partialclonefilter")) { + struct promisor_remote *r; + char *remote_name = xmemdupz(name, namelen); + + r = promisor_remote_lookup(remote_name, NULL); + if (!r) + r = promisor_remote_new(remote_name); + + free(remote_name); + + if (!r) + return 0; + + return git_config_string(&r->partial_clone_filter, var, value); + } return 0; } diff --git a/promisor-remote.h b/promisor-remote.h index dddd404..838cb09 100644 --- a/promisor-remote.h +++ b/promisor-remote.h @@ -5,10 +5,13 @@ struct object_id; /* * Promisor remote linked list - * Its information come from remote.XXX config entries. + * + * Information in its fields come from remote.XXX config entries or + * from extensions.partialclone or core.partialclonefilter. */ struct promisor_remote { struct promisor_remote *next; + const char *partial_clone_filter; const char name[FLEX_ARRAY]; }; diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index 3559313..3082eff 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -26,7 +26,7 @@ promise_and_delete () { test_expect_success 'extensions.partialclone without filter' ' test_create_repo server && git clone --filter="blob:none" "file://$(pwd)/server" client && - git -C client config --unset core.partialclonefilter && + git -C client config --unset remote.origin.partialclonefilter && git -C client fetch origin ' diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 534d03a..078cf48 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -655,6 +655,7 @@ partial_clone () { # Ensure that unneeded blobs are not inadvertently fetched. test_config -C client remote.origin.promisor "false" && + git -C client config --unset remote.origin.partialclonefilter && test_must_fail git -C client cat-file -e "$HASH1" && # But this blob was fetched, because clone performs an initial checkout diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 8f9a62a..8ae7ba9 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -43,7 +43,7 @@ test_expect_success 'do partial clone 1' ' test_cmp expect_1.oids observed.oids && test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" && test "$(git -C pc1 config --local remote.origin.promisor)" = "true" && - test "$(git -C pc1 config --local core.partialclonefilter)" = "blob:none" + test "$(git -C pc1 config --local remote.origin.partialclonefilter)" = "blob:none" ' # checkout master to force dynamic object fetch of blobs at HEAD. -- cgit v0.10.2-6-g49f6 From 5e46139376a4b8d31f02a521bdf801f3b8913e57 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:33 +0200 Subject: builtin/fetch: remove unique promisor remote limitation As the infrastructure for more than one promisor remote has been introduced in previous patches, we can remove code that forbids the registration of more than one promisor remote. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/builtin/fetch.c b/builtin/fetch.c index 13d8133..5657d05 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1465,27 +1465,17 @@ static inline void fetch_one_setup_partial(struct remote *remote) return; /* - * If this is the FIRST partial-fetch request, we enable partial - * on this repo and remember the given filter-spec as the default - * for subsequent fetches to this remote. + * If this is a partial-fetch request, we enable partial on + * this repo if not already enabled and remember the given + * filter-spec as the default for subsequent fetches to this + * remote. */ - if (!has_promisor_remote() && filter_options.choice) { + if (filter_options.choice) { partial_clone_register(remote->name, &filter_options); return; } /* - * We are currently limited to only ONE promisor remote and only - * allow partial-fetches from the promisor remote. - */ - if (!promisor_remote_find(remote->name)) { - if (filter_options.choice) - die(_("--filter can only be used with the remote " - "configured in extensions.partialClone")); - return; - } - - /* * Do a partial-fetch from the promisor remote using either the * explicitly given filter-spec or inherit the filter-spec from * the config. -- cgit v0.10.2-6-g49f6 From 9a4c50788641887510876ea1b40c77daf9b7f7cf Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:34 +0200 Subject: t0410: test fetching from many promisor remotes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This shows that it is now possible to fetch objects from more than one promisor remote, and that fetching from a new promisor remote can configure it as one. Helped-by: SZEDER Gábor Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index 3082eff..2498e72 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -183,8 +183,55 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled' grep "git< fetch=.*ref-in-want" trace ' +test_expect_success 'fetching of missing objects from another promisor remote' ' + git clone "file://$(pwd)/server" server2 && + test_commit -C server2 bar && + git -C server2 repack -a -d --write-bitmap-index && + HASH2=$(git -C server2 rev-parse bar) && + + git -C repo remote add server2 "file://$(pwd)/server2" && + git -C repo config remote.server2.promisor true && + git -C repo cat-file -p "$HASH2" && + + git -C repo fetch server2 && + rm -rf repo/.git/objects/* && + git -C repo cat-file -p "$HASH2" && + + # Ensure that the .promisor file is written, and check that its + # associated packfile contains the object + ls repo/.git/objects/pack/pack-*.promisor >promisorlist && + test_line_count = 1 promisorlist && + IDX=$(sed "s/promisor$/idx/" promisorlist) && + git verify-pack --verbose "$IDX" >out && + grep "$HASH2" out +' + +test_expect_success 'fetching of missing objects configures a promisor remote' ' + git clone "file://$(pwd)/server" server3 && + test_commit -C server3 baz && + git -C server3 repack -a -d --write-bitmap-index && + HASH3=$(git -C server3 rev-parse baz) && + git -C server3 config uploadpack.allowfilter 1 && + + rm repo/.git/objects/pack/pack-*.promisor && + + git -C repo remote add server3 "file://$(pwd)/server3" && + git -C repo fetch --filter="blob:none" server3 $HASH3 && + + test_cmp_config -C repo true remote.server3.promisor && + + # Ensure that the .promisor file is written, and check that its + # associated packfile contains the object + ls repo/.git/objects/pack/pack-*.promisor >promisorlist && + test_line_count = 1 promisorlist && + IDX=$(sed "s/promisor$/idx/" promisorlist) && + git verify-pack --verbose "$IDX" >out && + grep "$HASH3" out +' + test_expect_success 'fetching of missing blobs works' ' - rm -rf server repo && + rm -rf server server2 repo && + rm -rf server server3 repo && test_create_repo server && test_commit -C server foo && git -C server repack -a -d --write-bitmap-index && -- cgit v0.10.2-6-g49f6 From 7e154badc007db7c99f0fb507a3893d7e333b7b2 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:35 +0200 Subject: partial-clone: add multiple remotes in the doc While at it, let's remove a reference to ODB effort as the ODB effort has been replaced by directly enhancing partial clone and promisor remote features. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/partial-clone.txt b/Documentation/technical/partial-clone.txt index 896c7b3..210373e 100644 --- a/Documentation/technical/partial-clone.txt +++ b/Documentation/technical/partial-clone.txt @@ -30,12 +30,20 @@ advance* during clone and fetch operations and thereby reduce download times and disk usage. Missing objects can later be "demand fetched" if/when needed. +A remote that can later provide the missing objects is called a +promisor remote, as it promises to send the objects when +requested. Initialy Git supported only one promisor remote, the origin +remote from which the user cloned and that was configured in the +"extensions.partialClone" config option. Later support for more than +one promisor remote has been implemented. + Use of partial clone requires that the user be online and the origin -remote be available for on-demand fetching of missing objects. This may -or may not be problematic for the user. For example, if the user can -stay within the pre-selected subset of the source tree, they may not -encounter any missing objects. Alternatively, the user could try to -pre-fetch various objects if they know that they are going offline. +remote or other promisor remotes be available for on-demand fetching +of missing objects. This may or may not be problematic for the user. +For example, if the user can stay within the pre-selected subset of +the source tree, they may not encounter any missing objects. +Alternatively, the user could try to pre-fetch various objects if they +know that they are going offline. Non-Goals @@ -100,18 +108,18 @@ or commits that reference missing trees. Handling Missing Objects ------------------------ -- An object may be missing due to a partial clone or fetch, or missing due - to repository corruption. To differentiate these cases, the local - repository specially indicates such filtered packfiles obtained from the - promisor remote as "promisor packfiles". +- An object may be missing due to a partial clone or fetch, or missing + due to repository corruption. To differentiate these cases, the + local repository specially indicates such filtered packfiles + obtained from promisor remotes as "promisor packfiles". + These promisor packfiles consist of a ".promisor" file with arbitrary contents (like the ".keep" files), in addition to their ".pack" and ".idx" files. - The local repository considers a "promisor object" to be an object that - it knows (to the best of its ability) that the promisor remote has promised - that it has, either because the local repository has that object in one of + it knows (to the best of its ability) that promisor remotes have promised + that they have, either because the local repository has that object in one of its promisor packfiles, or because another promisor object refers to it. + When Git encounters a missing object, Git can see if it is a promisor object @@ -123,12 +131,12 @@ expensive-to-modify list of missing objects.[a] - Since almost all Git code currently expects any referenced object to be present locally and because we do not want to force every command to do a dry-run first, a fallback mechanism is added to allow Git to attempt - to dynamically fetch missing objects from the promisor remote. + to dynamically fetch missing objects from promisor remotes. + When the normal object lookup fails to find an object, Git invokes -fetch-object to try to get the object from the server and then retry -the object lookup. This allows objects to be "faulted in" without -complicated prediction algorithms. +promisor_remote_get_direct() to try to get the object from a promisor +remote and then retry the object lookup. This allows objects to be +"faulted in" without complicated prediction algorithms. + For efficiency reasons, no check as to whether the missing object is actually a promisor object is performed. @@ -157,8 +165,7 @@ and prefetch those objects in bulk. + We are not happy with this global variable and would like to remove it, but that requires significant refactoring of the object code to pass an -additional flag. We hope that concurrent efforts to add an ODB API can -encompass this. +additional flag. Fetching Missing Objects @@ -182,21 +189,63 @@ has been updated to not use any object flags when the corresponding argument though they are not necessary. +Using many promisor remotes +--------------------------- + +Many promisor remotes can be configured and used. + +This allows for example a user to have multiple geographically-close +cache servers for fetching missing blobs while continuing to do +filtered `git-fetch` commands from the central server. + +When fetching objects, promisor remotes are tried one after the other +until all the objects have been fetched. + +Remotes that are considered "promisor" remotes are those specified by +the following configuration variables: + +- `extensions.partialClone = ` + +- `remote..promisor = true` + +- `remote..partialCloneFilter = ...` + +Only one promisor remote can be configured using the +`extensions.partialClone` config variable. This promisor remote will +be the last one tried when fetching objects. + +We decided to make it the last one we try, because it is likely that +someone using many promisor remotes is doing so because the other +promisor remotes are better for some reason (maybe they are closer or +faster for some kind of objects) than the origin, and the origin is +likely to be the remote specified by extensions.partialClone. + +This justification is not very strong, but one choice had to be made, +and anyway the long term plan should be to make the order somehow +fully configurable. + +For now though the other promisor remotes will be tried in the order +they appear in the config file. + Current Limitations ------------------- -- The remote used for a partial clone (or the first partial fetch - following a regular clone) is marked as the "promisor remote". +- It is not possible to specify the order in which the promisor + remotes are tried in other ways than the order in which they appear + in the config file. + -We are currently limited to a single promisor remote and only that -remote may be used for subsequent partial fetches. +It is also not possible to specify an order to be used when fetching +from one remote and a different order when fetching from another +remote. + +- It is not possible to push only specific objects to a promisor + remote. + -We accept this limitation because we believe initial users of this -feature will be using it on repositories with a strong single central -server. +It is not possible to push at the same time to multiple promisor +remote in a specific order. -- Dynamic object fetching will only ask the promisor remote for missing - objects. We assume that the promisor remote has a complete view of the +- Dynamic object fetching will only ask promisor remotes for missing + objects. We assume that promisor remotes have a complete view of the repository and can satisfy all such requests. - Repack essentially treats promisor and non-promisor packfiles as 2 @@ -218,15 +267,17 @@ server. Future Work ----------- -- Allow more than one promisor remote and define a strategy for fetching - missing objects from specific promisor remotes or of iterating over the - set of promisor remotes until a missing object is found. +- Improve the way to specify the order in which promisor remotes are + tried. + -A user might want to have multiple geographically-close cache servers -for fetching missing blobs while continuing to do filtered `git-fetch` -commands from the central server, for example. +For example this could allow to specify explicitly something like: +"When fetching from this remote, I want to use these promisor remotes +in this order, though, when pushing or fetching to that remote, I want +to use those promisor remotes in that order." + +- Allow pushing to promisor remotes. + -Or the user might want to work in a triangular work flow with multiple +The user might want to work in a triangular work flow with multiple promisor remotes that each have an incomplete view of the repository. - Allow repack to work on promisor packfiles (while keeping them distinct -- cgit v0.10.2-6-g49f6 From 75de0852119bc73c60ace232ba799c3cada2608a Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:36 +0200 Subject: remote: add promisor and partial clone config to the doc Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/Documentation/config/remote.txt b/Documentation/config/remote.txt index 6c4cad8..a8e6437 100644 --- a/Documentation/config/remote.txt +++ b/Documentation/config/remote.txt @@ -76,3 +76,11 @@ remote..pruneTags:: + See also `remote..prune` and the PRUNING section of linkgit:git-fetch[1]. + +remote..promisor:: + When set to true, this remote will be used to fetch promisor + objects. + +remote..partialclonefilter:: + The filter that will be applied when fetching from this + promisor remote. -- cgit v0.10.2-6-g49f6 From db27dca5cfe41ffa62f3a34cff2f7fafb2547307 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:37 +0200 Subject: Remove fetch-object.{c,h} in favor of promisor-remote.{c,h} As fetch_objects() is now used only in promisor-remote.c and should't be used outside it, let's move it into promisor-remote.c, make it static there, and remove fetch-object.{c,h}. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index 049bc8c..9b0baa7 100644 --- a/Makefile +++ b/Makefile @@ -880,7 +880,6 @@ LIB_OBJS += ewah/ewah_io.o LIB_OBJS += ewah/ewah_rlw.o LIB_OBJS += exec-cmd.o LIB_OBJS += fetch-negotiator.o -LIB_OBJS += fetch-object.o LIB_OBJS += fetch-pack.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o diff --git a/fetch-object.c b/fetch-object.c deleted file mode 100644 index eac4d44..0000000 --- a/fetch-object.c +++ /dev/null @@ -1,43 +0,0 @@ -#include "cache.h" -#include "packfile.h" -#include "pkt-line.h" -#include "strbuf.h" -#include "transport.h" -#include "fetch-object.h" - -static int fetch_refs(const char *remote_name, struct ref *ref) -{ - struct remote *remote; - struct transport *transport; - int original_fetch_if_missing = fetch_if_missing; - int res; - - fetch_if_missing = 0; - remote = remote_get(remote_name); - if (!remote->url[0]) - die(_("Remote with no URL")); - transport = transport_get(remote, remote->url[0]); - - transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1"); - res = transport_fetch_refs(transport, ref); - fetch_if_missing = original_fetch_if_missing; - - return res; -} - -int fetch_objects(const char *remote_name, const struct object_id *oids, - int oid_nr) -{ - struct ref *ref = NULL; - int i; - - for (i = 0; i < oid_nr; i++) { - struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i])); - oidcpy(&new_ref->old_oid, &oids[i]); - new_ref->exact_oid = 1; - new_ref->next = ref; - ref = new_ref; - } - return fetch_refs(remote_name, ref); -} diff --git a/fetch-object.h b/fetch-object.h deleted file mode 100644 index 7bcc7ca..0000000 --- a/fetch-object.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef FETCH_OBJECT_H -#define FETCH_OBJECT_H - -struct object_id; - -int fetch_objects(const char *remote_name, const struct object_id *oids, - int oid_nr); - -#endif diff --git a/promisor-remote.c b/promisor-remote.c index 826890f..92c4c12 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -2,7 +2,45 @@ #include "object-store.h" #include "promisor-remote.h" #include "config.h" -#include "fetch-object.h" +#include "transport.h" + +static int fetch_refs(const char *remote_name, struct ref *ref) +{ + struct remote *remote; + struct transport *transport; + int original_fetch_if_missing = fetch_if_missing; + int res; + + fetch_if_missing = 0; + remote = remote_get(remote_name); + if (!remote->url[0]) + die(_("Remote with no URL")); + transport = transport_get(remote, remote->url[0]); + + transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); + transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1"); + res = transport_fetch_refs(transport, ref); + fetch_if_missing = original_fetch_if_missing; + + return res; +} + +static int fetch_objects(const char *remote_name, + const struct object_id *oids, + int oid_nr) +{ + struct ref *ref = NULL; + int i; + + for (i = 0; i < oid_nr; i++) { + struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i])); + oidcpy(&new_ref->old_oid, &oids[i]); + new_ref->exact_oid = 1; + new_ref->next = ref; + ref = new_ref; + } + return fetch_refs(remote_name, ref); +} static struct promisor_remote *promisors; static struct promisor_remote **promisors_tail = &promisors; -- cgit v0.10.2-6-g49f6 From 60b7a92d846a6cc542f96fa0045735cc0b9265fd Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:38 +0200 Subject: Move repository_format_partial_clone to promisor-remote.c Now that we have has_promisor_remote() and can use many promisor remotes, let's hide repository_format_partial_clone as a static in promisor-remote.c to avoid it being use for anything other than managing backward compatibility. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/cache.h b/cache.h index bf20337..e34b9e6 100644 --- a/cache.h +++ b/cache.h @@ -961,7 +961,6 @@ extern int grafts_replace_parents; #define GIT_REPO_VERSION 0 #define GIT_REPO_VERSION_READ 1 extern int repository_format_precious_objects; -extern char *repository_format_partial_clone; extern const char *core_partial_clone_filter_default; extern int repository_format_worktree_config; diff --git a/environment.c b/environment.c index 89af47c..8855d2f 100644 --- a/environment.c +++ b/environment.c @@ -31,7 +31,6 @@ int warn_ambiguous_refs = 1; int warn_on_object_refname_ambiguity = 1; int ref_paranoia = -1; int repository_format_precious_objects; -char *repository_format_partial_clone; const char *core_partial_clone_filter_default; int repository_format_worktree_config; const char *git_commit_encoding; diff --git a/promisor-remote.c b/promisor-remote.c index 92c4c12..31d51bb 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -4,6 +4,13 @@ #include "config.h" #include "transport.h" +static char *repository_format_partial_clone; + +void set_repository_format_partial_clone(char *partial_clone) +{ + repository_format_partial_clone = xstrdup_or_null(partial_clone); +} + static int fetch_refs(const char *remote_name, struct ref *ref) { struct remote *remote; diff --git a/promisor-remote.h b/promisor-remote.h index 838cb09..8200dfc 100644 --- a/promisor-remote.h +++ b/promisor-remote.h @@ -22,4 +22,10 @@ extern int promisor_remote_get_direct(struct repository *repo, const struct object_id *oids, int oid_nr); +/* + * This should be used only once from setup.c to set the value we got + * from the extensions.partialclone config option. + */ +extern void set_repository_format_partial_clone(char *partial_clone); + #endif /* PROMISOR_REMOTE_H */ diff --git a/setup.c b/setup.c index 8dcb463..25a3038 100644 --- a/setup.c +++ b/setup.c @@ -4,6 +4,7 @@ #include "dir.h" #include "string-list.h" #include "chdir-notify.h" +#include "promisor-remote.h" static int inside_git_dir = -1; static int inside_work_tree = -1; @@ -478,7 +479,7 @@ static int check_repository_format_gently(const char *gitdir, struct repository_ } repository_format_precious_objects = candidate->precious_objects; - repository_format_partial_clone = xstrdup_or_null(candidate->partial_clone); + set_repository_format_partial_clone(candidate->partial_clone); repository_format_worktree_config = candidate->worktree_config; string_list_clear(&candidate->unknown_extensions, 0); -- cgit v0.10.2-6-g49f6 From 4ca9474efa4fd431c07a470513e684c5b2eec34c Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 25 Jun 2019 15:40:39 +0200 Subject: Move core_partial_clone_filter_default to promisor-remote.c Now that we can have a different default partial clone filter for each promisor remote, let's hide core_partial_clone_filter_default as a static in promisor-remote.c to avoid it being use for anything other than managing backward compatibility. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano diff --git a/cache.h b/cache.h index e34b9e6..a4d7f84 100644 --- a/cache.h +++ b/cache.h @@ -961,7 +961,6 @@ extern int grafts_replace_parents; #define GIT_REPO_VERSION 0 #define GIT_REPO_VERSION_READ 1 extern int repository_format_precious_objects; -extern const char *core_partial_clone_filter_default; extern int repository_format_worktree_config; /* diff --git a/config.c b/config.c index 296a6d9..317b226 100644 --- a/config.c +++ b/config.c @@ -1344,11 +1344,6 @@ static int git_default_core_config(const char *var, const char *value, void *cb) return 0; } - if (!strcmp(var, "core.partialclonefilter")) { - return git_config_string(&core_partial_clone_filter_default, - var, value); - } - if (!strcmp(var, "core.usereplacerefs")) { read_replace_refs = git_config_bool(var, value); return 0; diff --git a/environment.c b/environment.c index 8855d2f..efa0726 100644 --- a/environment.c +++ b/environment.c @@ -31,7 +31,6 @@ int warn_ambiguous_refs = 1; int warn_on_object_refname_ambiguity = 1; int ref_paranoia = -1; int repository_format_precious_objects; -const char *core_partial_clone_filter_default; int repository_format_worktree_config; const char *git_commit_encoding; const char *git_log_output_encoding; diff --git a/promisor-remote.c b/promisor-remote.c index 31d51bb..9bc296c 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -5,6 +5,7 @@ #include "transport.h" static char *repository_format_partial_clone; +static const char *core_partial_clone_filter_default; void set_repository_format_partial_clone(char *partial_clone) { @@ -103,6 +104,10 @@ static int promisor_remote_config(const char *var, const char *value, void *data int namelen; const char *subkey; + if (!strcmp(var, "core.partialclonefilter")) + return git_config_string(&core_partial_clone_filter_default, + var, value); + if (parse_config_key(var, "remote", &name, &namelen, &subkey) < 0) return 0; -- cgit v0.10.2-6-g49f6