From ff03d9306c7dbc2011c0a6660359d9074e4a3ab3 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Fri, 3 Aug 2018 15:23:16 -0700 Subject: git-submodule.sh: align error reporting for update mode to use path All other error messages in cmd_update are reporting the submodule based on its path, so let's do that for invalid update modes, too. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/git-submodule.sh b/git-submodule.sh index 8b5ad59..5a58812 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -632,7 +632,7 @@ cmd_update() must_die_on_failure=yes ;; *) - die "$(eval_gettext "Invalid update mode '$update_module' for submodule '$name'")" + die "$(eval_gettext "Invalid update mode '$update_module' for submodule path '$path'")" esac if (sanitize_submodule_env; cd "$sm_path" && $command "$sha1") -- cgit v0.10.2-6-g49f6 From 9eca701f69b1dfb857a0445ba8a78e2445e9aa2b Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Fri, 3 Aug 2018 15:23:17 -0700 Subject: git-submodule.sh: rename unused variables The 'mode' variable is not used in cmd_update for its original purpose, rename it to 'dummy' as it only serves the purpose to abort quickly documenting this knowledge. The variable 'stage' is also not used any more in cmd_update, so remove it. This went unnoticed as first each function used the commonly used submodule listing, which was converted in 74703a1e4df (submodule: rewrite `module_list` shell function in C, 2015-09-02). When cmd_update was using its own function starting in 48308681b07 (git submodule update: have a dedicated helper for cloning, 2016-02-29), its removal was missed. A later patch in this series also touches the communication between the submodule helper and git-submodule.sh, but let's have this as a preparatory patch, as it eases the next patch, which stores the raw data instead of the line printed for this communication. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index a3c4564..da700c8 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1573,9 +1573,8 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce, needs_cloning = !file_exists(sb.buf); strbuf_reset(&sb); - strbuf_addf(&sb, "%06o %s %d %d\t%s\n", ce->ce_mode, - oid_to_hex(&ce->oid), ce_stage(ce), - needs_cloning, ce->name); + strbuf_addf(&sb, "dummy %s %d\t%s\n", + oid_to_hex(&ce->oid), needs_cloning, ce->name); string_list_append(&suc->projectlines, sb.buf); if (!needs_cloning) diff --git a/git-submodule.sh b/git-submodule.sh index 5a58812..8caaf27 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -531,9 +531,9 @@ cmd_update() "$@" || echo "#unmatched" $? } | { err= - while read -r mode sha1 stage just_cloned sm_path + while read -r quickabort sha1 just_cloned sm_path do - die_if_unmatched "$mode" "$sha1" + die_if_unmatched "$quickabort" "$sha1" name=$(git submodule--helper name "$sm_path") || exit if ! test -z "$update" -- cgit v0.10.2-6-g49f6 From 90efe595c53f4bb1851371344c35eff71f604d2b Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Fri, 3 Aug 2018 15:23:18 -0700 Subject: builtin/submodule--helper: factor out submodule updating Separate the command line parsing from the actual execution of the command within the repository. For now there is not a lot of execution as most of it is still in git-submodule.sh. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index da700c8..32f00ca 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1474,6 +1474,8 @@ struct submodule_update_clone { /* failed clones to be retried again */ const struct cache_entry **failed_clones; int failed_clones_nr, failed_clones_alloc; + + int max_jobs; }; #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ SUBMODULE_UPDATE_STRATEGY_INIT, 0, 0, -1, STRING_LIST_INIT_DUP, 0, \ @@ -1716,11 +1718,36 @@ static int git_update_clone_config(const char *var, const char *value, return 0; } +static int update_submodules(struct submodule_update_clone *suc) +{ + struct string_list_item *item; + + run_processes_parallel(suc->max_jobs, + update_clone_get_next_task, + update_clone_start_failure, + update_clone_task_finished, + suc); + + /* + * We saved the output and put it out all at once now. + * That means: + * - the listener does not have to interleave their (checkout) + * work with our fetching. The writes involved in a + * checkout involve more straightforward sequential I/O. + * - the listener can avoid doing any work if fetching failed. + */ + if (suc->quickstop) + return 1; + + for_each_string_list_item(item, &suc->projectlines) + fprintf(stdout, "%s", item->string); + + return 0; +} + static int update_clone(int argc, const char **argv, const char *prefix) { const char *update = NULL; - int max_jobs = 1; - struct string_list_item *item; struct pathspec pathspec; struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT; @@ -1742,7 +1769,7 @@ static int update_clone(int argc, const char **argv, const char *prefix) OPT_STRING(0, "depth", &suc.depth, "", N_("Create a shallow clone truncated to the " "specified number of revisions")), - OPT_INTEGER('j', "jobs", &max_jobs, + OPT_INTEGER('j', "jobs", &suc.max_jobs, N_("parallel jobs")), OPT_BOOL(0, "recommend-shallow", &suc.recommend_shallow, N_("whether the initial clone should follow the shallow recommendation")), @@ -1758,8 +1785,8 @@ static int update_clone(int argc, const char **argv, const char *prefix) }; suc.prefix = prefix; - update_clone_config_from_gitmodules(&max_jobs); - git_config(git_update_clone_config, &max_jobs); + update_clone_config_from_gitmodules(&suc.max_jobs); + git_config(git_update_clone_config, &suc.max_jobs); argc = parse_options(argc, argv, prefix, module_update_clone_options, git_submodule_helper_usage, 0); @@ -1774,27 +1801,7 @@ static int update_clone(int argc, const char **argv, const char *prefix) if (pathspec.nr) suc.warn_if_uninitialized = 1; - run_processes_parallel(max_jobs, - update_clone_get_next_task, - update_clone_start_failure, - update_clone_task_finished, - &suc); - - /* - * We saved the output and put it out all at once now. - * That means: - * - the listener does not have to interleave their (checkout) - * work with our fetching. The writes involved in a - * checkout involve more straightforward sequential I/O. - * - the listener can avoid doing any work if fetching failed. - */ - if (suc.quickstop) - return 1; - - for_each_string_list_item(item, &suc.projectlines) - fprintf(stdout, "%s", item->string); - - return 0; + return update_submodules(&suc); } static int resolve_relative_path(int argc, const char **argv, const char *prefix) -- cgit v0.10.2-6-g49f6 From f1d15713faef54c0bcc5d35c9089efffa4c914a1 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Fri, 3 Aug 2018 15:23:19 -0700 Subject: builtin/submodule--helper: store update_clone information in a struct The information that is printed for update_submodules in 'submodule--helper update-clone' and consumed by 'git submodule update' is stored as a string per submodule. This made sense at the time of 48308681b07 (git submodule update: have a dedicated helper for cloning, 2016-02-29), but as we want to migrate the rest of the submodule update into C, we're better off having access to the raw information in a helper struct. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 32f00ca..40b94dd 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1446,6 +1446,12 @@ static int module_clone(int argc, const char **argv, const char *prefix) return 0; } +struct update_clone_data { + const struct submodule *sub; + struct object_id oid; + unsigned just_cloned; +}; + struct submodule_update_clone { /* index into 'list', the list of submodules to look into for cloning */ int current; @@ -1465,8 +1471,9 @@ struct submodule_update_clone { const char *recursive_prefix; const char *prefix; - /* Machine-readable status lines to be consumed by git-submodule.sh */ - struct string_list projectlines; + /* to be consumed by git-submodule.sh */ + struct update_clone_data *update_clone; + int update_clone_nr; int update_clone_alloc; /* If we want to stop as fast as possible and return an error */ unsigned quickstop : 1; @@ -1480,7 +1487,7 @@ struct submodule_update_clone { #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ SUBMODULE_UPDATE_STRATEGY_INIT, 0, 0, -1, STRING_LIST_INIT_DUP, 0, \ NULL, NULL, NULL, \ - STRING_LIST_INIT_DUP, 0, NULL, 0, 0} + NULL, 0, 0, 0, NULL, 0, 0, 0} static void next_submodule_warn_missing(struct submodule_update_clone *suc, @@ -1574,10 +1581,12 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce, strbuf_addf(&sb, "%s/.git", ce->name); needs_cloning = !file_exists(sb.buf); - strbuf_reset(&sb); - strbuf_addf(&sb, "dummy %s %d\t%s\n", - oid_to_hex(&ce->oid), needs_cloning, ce->name); - string_list_append(&suc->projectlines, sb.buf); + ALLOC_GROW(suc->update_clone, suc->update_clone_nr + 1, + suc->update_clone_alloc); + oidcpy(&suc->update_clone[suc->update_clone_nr].oid, &ce->oid); + suc->update_clone[suc->update_clone_nr].just_cloned = needs_cloning; + suc->update_clone[suc->update_clone_nr].sub = sub; + suc->update_clone_nr++; if (!needs_cloning) goto cleanup; @@ -1720,7 +1729,8 @@ static int git_update_clone_config(const char *var, const char *value, static int update_submodules(struct submodule_update_clone *suc) { - struct string_list_item *item; + int i; + struct strbuf sb = STRBUF_INIT; run_processes_parallel(suc->max_jobs, update_clone_get_next_task, @@ -1739,9 +1749,16 @@ static int update_submodules(struct submodule_update_clone *suc) if (suc->quickstop) return 1; - for_each_string_list_item(item, &suc->projectlines) - fprintf(stdout, "%s", item->string); + for (i = 0; i < suc->update_clone_nr; i++) { + strbuf_addf(&sb, "dummy %s %d\t%s\n", + oid_to_hex(&suc->update_clone[i].oid), + suc->update_clone[i].just_cloned, + suc->update_clone[i].sub->path); + fprintf(stdout, "%s", sb.buf); + strbuf_reset(&sb); + } + strbuf_release(&sb); return 0; } -- cgit v0.10.2-6-g49f6 From c94d9dc286027e0343ffd58b5f7f2899691f6747 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Fri, 3 Aug 2018 15:23:20 -0700 Subject: builtin/submodule--helper: factor out method to update a single submodule In a later patch we'll find this method handy. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 40b94dd..8b1088a 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1727,10 +1727,17 @@ static int git_update_clone_config(const char *var, const char *value, return 0; } +static void update_submodule(struct update_clone_data *ucd) +{ + fprintf(stdout, "dummy %s %d\t%s\n", + oid_to_hex(&ucd->oid), + ucd->just_cloned, + ucd->sub->path); +} + static int update_submodules(struct submodule_update_clone *suc) { int i; - struct strbuf sb = STRBUF_INIT; run_processes_parallel(suc->max_jobs, update_clone_get_next_task, @@ -1749,16 +1756,9 @@ static int update_submodules(struct submodule_update_clone *suc) if (suc->quickstop) return 1; - for (i = 0; i < suc->update_clone_nr; i++) { - strbuf_addf(&sb, "dummy %s %d\t%s\n", - oid_to_hex(&suc->update_clone[i].oid), - suc->update_clone[i].just_cloned, - suc->update_clone[i].sub->path); - fprintf(stdout, "%s", sb.buf); - strbuf_reset(&sb); - } + for (i = 0; i < suc->update_clone_nr; i++) + update_submodule(&suc->update_clone[i]); - strbuf_release(&sb); return 0; } -- cgit v0.10.2-6-g49f6 From 74d4731da1fd61e3705e808bcd496979ef8ddf5a Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Mon, 13 Aug 2018 15:42:34 -0700 Subject: submodule--helper: replace connect-gitdir-workingtree by ensure-core-worktree e98317508c0 (submodule: ensure core.worktree is set after update, 2018-06-18) was overly aggressive in calling connect_work_tree_and_git_dir as that ensures both the 'core.worktree' configuration is set as well as setting up correct gitlink file pointing at the git directory. We do not need to check for the gitlink in this part of the cmd_update in git-submodule.sh, as the initial call to update-clone will have ensured that. So we can reduce the work to only (check and potentially) set the 'core.worktree' setting. While at it move the check from shell to C as that proves to be useful in a follow up patch, as we do not need the 'name' in shell now. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 8b1088a..648e133 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1964,6 +1964,45 @@ static int push_check(int argc, const char **argv, const char *prefix) return 0; } +static int ensure_core_worktree(int argc, const char **argv, const char *prefix) +{ + const struct submodule *sub; + const char *path; + char *cw; + struct repository subrepo; + + if (argc != 2) + BUG("submodule--helper connect-gitdir-workingtree "); + + path = argv[1]; + + sub = submodule_from_path(the_repository, &null_oid, path); + if (!sub) + BUG("We could get the submodule handle before?"); + + if (repo_submodule_init(&subrepo, the_repository, path)) + die(_("could not get a repository handle for submodule '%s'"), path); + + if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) { + char *cfg_file, *abs_path; + const char *rel_path; + struct strbuf sb = STRBUF_INIT; + + cfg_file = repo_git_path(&subrepo, "config"); + + abs_path = absolute_pathdup(path); + rel_path = relative_path(abs_path, subrepo.gitdir, &sb); + + git_config_set_in_file(cfg_file, "core.worktree", rel_path); + + free(cfg_file); + free(abs_path); + strbuf_release(&sb); + } + + return 0; +} + static int absorb_git_dirs(int argc, const char **argv, const char *prefix) { int i; @@ -2029,29 +2068,6 @@ static int check_name(int argc, const char **argv, const char *prefix) return 0; } -static int connect_gitdir_workingtree(int argc, const char **argv, const char *prefix) -{ - struct strbuf sb = STRBUF_INIT; - const char *name, *path; - char *sm_gitdir; - - if (argc != 3) - BUG("submodule--helper connect-gitdir-workingtree "); - - name = argv[1]; - path = argv[2]; - - strbuf_addf(&sb, "%s/modules/%s", get_git_dir(), name); - sm_gitdir = absolute_pathdup(sb.buf); - - connect_work_tree_and_git_dir(path, sm_gitdir, 0); - - strbuf_release(&sb); - free(sm_gitdir); - - return 0; -} - #define SUPPORT_SUPER_PREFIX (1<<0) struct cmd_struct { @@ -2065,7 +2081,7 @@ static struct cmd_struct commands[] = { {"name", module_name, 0}, {"clone", module_clone, 0}, {"update-clone", update_clone, 0}, - {"connect-gitdir-workingtree", connect_gitdir_workingtree, 0}, + {"ensure-core-worktree", ensure_core_worktree, 0}, {"relative-path", resolve_relative_path, 0}, {"resolve-relative-url", resolve_relative_url, 0}, {"resolve-relative-url-test", resolve_relative_url_test, 0}, diff --git a/git-submodule.sh b/git-submodule.sh index 8caaf27..19d010e 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -535,6 +535,8 @@ cmd_update() do die_if_unmatched "$quickabort" "$sha1" + git submodule--helper ensure-core-worktree "$sm_path" + name=$(git submodule--helper name "$sm_path") || exit if ! test -z "$update" then @@ -577,11 +579,6 @@ cmd_update() die "$(eval_gettext "Unable to find current \${remote_name}/\${branch} revision in submodule path '\$sm_path'")" fi - if ! $(git config -f "$(git rev-parse --git-common-dir)/modules/$name/config" core.worktree) 2>/dev/null - then - git submodule--helper connect-gitdir-workingtree "$name" "$sm_path" - fi - if test "$subsha1" != "$sha1" || test -n "$force" then subforce=$force -- cgit v0.10.2-6-g49f6 From ee69b2a90c5031bffb3341c5e50653a6ecca89ac Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Mon, 13 Aug 2018 15:42:35 -0700 Subject: submodule--helper: introduce new update-module-mode helper This chews off a bit of the shell part of the update command in git-submodule.sh. When writing the C code, keep in mind that the submodule--helper part will go away eventually and we want to have a C function that is able to determine the submodule update strategy, it as a nicety, make determine_submodule_update_strategy accessible for arbitrary repositories. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 648e133..5c9d1fb 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1446,6 +1446,66 @@ static int module_clone(int argc, const char **argv, const char *prefix) return 0; } +static void determine_submodule_update_strategy(struct repository *r, + int just_cloned, + const char *path, + const char *update, + struct submodule_update_strategy *out) +{ + const struct submodule *sub = submodule_from_path(r, &null_oid, path); + char *key; + const char *val; + + key = xstrfmt("submodule.%s.update", sub->name); + + if (update) { + trace_printf("parsing update"); + if (parse_submodule_update_strategy(update, out) < 0) + die(_("Invalid update mode '%s' for submodule path '%s'"), + update, path); + } else if (!repo_config_get_string_const(r, key, &val)) { + if (parse_submodule_update_strategy(val, out) < 0) + die(_("Invalid update mode '%s' configured for submodule path '%s'"), + val, path); + } else if (sub->update_strategy.type != SM_UPDATE_UNSPECIFIED) { + trace_printf("loaded thing"); + out->type = sub->update_strategy.type; + out->command = sub->update_strategy.command; + } else + out->type = SM_UPDATE_CHECKOUT; + + if (just_cloned && + (out->type == SM_UPDATE_MERGE || + out->type == SM_UPDATE_REBASE || + out->type == SM_UPDATE_NONE)) + out->type = SM_UPDATE_CHECKOUT; + + free(key); +} + +static int module_update_module_mode(int argc, const char **argv, const char *prefix) +{ + const char *path, *update = NULL; + int just_cloned; + struct submodule_update_strategy update_strategy = { .type = SM_UPDATE_CHECKOUT }; + + if (argc < 3 || argc > 4) + die("submodule--helper update-module-clone expects []"); + + just_cloned = git_config_int("just_cloned", argv[1]); + path = argv[2]; + + if (argc == 4) + update = argv[3]; + + determine_submodule_update_strategy(the_repository, + just_cloned, path, update, + &update_strategy); + fputs(submodule_strategy_to_string(&update_strategy), stdout); + + return 0; +} + struct update_clone_data { const struct submodule *sub; struct object_id oid; @@ -2080,6 +2140,7 @@ static struct cmd_struct commands[] = { {"list", module_list, 0}, {"name", module_name, 0}, {"clone", module_clone, 0}, + {"update-module-mode", module_update_module_mode, 0}, {"update-clone", update_clone, 0}, {"ensure-core-worktree", ensure_core_worktree, 0}, {"relative-path", resolve_relative_path, 0}, diff --git a/git-submodule.sh b/git-submodule.sh index 19d010e..19c9f12 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -537,27 +537,13 @@ cmd_update() git submodule--helper ensure-core-worktree "$sm_path" - name=$(git submodule--helper name "$sm_path") || exit - if ! test -z "$update" - then - update_module=$update - else - update_module=$(git config submodule."$name".update) - if test -z "$update_module" - then - update_module="checkout" - fi - fi + update_module=$(git submodule--helper update-module-mode $just_cloned "$sm_path" $update) displaypath=$(git submodule--helper relative-path "$prefix$sm_path" "$wt_prefix") if test $just_cloned -eq 1 then subsha1= - case "$update_module" in - merge | rebase | none) - update_module=checkout ;; - esac else subsha1=$(sanitize_submodule_env; cd "$sm_path" && git rev-parse --verify HEAD) || -- cgit v0.10.2-6-g49f6