From 74703a1e4dfc5affcb8944e78b53f0817b492246 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Wed, 2 Sep 2015 14:42:24 -0700 Subject: submodule: rewrite `module_list` shell function in C Most of the submodule operations work on a set of submodules. Calculating and using this set is usually done via: module_list "$@" | { while read mode sha1 stage sm_path do # the actual operation done } Currently the function `module_list` is implemented in the git-submodule.sh as a shell script wrapping a perl script. The rewrite is in C, such that it is faster and can later be easily adapted when other functions are rewritten in C. git-submodule.sh, similar to the builtin commands, will navigate to the top-most directory of the repository and keep the subdirectory as a variable. As the helper is called from within the git-submodule.sh script, we are already navigated to the root level, but the path arguments are still relative to the subdirectory we were in when calling git-submodule.sh. That's why there is a `--prefix` option pointing to an alternative path which to anchor relative path arguments. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/.gitignore b/.gitignore index 4fd81ba..1c2f832 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,7 @@ /git-status /git-stripspace /git-submodule +/git-submodule--helper /git-svn /git-symbolic-ref /git-tag diff --git a/Makefile b/Makefile index 24b636d..d434e73 100644 --- a/Makefile +++ b/Makefile @@ -901,6 +901,7 @@ BUILTIN_OBJS += builtin/shortlog.o BUILTIN_OBJS += builtin/show-branch.o BUILTIN_OBJS += builtin/show-ref.o BUILTIN_OBJS += builtin/stripspace.o +BUILTIN_OBJS += builtin/submodule--helper.o BUILTIN_OBJS += builtin/symbolic-ref.o BUILTIN_OBJS += builtin/tag.o BUILTIN_OBJS += builtin/unpack-file.o diff --git a/builtin.h b/builtin.h index 839483d..924e6c4 100644 --- a/builtin.h +++ b/builtin.h @@ -119,6 +119,7 @@ extern int cmd_show(int argc, const char **argv, const char *prefix); extern int cmd_show_branch(int argc, const char **argv, const char *prefix); extern int cmd_status(int argc, const char **argv, const char *prefix); extern int cmd_stripspace(int argc, const char **argv, const char *prefix); +extern int cmd_submodule__helper(int argc, const char **argv, const char *prefix); extern int cmd_symbolic_ref(int argc, const char **argv, const char *prefix); extern int cmd_tag(int argc, const char **argv, const char *prefix); extern int cmd_tar_tree(int argc, const char **argv, const char *prefix); diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c new file mode 100644 index 0000000..10db4e6 --- /dev/null +++ b/builtin/submodule--helper.c @@ -0,0 +1,128 @@ +#include "builtin.h" +#include "cache.h" +#include "parse-options.h" +#include "quote.h" +#include "pathspec.h" +#include "dir.h" +#include "utf8.h" + +struct module_list { + const struct cache_entry **entries; + int alloc, nr; +}; +#define MODULE_LIST_INIT { NULL, 0, 0 } + +static int module_list_compute(int argc, const char **argv, + const char *prefix, + struct pathspec *pathspec, + struct module_list *list) +{ + int i, result = 0; + char *max_prefix, *ps_matched = NULL; + int max_prefix_len; + parse_pathspec(pathspec, 0, + PATHSPEC_PREFER_FULL | + PATHSPEC_STRIP_SUBMODULE_SLASH_CHEAP, + prefix, argv); + + /* Find common prefix for all pathspec's */ + max_prefix = common_prefix(pathspec); + max_prefix_len = max_prefix ? strlen(max_prefix) : 0; + + if (pathspec->nr) + ps_matched = xcalloc(pathspec->nr, 1); + + if (read_cache() < 0) + die(_("index file corrupt")); + + for (i = 0; i < active_nr; i++) { + const struct cache_entry *ce = active_cache[i]; + + if (!S_ISGITLINK(ce->ce_mode) || + !match_pathspec(pathspec, ce->name, ce_namelen(ce), + max_prefix_len, ps_matched, 1)) + continue; + + ALLOC_GROW(list->entries, list->nr + 1, list->alloc); + list->entries[list->nr++] = ce; + while (i + 1 < active_nr && + !strcmp(ce->name, active_cache[i + 1]->name)) + /* + * Skip entries with the same name in different stages + * to make sure an entry is returned only once. + */ + i++; + } + free(max_prefix); + + if (ps_matched && report_path_error(ps_matched, pathspec, prefix)) + result = -1; + + free(ps_matched); + + return result; +} + +static int module_list(int argc, const char **argv, const char *prefix) +{ + int i; + struct pathspec pathspec; + struct module_list list = MODULE_LIST_INIT; + + struct option module_list_options[] = { + OPT_STRING(0, "prefix", &prefix, + N_("path"), + N_("alternative anchor for relative paths")), + OPT_END() + }; + + const char *const git_submodule_helper_usage[] = { + N_("git submodule--helper list [--prefix=] [...]"), + NULL + }; + + argc = parse_options(argc, argv, prefix, module_list_options, + git_submodule_helper_usage, 0); + + if (module_list_compute(argc, argv, prefix, &pathspec, &list) < 0) { + printf("#unmatched\n"); + return 1; + } + + for (i = 0; i < list.nr; i++) { + const struct cache_entry *ce = list.entries[i]; + + if (ce_stage(ce)) + printf("%06o %s U\t", ce->ce_mode, sha1_to_hex(null_sha1)); + else + printf("%06o %s %d\t", ce->ce_mode, sha1_to_hex(ce->sha1), ce_stage(ce)); + + utf8_fprintf(stdout, "%s\n", ce->name); + } + return 0; +} + + +struct cmd_struct { + const char *cmd; + int (*fn)(int, const char **, const char *); +}; + +static struct cmd_struct commands[] = { + {"list", module_list}, +}; + +int cmd_submodule__helper(int argc, const char **argv, const char *prefix) +{ + int i; + if (argc < 2) + die(_("fatal: submodule--helper subcommand must be " + "called with a subcommand")); + + for (i = 0; i < ARRAY_SIZE(commands); i++) + if (!strcmp(argv[1], commands[i].cmd)) + return commands[i].fn(argc - 1, argv + 1, prefix); + + die(_("fatal: '%s' is not a valid submodule--helper " + "subcommand"), argv[1]); +} diff --git a/git-submodule.sh b/git-submodule.sh index 36797c3..95c04fc 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -145,48 +145,6 @@ relative_path () echo "$result$target" } -# -# Get submodule info for registered submodules -# $@ = path to limit submodule list -# -module_list() -{ - eval "set $(git rev-parse --sq --prefix "$wt_prefix" -- "$@")" - ( - git ls-files -z --error-unmatch --stage -- "$@" || - echo "unmatched pathspec exists" - ) | - @@PERL@@ -e ' - my %unmerged = (); - my ($null_sha1) = ("0" x 40); - my @out = (); - my $unmatched = 0; - $/ = "\0"; - while () { - if (/^unmatched pathspec/) { - $unmatched = 1; - next; - } - chomp; - my ($mode, $sha1, $stage, $path) = - /^([0-7]+) ([0-9a-f]{40}) ([0-3])\t(.*)$/; - next unless $mode eq "160000"; - if ($stage ne "0") { - if (!$unmerged{$path}++) { - push @out, "$mode $null_sha1 U\t$path\n"; - } - next; - } - push @out, "$_\n"; - } - if ($unmatched) { - print "#unmatched\n"; - } else { - print for (@out); - } - ' -} - die_if_unmatched () { if test "$1" = "#unmatched" @@ -532,7 +490,7 @@ cmd_foreach() # command in the subshell (and a recursive call to this function) exec 3<&0 - module_list | + git submodule--helper list --prefix "$wt_prefix"| while read mode sha1 stage sm_path do die_if_unmatched "$mode" @@ -592,7 +550,7 @@ cmd_init() shift done - module_list "$@" | + git submodule--helper list --prefix "$wt_prefix" "$@" | while read mode sha1 stage sm_path do die_if_unmatched "$mode" @@ -674,7 +632,7 @@ cmd_deinit() die "$(eval_gettext "Use '.' if you really want to deinitialize all submodules")" fi - module_list "$@" | + git submodule--helper list --prefix "$wt_prefix" "$@" | while read mode sha1 stage sm_path do die_if_unmatched "$mode" @@ -790,7 +748,7 @@ cmd_update() fi cloned_modules= - module_list "$@" | { + git submodule--helper list --prefix "$wt_prefix" "$@" | { err= while read mode sha1 stage sm_path do @@ -1222,7 +1180,7 @@ cmd_status() shift done - module_list "$@" | + git submodule--helper list --prefix "$wt_prefix" "$@" | while read mode sha1 stage sm_path do die_if_unmatched "$mode" @@ -1299,7 +1257,7 @@ cmd_sync() esac done cd_to_toplevel - module_list "$@" | + git submodule--helper list --prefix "$wt_prefix" "$@" | while read mode sha1 stage sm_path do die_if_unmatched "$mode" diff --git a/git.c b/git.c index 55c327c..deecba0 100644 --- a/git.c +++ b/git.c @@ -469,6 +469,7 @@ static struct cmd_struct commands[] = { { "stage", cmd_add, RUN_SETUP | NEED_WORK_TREE }, { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, + { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, { "tag", cmd_tag, RUN_SETUP }, { "unpack-file", cmd_unpack_file, RUN_SETUP }, -- cgit v0.10.2-6-g49f6 From 0ea306ef1701d6f42e74d3c33addfcd630248904 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Wed, 2 Sep 2015 14:42:25 -0700 Subject: submodule: rewrite `module_name` shell function in C This implements the helper `name` in C instead of shell, yielding a nice performance boost. Before this patch, I measured a time (best out of three): $ time ./t7400-submodule-basic.sh >/dev/null real 0m11.066s user 0m3.348s sys 0m8.534s With this patch applied I measured (also best out of three) $ time ./t7400-submodule-basic.sh >/dev/null real 0m10.063s user 0m3.044s sys 0m7.487s Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 10db4e6..bc79c41 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -5,6 +5,9 @@ #include "pathspec.h" #include "dir.h" #include "utf8.h" +#include "submodule.h" +#include "submodule-config.h" +#include "string-list.h" struct module_list { const struct cache_entry **entries; @@ -102,6 +105,24 @@ static int module_list(int argc, const char **argv, const char *prefix) return 0; } +static int module_name(int argc, const char **argv, const char *prefix) +{ + const struct submodule *sub; + + if (argc != 2) + usage(_("git submodule--helper name ")); + + gitmodules_config(); + sub = submodule_from_path(null_sha1, argv[1]); + + if (!sub) + die(_("no submodule mapping found in .gitmodules for path '%s'"), + argv[1]); + + printf("%s\n", sub->name); + + return 0; +} struct cmd_struct { const char *cmd; @@ -110,6 +131,7 @@ struct cmd_struct { static struct cmd_struct commands[] = { {"list", module_list}, + {"name", module_name}, }; int cmd_submodule__helper(int argc, const char **argv, const char *prefix) diff --git a/git-submodule.sh b/git-submodule.sh index 95c04fc..2be8da2 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -178,24 +178,6 @@ get_submodule_config () { printf '%s' "${value:-$default}" } - -# -# Map submodule path to submodule name -# -# $1 = path -# -module_name() -{ - # Do we have "submodule..path = $1" defined in .gitmodules file? - sm_path="$1" - re=$(printf '%s\n' "$1" | sed -e 's/[].[^$\\*]/\\&/g') - name=$( git config -f .gitmodules --get-regexp '^submodule\..*\.path$' | - sed -n -e 's|^submodule\.\(.*\)\.path '"$re"'$|\1|p' ) - test -z "$name" && - die "$(eval_gettext "No submodule mapping found in .gitmodules for path '\$sm_path'")" - printf '%s\n' "$name" -} - # # Clone a submodule # @@ -498,7 +480,7 @@ cmd_foreach() then displaypath=$(relative_path "$sm_path") say "$(eval_gettext "Entering '\$prefix\$displaypath'")" - name=$(module_name "$sm_path") + name=$(git submodule--helper name "$sm_path") ( prefix="$prefix$sm_path/" clear_local_git_env @@ -554,7 +536,7 @@ cmd_init() while read mode sha1 stage sm_path do die_if_unmatched "$mode" - name=$(module_name "$sm_path") || exit + name=$(git submodule--helper name "$sm_path") || exit displaypath=$(relative_path "$sm_path") @@ -636,7 +618,7 @@ cmd_deinit() while read mode sha1 stage sm_path do die_if_unmatched "$mode" - name=$(module_name "$sm_path") || exit + name=$(git submodule--helper name "$sm_path") || exit displaypath=$(relative_path "$sm_path") @@ -758,7 +740,7 @@ cmd_update() echo >&2 "Skipping unmerged submodule $prefix$sm_path" continue fi - name=$(module_name "$sm_path") || exit + name=$(git submodule--helper name "$sm_path") || exit url=$(git config submodule."$name".url) branch=$(get_submodule_config "$name" branch master) if ! test -z "$update" @@ -1022,7 +1004,7 @@ cmd_summary() { # Respect the ignore setting for --for-status. if test -n "$for_status" then - name=$(module_name "$sm_path") + name=$(git submodule--helper name "$sm_path") ignore_config=$(get_submodule_config "$name" ignore none) test $status != A && test $ignore_config = all && continue fi @@ -1184,7 +1166,7 @@ cmd_status() while read mode sha1 stage sm_path do die_if_unmatched "$mode" - name=$(module_name "$sm_path") || exit + name=$(git submodule--helper name "$sm_path") || exit url=$(git config submodule."$name".url) displaypath=$(relative_path "$prefix$sm_path") if test "$stage" = U @@ -1261,7 +1243,7 @@ cmd_sync() while read mode sha1 stage sm_path do die_if_unmatched "$mode" - name=$(module_name "$sm_path") + name=$(git submodule--helper name "$sm_path") url=$(git config -f .gitmodules --get submodule."$name".url) # Possibly a url relative to parent -- cgit v0.10.2-6-g49f6 From ee8838d157761acf4cc38f2378277dc894c10eb0 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Tue, 8 Sep 2015 11:57:45 -0700 Subject: submodule: rewrite `module_clone` shell function in C This reimplements the helper function `module_clone` in shell in C as `clone`. This functionality is needed for converting `git submodule update` later on, which we want to add threading to. Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index bc79c41..f4c3eff 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -8,6 +8,7 @@ #include "submodule.h" #include "submodule-config.h" #include "string-list.h" +#include "run-command.h" struct module_list { const struct cache_entry **entries; @@ -123,6 +124,136 @@ static int module_name(int argc, const char **argv, const char *prefix) return 0; } +static int clone_submodule(const char *path, const char *gitdir, const char *url, + const char *depth, const char *reference, int quiet) +{ + struct child_process cp; + child_process_init(&cp); + + argv_array_push(&cp.args, "clone"); + argv_array_push(&cp.args, "--no-checkout"); + if (quiet) + argv_array_push(&cp.args, "--quiet"); + if (depth && *depth) + argv_array_pushl(&cp.args, "--depth", depth, NULL); + if (reference && *reference) + argv_array_pushl(&cp.args, "--reference", reference, NULL); + if (gitdir && *gitdir) + argv_array_pushl(&cp.args, "--separate-git-dir", gitdir, NULL); + + argv_array_push(&cp.args, url); + argv_array_push(&cp.args, path); + + cp.git_cmd = 1; + cp.env = local_repo_env; + cp.no_stdin = 1; + + return run_command(&cp); +} + +static int module_clone(int argc, const char **argv, const char *prefix) +{ + const char *path = NULL, *name = NULL, *url = NULL; + const char *reference = NULL, *depth = NULL; + int quiet = 0; + FILE *submodule_dot_git; + char *sm_gitdir, *cwd, *p; + struct strbuf rel_path = STRBUF_INIT; + struct strbuf sb = STRBUF_INIT; + + struct option module_clone_options[] = { + OPT_STRING(0, "prefix", &prefix, + N_("path"), + N_("alternative anchor for relative paths")), + OPT_STRING(0, "path", &path, + N_("path"), + N_("where the new submodule will be cloned to")), + OPT_STRING(0, "name", &name, + N_("string"), + N_("name of the new submodule")), + OPT_STRING(0, "url", &url, + N_("string"), + N_("url where to clone the submodule from")), + OPT_STRING(0, "reference", &reference, + N_("string"), + N_("reference repository")), + OPT_STRING(0, "depth", &depth, + N_("string"), + N_("depth for shallow clones")), + OPT__QUIET(&quiet, "Suppress output for cloning a submodule"), + OPT_END() + }; + + const char *const git_submodule_helper_usage[] = { + N_("git submodule--helper clone [--prefix=] [--quiet] " + "[--reference ] [--name ] [--url ]" + "[--depth ] [--] [...]"), + NULL + }; + + argc = parse_options(argc, argv, prefix, module_clone_options, + git_submodule_helper_usage, 0); + + strbuf_addf(&sb, "%s/modules/%s", get_git_dir(), name); + sm_gitdir = strbuf_detach(&sb, NULL); + + if (!file_exists(sm_gitdir)) { + if (safe_create_leading_directories_const(sm_gitdir) < 0) + die(_("could not create directory '%s'"), sm_gitdir); + if (clone_submodule(path, sm_gitdir, url, depth, reference, quiet)) + die(_("clone of '%s' into submodule path '%s' failed"), + url, path); + } else { + if (safe_create_leading_directories_const(path) < 0) + die(_("could not create directory '%s'"), path); + strbuf_addf(&sb, "%s/index", sm_gitdir); + unlink_or_warn(sb.buf); + strbuf_reset(&sb); + } + + /* Write a .git file in the submodule to redirect to the superproject. */ + if (safe_create_leading_directories_const(path) < 0) + die(_("could not create directory '%s'"), path); + + if (path && *path) + strbuf_addf(&sb, "%s/.git", path); + else + strbuf_addstr(&sb, ".git"); + + if (safe_create_leading_directories_const(sb.buf) < 0) + die(_("could not create leading directories of '%s'"), sb.buf); + submodule_dot_git = fopen(sb.buf, "w"); + if (!submodule_dot_git) + die_errno(_("cannot open file '%s'"), sb.buf); + + fprintf(submodule_dot_git, "gitdir: %s\n", + relative_path(sm_gitdir, path, &rel_path)); + if (fclose(submodule_dot_git)) + die(_("could not close file %s"), sb.buf); + strbuf_reset(&sb); + strbuf_reset(&rel_path); + + cwd = xgetcwd(); + /* Redirect the worktree of the submodule in the superproject's config */ + if (!is_absolute_path(sm_gitdir)) { + strbuf_addf(&sb, "%s/%s", cwd, sm_gitdir); + free(sm_gitdir); + sm_gitdir = strbuf_detach(&sb, NULL); + } + + strbuf_addf(&sb, "%s/%s", cwd, path); + p = git_pathdup_submodule(path, "config"); + if (!p) + die(_("could not get submodule directory for '%s'"), path); + git_config_set_in_file(p, "core.worktree", + relative_path(sb.buf, sm_gitdir, &rel_path)); + strbuf_release(&sb); + strbuf_release(&rel_path); + free(sm_gitdir); + free(cwd); + free(p); + return 0; +} struct cmd_struct { const char *cmd; @@ -132,6 +263,7 @@ struct cmd_struct { static struct cmd_struct commands[] = { {"list", module_list}, {"name", module_name}, + {"clone", module_clone}, }; int cmd_submodule__helper(int argc, const char **argv, const char *prefix) diff --git a/git-submodule.sh b/git-submodule.sh index 2be8da2..8b0eb9a 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -178,80 +178,6 @@ get_submodule_config () { printf '%s' "${value:-$default}" } -# -# Clone a submodule -# -# $1 = submodule path -# $2 = submodule name -# $3 = URL to clone -# $4 = reference repository to reuse (empty for independent) -# $5 = depth argument for shallow clones (empty for deep) -# -# Prior to calling, cmd_update checks that a possibly existing -# path is not a git repository. -# Likewise, cmd_add checks that path does not exist at all, -# since it is the location of a new submodule. -# -module_clone() -{ - sm_path=$1 - name=$2 - url=$3 - reference="$4" - depth="$5" - quiet= - if test -n "$GIT_QUIET" - then - quiet=-q - fi - - gitdir= - gitdir_base= - base_name=$(dirname "$name") - - gitdir=$(git rev-parse --git-dir) - gitdir_base="$gitdir/modules/$base_name" - gitdir="$gitdir/modules/$name" - - if test -d "$gitdir" - then - mkdir -p "$sm_path" - rm -f "$gitdir/index" - else - mkdir -p "$gitdir_base" - ( - clear_local_git_env - git clone $quiet ${depth:+"$depth"} -n ${reference:+"$reference"} \ - --separate-git-dir "$gitdir" "$url" "$sm_path" - ) || - die "$(eval_gettext "Clone of '\$url' into submodule path '\$sm_path' failed")" - fi - - # We already are at the root of the work tree but cd_to_toplevel will - # resolve any symlinks that might be present in $PWD - a=$(cd_to_toplevel && cd "$gitdir" && pwd)/ - b=$(cd_to_toplevel && cd "$sm_path" && pwd)/ - # Remove all common leading directories after a sanity check - if test "${a#$b}" != "$a" || test "${b#$a}" != "$b"; then - die "$(eval_gettext "Gitdir '\$a' is part of the submodule path '\$b' or vice versa")" - fi - while test "${a%%/*}" = "${b%%/*}" - do - a=${a#*/} - b=${b#*/} - done - # Now chop off the trailing '/'s that were added in the beginning - a=${a%/} - b=${b%/} - - # Turn each leading "*/" component into "../" - rel=$(printf '%s\n' "$b" | sed -e 's|[^/][^/]*|..|g') - printf '%s\n' "gitdir: $rel/$a" >"$sm_path/.git" - - rel=$(printf '%s\n' "$a" | sed -e 's|[^/][^/]*|..|g') - (clear_local_git_env; cd "$sm_path" && GIT_WORK_TREE=. git config core.worktree "$rel/$b") -} - isnumber() { n=$(($1 + 0)) 2>/dev/null && test "$n" = "$1" @@ -412,7 +338,7 @@ Use -f if you really want to add it." >&2 echo "$(eval_gettext "Reactivating local git directory for submodule '\$sm_name'.")" fi fi - module_clone "$sm_path" "$sm_name" "$realrepo" "$reference" "$depth" || exit + git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$wt_prefix" --path "$sm_path" --name "$sm_name" --url "$realrepo" "$reference" "$depth" || exit ( clear_local_git_env cd "$sm_path" && @@ -774,7 +700,7 @@ Maybe you want to use 'update --init'?")" if ! test -d "$sm_path"/.git && ! test -f "$sm_path"/.git then - module_clone "$sm_path" "$name" "$url" "$reference" "$depth" || exit + git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$prefix" --path "$sm_path" --name "$name" --url "$url" "$reference" "$depth" || exit cloned_modules="$cloned_modules;$name" subsha1= else -- cgit v0.10.2-6-g49f6