From 9b8dc263e1b0d470cc67a824837d8884ae3e7136 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:08:43 +0100 Subject: upload-pack: no longer call rev-list It is trivial to do now, and it is needed for the upcoming shallow clone stuff. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/upload-pack.c b/upload-pack.c index 4572fff..7f7df2a 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -9,6 +9,9 @@ #include "object.h" #include "commit.h" #include "exec_cmd.h" +#include "diff.h" +#include "revision.h" +#include "list-objects.h" static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=nn] "; @@ -57,6 +60,40 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz) return safe_write(fd, data, sz); } +FILE *pack_pipe = NULL; +static void show_commit(struct commit *commit) +{ + if (commit->object.flags & BOUNDARY) + fputc('-', pack_pipe); + if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0) + die("broken output pipe"); + fputc('\n', pack_pipe); + fflush(pack_pipe); + free(commit->buffer); + commit->buffer = NULL; +} + +static void show_object(struct object_array_entry *p) +{ + /* An object with name "foo\n0000000..." can be used to + * confuse downstream git-pack-objects very badly. + */ + const char *ep = strchr(p->name, '\n'); + if (ep) { + fprintf(pack_pipe, "%s %.*s\n", sha1_to_hex(p->item->sha1), + (int) (ep - p->name), + p->name); + } + else + fprintf(pack_pipe, "%s %s\n", + sha1_to_hex(p->item->sha1), p->name); +} + +static void show_edge(struct commit *commit) +{ + fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1)); +} + static void create_pack_file(void) { /* Pipes between rev-list to pack-objects, pack-objects to us @@ -78,48 +115,38 @@ static void create_pack_file(void) if (!pid_rev_list) { int i; - int args; - const char **argv; - const char **p; - char *buf; + struct rev_info revs; - if (create_full_pack) { - args = 10; - use_thin_pack = 0; /* no point doing it */ - } - else - args = have_obj.nr + want_obj.nr + 5; - p = xmalloc(args * sizeof(char *)); - argv = (const char **) p; - buf = xmalloc(args * 45); + pack_pipe = fdopen(lp_pipe[1], "w"); - dup2(lp_pipe[1], 1); - close(0); - close(lp_pipe[0]); - close(lp_pipe[1]); - *p++ = "rev-list"; - *p++ = use_thin_pack ? "--objects-edge" : "--objects"; if (create_full_pack) - *p++ = "--all"; - else { + use_thin_pack = 0; /* no point doing it */ + init_revisions(&revs, NULL); + revs.tag_objects = 1; + revs.tree_objects = 1; + revs.blob_objects = 1; + if (use_thin_pack) + revs.edge_hint = 1; + + if (create_full_pack) { + const char *args[] = {"rev-list", "--all", NULL}; + setup_revisions(2, args, &revs, NULL); + } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; - *p++ = buf; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + add_pending_object(&revs, o, NULL); } - } - if (!create_full_pack) for (i = 0; i < have_obj.nr; i++) { struct object *o = have_obj.objects[i].item; - *p++ = buf; - *buf++ = '^'; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + o->flags |= UNINTERESTING; + add_pending_object(&revs, o, NULL); } - *p++ = NULL; - execv_git_cmd(argv); - die("git-upload-pack: unable to exec git-rev-list"); + setup_revisions(0, NULL, &revs, NULL); + } + prepare_revision_walk(&revs); + mark_edges_uninteresting(revs.commits, &revs, show_edge); + traverse_commit_list(&revs, show_commit, show_object); + exit(0); } if (pipe(pu_pipe) < 0) -- cgit v0.10.2-6-g49f6 From ed09aef06fda2ba06a7412e3fa43ab1c3449f723 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:06 +0100 Subject: support fetching into a shallow repository A shallow commit is a commit which has parents, which in turn are "grafted away", i.e. the commit appears as if it were a root. Since these shallow commits should not be edited by the user, but only by core git, they are recorded in the file $GIT_DIR/shallow. A repository containing shallow commits is called shallow. The advantage of a shallow repository is that even if the upstream contains lots of history, your local (shallow) repository needs not occupy much disk space. The disadvantage is that you might miss a merge base when pulling some remote branch. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/Makefile b/Makefile index 36ce8cd..ea5d2cf 100644 --- a/Makefile +++ b/Makefile @@ -260,7 +260,7 @@ LIB_OBJS = \ revision.o pager.o tree-walk.o xdiff-interface.o \ write_or_die.o trace.o list-objects.o grep.o \ alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \ - color.o wt-status.o archive-zip.o archive-tar.o + color.o wt-status.o archive-zip.o archive-tar.o shallow.o BUILTIN_OBJS = \ builtin-add.o \ diff --git a/commit.c b/commit.c index a6d543e..bffa278 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #include "cache.h" #include "tag.h" #include "commit.h" +#include "pkt-line.h" int save_commit_buffer = 1; @@ -221,6 +222,8 @@ static void prepare_commit_graft(void) return; graft_file = get_graft_file(); read_graft_file(graft_file); + /* make sure shallows are read */ + is_repository_shallow(); commit_graft_prepared = 1; } @@ -234,6 +237,24 @@ static struct commit_graft *lookup_commit_graft(const unsigned char *sha1) return commit_graft[pos]; } +int write_shallow_commits(int fd, int use_pack_protocol) +{ + int i, count = 0; + for (i = 0; i < commit_graft_nr; i++) + if (commit_graft[i]->nr_parent < 0) { + const char *hex = + sha1_to_hex(commit_graft[i]->sha1); + count++; + if (use_pack_protocol) + packet_write(fd, "shallow %s", hex); + else { + write(fd, hex, 40); + write(fd, "\n", 1); + } + } + return count; +} + int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size) { char *tail = buffer; diff --git a/commit.h b/commit.h index fc13de9..c559510 100644 --- a/commit.h +++ b/commit.h @@ -97,7 +97,7 @@ void sort_in_topological_order_fn(struct commit_list ** list, int lifo, struct commit_graft { unsigned char sha1[20]; - int nr_parent; + int nr_parent; /* < 0 if shallow commit */ unsigned char parent[FLEX_ARRAY][20]; /* more */ }; @@ -107,4 +107,10 @@ int read_graft_file(const char *graft_file); extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup); +extern int register_shallow(const unsigned char *sha1); +extern int write_shallow_commits(int fd, int use_pack_protocol); +extern int is_repository_shallow(); +extern struct commit_list *get_shallow_commits(struct object_array *heads, + int depth); + #endif /* COMMIT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index 0a169dc..bc5e725 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -180,6 +180,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, packet_write(fd[1], "want %s\n", sha1_to_hex(remote)); fetching++; } + if (is_repository_shallow()) + write_shallow_commits(fd[1], 1); packet_flush(fd[1]); if (!fetching) return 1; @@ -523,6 +525,8 @@ static int fetch_pack(int fd[2], int nr_match, char **match) int status; get_remote_heads(fd[0], &ref, 0, NULL, 0); + if (is_repository_shallow() && !server_supports("shallow")) + die("Server does not support shallow clients"); if (server_supports("multi_ack")) { if (verbose) fprintf(stderr, "Server supports multi_ack\n"); diff --git a/shallow.c b/shallow.c new file mode 100644 index 0000000..3cf2127 --- /dev/null +++ b/shallow.c @@ -0,0 +1,97 @@ +#include "cache.h" +#include "commit.h" + +static int is_shallow = -1; + +int register_shallow(const unsigned char *sha1) +{ + struct commit_graft *graft = + xmalloc(sizeof(struct commit_graft)); + struct commit *commit = lookup_commit(sha1); + + hashcpy(graft->sha1, sha1); + graft->nr_parent = -1; + if (commit && commit->object.parsed) + commit->parents = NULL; + return register_commit_graft(graft, 0); +} + +int is_repository_shallow() +{ + FILE *fp; + char buf[1024]; + + if (is_shallow >= 0) + return is_shallow; + + fp = fopen(git_path("shallow"), "r"); + if (!fp) { + is_shallow = 0; + return is_shallow; + } + is_shallow = 1; + + while (fgets(buf, sizeof(buf), fp)) { + unsigned char sha1[20]; + if (get_sha1_hex(buf, sha1)) + die("bad shallow line: %s", buf); + register_shallow(sha1); + } + fclose(fp); + return is_shallow; +} + +struct commit_list *get_shallow_commits(struct object_array *heads, int depth) +{ + int i = 0, cur_depth = 0; + struct commit_list *result = NULL; + struct object_array stack = {0, 0, NULL}; + struct commit *commit = NULL; + + while (commit || i < heads->nr || stack.nr) { + struct commit_list *p; + if (!commit) { + if (i < heads->nr) { + commit = (struct commit *) + heads->objects[i++].item; + if (commit->object.type != OBJ_COMMIT) { + commit = NULL; + continue; + } + commit->util = xcalloc(1, sizeof(int)); + cur_depth = 0; + } else { + commit = (struct commit *) + stack.objects[--stack.nr].item; + cur_depth = *(int *)commit->util; + } + } + parse_commit(commit); + cur_depth++; + for (p = commit->parents, commit = NULL; p; p = p->next) { + if (!p->item->util) { + int *pointer = xmalloc(sizeof(int)); + p->item->util = pointer; + *pointer = cur_depth; + } else { + int *pointer = p->item->util; + if (cur_depth >= *pointer) + continue; + *pointer = cur_depth; + } + if (cur_depth < depth) { + if (p->next) + add_object_array(&p->item->object, + NULL, &stack); + else { + commit = p->item; + cur_depth = *(int *)commit->util; + } + } else + commit_list_insert(p->item, &result); + } + } + + return result; +} + diff --git a/upload-pack.c b/upload-pack.c index 7f7df2a..8dd6121 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -486,6 +486,7 @@ static int get_common_commits(void) static void receive_needs(void) { + struct object_array shallows = {0, 0, NULL}; static char line[1000]; int len; @@ -495,8 +496,19 @@ static void receive_needs(void) len = packet_read_line(0, line, sizeof(line)); reset_timeout(); if (!len) - return; + break; + if (!strncmp("shallow ", line, 8)) { + unsigned char sha1[20]; + struct object *object; + if (get_sha1(line + 8, sha1)) + die("invalid shallow line: %s", line); + object = parse_object(sha1); + if (!object) + die("did not find object for %s", line); + add_object_array(object, NULL, &shallows); + continue; + } if (strncmp("want ", line, 5) || get_sha1_hex(line+5, sha1_buf)) die("git-upload-pack: protocol error, " @@ -528,11 +540,17 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (shallows.nr > 0) { + int i; + for (i = 0; i < shallows.nr; i++) + register_shallow(shallows.objects[i].item->sha1); + } } static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data) { - static const char *capabilities = "multi_ack thin-pack side-band side-band-64k ofs-delta"; + static const char *capabilities = "multi_ack thin-pack side-band" + " side-band-64k ofs-delta shallow"; struct object *o = parse_object(sha1); if (!o) -- cgit v0.10.2-6-g49f6 From 016e6ccbe03438454777e43dd73d67844296a3fd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:29 +0100 Subject: allow cloning a repository "shallowly" By specifying a depth, you can now clone a repository such that all fetched ancestor-chains' length is at most "depth". For example, if the upstream repository has only 2 branches ("A" and "B"), which are linear, and you specify depth 3, you will get A, A~1, A~2, A~3, B, B~1, B~2, and B~3. The ends are automatically made shallow commits. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index bc5e725..f335bd4 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -11,8 +11,9 @@ static int keep_pack; static int quiet; static int verbose; static int fetch_all; +static int depth; static const char fetch_pack_usage[] = -"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory ..."; +"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=] [host:]directory ..."; static const char *exec = "git-upload-pack"; #define COMPLETE (1U << 0) @@ -182,10 +183,29 @@ static int find_common(int fd[2], unsigned char *result_sha1, } if (is_repository_shallow()) write_shallow_commits(fd[1], 1); + if (depth > 0) + packet_write(fd[1], "deepen %d", depth); packet_flush(fd[1]); if (!fetching) return 1; + if (depth > 0) { + char line[1024]; + unsigned char sha1[20]; + int len; + + while ((len = packet_read_line(fd[0], line, sizeof(line)))) { + if (!strncmp("shallow ", line, 8)) { + if (get_sha1_hex(line + 8, sha1)) + die("invalid shallow line: %s", line); + /* no need making it shallow if we have it already */ + if (lookup_object(sha1)) + continue; + register_shallow(sha1); + } + } + } + flushes = 0; retval = -1; while ((sha1 = get_rev())) { @@ -576,6 +596,8 @@ int main(int argc, char **argv) char *dest = NULL, **heads; int fd[2]; pid_t pid; + struct stat st; + struct lock_file lock; setup_git_directory(); @@ -609,6 +631,12 @@ int main(int argc, char **argv) verbose = 1; continue; } + if (!strncmp("--depth=", arg, 8)) { + depth = strtol(arg + 8, NULL, 0); + if (stat(git_path("shallow"), &st)) + st.st_mtime = 0; + continue; + } usage(fetch_pack_usage); } dest = arg; @@ -618,6 +646,8 @@ int main(int argc, char **argv) } if (!dest) usage(fetch_pack_usage); + if (is_repository_shallow() && depth > 0) + die("Deepening of a shallow repository not yet supported!"); pid = git_connect(fd, dest, exec); if (pid < 0) return 1; @@ -639,5 +669,34 @@ int main(int argc, char **argv) } } + if (!ret && depth > 0) { + struct cache_time mtime; + char *shallow = git_path("shallow"); + int fd; + + mtime.sec = st.st_mtime; +#ifdef USE_NSEC + mtime.usec = st.st_mtim.usec; +#endif + if (stat(shallow, &st)) { + if (mtime.sec) + die("shallow file was removed during fetch"); + } else if (st.st_mtime != mtime.sec +#ifdef USE_NSEC + || st.st_mtim.usec != mtime.usec +#endif + ) + die("shallow file was changed during fetch"); + + fd = hold_lock_file_for_update(&lock, shallow, 1); + if (!write_shallow_commits(fd, 0)) { + unlink(lock.filename); + rollback_lock_file(&lock); + } else { + close(fd); + commit_lock_file(&lock); + } + } + return !!ret; } diff --git a/git-clone.sh b/git-clone.sh index 9ed4135..8c0a93e 100755 --- a/git-clone.sh +++ b/git-clone.sh @@ -14,7 +14,7 @@ die() { } usage() { - die "Usage: $0 [--template=] [--use-immingled-remote] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [-n] []" + die "Usage: $0 [--template=] [--use-immingled-remote] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [--depth ] [-n] []" } get_repo_base() { @@ -116,6 +116,7 @@ reference= origin= origin_override= use_separate_remote=t +depth= while case "$#,$1" in 0,*) break ;; @@ -161,6 +162,10 @@ while *,-u|*,--upload-pack) shift upload_pack="--exec=$1" ;; + 1,--depth) usage;; + *,--depth) + shift + depth="--depth=$1";; *,-*) usage ;; *) break ;; esac @@ -265,6 +270,10 @@ yes,yes) *) case "$repo" in rsync://*) + case "$depth" in + "") ;; + *) die "shallow over rsync not supported" ;; + esac rsync $quiet -av --ignore-existing \ --exclude info "$repo/objects/" "$GIT_DIR/objects/" || exit @@ -293,6 +302,10 @@ yes,yes) git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1 ;; https://*|http://*|ftp://*) + case "$depth" in + "") ;; + *) die "shallow over http or ftp not supported" ;; + esac if test -z "@@NO_CURL@@" then clone_dumb_http "$repo" "$D" @@ -302,8 +315,8 @@ yes,yes) ;; *) case "$upload_pack" in - '') git-fetch-pack --all -k $quiet "$repo" ;; - *) git-fetch-pack --all -k $quiet "$upload_pack" "$repo" ;; + '') git-fetch-pack --all -k $quiet $depth "$repo" ;; + *) git-fetch-pack --all -k $quiet "$upload_pack" $depth "$repo" ;; esac >"$GIT_DIR/CLONE_HEAD" || die "fetch-pack from '$repo' failed." ;; diff --git a/upload-pack.c b/upload-pack.c index 8dd6121..ebe1e5a 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -488,7 +488,7 @@ static void receive_needs(void) { struct object_array shallows = {0, 0, NULL}; static char line[1000]; - int len; + int len, depth = 0; for (;;) { struct object *o; @@ -509,6 +509,13 @@ static void receive_needs(void) add_object_array(object, NULL, &shallows); continue; } + if (!strncmp("deepen ", line, 7)) { + char *end; + depth = strtol(line + 7, &end, 0); + if (end == line + 7 || depth <= 0) + die("Invalid deepen: %s", line); + continue; + } if (strncmp("want ", line, 5) || get_sha1_hex(line+5, sha1_buf)) die("git-upload-pack: protocol error, " @@ -540,6 +547,18 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (depth > 0) { + struct commit_list *result, *backup; + if (shallows.nr > 0) + die("Deepening a shallow repository not yet supported"); + backup = result = get_shallow_commits(&want_obj, depth); + while (result) { + packet_write(1, "shallow %s", + sha1_to_hex(result->item->object.sha1)); + result = result->next; + } + free_commit_list(backup); + } if (shallows.nr > 0) { int i; for (i = 0; i < shallows.nr; i++) -- cgit v0.10.2-6-g49f6 From f53514bc2d82f2f5cc7b447575e74aa266ed46f0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:53 +0100 Subject: allow deepening of a shallow repository Now, by saying "git fetch -depth " you can deepen a shallow repository. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/commit.c b/commit.c index bffa278..d5103cd 100644 --- a/commit.c +++ b/commit.c @@ -255,6 +255,19 @@ int write_shallow_commits(int fd, int use_pack_protocol) return count; } +int unregister_shallow(const unsigned char *sha1) +{ + int pos = commit_graft_pos(sha1); + if (pos < 0) + return -1; + if (pos + 1 < commit_graft_nr) + memcpy(commit_graft + pos, commit_graft + pos + 1, + sizeof(struct commit_graft *) + * (commit_graft_nr - pos - 1)); + commit_graft_nr--; + return 0; +} + int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size) { char *tail = buffer; diff --git a/commit.h b/commit.h index c559510..e9e158f 100644 --- a/commit.h +++ b/commit.h @@ -108,9 +108,10 @@ int read_graft_file(const char *graft_file); extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup); extern int register_shallow(const unsigned char *sha1); +extern int unregister_shallow(const unsigned char *sha1); extern int write_shallow_commits(int fd, int use_pack_protocol); extern int is_repository_shallow(); extern struct commit_list *get_shallow_commits(struct object_array *heads, - int depth); + int depth, int shallow_flag, int not_shallow_flag); #endif /* COMMIT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index f335bd4..c3064b9 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -202,7 +202,17 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (lookup_object(sha1)) continue; register_shallow(sha1); - } + } else if (!strncmp("unshallow ", line, 10)) { + if (get_sha1_hex(line + 10, sha1)) + die("invalid unshallow line: %s", line); + if (!lookup_object(sha1)) + die("object not found: %s", line); + /* make sure that it is parsed as shallow */ + parse_object(sha1); + if (unregister_shallow(sha1)) + die("no shallow found: %s", line); + } else + die("expected shallow/unshallow, got %s", line); } } @@ -391,9 +401,11 @@ static int everything_local(struct ref **refs, int nr_match, char **match) } } - for_each_ref(mark_complete, NULL); - if (cutoff) - mark_recent_complete_commits(cutoff); + if (!depth) { + for_each_ref(mark_complete, NULL); + if (cutoff) + mark_recent_complete_commits(cutoff); + } /* * Mark all complete remote refs as common refs. @@ -646,8 +658,6 @@ int main(int argc, char **argv) } if (!dest) usage(fetch_pack_usage); - if (is_repository_shallow() && depth > 0) - die("Deepening of a shallow repository not yet supported!"); pid = git_connect(fd, dest, exec); if (pid < 0) return 1; diff --git a/git-fetch.sh b/git-fetch.sh index eb32476..0b1e6d1 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -21,6 +21,7 @@ update_head_ok= exec= upload_pack= keep= +shallow_depth= while case "$#" in 0) break ;; esac do case "$1" in @@ -56,6 +57,13 @@ do --reflog-action=*) rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'` ;; + --depth=*) + shallow_depth="--depth=`expr "z$1" : 'z-[^=]*=\(.*\)'`" + ;; + --depth) + shift + shallow_depth="--depth=$1" + ;; -*) usage ;; @@ -296,6 +304,8 @@ fetch_main () { # There are transports that can fetch only one head at a time... case "$remote" in http://* | https://* | ftp://*) + test -n "$shallow_depth" && + die "shallow clone with http not supported" proto=`expr "$remote" : '\([^:]*\):'` if [ -n "$GIT_SSL_NO_VERIFY" ]; then curl_extra_args="-k" @@ -324,6 +334,8 @@ fetch_main () { git-http-fetch -v -a "$head" "$remote/" || exit ;; rsync://*) + test -n "$shallow_depth" && + die "shallow clone with rsync not supported" TMP_HEAD="$GIT_DIR/TMP_HEAD" rsync -L -q "$remote/$remote_name" "$TMP_HEAD" || exit 1 head=$(git-rev-parse --verify TMP_HEAD) @@ -371,7 +383,7 @@ fetch_main () { pack_lockfile= IFS=" $LF" ( - git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote" + git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || echo failed "$remote" ) | while read sha1 remote_name do diff --git a/shallow.c b/shallow.c index 3cf2127..58a7b20 100644 --- a/shallow.c +++ b/shallow.c @@ -41,7 +41,8 @@ int is_repository_shallow() return is_shallow; } -struct commit_list *get_shallow_commits(struct object_array *heads, int depth) +struct commit_list *get_shallow_commits(struct object_array *heads, int depth, + int shallow_flag, int not_shallow_flag) { int i = 0, cur_depth = 0; struct commit_list *result = NULL; @@ -67,6 +68,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth) } } parse_commit(commit); + commit->object.flags |= not_shallow_flag; cur_depth++; for (p = commit->parents, commit = NULL; p; p = p->next) { if (!p->item->util) { @@ -87,8 +89,10 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth) commit = p->item; cur_depth = *(int *)commit->util; } - } else + } else { commit_list_insert(p->item, &result); + p->item->object.flags |= shallow_flag; + } } } diff --git a/upload-pack.c b/upload-pack.c index ebe1e5a..4a9d672 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -22,6 +22,10 @@ static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=n #define COMMON_KNOWN (1u << 14) #define REACHABLE (1u << 15) +#define SHALLOW (1u << 16) +#define NOT_SHALLOW (1u << 17) +#define CLIENT_SHALLOW (1u << 18) + static unsigned long oldest_have; static int multi_ack, nr_our_refs; @@ -134,6 +138,7 @@ static void create_pack_file(void) } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; + o->flags &= ~UNINTERESTING; add_pending_object(&revs, o, NULL); } for (i = 0; i < have_obj.nr; i++) { @@ -501,16 +506,19 @@ static void receive_needs(void) if (!strncmp("shallow ", line, 8)) { unsigned char sha1[20]; struct object *object; + use_thin_pack = 0; if (get_sha1(line + 8, sha1)) die("invalid shallow line: %s", line); object = parse_object(sha1); if (!object) die("did not find object for %s", line); + object->flags |= CLIENT_SHALLOW; add_object_array(object, NULL, &shallows); continue; } if (!strncmp("deepen ", line, 7)) { char *end; + use_thin_pack = 0; depth = strtol(line + 7, &end, 0); if (end == line + 7 || depth <= 0) die("Invalid deepen: %s", line); @@ -547,23 +555,51 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (depth == 0 && shallows.nr == 0) + return; if (depth > 0) { struct commit_list *result, *backup; - if (shallows.nr > 0) - die("Deepening a shallow repository not yet supported"); - backup = result = get_shallow_commits(&want_obj, depth); + int i; + backup = result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); while (result) { - packet_write(1, "shallow %s", - sha1_to_hex(result->item->object.sha1)); + struct object *object = &result->item->object; + if (!(object->flags & CLIENT_SHALLOW)) { + packet_write(1, "shallow %s", + sha1_to_hex(object->sha1)); + register_shallow(object->sha1); + } result = result->next; } free_commit_list(backup); - } - if (shallows.nr > 0) { - int i; - for (i = 0; i < shallows.nr; i++) - register_shallow(shallows.objects[i].item->sha1); - } + for (i = 0; i < shallows.nr; i++) { + struct object *object = shallows.objects[i].item; + if (object->flags & NOT_SHALLOW) { + struct commit_list *parents; + packet_write(1, "unshallow %s", + sha1_to_hex(object->sha1)); + object->flags &= ~CLIENT_SHALLOW; + /* make sure the real parents are parsed */ + unregister_shallow(object->sha1); + parse_commit((struct commit *)object); + parents = ((struct commit *)object)->parents; + while (parents) { + add_object_array(&parents->item->object, + NULL, &want_obj); + parents = parents->next; + } + } + /* make sure commit traversal conforms to client */ + register_shallow(object->sha1); + } + packet_flush(1); + } else + if (shallows.nr > 0) { + int i; + for (i = 0; i < shallows.nr; i++) + register_shallow(shallows.objects[i].item->sha1); + } + free(shallows.objects); } static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data) -- cgit v0.10.2-6-g49f6 From abef3a1625b7aff168a5fa062432dc6ef2006963 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Sat, 11 Nov 2006 14:57:23 +0100 Subject: Shallow clone: do not ignore shallowness when following tags Tags should be considered when truncating the commit list. The patch below fixes it, and fetches the right number of commits for each tag. However the correct fix is probably to not fetch historical tags at all. Signed-off-by: Junio C Hamano diff --git a/shallow.c b/shallow.c index 58a7b20..2db1dc4 100644 --- a/shallow.c +++ b/shallow.c @@ -1,5 +1,6 @@ #include "cache.h" #include "commit.h" +#include "tag.h" static int is_shallow = -1; @@ -54,7 +55,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, if (!commit) { if (i < heads->nr) { commit = (struct commit *) - heads->objects[i++].item; + deref_tag(heads->objects[i++].item, NULL, 0); if (commit->object.type != OBJ_COMMIT) { commit = NULL; continue; -- cgit v0.10.2-6-g49f6 From 16ad35791099e5da118a0b6cfa0c3db4e2feff81 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:10:13 +0100 Subject: add tests for shallow stuff Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index f7625a6..fa10840 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -128,4 +128,49 @@ pull_to_client 2nd "B" $((64*3)) pull_to_client 3rd "A" $((1*3)) # old fails +test_expect_success "clone shallow" "git-clone --depth 2 . shallow" + +(cd shallow; git-count-objects -v) > count.shallow + +test_expect_success "clone shallow object count" \ + "test \"in-pack: 18\" = \"$(grep in-pack count.shallow)\"" + +test_expect_success "clone shallow object count (part 2)" \ + "test -z \"$(grep -v in-pack count.shallow | sed "s/^.*: 0//")\"" + +test_expect_success "fsck in shallow repo" \ + "(cd shallow; git-fsck-objects --full)" + +#test_done; exit + +add B66 $B65 +add B67 $B66 + +test_expect_success "pull in shallow repo" \ + "(cd shallow; git pull .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 6\" = \"$(grep count count.shallow)\"" + +add B68 $B67 +add B69 $B68 + +test_expect_success "deepening pull in shallow repo" \ + "(cd shallow; git pull --depth 4 .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 12\" = \"$(grep count count.shallow)\"" + +test_expect_success "deepening fetch in shallow repo" \ + "(cd shallow; git fetch --depth 4 .. A:A)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 18\" = \"$(grep count count.shallow)\"" + +test_expect_failure "pull in shallow repo with missing merge base" \ + "(cd shallow; git pull --depth 4 .. A)" + test_done -- cgit v0.10.2-6-g49f6 From cf01bd52efeb21084652b2b06b778ed458004e8f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:04:56 -0800 Subject: We should make sure that the protocol is still extensible. This just reformats if .. else if .. else chain to make it clear we are handling extended response from the other end. diff --git a/fetch-pack.c b/fetch-pack.c index c3064b9..6fd4570 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -189,7 +189,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (!fetching) return 1; - if (depth > 0) { + if (depth > 0) { char line[1024]; unsigned char sha1[20]; int len; @@ -202,7 +202,9 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (lookup_object(sha1)) continue; register_shallow(sha1); - } else if (!strncmp("unshallow ", line, 10)) { + continue; + } + if (!strncmp("unshallow ", line, 10)) { if (get_sha1_hex(line + 10, sha1)) die("invalid unshallow line: %s", line); if (!lookup_object(sha1)) @@ -211,8 +213,9 @@ static int find_common(int fd[2], unsigned char *result_sha1, parse_object(sha1); if (unregister_shallow(sha1)) die("no shallow found: %s", line); - } else - die("expected shallow/unshallow, got %s", line); + continue; + } + die("expected shallow/unshallow, got %s", line); } } -- cgit v0.10.2-6-g49f6 From fcd1e3190612fc59c2d820537280b5164d290d9d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:04:56 -0800 Subject: Why does it mean we do not have to register shallow if we have one? diff --git a/fetch-pack.c b/fetch-pack.c index 6fd4570..d00573d 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -198,9 +198,6 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (!strncmp("shallow ", line, 8)) { if (get_sha1_hex(line + 8, sha1)) die("invalid shallow line: %s", line); - /* no need making it shallow if we have it already */ - if (lookup_object(sha1)) - continue; register_shallow(sha1); continue; } -- cgit v0.10.2-6-g49f6 From c6702f4b95699c6bd47337516c32e3408dda0595 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:47:45 -0800 Subject: Why didn't we mark want_obj as ~UNINTERESTING in the old code? Is this something we would want to do regardless of shallow clone? diff --git a/upload-pack.c b/upload-pack.c index 4a9d672..7e3c437 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -138,6 +138,7 @@ static void create_pack_file(void) } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; + /* why??? */ o->flags &= ~UNINTERESTING; add_pending_object(&revs, o, NULL); } -- cgit v0.10.2-6-g49f6 From 176d45cb20eb4ffc661d73383aec01943e056e10 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:47:46 -0800 Subject: shallow clone: unparse and reparse an unshallowed commit Otherwise we would not read the real parents from the commit object. diff --git a/upload-pack.c b/upload-pack.c index 7e3c437..d5b4750 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -582,6 +582,7 @@ static void receive_needs(void) object->flags &= ~CLIENT_SHALLOW; /* make sure the real parents are parsed */ unregister_shallow(object->sha1); + object->parsed = 0; parse_commit((struct commit *)object); parents = ((struct commit *)object)->parents; while (parents) { -- cgit v0.10.2-6-g49f6 From d6491e3a214541aeb016b0688d0920845b03932d Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:04 +0100 Subject: fetch-pack: Properly remove the shallow file when it becomes empty. The code was unlinking the lock file instead. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index d00573d..bb310b6 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -700,7 +700,7 @@ int main(int argc, char **argv) fd = hold_lock_file_for_update(&lock, shallow, 1); if (!write_shallow_commits(fd, 0)) { - unlink(lock.filename); + unlink(shallow); rollback_lock_file(&lock); } else { close(fd); -- cgit v0.10.2-6-g49f6 From 1f2de769812f0adea598269d788c9c75d23e80b8 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:25 +0100 Subject: upload-pack: Check for NOT_SHALLOW flag before sending a shallow to the client. A commit may have been put on the shallow list, and then reached from another branch and marked NOT_SHALLOW without being removed from the list. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano diff --git a/upload-pack.c b/upload-pack.c index d5b4750..d4a7b62 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -565,7 +565,7 @@ static void receive_needs(void) SHALLOW, NOT_SHALLOW); while (result) { struct object *object = &result->item->object; - if (!(object->flags & CLIENT_SHALLOW)) { + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { packet_write(1, "shallow %s", sha1_to_hex(object->sha1)); register_shallow(object->sha1); -- cgit v0.10.2-6-g49f6 From d158631549b87e11f2415622e421546539b09b67 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:59:12 +0100 Subject: git-fetch: Reset shallow_depth before auto-following tags. Otherwise fetching the tags could also fetch commits up to the specified depth, which isn't the expected behavior. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano diff --git a/git-fetch.sh b/git-fetch.sh index 0b1e6d1..f0645d9 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -461,6 +461,8 @@ case "$no_tags$tags" in case "$taglist" in '') ;; ?*) + # do not deepen a shallow tree when following tags + shallow_depth= fetch_main "$taglist" ;; esac esac -- cgit v0.10.2-6-g49f6 From d64d6c9fc712cf6fad9a3ec7f659cf843ee2e18d Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:50 +0100 Subject: get_shallow_commits: Avoid memory leak if a commit has been reached already. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano diff --git a/shallow.c b/shallow.c index 2db1dc4..3d53d17 100644 --- a/shallow.c +++ b/shallow.c @@ -60,7 +60,9 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, commit = NULL; continue; } - commit->util = xcalloc(1, sizeof(int)); + if (!commit->util) + commit->util = xmalloc(sizeof(int)); + *(int *)commit->util = 0; cur_depth = 0; } else { commit = (struct commit *) -- cgit v0.10.2-6-g49f6 From 4bcb310c2539b66d535e87508d1b7a90fe29c083 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 16:00:13 +0100 Subject: fetch-pack: Do not fetch tags for shallow clones. A better fix may be to only fetch tags that point to commits that we are downloading, but git-clone doesn't have support for following tags. This will happen automatically on the next git-fetch though. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano diff --git a/fetch-pack.c b/fetch-pack.c index bb310b6..80979b8 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -342,7 +342,8 @@ static void filter_refs(struct ref **refs, int nr_match, char **match) if (!memcmp(ref->name, "refs/", 5) && check_ref_format(ref->name + 5)) ; /* trash */ - else if (fetch_all) { + else if (fetch_all && + (!depth || strncmp(ref->name, "refs/tags/", 10) )) { *newtail = ref; ref->next = NULL; newtail = &ref->next; -- cgit v0.10.2-6-g49f6