From aecf567cbfb6ab46e82f7f5df36fb6a2dd5bee69 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 5 Jul 2014 21:06:56 -0700 Subject: cache-tree: create/update cache-tree on checkout When git checkout checks out a branch, create or update the cache-tree so that subsequent operations are faster. update_main_cache_tree learned a new flag, WRITE_TREE_REPAIR. When WRITE_TREE_REPAIR is set, portions of the cache-tree which do not correspond to existing tree objects are invalidated (and portions which do are marked as valid). No new tree objects are created. Signed-off-by: David Turner Signed-off-by: Junio C Hamano diff --git a/builtin/checkout.c b/builtin/checkout.c index 07cf555..054214f 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -553,6 +553,14 @@ static int merge_working_tree(const struct checkout_opts *opts, } } + if (!active_cache_tree) + active_cache_tree = cache_tree(); + + if (!cache_tree_fully_valid(active_cache_tree)) + cache_tree_update(active_cache_tree, + (const struct cache_entry * const *)active_cache, + active_nr, WRITE_TREE_SILENT | WRITE_TREE_REPAIR); + if (write_cache(newfd, active_cache, active_nr) || commit_locked_index(lock_file)) die(_("unable to write new index file")); diff --git a/cache-tree.c b/cache-tree.c index 7fa524a..f951d7d 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -239,9 +239,12 @@ static int update_one(struct cache_tree *it, struct strbuf buffer; int missing_ok = flags & WRITE_TREE_MISSING_OK; int dryrun = flags & WRITE_TREE_DRY_RUN; + int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; int i; + assert(!(dryrun && repair)); + *skip_count = 0; if (0 <= it->entry_count && has_sha1_file(it->sha1)) @@ -374,7 +377,14 @@ static int update_one(struct cache_tree *it, #endif } - if (dryrun) + if (repair) { + unsigned char sha1[20]; + hash_sha1_file(buffer.buf, buffer.len, tree_type, sha1); + if (has_sha1_file(sha1)) + hashcpy(it->sha1, sha1); + else + to_invalidate = 1; + } else if (dryrun) hash_sha1_file(buffer.buf, buffer.len, tree_type, it->sha1); else if (write_sha1_file(buffer.buf, buffer.len, tree_type, it->sha1)) { strbuf_release(&buffer); diff --git a/cache-tree.h b/cache-tree.h index f1923ad..666d18f 100644 --- a/cache-tree.h +++ b/cache-tree.h @@ -39,6 +39,7 @@ int update_main_cache_tree(int); #define WRITE_TREE_IGNORE_CACHE_TREE 2 #define WRITE_TREE_DRY_RUN 4 #define WRITE_TREE_SILENT 8 +#define WRITE_TREE_REPAIR 16 /* error return codes */ #define WRITE_TREE_UNREADABLE_INDEX (-1) diff --git a/t/t0090-cache-tree.sh b/t/t0090-cache-tree.sh index 6c33e28..98fb1ab 100755 --- a/t/t0090-cache-tree.sh +++ b/t/t0090-cache-tree.sh @@ -44,14 +44,14 @@ test_expect_success 'read-tree HEAD establishes cache-tree' ' test_expect_success 'git-add invalidates cache-tree' ' test_when_finished "git reset --hard; git read-tree HEAD" && - echo "I changed this file" > foo && + echo "I changed this file" >foo && git add foo && test_invalid_cache_tree ' test_expect_success 'update-index invalidates cache-tree' ' test_when_finished "git reset --hard; git read-tree HEAD" && - echo "I changed this file" > foo && + echo "I changed this file" >foo && git update-index --add foo && test_invalid_cache_tree ' @@ -85,9 +85,22 @@ test_expect_success 'reset --hard without index gives cache-tree' ' test_shallow_cache_tree ' -test_expect_failure 'checkout gives cache-tree' ' +test_expect_success 'checkout gives cache-tree' ' + git tag current && git checkout HEAD^ && test_shallow_cache_tree ' +test_expect_success 'checkout -b gives cache-tree' ' + git checkout current && + git checkout -b prev HEAD^ && + test_shallow_cache_tree +' + +test_expect_success 'checkout -B gives cache-tree' ' + git checkout current && + git checkout -B prev HEAD^ && + test_shallow_cache_tree +' + test_done -- cgit v0.10.2-6-g49f6 From 969dd8c612b3b22c4080ffd4448cef4accbf428b Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 7 Jul 2014 17:33:43 -0700 Subject: test-dump-cache-tree: invalid trees are not errors Do not treat known-invalid trees as errors even when their subtree_nr is incorrect. Because git already knows that these trees are invalid, an incorrect subtree_nr will not cause problems. Add a couple of comments. Signed-off-by: David Turner Signed-off-by: Junio C Hamano diff --git a/test-dump-cache-tree.c b/test-dump-cache-tree.c index 47eab97..cbbbd8e 100644 --- a/test-dump-cache-tree.c +++ b/test-dump-cache-tree.c @@ -26,16 +26,16 @@ static int dump_cache_tree(struct cache_tree *it, return 0; if (it->entry_count < 0) { + /* invalid */ dump_one(it, pfx, ""); dump_one(ref, pfx, "#(ref) "); - if (it->subtree_nr != ref->subtree_nr) - errs = 1; } else { dump_one(it, pfx, ""); if (hashcmp(it->sha1, ref->sha1) || ref->entry_count != it->entry_count || ref->subtree_nr != it->subtree_nr) { + /* claims to be valid but is lying */ dump_one(ref, pfx, "#(ref) "); errs = 1; } -- cgit v0.10.2-6-g49f6 From 59a8adb6fb4ed27868e7144f4531b1bfd065a9c6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 10 Jul 2014 17:31:25 -0700 Subject: cache-tree: subdirectory tests Add tests to confirm that invalidation of subdirectories neither over- nor under-invalidates. Signed-off-by: David Turner Signed-off-by: Junio C Hamano diff --git a/t/t0090-cache-tree.sh b/t/t0090-cache-tree.sh index 98fb1ab..3a3342e 100755 --- a/t/t0090-cache-tree.sh +++ b/t/t0090-cache-tree.sh @@ -22,9 +22,10 @@ test_shallow_cache_tree () { } test_invalid_cache_tree () { - echo "invalid (0 subtrees)" >expect && - printf "SHA #(ref) (%d entries, 0 subtrees)\n" $(git ls-files|wc -l) >>expect && - cmp_cache_tree expect + printf "invalid %s ()\n" "" "$@" >expect && + test-dump-cache-tree | \ + sed -n -e "s/[0-9]* subtrees//" -e '/#(ref)/d' -e '/^invalid /p' >actual && + test_cmp expect actual } test_no_cache_tree () { @@ -49,6 +50,25 @@ test_expect_success 'git-add invalidates cache-tree' ' test_invalid_cache_tree ' +test_expect_success 'git-add in subdir invalidates cache-tree' ' + test_when_finished "git reset --hard; git read-tree HEAD" && + mkdir dirx && + echo "I changed this file" >dirx/foo && + git add dirx/foo && + test_invalid_cache_tree +' + +test_expect_success 'git-add in subdir does not invalidate sibling cache-tree' ' + git tag no-children && + test_when_finished "git reset --hard no-children; git read-tree HEAD" && + mkdir dir1 dir2 && + test_commit dir1/a && + test_commit dir2/b && + echo "I changed this file" >dir1/a && + git add dir1/a && + test_invalid_cache_tree dir1/ +' + test_expect_success 'update-index invalidates cache-tree' ' test_when_finished "git reset --hard; git read-tree HEAD" && echo "I changed this file" >foo && -- cgit v0.10.2-6-g49f6 From 9c4d6c0297b7d6167bf330853139b0515b351b89 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 13 Jul 2014 13:28:19 -0700 Subject: cache-tree: Write updated cache-tree after commit During the commit process, update the cache-tree. Write this updated cache-tree so that it's ready for subsequent commands. Add test code which demonstrates that git commit now writes the cache tree. Make all tests test the entire cache-tree, not just the root level. Signed-off-by: David Turner Signed-off-by: Junio C Hamano diff --git a/builtin/commit.c b/builtin/commit.c index 12afc42..77570c4 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -342,6 +342,17 @@ static char *prepare_index(int argc, const char **argv, const char *prefix, discard_cache(); read_cache_from(index_lock.filename); + if (update_main_cache_tree(WRITE_TREE_SILENT) == 0) { + fd = open(index_lock.filename, O_WRONLY); + if (fd >= 0) + if (write_cache(fd, active_cache, active_nr) < 0) + die(_("unable to write index file")); + else + close_lock_file(&index_lock); + else + die(_("unable to write index file")); + } else + warning(_("Failed to update main cache tree")); commit_style = COMMIT_NORMAL; return index_lock.filename; @@ -383,8 +394,12 @@ static char *prepare_index(int argc, const char **argv, const char *prefix, if (!only && !pathspec.nr) { fd = hold_locked_index(&index_lock, 1); refresh_cache_or_die(refresh_flags); - if (active_cache_changed) { + if (active_cache_changed + || !cache_tree_fully_valid(active_cache_tree)) { update_main_cache_tree(WRITE_TREE_SILENT); + active_cache_changed = 1; + } + if (active_cache_changed) { if (write_cache(fd, active_cache, active_nr) || commit_locked_index(&index_lock)) die(_("unable to write new_index file")); @@ -435,6 +450,7 @@ static char *prepare_index(int argc, const char **argv, const char *prefix, fd = hold_locked_index(&index_lock, 1); add_remove_files(&partial); refresh_cache(REFRESH_QUIET); + update_main_cache_tree(WRITE_TREE_SILENT); if (write_cache(fd, active_cache, active_nr) || close_lock_file(&index_lock)) die(_("unable to write new_index file")); diff --git a/t/t0090-cache-tree.sh b/t/t0090-cache-tree.sh index 3a3342e..48c4240 100755 --- a/t/t0090-cache-tree.sh +++ b/t/t0090-cache-tree.sh @@ -8,7 +8,7 @@ cache-tree extension. . ./test-lib.sh cmp_cache_tree () { - test-dump-cache-tree >actual && + test-dump-cache-tree | sed -e '/#(ref)/d' >actual && sed "s/$_x40/SHA/" filtered && test_cmp "$1" filtered } @@ -16,14 +16,38 @@ cmp_cache_tree () { # We don't bother with actually checking the SHA1: # test-dump-cache-tree already verifies that all existing data is # correct. -test_shallow_cache_tree () { - printf "SHA (%d entries, 0 subtrees)\n" $(git ls-files|wc -l) >expect && +generate_expected_cache_tree_rec () { + dir="$1${1:+/}" && + parent="$2" && + # ls-files might have foo/bar, foo/bar/baz, and foo/bar/quux + # We want to count only foo because it's the only direct child + subtrees=$(git ls-files|grep /|cut -d / -f 1|uniq) && + subtree_count=$(echo "$subtrees"|awk '$1 {++c} END {print c}') && + entries=$(git ls-files|wc -l) && + printf "SHA $dir (%d entries, %d subtrees)\n" "$entries" "$subtree_count" && + for subtree in $subtrees + do + cd "$subtree" + generate_expected_cache_tree_rec "$dir$subtree" "$dir" || return 1 + cd .. + done && + dir=$parent +} + +generate_expected_cache_tree () { + ( + generate_expected_cache_tree_rec + ) +} + +test_cache_tree () { + generate_expected_cache_tree >expect && cmp_cache_tree expect } test_invalid_cache_tree () { printf "invalid %s ()\n" "" "$@" >expect && - test-dump-cache-tree | \ + test-dump-cache-tree | sed -n -e "s/[0-9]* subtrees//" -e '/#(ref)/d' -e '/^invalid /p' >actual && test_cmp expect actual } @@ -33,14 +57,14 @@ test_no_cache_tree () { cmp_cache_tree expect } -test_expect_failure 'initial commit has cache-tree' ' +test_expect_success 'initial commit has cache-tree' ' test_commit foo && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'read-tree HEAD establishes cache-tree' ' git read-tree HEAD && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'git-add invalidates cache-tree' ' @@ -58,6 +82,18 @@ test_expect_success 'git-add in subdir invalidates cache-tree' ' test_invalid_cache_tree ' +cat >before <<\EOF +SHA (3 entries, 2 subtrees) +SHA dir1/ (1 entries, 0 subtrees) +SHA dir2/ (1 entries, 0 subtrees) +EOF + +cat >expect <<\EOF +invalid (2 subtrees) +invalid dir1/ (0 subtrees) +SHA dir2/ (1 entries, 0 subtrees) +EOF + test_expect_success 'git-add in subdir does not invalidate sibling cache-tree' ' git tag no-children && test_when_finished "git reset --hard no-children; git read-tree HEAD" && @@ -65,8 +101,10 @@ test_expect_success 'git-add in subdir does not invalidate sibling cache-tree' ' test_commit dir1/a && test_commit dir2/b && echo "I changed this file" >dir1/a && + cmp_cache_tree before && + echo "I changed this file" >dir1/a && git add dir1/a && - test_invalid_cache_tree dir1/ + cmp_cache_tree expect ' test_expect_success 'update-index invalidates cache-tree' ' @@ -79,7 +117,7 @@ test_expect_success 'update-index invalidates cache-tree' ' test_expect_success 'write-tree establishes cache-tree' ' test-scrap-cache-tree && git write-tree && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'test-scrap-cache-tree works' ' @@ -90,37 +128,86 @@ test_expect_success 'test-scrap-cache-tree works' ' test_expect_success 'second commit has cache-tree' ' test_commit bar && - test_shallow_cache_tree + test_cache_tree +' + +test_expect_success 'commit --interactive gives cache-tree on partial commit' ' + cat <<-\EOT >foo.c && + int foo() + { + return 42; + } + int bar() + { + return 42; + } + EOT + git add foo.c && + test_invalid_cache_tree && + git commit -m "add a file" && + test_cache_tree && + cat <<-\EOT >foo.c && + int foo() + { + return 43; + } + int bar() + { + return 44; + } + EOT + (echo p; echo 1; echo; echo s; echo n; echo y; echo q) | + git commit --interactive -m foo && + test_cache_tree +' + +test_expect_success 'commit in child dir has cache-tree' ' + mkdir dir && + >dir/child.t && + git add dir/child.t && + git commit -m dir/child.t && + test_cache_tree ' test_expect_success 'reset --hard gives cache-tree' ' test-scrap-cache-tree && git reset --hard && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'reset --hard without index gives cache-tree' ' rm -f .git/index && git reset --hard && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'checkout gives cache-tree' ' git tag current && git checkout HEAD^ && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'checkout -b gives cache-tree' ' git checkout current && git checkout -b prev HEAD^ && - test_shallow_cache_tree + test_cache_tree ' test_expect_success 'checkout -B gives cache-tree' ' git checkout current && git checkout -B prev HEAD^ && - test_shallow_cache_tree + test_cache_tree +' + +test_expect_success 'partial commit gives cache-tree' ' + git checkout -b partial no-children && + test_commit one && + test_commit two && + echo "some change" >one.t && + git add one.t && + echo "some other change" >two.t && + git commit two.t -m partial && + test_cache_tree ' test_done -- cgit v0.10.2-6-g49f6 From 4ed115e9c545fb27a5b80c751b2c04ec7ecc4d97 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 2 Sep 2014 14:16:20 -0700 Subject: cache-tree: do not try to use an invalidated subtree info to build a tree We punt from repairing the cache-tree during a branch switching if it involves having to create a new tree object that does not yet exist in the object store. "mkdir dir && >dir/file && git add dir" followed by "git checkout" is one example, when a tree that records the state of such "dir/" is not in the object store. However, after discovering that we do not have a tree object that records the state of "dir/", the caller failed to remember the fact that it noticed the cache-tree entry it received for "dir/" is invalidated, it already knows it should not be populating the level that has "dir/" as its immediate subdirectory, and it is not an error at all for the sublevel cache-tree entry gave it a bogus object name it shouldn't even look at. This led the caller to detect and report a non-existent error. The end result was the same and we avoided stuffing a non-existent tree to the cache-tree, but we shouldn't have issued an alarming error message to the user. Signed-off-by: Junio C Hamano diff --git a/cache-tree.c b/cache-tree.c index f951d7d..57597ac 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -316,6 +316,7 @@ static int update_one(struct cache_tree *it, int pathlen, entlen; const unsigned char *sha1; unsigned mode; + int expected_missing = 0; path = ce->name; pathlen = ce_namelen(ce); @@ -332,8 +333,10 @@ static int update_one(struct cache_tree *it, i += sub->count; sha1 = sub->cache_tree->sha1; mode = S_IFDIR; - if (sub->cache_tree->entry_count < 0) + if (sub->cache_tree->entry_count < 0) { to_invalidate = 1; + expected_missing = 1; + } } else { sha1 = ce->sha1; @@ -343,6 +346,8 @@ static int update_one(struct cache_tree *it, } if (mode != S_IFGITLINK && !missing_ok && !has_sha1_file(sha1)) { strbuf_release(&buffer); + if (expected_missing) + return -1; return error("invalid object %06o %s for '%.*s'", mode, sha1_to_hex(sha1), entlen+baselen, path); } diff --git a/t/t0090-cache-tree.sh b/t/t0090-cache-tree.sh index 48c4240..f9648a8 100755 --- a/t/t0090-cache-tree.sh +++ b/t/t0090-cache-tree.sh @@ -210,4 +210,12 @@ test_expect_success 'partial commit gives cache-tree' ' test_cache_tree ' +test_expect_success 'no phantom error when switching trees' ' + mkdir newdir && + >newdir/one && + git add newdir/one && + git checkout 2>errors && + ! test -s errors +' + test_done -- cgit v0.10.2-6-g49f6