diff options
Diffstat (limited to 't/t5319-multi-pack-index.sh')
-rwxr-xr-x | t/t5319-multi-pack-index.sh | 408 |
1 files changed, 386 insertions, 22 deletions
diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 3d4d9f1..dd09134 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -2,6 +2,7 @@ test_description='multi-pack-indexes' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-chunk.sh GIT_TEST_MULTI_PACK_INDEX=0 objdir=.git/objects @@ -93,7 +94,7 @@ test_expect_success 'create objects' ' test_commit initial && for i in $(test_seq 1 5) do - generate_objects $i + generate_objects $i || return 1 done && commit_and_list_objects ' @@ -155,7 +156,7 @@ test_expect_success 'corrupt idx reports errors' ' test_expect_success 'add more objects' ' for i in $(test_seq 6 10) do - generate_objects $i + generate_objects $i || return 1 done && commit_and_list_objects ' @@ -168,18 +169,45 @@ test_expect_success 'write midx with two packs' ' compare_results_with_midx "two packs" +test_expect_success 'write midx with --stdin-packs' ' + rm -fr $objdir/pack/multi-pack-index && + + idx="$(find $objdir/pack -name "test-2-*.idx")" && + basename "$idx" >in && + + git multi-pack-index write --stdin-packs <in && + + test-tool read-midx $objdir | grep "\.idx$" >packs && + + test_cmp packs in +' + +compare_results_with_midx "mixed mode (one pack + extra)" + +test_expect_success 'write with no objects and preferred pack' ' + test_when_finished "rm -rf empty" && + git init empty && + test_must_fail git -C empty multi-pack-index write \ + --stdin-packs --preferred-pack=does-not-exist </dev/null 2>err && + cat >expect <<-EOF && + warning: unknown preferred pack: ${SQ}does-not-exist${SQ} + error: no pack files to index. + EOF + test_cmp expect err +' + test_expect_success 'write progress off for redirected stderr' ' git multi-pack-index --object-dir=$objdir write 2>err && test_line_count = 0 err ' test_expect_success 'write force progress on for stderr' ' - GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir --progress write 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir write --progress 2>err && test_file_not_empty err ' test_expect_success 'write with the --no-progress option' ' - GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir --no-progress write 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir write --no-progress 2>err && test_line_count = 0 err ' @@ -188,7 +216,7 @@ test_expect_success 'add more packs' ' do generate_objects $j && commit_and_list_objects && - git pack-objects --index-version=2 $objdir/pack/test-pack <obj-list + git pack-objects --index-version=2 $objdir/pack/test-pack <obj-list || return 1 done ' @@ -201,6 +229,34 @@ test_expect_success 'write midx with twelve packs' ' compare_results_with_midx "twelve packs" +test_expect_success 'multi-pack-index *.rev cleanup with --object-dir' ' + git init repo && + git clone -s repo alternate && + + test_when_finished "rm -rf repo alternate" && + + ( + cd repo && + test_commit base && + git repack -d + ) && + + ours="alternate/.git/objects/pack/multi-pack-index-123.rev" && + theirs="repo/.git/objects/pack/multi-pack-index-abc.rev" && + touch "$ours" "$theirs" && + + ( + cd alternate && + git multi-pack-index --object-dir ../repo/.git/objects write + ) && + + # writing a midx in "repo" should not remove the .rev file in the + # alternate + test_path_is_file repo/.git/objects/pack/multi-pack-index && + test_path_is_file $ours && + test_path_is_missing $theirs +' + test_expect_success 'warn on improper hash version' ' git init --object-format=sha1 sha1 && ( @@ -224,13 +280,13 @@ test_expect_success 'warn on improper hash version' ' cd sha1 && mv ../mpi-sha256 .git/objects/pack/multi-pack-index && git log -1 2>err && - test_i18ngrep "multi-pack-index hash version 2 does not match version 1" err + test_grep "multi-pack-index hash version 2 does not match version 1" err ) && ( cd sha256 && mv ../mpi-sha1 .git/objects/pack/multi-pack-index && git log -1 2>err && - test_i18ngrep "multi-pack-index hash version 1 does not match version 2" err + test_grep "multi-pack-index hash version 1 does not match version 2" err ) ' @@ -277,6 +333,23 @@ test_expect_success 'midx picks objects from preferred pack' ' ) ' +test_expect_success 'preferred packs must be non-empty' ' + test_when_finished rm -rf preferred.git && + git init preferred.git && + ( + cd preferred.git && + + test_commit base && + git repack -ad && + + empty="$(git pack-objects $objdir/pack/pack </dev/null)" && + + test_must_fail git multi-pack-index write \ + --preferred-pack=pack-$empty.pack 2>err && + grep "with no objects" err + ) +' + test_expect_success 'verify multi-pack-index success' ' git multi-pack-index verify --object-dir=$objdir ' @@ -314,7 +387,7 @@ corrupt_midx_and_verify() { printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc && test_must_fail $COMMAND 2>test_err && grep -v "^+" test_err >err && - test_i18ngrep "$GREPSTR" err + test_grep "$GREPSTR" err } test_expect_success 'verify bad signature' ' @@ -366,7 +439,7 @@ test_expect_success 'verify extended chunk count' ' test_expect_success 'verify missing required chunk' ' corrupt_midx_and_verify $MIDX_BYTE_CHUNK_ID "\01" $objdir \ - "missing required" + "required pack-name chunk missing" ' test_expect_success 'verify invalid chunk offset' ' @@ -407,20 +480,37 @@ test_expect_success 'verify incorrect offset' ' test_expect_success 'git-fsck incorrect offset' ' corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \ "incorrect object offset" \ - "git -c core.multipackindex=true fsck" + "git -c core.multiPackIndex=true fsck" && + test_unconfig core.multiPackIndex && + test_must_fail git fsck && + git -c core.multiPackIndex=false fsck +' + +test_expect_success 'git fsck shows MIDX output with --progress' ' + git fsck --progress 2>err && + grep "Verifying OID order in multi-pack-index" err && + grep "Verifying object offsets" err +' + +test_expect_success 'git fsck suppresses MIDX output with --no-progress' ' + git fsck --no-progress 2>err && + ! grep "Verifying OID order in multi-pack-index" err && + ! grep "Verifying object offsets" err ' test_expect_success 'corrupt MIDX is not reused' ' corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \ "incorrect object offset" && git multi-pack-index write 2>err && - test_i18ngrep checksum.mismatch err && + test_grep checksum.mismatch err && git multi-pack-index verify ' test_expect_success 'verify incorrect checksum' ' - pos=$(($(wc -c <$objdir/pack/multi-pack-index) - 1)) && - corrupt_midx_and_verify $pos "\377" $objdir "incorrect checksum" + pos=$(($(wc -c <$objdir/pack/multi-pack-index) - 10)) && + corrupt_midx_and_verify $pos \ + "\377\377\377\377\377\377\377\377\377\377" \ + $objdir "incorrect checksum" ' test_expect_success 'repack progress off for redirected stderr' ' @@ -429,12 +519,12 @@ test_expect_success 'repack progress off for redirected stderr' ' ' test_expect_success 'repack force progress on for stderr' ' - GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir --progress repack 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir repack --progress 2>err && test_file_not_empty err ' test_expect_success 'repack with the --no-progress option' ' - GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir --no-progress repack 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir repack --no-progress 2>err && test_line_count = 0 err ' @@ -487,7 +577,8 @@ test_expect_success 'repack preserves multi-pack-index when creating packs' ' compare_results_with_midx "after repack" test_expect_success 'multi-pack-index and pack-bitmap' ' - git -c repack.writeBitmaps=true repack -ad && + GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 \ + git -c repack.writeBitmaps=true repack -ad && git multi-pack-index write && git rev-list --test-bitmap HEAD ' @@ -530,14 +621,22 @@ test_expect_success 'force some 64-bit offsets with pack-objects' ' mkdir objects64/pack && for i in $(test_seq 1 11) do - generate_objects 11 + generate_objects 11 || return 1 done && commit_and_list_objects && pack64=$(git pack-objects --index-version=2,0x40 objects64/pack/test-64 <obj-list) && idx64=objects64/pack/test-64-$pack64.idx && chmod u+w $idx64 && corrupt_data $idx64 $(test_oid idxoff) "\02" && - midx64=$(git multi-pack-index --object-dir=objects64 write) && + # objects64 is not a real repository, but can serve as an alternate + # anyway so we can write a MIDX into it + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + ( cd ../objects64 && pwd ) >.git/objects/info/alternates && + midx64=$(git multi-pack-index --object-dir=../objects64 write) + ) && midx_read_expect 1 63 5 objects64 " large-offsets" ' @@ -566,7 +665,7 @@ test_expect_success 'setup expire tests' ' git update-index --add large_file.txt && for i in $(test_seq 1 20) do - test_commit $i + test_commit $i || exit 1 done && git branch A HEAD && git branch B HEAD~8 && @@ -618,7 +717,7 @@ test_expect_success 'expire progress off for redirected stderr' ' test_expect_success 'expire force progress on for stderr' ' ( cd dup && - GIT_PROGRESS_DELAY=0 git multi-pack-index --progress expire 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index expire --progress 2>err && test_file_not_empty err ) ' @@ -626,7 +725,7 @@ test_expect_success 'expire force progress on for stderr' ' test_expect_success 'expire with the --no-progress option' ' ( cd dup && - GIT_PROGRESS_DELAY=0 git multi-pack-index --no-progress expire 2>err && + GIT_PROGRESS_DELAY=0 git multi-pack-index expire --no-progress 2>err && test_line_count = 0 err ) ' @@ -710,6 +809,70 @@ test_expect_success 'repack creates a new pack' ' ) ' +test_expect_success 'repack (all) ignores cruft pack' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit base && + test_commit --no-tag unreachable && + + git reset --hard base && + git reflog expire --all --expire=all && + git repack --cruft -d && + + git multi-pack-index write && + + find $objdir/pack | sort >before && + git multi-pack-index repack --batch-size=0 && + find $objdir/pack | sort >after && + + test_cmp before after + ) +' + +test_expect_success 'repack (--batch-size) ignores cruft pack' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit_bulk 5 && + test_commit --no-tag unreachable && + + git reset --hard HEAD^ && + git reflog expire --all --expire=all && + git repack --cruft -d && + + test_commit four && + + find $objdir/pack -type f -name "*.pack" | sort >before && + git repack -d && + find $objdir/pack -type f -name "*.pack" | sort >after && + + pack="$(comm -13 before after)" && + test_file_size "$pack" >sz && + # Set --batch-size to twice the size of the pack created + # in the previous step, since this is enough to + # accommodate it and the cruft pack. + # + # This means that the MIDX machinery *could* combine the + # new and cruft packs together. + # + # We ensure that it does not below. + batch="$((($(cat sz) * 2)))" && + + git multi-pack-index write && + + find $objdir/pack | sort >before && + git multi-pack-index repack --batch-size=$batch && + find $objdir/pack | sort >after && + + test_cmp before after + ) +' + test_expect_success 'expire removes repacked packs' ' ( cd dup && @@ -773,6 +936,36 @@ test_expect_success 'expire respects .keep files' ' ) ' +test_expect_success 'expiring unreferenced cruft pack retains pack' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit base && + test_commit --no-tag unreachable && + unreachable=$(git rev-parse HEAD) && + + git reset --hard base && + git reflog expire --all --expire=all && + git repack --cruft -d && + mtimes="$(ls $objdir/pack/pack-*.mtimes)" && + + echo "base..$unreachable" >in && + pack="$(git pack-objects --revs --delta-base-offset \ + $objdir/pack/pack <in)" && + + # Preferring the contents of "$pack" will leave the + # cruft pack unreferenced (ie., none of the objects + # contained in the cruft pack will have their MIDX copy + # selected from the cruft pack). + git multi-pack-index write --preferred-pack="pack-$pack.pack" && + git multi-pack-index expire && + + test_path_is_file "$mtimes" + ) +' + test_expect_success 'repack --batch-size=0 repacks everything' ' cp -r dup dup2 && ( @@ -839,7 +1032,178 @@ test_expect_success 'load reverse index when missing .idx, .pack' ' test_expect_success 'usage shown without sub-command' ' test_expect_code 129 git multi-pack-index 2>err && - ! test_i18ngrep "unrecognized subcommand" err + ! test_grep "unrecognized subcommand" err +' + +test_expect_success 'complains when run outside of a repository' ' + nongit test_must_fail git multi-pack-index write 2>err && + grep "not a git repository" err +' + +test_expect_success 'repack with delta islands' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit first && + git repack && + test_commit second && + git repack && + + git multi-pack-index write && + git -c repack.useDeltaIslands=true multi-pack-index repack + ) +' + +corrupt_chunk () { + midx=.git/objects/pack/multi-pack-index && + test_when_finished "rm -rf $midx" && + git repack -ad --write-midx && + corrupt_chunk_file $midx "$@" +} + +test_expect_success 'reader notices too-small oid fanout chunk' ' + corrupt_chunk OIDF clear 00000000 && + test_must_fail git log 2>err && + cat >expect <<-\EOF && + error: multi-pack-index OID fanout is of the wrong size + fatal: multi-pack-index required OID fanout chunk missing or corrupted + EOF + test_cmp expect err +' + +test_expect_success 'reader notices too-small oid lookup chunk' ' + corrupt_chunk OIDL clear 00000000 && + test_must_fail git log 2>err && + cat >expect <<-\EOF && + error: multi-pack-index OID lookup chunk is the wrong size + fatal: multi-pack-index required OID lookup chunk missing or corrupted + EOF + test_cmp expect err +' + +test_expect_success 'reader notices too-small pack names chunk' ' + # There is no NUL to terminate the name here, so the + # chunk is too short. + corrupt_chunk PNAM clear 70656666 && + test_must_fail git log 2>err && + cat >expect <<-\EOF && + fatal: multi-pack-index pack-name chunk is too short + EOF + test_cmp expect err +' + +test_expect_success 'reader handles unaligned chunks' ' + # A 9-byte PNAM means all of the subsequent chunks + # will no longer be 4-byte aligned, but it is still + # a valid one-pack chunk on its own (it is "foo.pack\0"). + corrupt_chunk PNAM clear 666f6f2e7061636b00 && + git -c core.multipackindex=false log >expect.out && + git -c core.multipackindex=true log >out 2>err && + test_cmp expect.out out && + cat >expect.err <<-\EOF && + error: chunk id 4f494446 not 4-byte aligned + EOF + test_cmp expect.err err +' + +test_expect_success 'reader notices too-small object offset chunk' ' + corrupt_chunk OOFF clear 00000000 && + test_must_fail git log 2>err && + cat >expect <<-\EOF && + error: multi-pack-index object offset chunk is the wrong size + fatal: multi-pack-index required object offsets chunk missing or corrupted + EOF + test_cmp expect err +' + +test_expect_success 'reader bounds-checks large offset table' ' + # re-use the objects64 dir here to cheaply get access to a midx + # with large offsets. + git init repo && + test_when_finished "rm -rf repo" && + ( + cd repo && + (cd ../objects64 && pwd) >.git/objects/info/alternates && + git multi-pack-index --object-dir=../objects64 write && + midx=../objects64/pack/multi-pack-index && + corrupt_chunk_file $midx LOFF clear && + # using only %(objectsize) is important here; see the commit + # message for more details + test_must_fail git cat-file --batch-all-objects \ + --batch-check="%(objectsize)" 2>err && + cat >expect <<-\EOF && + fatal: multi-pack-index large offset out of bounds + EOF + test_cmp expect err + ) +' + +test_expect_success 'reader notices too-small revindex chunk' ' + # We only get a revindex with bitmaps (and likewise only + # load it when they are asked for). + test_config repack.writeBitmaps true && + corrupt_chunk RIDX clear 00000000 && + git -c core.multipackIndex=false rev-list \ + --all --use-bitmap-index >expect.out && + git -c core.multipackIndex=true rev-list \ + --all --use-bitmap-index >out 2>err && + test_cmp expect.out out && + cat >expect.err <<-\EOF && + error: multi-pack-index reverse-index chunk is the wrong size + warning: multi-pack bitmap is missing required reverse index + EOF + test_cmp expect.err err +' + +test_expect_success 'reader notices out-of-bounds fanout' ' + # This is similar to the out-of-bounds fanout test in t5318. The values + # in adjacent entries should be large but not identical (they + # are used as hi/lo starts for a binary search, which would then abort + # immediately). + corrupt_chunk OIDF 0 $(printf "%02x000000" $(test_seq 0 254)) && + test_must_fail git log 2>err && + cat >expect <<-\EOF && + error: oid fanout out of order: fanout[254] = fe000000 > 5c = fanout[255] + fatal: multi-pack-index required OID fanout chunk missing or corrupted + EOF + test_cmp expect err +' + +test_expect_success 'bitmapped packs are stored via the BTMP chunk' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + + for i in 1 2 3 4 5 + do + test_commit "$i" && + git repack -d || return 1 + done && + + find $objdir/pack -type f -name "*.idx" | xargs -n 1 basename | + sort >packs && + + git multi-pack-index write --stdin-packs <packs && + test_must_fail test-tool read-midx --bitmap $objdir 2>err && + cat >expect <<-\EOF && + error: MIDX does not contain the BTMP chunk + EOF + test_cmp expect err && + + git multi-pack-index write --stdin-packs --bitmap \ + --preferred-pack="$(head -n1 <packs)" <packs && + test-tool read-midx --bitmap $objdir >actual && + for i in $(test_seq $(wc -l <packs)) + do + sed -ne "${i}s/\.idx$/\.pack/p" packs && + echo " bitmap_pos: $((($i - 1) * 3))" && + echo " bitmap_nr: 3" || return 1 + done >expect && + test_cmp expect actual + ) ' test_done |