Merge branch 'jx/bundle'

"git bundle" learns "--stdin" option to read its refs from the
standard input.  Also, it now does not lose refs whey they point
at the same object.

* jx/bundle:
  bundle: arguments can be read from stdin
  bundle: lost objects when removing duplicate pendings
  test: add helper functions for git-bundle

4 files changed, 511 insertions, 24 deletions

diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh
index 8da907e..42f5503 100755
--- a/t/t5510-fetch.sh
+++ b/t/t5510-fetch.sh
@@ -9,22 +9,10 @@ GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
 
 . ./test-lib.sh
+. "$TEST_DIRECTORY"/test-bundle-functions.sh
 
 D=$(pwd)
 
-test_bundle_object_count () {
-	git verify-pack -v "$1" >verify.out &&
-	test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l)
-}
-
-convert_bundle_to_pack () {
-	while read x && test -n "$x"
-	do
-		:;
-	done
-	cat
-}
-
 test_expect_success setup '
 	echo >file original &&
 	git add file &&
@@ -483,9 +471,7 @@ test_expect_success 'unbundle 1' '
 
 test_expect_success 'bundle 1 has only 3 files ' '
 	cd "$D" &&
-	convert_bundle_to_pack <bundle1 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle1 3
 '
 
 test_expect_success 'unbundle 2' '
@@ -500,9 +486,7 @@ test_expect_success 'bundle does not prerequisite objects' '
 	git add file2 &&
 	git commit -m add.file2 file2 &&
 	git bundle create bundle3 -1 HEAD &&
-	convert_bundle_to_pack <bundle3 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle3 3
 '
 
 test_expect_success 'bundle should be able to create a full history' '
@@ -1055,9 +1039,7 @@ test_expect_success 'all boundary commits are excluded' '
 	git merge otherside &&
 	ad=$(git log --no-walk --format=%ad HEAD) &&
 	git bundle create twoside-boundary.bdl main --since="$ad" &&
-	convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack &&
-	pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) &&
-	test_bundle_object_count .git/objects/pack/pack-${pack##pack	}.pack 3
+	test_bundle_object_count --thin twoside-boundary.bdl 3
 '
 
 test_expect_success 'fetch --prune prints the remotes url' '
diff --git a/t/t5607-clone-bundle.sh b/t/t5607-clone-bundle.sh
index 5b39f4a..f4c383c 100755
--- a/t/t5607-clone-bundle.sh
+++ b/t/t5607-clone-bundle.sh
@@ -41,13 +41,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo main | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep main output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo main | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep main output
diff --git a/t/t6020-bundle-misc.sh b/t/t6020-bundle-misc.sh
new file mode 100755
index 0000000..6249420
--- /dev/null
+++ b/t/t6020-bundle-misc.sh
@@ -0,0 +1,463 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Jiang Xin
+#
+
+test_description='Test git-bundle'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/test-bundle-functions.sh
+
+# Create a commit or tag and set the variable with the object ID.
+test_commit_setvar () {
+	notick=
+	signoff=
+	indir=
+	merge=
+	tag=
+	var=
+
+	while test $# != 0
+	do
+		case "$1" in
+		--merge)
+			merge=t
+			;;
+		--tag)
+			tag=t
+			;;
+		--notick)
+			notick=t
+			;;
+		--signoff)
+			signoff="$1"
+			;;
+		-C)
+			shift
+			indir="$1"
+			;;
+		-*)
+			echo >&2 "error: unknown option $1"
+			return 1
+			;;
+		*)
+			break
+			;;
+		esac
+		shift
+	done
+	if test $# -lt 2
+	then
+		echo >&2 "error: test_commit_setvar must have at least 2 arguments"
+		return 1
+	fi
+	var=$1
+	shift
+	indir=${indir:+"$indir"/}
+	if test -z "$notick"
+	then
+		test_tick
+	fi &&
+	if test -n "$merge"
+	then
+		git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
+			${2:+-m "$2"} "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	elif test -n "$tag"
+	then
+		git ${indir:+ -C "$indir"} tag -m "$1" "$1" "${2:-HEAD}" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
+	else
+		file=${2:-"$1.t"} &&
+		echo "${3-$1}" >"$indir$file" &&
+		git ${indir:+ -C "$indir"} add "$file" &&
+		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	fi &&
+	eval $var=$oid
+}
+
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+	sed \
+		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
+		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
+		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
+		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
+		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
+		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
+		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
+		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
+		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
+		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
+		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
+		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
+		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
+		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
+		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
+		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
+		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+		-e "s/ *\$//"
+}
+
+#            (C)   (D, pull/1/head, topic/1)
+#             o --- o
+#            /       \                              (L)
+#           /         \        o (H, topic/2)             (M, tag:v2)
+#          /    (F)    \      /                                 (N, tag:v3)
+#         /      o --------- o (G, pull/2/head)      o --- o --- o (release)
+#        /      /        \    \                      /       \
+#  o --- o --- o -------- o -- o ------------------ o ------- o --- o (main)
+# (A)   (B)  (E, tag:v1) (I)  (J)                  (K)       (O)   (P)
+#
+test_expect_success 'setup' '
+	# Try to make a stable fixed width for abbreviated commit ID,
+	# this fixed-width oid will be replaced with "<OID>".
+	git config core.abbrev 7 &&
+
+	# branch main: commit A & B
+	test_commit_setvar A "Commit A" main.txt &&
+	test_commit_setvar B "Commit B" main.txt &&
+
+	# branch topic/1: commit C & D, refs/pull/1/head
+	git checkout -b topic/1 &&
+	test_commit_setvar C "Commit C" topic-1.txt &&
+	test_commit_setvar D "Commit D" topic-1.txt &&
+	git update-ref refs/pull/1/head HEAD &&
+
+	# branch topic/1: commit E, tag v1
+	git checkout main &&
+	test_commit_setvar E "Commit E" main.txt &&
+	test_commit_setvar --tag TAG1 v1 &&
+
+	# branch topic/2: commit F & G, refs/pull/2/head
+	git checkout -b topic/2 &&
+	test_commit_setvar F "Commit F" topic-2.txt &&
+	test_commit_setvar G "Commit G" topic-2.txt &&
+	git update-ref refs/pull/2/head HEAD &&
+	test_commit_setvar H "Commit H" topic-2.txt &&
+
+	# branch main: merge commit I & J
+	git checkout main &&
+	test_commit_setvar --merge I topic/1 "Merge commit I" &&
+	test_commit_setvar --merge J refs/pull/2/head "Merge commit J" &&
+
+	# branch main: commit K
+	git checkout main &&
+	test_commit_setvar K "Commit K" main.txt &&
+
+	# branch release:
+	git checkout -b release &&
+	test_commit_setvar L "Commit L" release.txt &&
+	test_commit_setvar M "Commit M" release.txt &&
+	test_commit_setvar --tag TAG2 v2 &&
+	test_commit_setvar N "Commit N" release.txt &&
+	test_commit_setvar --tag TAG3 v3 &&
+
+	# branch main: merge commit O, commit P
+	git checkout main &&
+	test_commit_setvar --merge O tags/v2 "Merge commit O" &&
+	test_commit_setvar P "Commit P" main.txt
+'
+
+test_expect_success 'create bundle from special rev: main^!' '
+	git bundle create special-rev.bdl "main^!" &&
+
+	git bundle list-heads special-rev.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	<COMMIT-P> refs/heads/main
+	EOF
+	test_i18ncmp expect actual &&
+
+	git bundle verify special-rev.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	The bundle contains this ref:
+	<COMMIT-P> refs/heads/main
+	The bundle requires this ref:
+	<COMMIT-O>
+	EOF
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count special-rev.bdl 3
+'
+
+test_expect_success 'create bundle with --max-count option' '
+	git bundle create max-count.bdl --max-count 1 \
+		main \
+		"^release" \
+		refs/tags/v1 \
+		refs/pull/1/head \
+		refs/pull/2/head &&
+
+	git bundle verify max-count.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	The bundle contains these 2 refs:
+	<COMMIT-P> refs/heads/main
+	<TAG-1> refs/tags/v1
+	The bundle requires this ref:
+	<COMMIT-O>
+	EOF
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count max-count.bdl 4
+'
+
+test_expect_success 'create bundle with --since option' '
+	git log -1 --pretty="%ad" $M >actual &&
+	cat >expect <<-\EOF &&
+	Thu Apr 7 15:26:13 2005 -0700
+	EOF
+	test_cmp expect actual &&
+
+	git bundle create since.bdl \
+		--since "Thu Apr 7 15:27:00 2005 -0700" \
+		--all &&
+
+	git bundle verify since.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	The bundle contains these 5 refs:
+	<COMMIT-P> refs/heads/main
+	<COMMIT-N> refs/heads/release
+	<TAG-2> refs/tags/v2
+	<TAG-3> refs/tags/v3
+	<COMMIT-P> HEAD
+	The bundle requires these 2 refs:
+	<COMMIT-M>
+	<COMMIT-K>
+	EOF
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count --thin since.bdl 13
+'
+
+test_expect_success 'create bundle 1 - no prerequisites' '
+	# create bundle from args
+	git bundle create 1.bdl topic/1 topic/2 &&
+
+	# create bundle from stdin
+	cat >input <<-\EOF &&
+	topic/1
+	topic/2
+	EOF
+	git bundle create stdin-1.bdl --stdin <input &&
+
+	cat >expect <<-\EOF &&
+	The bundle contains these 2 refs:
+	<COMMIT-D> refs/heads/topic/1
+	<COMMIT-H> refs/heads/topic/2
+	The bundle records a complete history.
+	EOF
+
+	# verify bundle, which has no prerequisites
+	git bundle verify 1.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	git bundle verify stdin-1.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count       1.bdl 24 &&
+	test_bundle_object_count stdin-1.bdl 24
+'
+
+test_expect_success 'create bundle 2 - has prerequisites' '
+	# create bundle from args
+	git bundle create 2.bdl \
+		--ignore-missing \
+		^topic/deleted \
+		^$D \
+		^topic/2 \
+		release &&
+
+	# create bundle from stdin
+	# input has a non-exist reference: "topic/deleted"
+	cat >input <<-EOF &&
+	^topic/deleted
+	^$D
+	^topic/2
+	EOF
+	git bundle create stdin-2.bdl \
+		--ignore-missing \
+		--stdin \
+		release <input &&
+
+	cat >expect <<-\EOF &&
+	The bundle contains this ref:
+	<COMMIT-N> refs/heads/release
+	The bundle requires these 3 refs:
+	<COMMIT-D>
+	<COMMIT-E>
+	<COMMIT-G>
+	EOF
+
+	git bundle verify 2.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	git bundle verify stdin-2.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count       2.bdl 16 &&
+	test_bundle_object_count stdin-2.bdl 16
+'
+
+test_expect_success 'fail to verify bundle without prerequisites' '
+	git init --bare test1.git &&
+
+	cat >expect <<-\EOF &&
+	error: Repository lacks these prerequisite commits:
+	error: <COMMIT-D>
+	error: <COMMIT-E>
+	error: <COMMIT-G>
+	EOF
+
+	test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_must_fail git -C test1.git bundle verify ../stdin-2.bdl 2>&1 |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual
+'
+
+test_expect_success 'create bundle 3 - two refs, same object' '
+	# create bundle from args
+	git bundle create --version=3 3.bdl \
+		^release \
+		^topic/1 \
+		^topic/2 \
+		main \
+		HEAD &&
+
+	# create bundle from stdin
+	cat >input <<-\EOF &&
+	^release
+	^topic/1
+	^topic/2
+	EOF
+	git bundle create --version=3 stdin-3.bdl \
+		--stdin \
+		main HEAD <input &&
+
+	cat >expect <<-\EOF &&
+	The bundle contains these 2 refs:
+	<COMMIT-P> refs/heads/main
+	<COMMIT-P> HEAD
+	The bundle requires these 2 refs:
+	<COMMIT-M>
+	<COMMIT-K>
+	EOF
+
+	git bundle verify 3.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	git bundle verify stdin-3.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count       3.bdl 4 &&
+	test_bundle_object_count stdin-3.bdl 4
+'
+
+test_expect_success 'create bundle 4 - with tags' '
+	# create bundle from args
+	git bundle create 4.bdl \
+		^main \
+		^release \
+		^topic/1 \
+		^topic/2 \
+		--all &&
+
+	# create bundle from stdin
+	cat >input <<-\EOF &&
+	^main
+	^release
+	^topic/1
+	^topic/2
+	EOF
+	git bundle create stdin-4.bdl \
+		--ignore-missing \
+		--stdin \
+		--all <input &&
+
+	cat >expect <<-\EOF &&
+	The bundle contains these 3 refs:
+	<TAG-1> refs/tags/v1
+	<TAG-2> refs/tags/v2
+	<TAG-3> refs/tags/v3
+	The bundle records a complete history.
+	EOF
+
+	git bundle verify 4.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	git bundle verify stdin-4.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count       4.bdl 3 &&
+	test_bundle_object_count stdin-4.bdl 3
+'
+
+test_expect_success 'clone from bundle' '
+	git clone --mirror 1.bdl mirror.git &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	<COMMIT-D> refs/heads/topic/1
+	<COMMIT-H> refs/heads/topic/2
+	EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../2.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	<COMMIT-N> refs/heads/release
+	<COMMIT-D> refs/heads/topic/1
+	<COMMIT-H> refs/heads/topic/2
+	EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../3.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	<COMMIT-P> refs/heads/main
+	<COMMIT-N> refs/heads/release
+	<COMMIT-D> refs/heads/topic/1
+	<COMMIT-H> refs/heads/topic/2
+	EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../4.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-\EOF &&
+	<COMMIT-P> refs/heads/main
+	<COMMIT-N> refs/heads/release
+	<COMMIT-D> refs/heads/topic/1
+	<COMMIT-H> refs/heads/topic/2
+	<TAG-1> refs/tags/v1
+	<TAG-2> refs/tags/v2
+	<TAG-3> refs/tags/v3
+	EOF
+	test_cmp expect actual
+'
+
+test_done
diff --git a/t/test-bundle-functions.sh b/t/test-bundle-functions.sh
new file mode 100644
index 0000000..cf7ed81
--- /dev/null
+++ b/t/test-bundle-functions.sh
@@ -0,0 +1,42 @@
+# Library of git-bundle related functions.
+
+# Display the pack data contained in the bundle file, bypassing the
+# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+	while read x && test -n "$x"
+	do
+		:;
+	done
+	cat
+}
+
+# Check count of objects in a bundle file.
+# We can use "--thin" opiton to check thin pack, which must be fixed by
+# command `git-index-pack --fix-thin --stdin`.
+test_bundle_object_count () {
+	thin=
+	if test "$1" = "--thin"
+	then
+		thin=t
+		shift
+	fi
+	if test $# -ne 2
+	then
+		echo >&2 "args should be: <bundle> <count>"
+		return 1
+	fi
+	bundle=$1
+	pack=$bundle.pack
+	convert_bundle_to_pack <"$bundle" >"$pack" &&
+	if test -n "$thin"
+	then
+		mv "$pack" "$bundle.thin.pack" &&
+		git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack"
+	else
+		git index-pack "$pack"
+	fi || return 1
+	count=$(git show-index <"${pack%pack}idx" | wc -l) &&
+	test $2 = $count && return 0
+	echo >&2 "error: object count for $bundle is $count, not $2"
+	return 1
+}