Merge branch 'am/p4-branches-excludes'

"git p4" update.

* am/p4-branches-excludes:
  git-p4: respect excluded paths when detecting branches
  git-p4: add failing test for "git-p4: respect excluded paths when detecting branches"
  git-p4: don't exclude other files with same prefix
  git-p4: add failing test for "don't exclude other files with same prefix"
  git-p4: don't groom exclude path list on every commit
  git-p4: match branches case insensitively if configured
  git-p4: add failing test for "git-p4: match branches case insensitively if configured"
  git-p4: detect/prevent infinite loop in gitCommitByP4Change()

3 files changed, 205 insertions, 22 deletions

diff --git a/git-p4.py b/git-p4.py
index c71a683..3991e7d 100755
--- a/git-p4.py
+++ b/git-p4.py
@@ -1316,7 +1316,7 @@ class Command:
         self.needsGit = True
         self.verbose = False
 
-    # This is required for the "append" cloneExclude action
+    # This is required for the "append" update_shelve action
     def ensure_value(self, attr, value):
         if not hasattr(self, attr) or getattr(self, attr) is None:
             setattr(self, attr, value)
@@ -2530,6 +2530,11 @@ class View(object):
         die( "Error: %s is not found in client spec path" % depot_path )
         return ""
 
+def cloneExcludeCallback(option, opt_str, value, parser):
+    # prepend "/" because the first "/" was consumed as part of the option itself.
+    # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
+    parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
+
 class P4Sync(Command, P4UserMap):
 
     def __init__(self):
@@ -2553,7 +2558,7 @@ class P4Sync(Command, P4UserMap):
                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
                                      help="Only sync files that are included in the Perforce Client Spec"),
                 optparse.make_option("-/", dest="cloneExclude",
-                                     action="append", type="string",
+                                     action="callback", callback=cloneExcludeCallback, type="string",
                                      help="exclude depot path"),
         ]
         self.description = """Imports from Perforce into a git repository.\n
@@ -2618,20 +2623,25 @@ class P4Sync(Command, P4UserMap):
         if self.verbose:
             print("checkpoint finished: " + out)
 
+    def isPathWanted(self, path):
+        for p in self.cloneExclude:
+            if p.endswith("/"):
+                if p4PathStartsWith(path, p):
+                    return False
+            # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
+            elif path.lower() == p.lower():
+                return False
+        for p in self.depotPaths:
+            if p4PathStartsWith(path, p):
+                return True
+        return False
+
     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
-        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
-                             for path in self.cloneExclude]
         files = []
         fnum = 0
         while "depotFile%s" % fnum in commit:
             path =  commit["depotFile%s" % fnum]
-
-            if [p for p in self.cloneExclude
-                if p4PathStartsWith(path, p)]:
-                found = False
-            else:
-                found = [p for p in self.depotPaths
-                         if p4PathStartsWith(path, p)]
+            found = self.isPathWanted(path)
             if not found:
                 fnum = fnum + 1
                 continue
@@ -2668,7 +2678,7 @@ class P4Sync(Command, P4UserMap):
             path = self.clientSpecDirs.map_in_client(path)
             if self.detectBranches:
                 for b in self.knownBranches:
-                    if path.startswith(b + "/"):
+                    if p4PathStartsWith(path, b + "/"):
                         path = path[len(b)+1:]
 
         elif self.keepRepoPath:
@@ -2700,8 +2710,7 @@ class P4Sync(Command, P4UserMap):
         fnum = 0
         while "depotFile%s" % fnum in commit:
             path =  commit["depotFile%s" % fnum]
-            found = [p for p in self.depotPaths
-                     if p4PathStartsWith(path, p)]
+            found = self.isPathWanted(path)
             if not found:
                 fnum = fnum + 1
                 continue
@@ -2723,7 +2732,7 @@ class P4Sync(Command, P4UserMap):
             for branch in self.knownBranches.keys():
                 # add a trailing slash so that a commit into qt/4.2foo
                 # doesn't end up in qt/4.2, e.g.
-                if relPath.startswith(branch + "/"):
+                if p4PathStartsWith(relPath, branch + "/"):
                     if branch not in branches:
                         branches[branch] = []
                     branches[branch].append(file)
@@ -3325,7 +3334,9 @@ class P4Sync(Command, P4UserMap):
             if currentChange < change:
                 earliestCommit = "^%s" % next
             else:
-                latestCommit = "%s" % next
+                if next == latestCommit:
+                    die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
+                latestCommit = "%s^@" % next
 
         return ""
 
@@ -3888,7 +3899,6 @@ class P4Clone(P4Sync):
             self.cloneDestination = depotPaths[-1]
             depotPaths = depotPaths[:-1]
 
-        self.cloneExclude = ["/"+p for p in self.cloneExclude]
         for p in depotPaths:
             if not p.startswith("//"):
                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
diff --git a/t/t9801-git-p4-branch.sh b/t/t9801-git-p4-branch.sh
index 38d6b90..67ff271 100755
--- a/t/t9801-git-p4-branch.sh
+++ b/t/t9801-git-p4-branch.sh
@@ -411,6 +411,46 @@ test_expect_failure 'git p4 clone file subset branch' '
 	)
 '
 
+# Check that excluded files are omitted during import
+test_expect_success 'git p4 clone complex branches with excluded files' '
+	test_when_finished cleanup_git &&
+	test_create_repo "$git" &&
+	(
+		cd "$git" &&
+		git config git-p4.branchList branch1:branch2 &&
+		git config --add git-p4.branchList branch1:branch3 &&
+		git config --add git-p4.branchList branch1:branch4 &&
+		git config --add git-p4.branchList branch1:branch5 &&
+		git config --add git-p4.branchList branch1:branch6 &&
+		git p4 clone --dest=. --detect-branches -//depot/branch1/file2 -//depot/branch2/file2 -//depot/branch3/file2 -//depot/branch4/file2 -//depot/branch5/file2 -//depot/branch6/file2 //depot@all &&
+		git log --all --graph --decorate --stat &&
+		git reset --hard p4/depot/branch1 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_file file3 &&
+		git reset --hard p4/depot/branch2 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_missing file3 &&
+		git reset --hard p4/depot/branch3 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_missing file3 &&
+		git reset --hard p4/depot/branch4 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_file file3 &&
+		git reset --hard p4/depot/branch5 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_file file3 &&
+		git reset --hard p4/depot/branch6 &&
+		test_path_is_file file1 &&
+		test_path_is_missing file2 &&
+		test_path_is_missing file3
+	)
+'
+
 # From a report in http://stackoverflow.com/questions/11893688
 # where --use-client-spec caused branch prefixes not to be removed;
 # every file in git appeared into a subdirectory of the branch name.
@@ -610,4 +650,96 @@ test_expect_success 'Update a file in git side and submit to P4 using client vie
 	)
 '
 
+test_expect_success 'restart p4d (case folding enabled)' '
+	stop_and_cleanup_p4d &&
+	start_p4d -C1
+'
+
+#
+# 1: //depot/main/mf1
+# 2: integrate //depot/main/... -> //depot/branch1/...
+# 3: //depot/main/mf2
+# 4: //depot/BRANCH1/B1f3
+# 5: //depot/branch1/b1f4
+#
+test_expect_success !CASE_INSENSITIVE_FS 'basic p4 branches for case folding' '
+	(
+		cd "$cli" &&
+		mkdir -p main &&
+
+		echo mf1 >main/mf1 &&
+		p4 add main/mf1 &&
+		p4 submit -d "main/mf1" &&
+
+		p4 integrate //depot/main/... //depot/branch1/... &&
+		p4 submit -d "integrate main to branch1" &&
+
+		echo mf2 >main/mf2 &&
+		p4 add main/mf2 &&
+		p4 submit -d "main/mf2" &&
+
+		mkdir BRANCH1 &&
+		echo B1f3 >BRANCH1/B1f3 &&
+		p4 add BRANCH1/B1f3 &&
+		p4 submit -d "BRANCH1/B1f3" &&
+
+		echo b1f4 >branch1/b1f4 &&
+		p4 add branch1/b1f4 &&
+		p4 submit -d "branch1/b1f4"
+	)
+'
+
+# Check that files are properly split across branches when ignorecase is set
+test_expect_success !CASE_INSENSITIVE_FS 'git p4 clone, branchList branch definition, ignorecase' '
+	test_when_finished cleanup_git &&
+	test_create_repo "$git" &&
+	(
+		cd "$git" &&
+		git config git-p4.branchList main:branch1 &&
+		git config --type=bool core.ignoreCase true &&
+		git p4 clone --dest=. --detect-branches //depot@all &&
+
+		git log --all --graph --decorate --stat &&
+
+		git reset --hard p4/master &&
+		test_path_is_file mf1 &&
+		test_path_is_file mf2 &&
+		test_path_is_missing B1f3 &&
+		test_path_is_missing b1f4 &&
+
+		git reset --hard p4/depot/branch1 &&
+		test_path_is_file mf1 &&
+		test_path_is_missing mf2 &&
+		test_path_is_file B1f3 &&
+		test_path_is_file b1f4
+	)
+'
+
+# Check that files are properly split across branches when ignorecase is set, use-client-spec case
+test_expect_success !CASE_INSENSITIVE_FS 'git p4 clone with client-spec, branchList branch definition, ignorecase' '
+	client_view "//depot/... //client/..." &&
+	test_when_finished cleanup_git &&
+	test_create_repo "$git" &&
+	(
+		cd "$git" &&
+		git config git-p4.branchList main:branch1 &&
+		git config --type=bool core.ignoreCase true &&
+		git p4 clone --dest=. --use-client-spec --detect-branches //depot@all &&
+
+		git log --all --graph --decorate --stat &&
+
+		git reset --hard p4/master &&
+		test_path_is_file mf1 &&
+		test_path_is_file mf2 &&
+		test_path_is_missing B1f3 &&
+		test_path_is_missing b1f4 &&
+
+		git reset --hard p4/depot/branch1 &&
+		test_path_is_file mf1 &&
+		test_path_is_missing mf2 &&
+		test_path_is_file B1f3 &&
+		test_path_is_file b1f4
+	)
+'
+
 test_done
diff --git a/t/t9817-git-p4-exclude.sh b/t/t9817-git-p4-exclude.sh
index 96d25f0..ec3d937 100755
--- a/t/t9817-git-p4-exclude.sh
+++ b/t/t9817-git-p4-exclude.sh
@@ -22,7 +22,9 @@ test_expect_success 'create exclude repo' '
 		mkdir -p wanted discard &&
 		echo wanted >wanted/foo &&
 		echo discard >discard/foo &&
-		p4 add wanted/foo discard/foo &&
+		echo discard_file >discard_file &&
+		echo discard_file_not >discard_file_not &&
+		p4 add wanted/foo discard/foo discard_file discard_file_not &&
 		p4 submit -d "initial revision"
 	)
 '
@@ -33,7 +35,9 @@ test_expect_success 'check the repo was created correctly' '
 	(
 		cd "$git" &&
 		test_path_is_file wanted/foo &&
-		test_path_is_file discard/foo
+		test_path_is_file discard/foo &&
+		test_path_is_file discard_file &&
+		test_path_is_file discard_file_not
 	)
 '
 
@@ -43,7 +47,21 @@ test_expect_success 'clone, excluding part of repo' '
 	(
 		cd "$git" &&
 		test_path_is_file wanted/foo &&
-		test_path_is_missing discard/foo
+		test_path_is_missing discard/foo &&
+		test_path_is_file discard_file &&
+		test_path_is_file discard_file_not
+	)
+'
+
+test_expect_success 'clone, excluding single file, no trailing /' '
+	test_when_finished cleanup_git &&
+	git p4 clone -//depot/discard_file --dest="$git" //depot/...@all &&
+	(
+		cd "$git" &&
+		test_path_is_file wanted/foo &&
+		test_path_is_file discard/foo &&
+		test_path_is_missing discard_file &&
+		test_path_is_file discard_file_not
 	)
 '
 
@@ -52,15 +70,38 @@ test_expect_success 'clone, then sync with exclude' '
 	git p4 clone -//depot/discard/... --dest="$git" //depot/...@all &&
 	(
 		cd "$cli" &&
-		p4 edit wanted/foo discard/foo &&
+		p4 edit wanted/foo discard/foo discard_file_not &&
 		date >>wanted/foo &&
 		date >>discard/foo &&
+		date >>discard_file_not &&
 		p4 submit -d "updating" &&
 
 		cd "$git" &&
 		git p4 sync -//depot/discard/... &&
 		test_path_is_file wanted/foo &&
-		test_path_is_missing discard/foo
+		test_path_is_missing discard/foo &&
+		test_path_is_file discard_file &&
+		test_path_is_file discard_file_not
+	)
+'
+
+test_expect_success 'clone, then sync with exclude, no trailing /' '
+	test_when_finished cleanup_git &&
+	git p4 clone -//depot/discard/... -//depot/discard_file --dest="$git" //depot/...@all &&
+	(
+		cd "$cli" &&
+		p4 edit wanted/foo discard/foo discard_file_not &&
+		date >>wanted/foo &&
+		date >>discard/foo &&
+		date >>discard_file_not &&
+		p4 submit -d "updating" &&
+
+		cd "$git" &&
+		git p4 sync -//depot/discard/... -//depot/discard_file &&
+		test_path_is_file wanted/foo &&
+		test_path_is_missing discard/foo &&
+		test_path_is_missing discard_file &&
+		test_path_is_file discard_file_not
 	)
 '