diff options
Diffstat (limited to 'git-p4.py')
| -rwxr-xr-x | git-p4.py | 280 |
1 files changed, 195 insertions, 85 deletions
@@ -25,6 +25,7 @@ import stat import zipfile import zlib import ctypes +import errno try: from subprocess import CalledProcessError @@ -78,6 +79,13 @@ def p4_build_cmd(cmd): if len(client) > 0: real_cmd += ["-c", client] + retries = gitConfigInt("git-p4.retries") + if retries is None: + # Perform 3 retries by default + retries = 3 + if retries > 0: + # Provide a way to not pass this option by setting git-p4.retries to 0 + real_cmd += ["-r", str(retries)] if isinstance(cmd,basestring): real_cmd = ' '.join(real_cmd) + ' ' + cmd @@ -85,6 +93,16 @@ def p4_build_cmd(cmd): real_cmd += cmd return real_cmd +def git_dir(path): + """ Return TRUE if the given path is a git directory (/path/to/dir/.git). + This won't automatically add ".git" to a directory. + """ + d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip() + if not d or len(d) == 0: + return None + else: + return d + def chdir(path, is_client_path=False): """Do chdir to the given path, and set the PWD environment variable for use by P4. It does not look at getcwd() output. @@ -253,8 +271,8 @@ def p4_add(f): def p4_delete(f): p4_system(["delete", wildcard_encode(f)]) -def p4_edit(f): - p4_system(["edit", wildcard_encode(f)]) +def p4_edit(f, *options): + p4_system(["edit"] + list(options) + [wildcard_encode(f)]) def p4_revert(f): p4_system(["revert", wildcard_encode(f)]) @@ -262,6 +280,10 @@ def p4_revert(f): def p4_reopen(type, f): p4_system(["reopen", "-t", type, wildcard_encode(f)]) +def p4_reopen_in_change(changelist, files): + cmd = ["reopen", "-c", str(changelist)] + files + p4_system(cmd) + def p4_move(src, dest): p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)]) @@ -563,10 +585,7 @@ def currentGitBranch(): return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip() def isValidGitDir(path): - if (os.path.exists(path + "/HEAD") - and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")): - return True; - return False + return git_dir(path) != None def parseRevision(ref): return read_pipe("git rev-parse %s" % ref).strip() @@ -655,7 +674,7 @@ def gitConfigInt(key): def gitConfigList(key): if not _gitConfig.has_key(key): s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) - _gitConfig[key] = s.strip().split(os.linesep) + _gitConfig[key] = s.strip().splitlines() if _gitConfig[key] == ['']: _gitConfig[key] = [] return _gitConfig[key] @@ -822,39 +841,37 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): die("cannot use --changes-block-size with non-numeric revisions") block_size = None - # Accumulate change numbers in a dictionary to avoid duplicates - changes = {} + changes = set() - for p in depotPaths: - # Retrieve changes a block at a time, to prevent running - # into a MaxResults/MaxScanRows error from the server. + # Retrieve changes a block at a time, to prevent running + # into a MaxResults/MaxScanRows error from the server. - while True: - cmd = ['changes'] + while True: + cmd = ['changes'] - if block_size: - end = min(changeEnd, changeStart + block_size) - revisionRange = "%d,%d" % (changeStart, end) - else: - revisionRange = "%s,%s" % (changeStart, changeEnd) + if block_size: + end = min(changeEnd, changeStart + block_size) + revisionRange = "%d,%d" % (changeStart, end) + else: + revisionRange = "%s,%s" % (changeStart, changeEnd) + for p in depotPaths: cmd += ["%s...@%s" % (p, revisionRange)] - for line in p4_read_pipe_lines(cmd): - changeNum = int(line.split(" ")[1]) - changes[changeNum] = True + # Insert changes in chronological order + for line in reversed(p4_read_pipe_lines(cmd)): + changes.add(int(line.split(" ")[1])) - if not block_size: - break + if not block_size: + break - if end >= changeEnd: - break + if end >= changeEnd: + break - changeStart = end + 1 + changeStart = end + 1 - changelist = changes.keys() - changelist.sort() - return changelist + changes = sorted(changes) + return changes def p4PathStartsWith(path, prefix): # This method tries to remedy a potential mixed-case issue: @@ -1007,18 +1024,20 @@ class LargeFileSystem(object): steps.""" if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath): contentTempFile = self.generateTempFile(contents) - (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile) - - # Move temp file to final location in large file system - largeFileDir = os.path.dirname(localLargeFile) - if not os.path.isdir(largeFileDir): - os.makedirs(largeFileDir) - shutil.move(contentTempFile, localLargeFile) - self.addLargeFile(relPath) - if gitConfigBool('git-p4.largeFilePush'): - self.pushFile(localLargeFile) - if verbose: - sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile)) + (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile) + if pointer_git_mode: + git_mode = pointer_git_mode + if localLargeFile: + # Move temp file to final location in large file system + largeFileDir = os.path.dirname(localLargeFile) + if not os.path.isdir(largeFileDir): + os.makedirs(largeFileDir) + shutil.move(contentTempFile, localLargeFile) + self.addLargeFile(relPath) + if gitConfigBool('git-p4.largeFilePush'): + self.pushFile(localLargeFile) + if verbose: + sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile)) return (git_mode, contents) class MockLFS(LargeFileSystem): @@ -1058,6 +1077,9 @@ class GitLFS(LargeFileSystem): the actual content. Return also the new location of the actual content. """ + if os.path.getsize(contentFile) == 0: + return (None, '', None) + pointerProcess = subprocess.Popen( ['git', 'lfs', 'pointer', '--file=' + contentFile], stdout=subprocess.PIPE @@ -1066,8 +1088,15 @@ class GitLFS(LargeFileSystem): if pointerProcess.wait(): os.remove(contentFile) die('git-lfs pointer command failed. Did you install the extension?') - pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]] - oid = pointerContents[1].split(' ')[1].split(':')[1][:-1] + + # Git LFS removed the preamble in the output of the 'pointer' command + # starting from version 1.2.0. Check for the preamble here to support + # earlier versions. + # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43 + if pointerFile.startswith('Git LFS pointer for'): + pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile) + + oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1) localLargeFile = os.path.join( os.getcwd(), '.git', 'lfs', 'objects', oid[:2], oid[2:4], @@ -1075,7 +1104,7 @@ class GitLFS(LargeFileSystem): ) # LFS Spec states that pointer files should not have the executable bit set. gitMode = '100644' - return (gitMode, pointerContents, localLargeFile) + return (gitMode, pointerFile, localLargeFile) def pushFile(self, localLargeFile): uploadProcess = subprocess.Popen( @@ -1093,10 +1122,10 @@ class GitLFS(LargeFileSystem): '# Git LFS (see https://git-lfs.github.com/)\n', '#\n', ] + - ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n' + ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n' for f in sorted(gitConfigList('git-p4.largeFileExtensions')) ] + - ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n' + ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n' for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f) ] ) @@ -1162,6 +1191,15 @@ class P4UserMap: self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" self.emails[output["Email"]] = output["User"] + mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE) + for mapUserConfig in gitConfigList("git-p4.mapUser"): + mapUser = mapUserConfigRegex.findall(mapUserConfig) + if mapUser and len(mapUser[0]) == 3: + user = mapUser[0][0] + fullname = mapUser[0][1] + email = mapUser[0][2] + self.users[user] = fullname + " <" + email + ">" + self.emails[email] = user s = '' for (key, val) in self.users.items(): @@ -1275,6 +1313,12 @@ class P4Submit(Command, P4UserMap): optparse.make_option("--conflict", dest="conflict_behavior", choices=self.conflict_behavior_choices), optparse.make_option("--branch", dest="branch"), + optparse.make_option("--shelve", dest="shelve", action="store_true", + help="Shelve instead of submit. Shelved files are reverted, " + "restoring the workspace to the state before the shelve"), + optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int", + metavar="CHANGELIST", + help="update an existing shelved changelist, implies --shelve") ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" @@ -1282,6 +1326,8 @@ class P4Submit(Command, P4UserMap): self.detectRenames = False self.preserveUser = gitConfigBool("git-p4.preserveUser") self.dry_run = False + self.shelve = False + self.update_shelve = None self.prepare_p4_only = False self.conflict_behavior = None self.isWindows = (platform.system() == "Windows") @@ -1450,7 +1496,7 @@ class P4Submit(Command, P4UserMap): return 1 return 0 - def prepareSubmitTemplate(self): + def prepareSubmitTemplate(self, changelist=None): """Run "p4 change -o" to grab a change specification template. This does not use "p4 -G", as it is nice to keep the submission template in original order, since a human might edit it. @@ -1458,9 +1504,15 @@ class P4Submit(Command, P4UserMap): Remove lines in the Files section that show changes to files outside the depot path we're committing into.""" + [upstream, settings] = findUpstreamBranchPoint() + template = "" inFilesSection = False - for line in p4_read_pipe_lines(['change', '-o']): + args = ['change', '-o'] + if changelist: + args.append(str(changelist)) + + for line in p4_read_pipe_lines(args): if line.endswith("\r\n"): line = line[:-2] + "\n" if inFilesSection: @@ -1470,8 +1522,13 @@ class P4Submit(Command, P4UserMap): lastTab = path.rfind("\t") if lastTab != -1: path = path[:lastTab] - if not p4PathStartsWith(path, self.depotPath): - continue + if settings.has_key('depot-paths'): + if not [p for p in settings['depot-paths'] + if p4PathStartsWith(path, p)]: + continue + else: + if not p4PathStartsWith(path, self.depotPath): + continue else: inFilesSection = False else: @@ -1517,7 +1574,7 @@ class P4Submit(Command, P4UserMap): if response == 'n': return False - def get_diff_description(self, editedFiles, filesToAdd): + def get_diff_description(self, editedFiles, filesToAdd, symlinks): # diff if os.environ.has_key("P4DIFF"): del(os.environ["P4DIFF"]) @@ -1532,10 +1589,17 @@ class P4Submit(Command, P4UserMap): newdiff += "==== new file ====\n" newdiff += "--- /dev/null\n" newdiff += "+++ %s\n" % newFile - f = open(newFile, "r") - for line in f.readlines(): - newdiff += "+" + line - f.close() + + is_link = os.path.islink(newFile) + expect_link = newFile in symlinks + + if is_link and expect_link: + newdiff += "+%s\n" % os.readlink(newFile) + else: + f = open(newFile, "r") + for line in f.readlines(): + newdiff += "+" + line + f.close() return (diff + newdiff).replace('\r\n', '\n') @@ -1549,15 +1613,20 @@ class P4Submit(Command, P4UserMap): diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id)) filesToAdd = set() + filesToChangeType = set() filesToDelete = set() editedFiles = set() pureRenameCopy = set() + symlinks = set() filesToChangeExecBit = {} + all_files = list() for line in diff: diff = parseDiffTreeEntry(line) modifier = diff['status'] path = diff['src'] + all_files.append(path) + if modifier == "M": p4_edit(path) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): @@ -1568,6 +1637,11 @@ class P4Submit(Command, P4UserMap): filesToChangeExecBit[path] = diff['dst_mode'] if path in filesToDelete: filesToDelete.remove(path) + + dst_mode = int(diff['dst_mode'], 8) + if dst_mode == 0120000: + symlinks.add(path) + elif modifier == "D": filesToDelete.add(path) if path in filesToAdd: @@ -1609,6 +1683,8 @@ class P4Submit(Command, P4UserMap): os.unlink(dest) filesToDelete.add(src) editedFiles.add(dest) + elif modifier == "T": + filesToChangeType.add(path) else: die("unknown modifier %s for %s" % (modifier, path)) @@ -1668,6 +1744,8 @@ class P4Submit(Command, P4UserMap): # system(applyPatchCmd) + for f in filesToChangeType: + p4_edit(f, "-t", "auto") for f in filesToAdd: p4_add(f) for f in filesToDelete: @@ -1679,6 +1757,10 @@ class P4Submit(Command, P4UserMap): mode = filesToChangeExecBit[f] setP4ExecBit(f, mode) + if self.update_shelve: + print("all_files = %s" % str(all_files)) + p4_reopen_in_change(self.update_shelve, all_files) + # # Build p4 change description, starting with the contents # of the git commit message. @@ -1687,7 +1769,7 @@ class P4Submit(Command, P4UserMap): logMessage = logMessage.strip() (logMessage, jobs) = self.separate_jobs_from_description(logMessage) - template = self.prepareSubmitTemplate() + template = self.prepareSubmitTemplate(self.update_shelve) submitTemplate = self.prepareLogMessage(template, logMessage, jobs) if self.preserveUser: @@ -1701,7 +1783,7 @@ class P4Submit(Command, P4UserMap): separatorLine = "######## everything below this line is just the diff #######\n" if not self.prepare_p4_only: submitTemplate += separatorLine - submitTemplate += self.get_diff_description(editedFiles, filesToAdd) + submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks) (handle, fileName) = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+b") @@ -1759,7 +1841,17 @@ class P4Submit(Command, P4UserMap): if self.isWindows: message = message.replace("\r\n", "\n") submitTemplate = message[:message.index(separatorLine)] - p4_write_pipe(['submit', '-i'], submitTemplate) + + if self.update_shelve: + p4_write_pipe(['shelve', '-r', '-i'], submitTemplate) + elif self.shelve: + p4_write_pipe(['shelve', '-i'], submitTemplate) + else: + p4_write_pipe(['submit', '-i'], submitTemplate) + # The rename/copy happened by applying a patch that created a + # new file. This leaves it writable, which confuses p4. + for f in pureRenameCopy: + p4_sync(f, "-f") if self.preserveUser: if p4User: @@ -1769,23 +1861,20 @@ class P4Submit(Command, P4UserMap): changelist = self.lastP4Changelist() self.modifyChangelistUser(changelist, p4User) - # The rename/copy happened by applying a patch that created a - # new file. This leaves it writable, which confuses p4. - for f in pureRenameCopy: - p4_sync(f, "-f") submitted = True finally: # skip this patch - if not submitted: - print "Submission cancelled, undoing p4 changes." - for f in editedFiles: + if not submitted or self.shelve: + if self.shelve: + print ("Reverting shelved files.") + else: + print ("Submission cancelled, undoing p4 changes.") + for f in editedFiles | filesToDelete: p4_revert(f) for f in filesToAdd: p4_revert(f) os.remove(f) - for f in filesToDelete: - p4_revert(f) os.remove(fileName) return submitted @@ -1881,6 +1970,9 @@ class P4Submit(Command, P4UserMap): if len(self.origin) == 0: self.origin = upstream + if self.update_shelve: + self.shelve = True + if self.preserveUser: if not self.canChangeChangelists(): die("Cannot preserve user names without p4 super-user or admin permissions") @@ -1908,7 +2000,7 @@ class P4Submit(Command, P4UserMap): if self.useClientSpec: self.clientSpecDirs = getClientSpec() - # Check for the existance of P4 branches + # Check for the existence of P4 branches branchesDetected = (len(p4BranchesInGit().keys()) > 1) if self.useClientSpec and not branchesDetected: @@ -2041,13 +2133,13 @@ class P4Submit(Command, P4UserMap): break chdir(self.oldWorkingDirectory) - + shelved_applied = "shelved" if self.shelve else "applied" if self.dry_run: pass elif self.prepare_p4_only: pass elif len(commits) == len(applied): - print "All commits applied!" + print ("All commits {0}!".format(shelved_applied)) sync = P4Sync() if self.branch: @@ -2059,9 +2151,9 @@ class P4Submit(Command, P4UserMap): else: if len(applied) == 0: - print "No commits applied." + print ("No commits {0}.".format(shelved_applied)) else: - print "Applied only the commits marked with '*':" + print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize())) for c in commits: if c in applied: star = "*" @@ -2248,7 +2340,7 @@ class P4Sync(Command, P4UserMap): self.useClientSpec_from_options = False self.clientSpecDirs = None self.tempBranches = [] - self.tempBranchLocation = "git-p4-tmp" + self.tempBranchLocation = "refs/git-p4-tmp" self.largeFileSystem = None if gitConfig('git-p4.largeFileSystem'): @@ -2301,6 +2393,15 @@ class P4Sync(Command, P4UserMap): fnum = fnum + 1 return files + def extractJobsFromCommit(self, commit): + jobs = [] + jnum = 0 + while commit.has_key("job%s" % jnum): + job = commit["job%s" % jnum] + jobs.append(job) + jnum = jnum + 1 + return jobs + def stripRepoPath(self, path, prefixes): """When streaming files, this is called to map a p4 depot path to where it should go in git. The prefixes are either @@ -2383,11 +2484,24 @@ class P4Sync(Command, P4UserMap): self.gitStream.write(d) self.gitStream.write('\n') + def encodeWithUTF8(self, path): + try: + path.decode('ascii') + except: + encoding = 'utf8' + if gitConfig('git-p4.pathEncoding'): + encoding = gitConfig('git-p4.pathEncoding') + path = path.decode(encoding, 'replace').encode('utf8', 'replace') + if self.verbose: + print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path) + return path + # output one file from the P4 stream # - helper for streamP4Files def streamOneP4File(self, file, contents): relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) + relPath = self.encodeWithUTF8(relPath) if verbose: size = int(self.stream_file['fileSize']) sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024)) @@ -2460,16 +2574,6 @@ class P4Sync(Command, P4UserMap): text = regexp.sub(r'$\1$', text) contents = [ text ] - try: - relPath.decode('ascii') - except: - encoding = 'utf8' - if gitConfig('git-p4.pathEncoding'): - encoding = gitConfig('git-p4.pathEncoding') - relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace') - if self.verbose: - print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath) - if self.largeFileSystem: (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents) @@ -2477,6 +2581,7 @@ class P4Sync(Command, P4UserMap): def streamOneP4Deletion(self, file): relPath = self.stripRepoPath(file['path'], self.branchPrefixes) + relPath = self.encodeWithUTF8(relPath) if verbose: sys.stdout.write("delete %s\n" % relPath) sys.stdout.flush() @@ -2639,13 +2744,14 @@ class P4Sync(Command, P4UserMap): return True hasPrefix = [p for p in self.branchPrefixes if p4PathStartsWith(path, p)] - if hasPrefix and self.verbose: + if not hasPrefix and self.verbose: print('Ignoring file outside of prefix: {0}'.format(path)) return hasPrefix def commit(self, details, files, branch, parent = ""): epoch = details["time"] author = details["user"] + jobs = self.extractJobsFromCommit(details) if self.verbose: print('commit into {0}'.format(branch)) @@ -2673,6 +2779,8 @@ class P4Sync(Command, P4UserMap): self.gitStream.write("data <<EOT\n") self.gitStream.write(details["desc"]) + if len(jobs) > 0: + self.gitStream.write("\nJobs: %s" % (' '.join(jobs))) self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" % (','.join(self.branchPrefixes), details["change"])) if len(details['options']) > 0: @@ -3644,6 +3752,7 @@ def main(): if cmd.gitdir == None: cmd.gitdir = os.path.abspath(".git") if not isValidGitDir(cmd.gitdir): + # "rev-parse --git-dir" without arguments will try $PWD/.git cmd.gitdir = read_pipe("git rev-parse --git-dir").strip() if os.path.exists(cmd.gitdir): cdup = read_pipe("git rev-parse --show-cdup").strip() @@ -3656,6 +3765,7 @@ def main(): else: die("fatal: cannot locate git repository at %s" % cmd.gitdir) + # so git commands invoked from the P4 workspace will succeed os.environ["GIT_DIR"] = cmd.gitdir if not cmd.run(args): |