diff options
Diffstat (limited to 'git-p4.py')
| -rwxr-xr-x | git-p4.py | 1122 |
1 files changed, 726 insertions, 396 deletions
@@ -7,34 +7,52 @@ # 2007 Trolltech ASA # License: MIT <http://www.opensource.org/licenses/mit-license.php> # -# pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except -# pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary -# pylint: disable=wrong-import-order,unused-import,too-few-public-methods -# pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals -# pylint: disable=line-too-long,bad-whitespace,superfluous-parens -# pylint: disable=too-many-statements,too-many-instance-attributes -# pylint: disable=too-many-branches,too-many-nested-blocks +# pylint: disable=bad-whitespace +# pylint: disable=broad-except +# pylint: disable=consider-iterating-dictionary +# pylint: disable=disable +# pylint: disable=fixme +# pylint: disable=invalid-name +# pylint: disable=line-too-long +# pylint: disable=missing-docstring +# pylint: disable=no-self-use +# pylint: disable=superfluous-parens +# pylint: disable=too-few-public-methods +# pylint: disable=too-many-arguments +# pylint: disable=too-many-branches +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-lines +# pylint: disable=too-many-locals +# pylint: disable=too-many-nested-blocks +# pylint: disable=too-many-statements +# pylint: disable=ungrouped-imports +# pylint: disable=unused-import +# pylint: disable=wrong-import-order +# pylint: disable=wrong-import-position # + +import struct import sys if sys.version_info.major < 3 and sys.version_info.minor < 7: sys.stderr.write("git-p4: requires Python 2.7 or later.\n") sys.exit(1) -import os -import optparse + +import ctypes +import errno import functools +import glob import marshal -import subprocess -import tempfile -import time +import optparse +import os import platform import re import shutil import stat +import subprocess +import tempfile +import time import zipfile import zlib -import ctypes -import errno -import glob # On python2.7 where raw_input() and input() are both availble, # we want raw_input's semantics, but aliased to input for python3 @@ -52,17 +70,21 @@ verbose = False defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' # The block size is reduced automatically if required -defaultBlockSize = 1<<20 +defaultBlockSize = 1 << 20 + +defaultMetadataDecodingStrategy = 'passthrough' if sys.version_info.major == 2 else 'fallback' +defaultFallbackMetadataEncoding = 'cp1252' p4_access_checked = False re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$') re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$') + def format_size_human_readable(num): - """ Returns a number of units (typically bytes) formatted as a human-readable - string. - """ + """Returns a number of units (typically bytes) formatted as a + human-readable string. + """ if num < 1024: return '{:d} B'.format(num) for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: @@ -71,18 +93,19 @@ def format_size_human_readable(num): return "{:3.1f} {}B".format(num, unit) return "{:.1f} YiB".format(num) + def p4_build_cmd(cmd): """Build a suitable p4 command line. - This consolidates building and returning a p4 command line into one - location. It means that hooking into the environment, or other configuration - can be done more easily. - """ + This consolidates building and returning a p4 command line into one + location. It means that hooking into the environment, or other + configuration can be done more easily. + """ real_cmd = ["p4"] user = gitConfig("git-p4.user") if len(user) > 0: - real_cmd += ["-u",user] + real_cmd += ["-u", user] password = gitConfig("git-p4.password") if len(password) > 0: @@ -118,31 +141,31 @@ def p4_build_cmd(cmd): return real_cmd + def git_dir(path): - """ Return TRUE if the given path is a git directory (/path/to/dir/.git). - This won't automatically add ".git" to a directory. - """ + """Return TRUE if the given path is a git directory (/path/to/dir/.git). + This won't automatically add ".git" to a directory. + """ d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip() if not d or len(d) == 0: return None else: return d + def chdir(path, is_client_path=False): - """Do chdir to the given path, and set the PWD environment - variable for use by P4. It does not look at getcwd() output. - Since we're not using the shell, it is necessary to set the - PWD environment variable explicitly. - - Normally, expand the path to force it to be absolute. This - addresses the use of relative path names inside P4 settings, - e.g. P4CONFIG=.p4config. P4 does not simply open the filename - as given; it looks for .p4config using PWD. - - If is_client_path, the path was handed to us directly by p4, - and may be a symbolic link. Do not call os.getcwd() in this - case, because it will cause p4 to think that PWD is not inside - the client path. + """Do chdir to the given path, and set the PWD environment variable for use + by P4. It does not look at getcwd() output. Since we're not using the + shell, it is necessary to set the PWD environment variable explicitly. + + Normally, expand the path to force it to be absolute. This addresses + the use of relative path names inside P4 settings, e.g. + P4CONFIG=.p4config. P4 does not simply open the filename as given; it + looks for .p4config using PWD. + + If is_client_path, the path was handed to us directly by p4, and may be + a symbolic link. Do not call os.getcwd() in this case, because it will + cause p4 to think that PWD is not inside the client path. """ os.chdir(path) @@ -150,6 +173,7 @@ def chdir(path, is_client_path=False): path = os.getcwd() os.environ['PWD'] = path + def calcDiskFree(): """Return free space in bytes on the disk of the given dirname.""" if platform.system() == 'Windows': @@ -160,52 +184,124 @@ def calcDiskFree(): st = os.statvfs(os.getcwd()) return st.f_bavail * st.f_frsize + def die(msg): - """ Terminate execution. Make sure that any running child processes have been wait()ed for before - calling this. - """ + """Terminate execution. Make sure that any running child processes have + been wait()ed for before calling this. + """ if verbose: raise Exception(msg) else: sys.stderr.write(msg + "\n") sys.exit(1) + def prompt(prompt_text): - """ Prompt the user to choose one of the choices + """Prompt the user to choose one of the choices. - Choices are identified in the prompt_text by square brackets around - a single letter option. - """ + Choices are identified in the prompt_text by square brackets around a + single letter option. + """ choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text)) while True: sys.stderr.flush() sys.stdout.write(prompt_text) sys.stdout.flush() - response=sys.stdin.readline().strip().lower() + response = sys.stdin.readline().strip().lower() if not response: continue response = response[0] if response in choices: return response + # We need different encoding/decoding strategies for text data being passed # around in pipes depending on python version if bytes is not str: # For python3, always encode and decode as appropriate def decode_text_stream(s): return s.decode() if isinstance(s, bytes) else s + def encode_text_stream(s): return s.encode() if isinstance(s, str) else s else: # For python2.7, pass read strings as-is, but also allow writing unicode def decode_text_stream(s): return s + def encode_text_stream(s): return s.encode('utf_8') if isinstance(s, unicode) else s + +class MetadataDecodingException(Exception): + def __init__(self, input_string): + self.input_string = input_string + + def __str__(self): + return """Decoding perforce metadata failed! +The failing string was: +--- +{} +--- +Consider setting the git-p4.metadataDecodingStrategy config option to +'fallback', to allow metadata to be decoded using a fallback encoding, +defaulting to cp1252.""".format(self.input_string) + + +encoding_fallback_warning_issued = False +encoding_escape_warning_issued = False +def metadata_stream_to_writable_bytes(s): + encodingStrategy = gitConfig('git-p4.metadataDecodingStrategy') or defaultMetadataDecodingStrategy + fallbackEncoding = gitConfig('git-p4.metadataFallbackEncoding') or defaultFallbackMetadataEncoding + if not isinstance(s, bytes): + return s.encode('utf_8') + if encodingStrategy == 'passthrough': + return s + try: + s.decode('utf_8') + return s + except UnicodeDecodeError: + if encodingStrategy == 'fallback' and fallbackEncoding: + global encoding_fallback_warning_issued + global encoding_escape_warning_issued + try: + if not encoding_fallback_warning_issued: + print("\nCould not decode value as utf-8; using configured fallback encoding %s: %s" % (fallbackEncoding, s)) + print("\n(this warning is only displayed once during an import)") + encoding_fallback_warning_issued = True + return s.decode(fallbackEncoding).encode('utf_8') + except Exception as exc: + if not encoding_escape_warning_issued: + print("\nCould not decode value with configured fallback encoding %s; escaping bytes over 127: %s" % (fallbackEncoding, s)) + print("\n(this warning is only displayed once during an import)") + encoding_escape_warning_issued = True + escaped_bytes = b'' + # bytes and strings work very differently in python2 vs python3... + if str is bytes: + for byte in s: + byte_number = struct.unpack('>B', byte)[0] + if byte_number > 127: + escaped_bytes += b'%' + escaped_bytes += hex(byte_number)[2:].upper() + else: + escaped_bytes += byte + else: + for byte_number in s: + if byte_number > 127: + escaped_bytes += b'%' + escaped_bytes += hex(byte_number).upper().encode()[2:] + else: + escaped_bytes += bytes([byte_number]) + return escaped_bytes + + raise MetadataDecodingException(s) + + def decode_path(path): - """Decode a given string (bytes or otherwise) using configured path encoding options - """ + """Decode a given string (bytes or otherwise) using configured path + encoding options. + """ + encoding = gitConfig('git-p4.pathEncoding') or 'utf_8' if bytes is not str: return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path @@ -218,6 +314,7 @@ def decode_path(path): print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path)) return path + def run_git_hook(cmd, param=[]): """Execute a hook if the hook exists.""" args = ['git', 'hook', 'run', '--ignore-missing', cmd] @@ -227,6 +324,7 @@ def run_git_hook(cmd, param=[]): args.append(p) return subprocess.call(args) == 0 + def write_pipe(c, stdin, *k, **kw): if verbose: sys.stderr.write('Writing pipe: {}\n'.format(' '.join(c))) @@ -240,33 +338,35 @@ def write_pipe(c, stdin, *k, **kw): return val + def p4_write_pipe(c, stdin, *k, **kw): real_cmd = p4_build_cmd(c) if bytes is not str and isinstance(stdin, str): stdin = encode_text_stream(stdin) return write_pipe(real_cmd, stdin, *k, **kw) + def read_pipe_full(c, *k, **kw): - """ Read output from command. Returns a tuple - of the return status, stdout text and stderr - text. - """ + """Read output from command. Returns a tuple of the return status, stdout + text and stderr text. + """ if verbose: sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c))) p = subprocess.Popen( c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, *k, **kw) - (out, err) = p.communicate() + out, err = p.communicate() return (p.returncode, out, decode_text_stream(err)) + def read_pipe(c, ignore_error=False, raw=False, *k, **kw): - """ Read output from command. Returns the output text on - success. On failure, terminates execution, unless - ignore_error is True, when it returns an empty string. + """Read output from command. Returns the output text on success. On + failure, terminates execution, unless ignore_error is True, when it + returns an empty string. - If raw is True, do not attempt to decode output text. - """ - (retcode, out, err) = read_pipe_full(c, *k, **kw) + If raw is True, do not attempt to decode output text. + """ + retcode, out, err = read_pipe_full(c, *k, **kw) if retcode != 0: if ignore_error: out = "" @@ -276,20 +376,23 @@ def read_pipe(c, ignore_error=False, raw=False, *k, **kw): out = decode_text_stream(out) return out + def read_pipe_text(c, *k, **kw): - """ Read output from a command with trailing whitespace stripped. - On error, returns None. - """ - (retcode, out, err) = read_pipe_full(c, *k, **kw) + """Read output from a command with trailing whitespace stripped. On error, + returns None. + """ + retcode, out, err = read_pipe_full(c, *k, **kw) if retcode != 0: return None else: return decode_text_stream(out).rstrip() + def p4_read_pipe(c, ignore_error=False, raw=False, *k, **kw): real_cmd = p4_build_cmd(c) return read_pipe(real_cmd, ignore_error, raw=raw, *k, **kw) + def read_pipe_lines(c, raw=False, *k, **kw): if verbose: sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c))) @@ -303,31 +406,36 @@ def read_pipe_lines(c, raw=False, *k, **kw): die('Command failed: {}'.format(' '.join(c))) return lines + def p4_read_pipe_lines(c, *k, **kw): - """Specifically invoke p4 on the command supplied. """ + """Specifically invoke p4 on the command supplied.""" real_cmd = p4_build_cmd(c) return read_pipe_lines(real_cmd, *k, **kw) + def p4_has_command(cmd): - """Ask p4 for help on this command. If it returns an error, the - command does not exist in this version of p4.""" + """Ask p4 for help on this command. If it returns an error, the command + does not exist in this version of p4. + """ real_cmd = p4_build_cmd(["help", cmd]) p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() return p.returncode == 0 + def p4_has_move_command(): - """See if the move command exists, that it supports -k, and that - it has not been administratively disabled. The arguments - must be correct, but the filenames do not have to exist. Use - ones with wildcards so even if they exist, it will fail.""" + """See if the move command exists, that it supports -k, and that it has not + been administratively disabled. The arguments must be correct, but the + filenames do not have to exist. Use ones with wildcards so even if they + exist, it will fail. + """ if not p4_has_command("move"): return False cmd = p4_build_cmd(["move", "-k", "@from", "@to"]) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = p.communicate() + out, err = p.communicate() err = decode_text_stream(err) # return code will be 1 in either case if err.find("Invalid option") >= 0: @@ -337,6 +445,7 @@ def p4_has_move_command(): # assume it failed because @... was invalid changelist return True + def system(cmd, ignore_error=False, *k, **kw): if verbose: sys.stderr.write("executing {}\n".format( @@ -347,19 +456,22 @@ def system(cmd, ignore_error=False, *k, **kw): return retcode + def p4_system(cmd, *k, **kw): - """Specifically invoke p4 as the system command. """ + """Specifically invoke p4 as the system command.""" real_cmd = p4_build_cmd(cmd) retcode = subprocess.call(real_cmd, *k, **kw) if retcode: raise subprocess.CalledProcessError(retcode, real_cmd) + def die_bad_access(s): die("failure accessing depot: {0}".format(s.rstrip())) + def p4_check_access(min_expiration=1): - """ Check if we can access Perforce - account still logged in - """ + """Check if we can access Perforce - account still logged in.""" + results = p4CmdList(["login", "-s"]) if len(results) == 0: @@ -402,62 +514,78 @@ def p4_check_access(min_expiration=1): else: die_bad_access("unknown error code {0}".format(code)) + _p4_version_string = None + + def p4_version_string(): - """Read the version string, showing just the last line, which - hopefully is the interesting version bit. + """Read the version string, showing just the last line, which hopefully is + the interesting version bit. $ p4 -V Perforce - The Fast Software Configuration Management System. Copyright 1995-2011 Perforce Software. All rights reserved. Rev. P4/NTX86/2011.1/393975 (2011/12/16). - """ + """ global _p4_version_string if not _p4_version_string: a = p4_read_pipe_lines(["-V"]) _p4_version_string = a[-1].rstrip() return _p4_version_string + def p4_integrate(src, dest): p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)]) + def p4_sync(f, *options): p4_system(["sync"] + list(options) + [wildcard_encode(f)]) + def p4_add(f): - # forcibly add file names with wildcards + """Forcibly add file names with wildcards.""" if wildcard_present(f): p4_system(["add", "-f", f]) else: p4_system(["add", f]) + def p4_delete(f): p4_system(["delete", wildcard_encode(f)]) + def p4_edit(f, *options): p4_system(["edit"] + list(options) + [wildcard_encode(f)]) + def p4_revert(f): p4_system(["revert", wildcard_encode(f)]) + def p4_reopen(type, f): p4_system(["reopen", "-t", type, wildcard_encode(f)]) + def p4_reopen_in_change(changelist, files): cmd = ["reopen", "-c", str(changelist)] + files p4_system(cmd) + def p4_move(src, dest): p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)]) + def p4_last_change(): results = p4CmdList(["changes", "-m", "1"], skip_info=True) return int(results[0]['change']) + def p4_describe(change, shelved=False): - """Make sure it returns a valid result by checking for - the presence of field "time". Return a dict of the - results.""" + """Make sure it returns a valid result by checking for the presence of + field "time". + + Return a dict of the results. + """ cmd = ["describe", "-s"] if shelved: @@ -482,12 +610,11 @@ def p4_describe(change, shelved=False): return d -# -# Canonicalize the p4 type and return a tuple of the -# base type, plus any modifiers. See "p4 help filetypes" -# for a list and explanation. -# + def split_p4_type(p4type): + """Canonicalize the p4 type and return a tuple of the base type, plus any + modifiers. See "p4 help filetypes" for a list and explanation. + """ p4_filetypes_historical = { "ctempobj": "binary+Sw", @@ -517,18 +644,19 @@ def split_p4_type(p4type): mods = s[1] return (base, mods) -# -# return the raw p4 type of a file (text, text+ko, etc) -# + def p4_type(f): + """Return the raw p4 type of a file (text, text+ko, etc).""" + results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)]) return results[0]['headType'] -# -# Given a type base and modifier, return a regexp matching -# the keywords that can be expanded in the file -# + def p4_keywords_regexp_for_type(base, type_mods): + """Given a type base and modifier, return a regexp matching the keywords + that can be expanded in the file. + """ + if base in ("text", "unicode", "binary"): if "ko" in type_mods: return re_ko_keywords @@ -539,21 +667,23 @@ def p4_keywords_regexp_for_type(base, type_mods): else: return None -# -# Given a file, return a regexp matching the possible -# RCS keywords that will be expanded, or None for files -# with kw expansion turned off. -# + def p4_keywords_regexp_for_file(file): + """Given a file, return a regexp matching the possible RCS keywords that + will be expanded, or None for files with kw expansion turned off. + """ + if not os.path.exists(file): return None else: - (type_base, type_mods) = split_p4_type(p4_type(file)) + type_base, type_mods = split_p4_type(p4_type(file)) return p4_keywords_regexp_for_type(type_base, type_mods) + def setP4ExecBit(file, mode): - # Reopens an already open file and changes the execute bit to match - # the execute bit setting in the passed in mode. + """Reopens an already open file and changes the execute bit to match the + execute bit setting in the passed in mode. + """ p4Type = "+x" @@ -566,8 +696,9 @@ def setP4ExecBit(file, mode): p4_reopen(p4Type, file) + def getP4OpenedType(file): - # Returns the perforce file type for the given file. + """Returns the perforce file type for the given file.""" result = p4_read_pipe(["opened", wildcard_encode(file)]) match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) @@ -576,8 +707,10 @@ def getP4OpenedType(file): else: die("Could not determine file type for %s (result: '%s')" % (file, result)) -# Return the set of all p4 labels + def getP4Labels(depotPaths): + """Return the set of all p4 labels.""" + labels = set() if not isinstance(depotPaths, list): depotPaths = [depotPaths] @@ -588,34 +721,39 @@ def getP4Labels(depotPaths): return labels -# Return the set of all git tags + def getGitTags(): + """Return the set of all git tags.""" + gitTags = set() for line in read_pipe_lines(["git", "tag"]): tag = line.strip() gitTags.add(tag) return gitTags + _diff_tree_pattern = None + def parseDiffTreeEntry(entry): """Parses a single diff tree entry into its component elements. - See git-diff-tree(1) manpage for details about the format of the diff - output. This method returns a dictionary with the following elements: - - src_mode - The mode of the source file - dst_mode - The mode of the destination file - src_sha1 - The sha1 for the source file - dst_sha1 - The sha1 fr the destination file - status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) - status_score - The score for the status (applicable for 'C' and 'R' - statuses). This is None if there is no score. - src - The path for the source file. - dst - The path for the destination file. This is only present for - copy or renames. If it is not present, this is None. - - If the pattern is not matched, None is returned.""" + See git-diff-tree(1) manpage for details about the format of the diff + output. This method returns a dictionary with the following elements: + + src_mode - The mode of the source file + dst_mode - The mode of the destination file + src_sha1 - The sha1 for the source file + dst_sha1 - The sha1 fr the destination file + status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) + status_score - The score for the status (applicable for 'C' and 'R' + statuses). This is None if there is no score. + src - The path for the source file. + dst - The path for the destination file. This is only present for + copy or renames. If it is not present, this is None. + + If the pattern is not matched, None is returned. + """ global _diff_tree_pattern if not _diff_tree_pattern: @@ -635,41 +773,91 @@ def parseDiffTreeEntry(entry): } return None + def isModeExec(mode): - # Returns True if the given git mode represents an executable file, - # otherwise False. + """Returns True if the given git mode represents an executable file, + otherwise False. + """ return mode[-3:] == "755" + class P4Exception(Exception): - """ Base class for exceptions from the p4 client """ + """Base class for exceptions from the p4 client.""" + def __init__(self, exit_code): self.p4ExitCode = exit_code + class P4ServerException(P4Exception): - """ Base class for exceptions where we get some kind of marshalled up result from the server """ + """Base class for exceptions where we get some kind of marshalled up result + from the server. + """ + def __init__(self, exit_code, p4_result): super(P4ServerException, self).__init__(exit_code) self.p4_result = p4_result self.code = p4_result[0]['code'] self.data = p4_result[0]['data'] + class P4RequestSizeException(P4ServerException): - """ One of the maxresults or maxscanrows errors """ + """One of the maxresults or maxscanrows errors.""" + def __init__(self, exit_code, p4_result, limit): super(P4RequestSizeException, self).__init__(exit_code, p4_result) self.limit = limit + class P4CommandException(P4Exception): - """ Something went wrong calling p4 which means we have to give up """ + """Something went wrong calling p4 which means we have to give up.""" + def __init__(self, msg): self.msg = msg def __str__(self): return self.msg + def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) + +def p4KeysContainingNonUtf8Chars(): + """Returns all keys which may contain non UTF-8 encoded strings + for which a fallback strategy has to be applied. + """ + return ['desc', 'client', 'FullName'] + + +def p4KeysContainingBinaryData(): + """Returns all keys which may contain arbitrary binary data + """ + return ['data'] + + +def p4KeyContainsFilePaths(key): + """Returns True if the key contains file paths. These are handled by decode_path(). + Otherwise False. + """ + return key.startswith('depotFile') or key in ['path', 'clientFile'] + + +def p4KeyWhichCanBeDirectlyDecoded(key): + """Returns True if the key can be directly decoded as UTF-8 string + Otherwise False. + + Keys which can not be encoded directly: + - `data` which may contain arbitrary binary data + - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text + - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path() + """ + if key in p4KeysContainingNonUtf8Chars() or \ + key in p4KeysContainingBinaryData() or \ + p4KeyContainsFilePaths(key): + return False + return True + + def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, errors_as_exceptions=False, *k, **kw): @@ -699,14 +887,13 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, try: while True: entry = marshal.load(p4.stdout) + if bytes is not str: - # Decode unmarshalled dict to use str keys and values, except for: - # - `data` which may contain arbitrary binary data - # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text + # Decode unmarshalled dict to use str keys and values. Special cases are handled below. decoded_entry = {} for key, value in entry.items(): key = key.decode() - if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')): + if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key): value = value.decode() decoded_entry[key] = value # Parse out data if it's an error response @@ -716,6 +903,9 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, if skip_info: if 'code' in entry and entry['code'] == 'info': continue + for key in p4KeysContainingNonUtf8Chars(): + if key in entry: + entry[key] = metadata_stream_to_writable_bytes(entry[key]) if cb is not None: cb(entry) else: @@ -746,12 +936,14 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, return result + def p4Cmd(cmd, *k, **kw): list = p4CmdList(cmd, *k, **kw) result = {} for entry in list: result.update(entry) - return result; + return result + def p4Where(depotPath): if not depotPath.endswith("/"): @@ -773,7 +965,7 @@ def p4Where(depotPath): if data[:space] == depotPath: output = entry break - if output == None: + if output is None: return "" if output["code"] == "error": return "" @@ -789,48 +981,54 @@ def p4Where(depotPath): clientPath = clientPath[:-3] return clientPath + def currentGitBranch(): return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"]) + def isValidGitDir(path): - return git_dir(path) != None + return git_dir(path) is not None + def parseRevision(ref): return read_pipe(["git", "rev-parse", ref]).strip() + def branchExists(ref): rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref], ignore_error=True) return len(rev) > 0 + def extractLogMessageFromGitCommit(commit): logMessage = "" - ## fixme: title is first line of commit, not 1st paragraph. + # fixme: title is first line of commit, not 1st paragraph. foundTitle = False for log in read_pipe_lines(["git", "cat-file", "commit", commit]): - if not foundTitle: - if len(log) == 1: - foundTitle = True - continue + if not foundTitle: + if len(log) == 1: + foundTitle = True + continue - logMessage += log + logMessage += log return logMessage + def extractSettingsGitLog(log): values = {} for line in log.split("\n"): line = line.strip() - m = re.search (r"^ *\[git-p4: (.*)\]$", line) + m = re.search(r"^ *\[git-p4: (.*)\]$", line) if not m: continue - assignments = m.group(1).split (':') + assignments = m.group(1).split(':') for a in assignments: - vals = a.split ('=') + vals = a.split('=') key = vals[0].strip() - val = ('='.join (vals[1:])).strip() - if val.endswith ('\"') and val.startswith('"'): + val = ('='.join(vals[1:])).strip() + if val.endswith('\"') and val.startswith('"'): val = val[1:-1] values[key] = val @@ -842,41 +1040,49 @@ def extractSettingsGitLog(log): values['depot-paths'] = paths.split(',') return values + def gitBranchExists(branch): proc = subprocess.Popen(["git", "rev-parse", branch], - stderr=subprocess.PIPE, stdout=subprocess.PIPE); - return proc.wait() == 0; + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + return proc.wait() == 0 + def gitUpdateRef(ref, newvalue): subprocess.check_call(["git", "update-ref", ref, newvalue]) + def gitDeleteRef(ref): subprocess.check_call(["git", "update-ref", "-d", ref]) + _gitConfig = {} + def gitConfig(key, typeSpecifier=None): if key not in _gitConfig: - cmd = [ "git", "config" ] + cmd = ["git", "config"] if typeSpecifier: - cmd += [ typeSpecifier ] - cmd += [ key ] + cmd += [typeSpecifier] + cmd += [key] s = read_pipe(cmd, ignore_error=True) _gitConfig[key] = s.strip() return _gitConfig[key] + def gitConfigBool(key): """Return a bool, using git config --bool. It is True only if the variable is set to true, and False if set to false or not present - in the config.""" + in the config. + """ if key not in _gitConfig: _gitConfig[key] = gitConfig(key, '--bool') == "true" return _gitConfig[key] + def gitConfigInt(key): if key not in _gitConfig: - cmd = [ "git", "config", "--int", key ] + cmd = ["git", "config", "--int", key] s = read_pipe(cmd, ignore_error=True) v = s.strip() try: @@ -885,6 +1091,7 @@ def gitConfigInt(key): _gitConfig[key] = None return _gitConfig[key] + def gitConfigList(key): if key not in _gitConfig: s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) @@ -893,12 +1100,43 @@ def gitConfigList(key): _gitConfig[key] = [] return _gitConfig[key] +def fullP4Ref(incomingRef, importIntoRemotes=True): + """Standardize a given provided p4 ref value to a full git ref: + refs/foo/bar/branch -> use it exactly + p4/branch -> prepend refs/remotes/ or refs/heads/ + branch -> prepend refs/remotes/p4/ or refs/heads/p4/""" + if incomingRef.startswith("refs/"): + return incomingRef + if importIntoRemotes: + prepend = "refs/remotes/" + else: + prepend = "refs/heads/" + if not incomingRef.startswith("p4/"): + prepend += "p4/" + return prepend + incomingRef + +def shortP4Ref(incomingRef, importIntoRemotes=True): + """Standardize to a "short ref" if possible: + refs/foo/bar/branch -> ignore + refs/remotes/p4/branch or refs/heads/p4/branch -> shorten + p4/branch -> shorten""" + if importIntoRemotes: + longprefix = "refs/remotes/p4/" + else: + longprefix = "refs/heads/p4/" + if incomingRef.startswith(longprefix): + return incomingRef[len(longprefix):] + if incomingRef.startswith("p4/"): + return incomingRef[3:] + return incomingRef + def p4BranchesInGit(branchesAreInRemotes=True): """Find all the branches whose names start with "p4/", looking in remotes or heads as specified by the argument. Return a dictionary of { branch: revision } for each one found. The branch names are the short names, without any - "p4/" prefix.""" + "p4/" prefix. + """ branches = {} @@ -925,10 +1163,11 @@ def p4BranchesInGit(branchesAreInRemotes=True): return branches + def branch_exists(branch): """Make sure that the given ref name really exists.""" - cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ] + cmd = ["git", "rev-parse", "--symbolic", "--verify", branch] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = p.communicate() out = decode_text_stream(out) @@ -937,7 +1176,8 @@ def branch_exists(branch): # expect exactly one line of output: the branch name return out.rstrip() == branch -def findUpstreamBranchPoint(head = "HEAD"): + +def findUpstreamBranchPoint(head="HEAD"): branches = p4BranchesInGit() # map from depot-path to branch name branchByDepotPath = {} @@ -946,8 +1186,12 @@ def findUpstreamBranchPoint(head = "HEAD"): log = extractLogMessageFromGitCommit(tip) settings = extractSettingsGitLog(log) if "depot-paths" in settings: + git_branch = "remotes/p4/" + branch paths = ",".join(settings["depot-paths"]) - branchByDepotPath[paths] = "remotes/p4/" + branch + branchByDepotPath[paths] = git_branch + if "change" in settings: + paths = paths + ";" + settings["change"] + branchByDepotPath[paths] = git_branch settings = None parent = 0 @@ -957,6 +1201,10 @@ def findUpstreamBranchPoint(head = "HEAD"): settings = extractSettingsGitLog(log) if "depot-paths" in settings: paths = ",".join(settings["depot-paths"]) + if "change" in settings: + expaths = paths + ";" + settings["change"] + if expaths in branchByDepotPath: + return [branchByDepotPath[expaths], settings] if paths in branchByDepotPath: return [branchByDepotPath[paths], settings] @@ -964,7 +1212,8 @@ def findUpstreamBranchPoint(head = "HEAD"): return ["", settings] -def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True): + +def createOrUpdateBranchesFromOrigin(localRefPrefix="refs/remotes/p4/", silent=True): if not silent: print("Creating/updating branch(es) in %s based on origin branch(es)" % localRefPrefix) @@ -981,8 +1230,7 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent originHead = line original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) - if ('depot-paths' not in original - or 'change' not in original): + if 'depot-paths' not in original or 'change' not in original: continue update = False @@ -1011,8 +1259,9 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent if update: system(["git", "update-ref", remoteHead, originHead]) + def originP4BranchesExist(): - return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") + return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") def p4ParseNumericChangeRange(parts): @@ -1024,12 +1273,14 @@ def p4ParseNumericChangeRange(parts): return (changeStart, changeEnd) + def chooseBlockSize(blockSize): if blockSize: return blockSize else: return defaultBlockSize + def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): assert depotPaths @@ -1047,7 +1298,7 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): parts = changeRange.split(',') assert len(parts) == 2 try: - (changeStart, changeEnd) = p4ParseNumericChangeRange(parts) + changeStart, changeEnd = p4ParseNumericChangeRange(parts) block_size = chooseBlockSize(requestedBlockSize) except ValueError: changeStart = parts[0][1:] @@ -1085,7 +1336,8 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): else: block_size = max(2, block_size // 2) - if verbose: print("block size error, retrying with block size {0}".format(block_size)) + if verbose: + print("block size error, retrying with block size {0}".format(block_size)) continue except P4Exception as e: die('Error retrieving changes description ({0})'.format(e.p4ExitCode)) @@ -1107,21 +1359,25 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): changes = sorted(changes) return changes + def p4PathStartsWith(path, prefix): - # This method tries to remedy a potential mixed-case issue: - # - # If UserA adds //depot/DirA/file1 - # and UserB adds //depot/dira/file2 - # - # we may or may not have a problem. If you have core.ignorecase=true, - # we treat DirA and dira as the same directory + """This method tries to remedy a potential mixed-case issue: + + If UserA adds //depot/DirA/file1 + and UserB adds //depot/dira/file2 + + we may or may not have a problem. If you have core.ignorecase=true, + we treat DirA and dira as the same directory. + """ if gitConfigBool("core.ignorecase"): return path.lower().startswith(prefix.lower()) return path.startswith(prefix) + def getClientSpec(): """Look at the p4 client spec, create a View() object that contains - all the mappings, and return it.""" + all the mappings, and return it. + """ specList = p4CmdList(["client", "-o"]) if len(specList) != 1: @@ -1135,7 +1391,7 @@ def getClientSpec(): client_name = entry["Client"] # just the keys that start with "View" - view_keys = [ k for k in entry.keys() if k.startswith("View") ] + view_keys = [k for k in entry.keys() if k.startswith("View")] # hold this new View view = View(client_name) @@ -1149,6 +1405,7 @@ def getClientSpec(): return view + def getClientRoot(): """Grab the client directory.""" @@ -1162,12 +1419,15 @@ def getClientRoot(): return entry["Root"] -# -# P4 wildcards are not allowed in filenames. P4 complains -# if you simply add them, but you can force it with "-f", in -# which case it translates them into %xx encoding internally. -# + def wildcard_decode(path): + """Decode P4 wildcards into %xx encoding + + P4 wildcards are not allowed in filenames. P4 complains if you simply + add them, but you can force it with "-f", in which case it translates + them into %xx encoding internally. + """ + # Search for and fix just these four characters. Do % last so # that fixing it does not inadvertently create new %-escapes. # Cannot have * in a filename in windows; untested as to @@ -1179,7 +1439,10 @@ def wildcard_decode(path): .replace("%25", "%") return path + def wildcard_encode(path): + """Encode %xx coded wildcards into P4 coding.""" + # do % first to avoid double-encoding the %s introduced here path = path.replace("%", "%25") \ .replace("*", "%2A") \ @@ -1187,10 +1450,12 @@ def wildcard_encode(path): .replace("@", "%40") return path + def wildcard_present(path): m = re.search("[*#@%]", path) return m is not None + class LargeFileSystem(object): """Base class for large file system support.""" @@ -1199,13 +1464,15 @@ class LargeFileSystem(object): self.writeToGitStream = writeToGitStream def generatePointer(self, cloneDestination, contentFile): - """Return the content of a pointer file that is stored in Git instead of - the actual content.""" + """Return the content of a pointer file that is stored in Git instead + of the actual content. + """ assert False, "Method 'generatePointer' required in " + self.__class__.__name__ def pushFile(self, localLargeFile): """Push the actual content which is not stored in the Git repository to - a server.""" + a server. + """ assert False, "Method 'pushFile' required in " + self.__class__.__name__ def hasLargeFileExtension(self, relPath): @@ -1253,10 +1520,11 @@ class LargeFileSystem(object): def processContent(self, git_mode, relPath, contents): """Processes the content of git fast import. This method decides if a file is stored in the large file system and handles all necessary - steps.""" + steps. + """ if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath): contentTempFile = self.generateTempFile(contents) - (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile) + pointer_git_mode, contents, localLargeFile = self.generatePointer(contentTempFile) if pointer_git_mode: git_mode = pointer_git_mode if localLargeFile: @@ -1272,12 +1540,14 @@ class LargeFileSystem(object): sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile)) return (git_mode, contents) + class MockLFS(LargeFileSystem): """Mock large file system for testing.""" def generatePointer(self, contentFile): """The pointer content is the original content prefixed with "pointer-". - The local filename of the large file storage is derived from the file content. + The local filename of the large file storage is derived from the + file content. """ with open(contentFile, 'r') as f: content = next(f) @@ -1287,17 +1557,19 @@ class MockLFS(LargeFileSystem): return (gitMode, pointerContents, localLargeFile) def pushFile(self, localLargeFile): - """The remote filename of the large file storage is the same as the local - one but in a different directory. + """The remote filename of the large file storage is the same as the + local one but in a different directory. """ remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote') if not os.path.exists(remotePath): os.makedirs(remotePath) shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile))) + class GitLFS(LargeFileSystem): """Git LFS as backend for the git-p4 large file system. - See https://git-lfs.github.com/ for details.""" + See https://git-lfs.github.com/ for details. + """ def __init__(self, *args): LargeFileSystem.__init__(self, *args) @@ -1383,9 +1655,10 @@ class GitLFS(LargeFileSystem): else: return LargeFileSystem.processContent(self, git_mode, relPath, contents) + class Command: - delete_actions = ( "delete", "move/delete", "purge" ) - add_actions = ( "add", "branch", "move/add" ) + delete_actions = ("delete", "move/delete", "purge") + add_actions = ("add", "branch", "move/add") def __init__(self): self.usage = "usage: %prog [options]" @@ -1398,6 +1671,7 @@ class Command: setattr(self, attr, value) return getattr(self, attr) + class P4UserMap: def __init__(self): self.userMapFromPerforceServer = False @@ -1415,7 +1689,7 @@ class P4UserMap: die("Could not find your p4 user id") def p4UserIsMe(self, p4User): - # return True if the given p4 user is actually me + """Return True if the given p4 user is actually me.""" me = self.p4UserId() if not p4User or p4User != me: return False @@ -1435,7 +1709,13 @@ class P4UserMap: for output in p4CmdList(["users"]): if "User" not in output: continue - self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + # "FullName" is bytes. "Email" on the other hand might be bytes + # or unicode string depending on whether we are running under + # python2 or python3. To support + # git-p4.metadataDecodingStrategy=fallback, self.users dict values + # are always bytes, ready to be written to git. + emailbytes = metadata_stream_to_writable_bytes(output["Email"]) + self.users[output["User"]] = output["FullName"] + b" <" + emailbytes + b">" self.emails[output["Email"]] = output["User"] mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE) @@ -1445,29 +1725,32 @@ class P4UserMap: user = mapUser[0][0] fullname = mapUser[0][1] email = mapUser[0][2] - self.users[user] = fullname + " <" + email + ">" + fulluser = fullname + " <" + email + ">" + self.users[user] = metadata_stream_to_writable_bytes(fulluser) self.emails[email] = user - s = '' + s = b'' for (key, val) in self.users.items(): - s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) + keybytes = metadata_stream_to_writable_bytes(key) + s += b"%s\t%s\n" % (keybytes.expandtabs(1), val.expandtabs(1)) - open(self.getUserCacheFilename(), 'w').write(s) + open(self.getUserCacheFilename(), 'wb').write(s) self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} self.userMapFromPerforceServer = False try: - cache = open(self.getUserCacheFilename(), 'r') + cache = open(self.getUserCacheFilename(), 'rb') lines = cache.readlines() cache.close() for line in lines: - entry = line.strip().split("\t") - self.users[entry[0]] = entry[1] + entry = line.strip().split(b"\t") + self.users[entry[0].decode('utf_8')] = entry[1] except IOError: self.getUserMapFromPerforceServer() + class P4Submit(Command, P4UserMap): conflict_behavior_choices = ("ask", "skip", "quit") @@ -1560,20 +1843,20 @@ class P4Submit(Command, P4UserMap): die("You have files opened with perforce! Close them before starting the sync.") def separate_jobs_from_description(self, message): - """Extract and return a possible Jobs field in the commit - message. It goes into a separate section in the p4 change - specification. + """Extract and return a possible Jobs field in the commit message. It + goes into a separate section in the p4 change specification. - A jobs line starts with "Jobs:" and looks like a new field - in a form. Values are white-space separated on the same - line or on following lines that start with a tab. + A jobs line starts with "Jobs:" and looks like a new field in a + form. Values are white-space separated on the same line or on + following lines that start with a tab. - This does not parse and extract the full git commit message - like a p4 form. It just sees the Jobs: line as a marker - to pass everything from then on directly into the p4 form, - but outside the description section. + This does not parse and extract the full git commit message like a + p4 form. It just sees the Jobs: line as a marker to pass everything + from then on directly into the p4 form, but outside the description + section. - Return a tuple (stripped log message, jobs string).""" + Return a tuple (stripped log message, jobs string). + """ m = re.search(r'^Jobs:', message, re.MULTILINE) if m is None: @@ -1584,9 +1867,10 @@ class P4Submit(Command, P4UserMap): return (stripped_message, jobtext) def prepareLogMessage(self, template, message, jobs): - """Edits the template returned from "p4 change -o" to insert - the message in the Description field, and the jobs text in - the Jobs field.""" + """Edits the template returned from "p4 change -o" to insert the + message in the Description field, and the jobs text in the Jobs + field. + """ result = "" inDescriptionSection = False @@ -1616,8 +1900,10 @@ class P4Submit(Command, P4UserMap): return result def patchRCSKeywords(self, file, regexp): - # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex - (handle, outFileName) = tempfile.mkstemp(dir='.') + """Attempt to zap the RCS keywords in a p4 controlled file matching the + given regex. + """ + handle, outFileName = tempfile.mkstemp(dir='.') try: with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile: for line in inFile.readlines(): @@ -1633,21 +1919,23 @@ class P4Submit(Command, P4UserMap): print("Patched up RCS keywords in %s" % file) - def p4UserForCommit(self,id): - # Return the tuple (perforce user,git email) for a given git commit id + def p4UserForCommit(self, id): + """Return the tuple (perforce user,git email) for a given git commit + id. + """ self.getUserMapFromPerforceServer() gitEmail = read_pipe(["git", "log", "--max-count=1", "--format=%ae", id]) gitEmail = gitEmail.strip() if gitEmail not in self.emails: - return (None,gitEmail) + return (None, gitEmail) else: - return (self.emails[gitEmail],gitEmail) + return (self.emails[gitEmail], gitEmail) - def checkValidP4Users(self,commits): - # check if any git authors cannot be mapped to p4 users + def checkValidP4Users(self, commits): + """Check if any git authors cannot be mapped to p4 users.""" for id in commits: - (user,email) = self.p4UserForCommit(id) + user, email = self.p4UserForCommit(id) if not user: msg = "Cannot find p4 user for email %s in commit %s." % (email, id) if gitConfigBool("git-p4.allowMissingP4Users"): @@ -1656,10 +1944,12 @@ class P4Submit(Command, P4UserMap): die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg) def lastP4Changelist(self): - # Get back the last changelist number submitted in this client spec. This - # then gets used to patch up the username in the change. If the same - # client spec is being used by multiple processes then this might go - # wrong. + """Get back the last changelist number submitted in this client spec. + + This then gets used to patch up the username in the change. If the + same client spec is being used by multiple processes then this might + go wrong. + """ results = p4CmdList(["client", "-o"]) # find the current client client = None for r in results: @@ -1675,14 +1965,16 @@ class P4Submit(Command, P4UserMap): die("Could not get changelist number for last submit - cannot patch up user details") def modifyChangelistUser(self, changelist, newUser): - # fixup the user field of a changelist after it has been submitted. + """Fixup the user field of a changelist after it has been submitted.""" changes = p4CmdList(["change", "-o", changelist]) if len(changes) != 1: die("Bad output from p4 change modifying %s to user %s" % (changelist, newUser)) c = changes[0] - if c['User'] == newUser: return # nothing to do + if c['User'] == newUser: + # Nothing to do + return c['User'] = newUser # p4 does not understand format version 3 and above input = marshal.dumps(c, 2) @@ -1698,8 +1990,9 @@ class P4Submit(Command, P4UserMap): die("Could not modify user field of changelist %s to %s" % (changelist, newUser)) def canChangeChangelists(self): - # check to see if we have p4 admin or super-user permissions, either of - # which are required to modify changelists. + """Check to see if we have p4 admin or super-user permissions, either + of which are required to modify changelists. + """ results = p4CmdList(["protects", self.depotPath]) for r in results: if 'perm' in r: @@ -1711,13 +2004,15 @@ class P4Submit(Command, P4UserMap): def prepareSubmitTemplate(self, changelist=None): """Run "p4 change -o" to grab a change specification template. + This does not use "p4 -G", as it is nice to keep the submission template in original order, since a human might edit it. Remove lines in the Files section that show changes to files - outside the depot path we're committing into.""" + outside the depot path we're committing into. + """ - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() template = """\ # A Perforce Change Specification. @@ -1778,8 +2073,10 @@ class P4Submit(Command, P4UserMap): return template def edit_template(self, template_file): - """Invoke the editor to let the user change the submission - message. Return true if okay to continue with the submit.""" + """Invoke the editor to let the user change the submission message. + + Return true if okay to continue with the submit. + """ # if configured to skip the editing part, just submit if gitConfigBool("git-p4.skipSubmitEdit"): @@ -1838,7 +2135,9 @@ class P4Submit(Command, P4UserMap): for line in f.readlines(): newdiff += "+" + line except UnicodeDecodeError: - pass # Found non-text data and skip, since diff description should only include text + # Found non-text data and skip, since diff description + # should only include text + pass f.close() return (diff + newdiff).replace('\r\n', '\n') @@ -1849,7 +2148,7 @@ class P4Submit(Command, P4UserMap): print("Applying", read_pipe(["git", "show", "-s", "--format=format:%h %s", id])) - (p4User, gitEmail) = self.p4UserForCommit(id) + p4User, gitEmail = self.p4UserForCommit(id) diff = read_pipe_lines( ["git", "diff-tree", "-r"] + self.diffOpts + ["{}^".format(id), id]) @@ -1956,11 +2255,11 @@ class P4Submit(Command, P4UserMap): if regexp: # this file is a possibility...look for RCS keywords. for line in read_pipe_lines( - ["git", "diff", "%s^..%s" % (id, id), file], - raw=True): + ["git", "diff", "%s^..%s" % (id, id), file], + raw=True): if regexp.search(line): if verbose: - print("got keyword match on %s in %s in %s" % (regex.pattern, line, file)) + print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file)) kwfiles[file] = regexp break @@ -2014,13 +2313,13 @@ class P4Submit(Command, P4UserMap): # logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.strip() - (logMessage, jobs) = self.separate_jobs_from_description(logMessage) + logMessage, jobs = self.separate_jobs_from_description(logMessage) template = self.prepareSubmitTemplate(update_shelve) submitTemplate = self.prepareLogMessage(template, logMessage, jobs) if self.preserveUser: - submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User + submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User if self.checkAuthorship and not self.p4UserIsMe(p4User): submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail @@ -2032,7 +2331,7 @@ class P4Submit(Command, P4UserMap): submitTemplate += separatorLine submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks) - (handle, fileName) = tempfile.mkstemp() + handle, fileName = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+b") if self.isWindows: submitTemplate = submitTemplate.replace("\n", "\r\n") @@ -2059,13 +2358,13 @@ class P4Submit(Command, P4UserMap): print(" " + self.clientPath) print("") print("To submit, use \"p4 submit\" to write a new description,") - print("or \"p4 submit -i <%s\" to use the one prepared by" \ + print("or \"p4 submit -i <%s\" to use the one prepared by" " \"git p4\"." % fileName) print("You can delete the file \"%s\" when finished." % fileName) if self.preserveUser and p4User and not self.p4UserIsMe(p4User): - print("To preserve change ownership by user %s, you must\n" \ - "do \"p4 change -f <change>\" after submitting and\n" \ + print("To preserve change ownership by user %s, you must\n" + "do \"p4 change -f <change>\" after submitting and\n" "edit the User field.") if pureRenameCopy: print("After submitting, renamed files must be re-synced.") @@ -2133,9 +2432,9 @@ class P4Submit(Command, P4UserMap): # Revert changes if we skip this patch if not submitted or self.shelve: if self.shelve: - print ("Reverting shelved files.") + print("Reverting shelved files.") else: - print ("Submission cancelled, undoing p4 changes.") + print("Submission cancelled, undoing p4 changes.") sys.stdout.flush() for f in editedFiles | filesToDelete: p4_revert(f) @@ -2147,9 +2446,11 @@ class P4Submit(Command, P4UserMap): os.remove(fileName) return submitted - # Export git tags as p4 labels. Create a p4 label and then tag - # with that. def exportGitTags(self, gitTags): + """Export git tags as p4 labels. Create a p4 label and then tag with + that. + """ + validLabelRegexp = gitConfig("git-p4.labelExportRegexp") if len(validLabelRegexp) == 0: validLabelRegexp = defaultLabelRegexp @@ -2195,7 +2496,7 @@ class P4Submit(Command, P4UserMap): # Create the label - use the same view as the client spec we are using clientSpec = getClientSpec() - labelTemplate = "Label: %s\n" % name + labelTemplate = "Label: %s\n" % name labelTemplate += "Description:\n" for b in body: labelTemplate += "\t" + b + "\n" @@ -2206,7 +2507,7 @@ class P4Submit(Command, P4UserMap): if self.dry_run: print("Would create p4 label %s for tag" % name) elif self.prepare_p4_only: - print("Not creating p4 label %s for tag due to option" \ + print("Not creating p4 label %s for tag due to option" " --prepare-p4-only" % name) else: p4_write_pipe(["label", "-i"], labelTemplate) @@ -2237,7 +2538,7 @@ class P4Submit(Command, P4UserMap): if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","): die("%s is not in git-p4.allowSubmit" % self.master) - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() self.depotPath = settings['depot-paths'][0] if len(self.origin) == 0: self.origin = upstream @@ -2371,13 +2672,13 @@ class P4Submit(Command, P4UserMap): if not self.no_verify: try: if not run_git_hook("p4-pre-submit"): - print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \ - "this pre-submission check by adding\nthe command line option '--no-verify', " \ + print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " + "this pre-submission check by adding\nthe command line option '--no-verify', " "however,\nthis will also skip the p4-changelist hook as well.") sys.exit(1) except Exception as e: - print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\ - "with the error '{0}'".format(e.message) ) + print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed " + "with the error '{0}'".format(e.message)) sys.exit(1) # @@ -2399,7 +2700,7 @@ class P4Submit(Command, P4UserMap): applied.append(commit) if self.prepare_p4_only: if i < last: - print("Processing only the first commit due to option" \ + print("Processing only the first commit due to option" " --prepare-p4-only") break else: @@ -2469,13 +2770,15 @@ class P4Submit(Command, P4UserMap): # exit with error unless everything applied perfectly if len(commits) != len(applied): - sys.exit(1) + sys.exit(1) return True + class View(object): - """Represent a p4 view ("p4 help views"), and map files in a - repo according to the view.""" + """Represent a p4 view ("p4 help views"), and map files in a repo according + to the view. + """ def __init__(self, client_name): self.mappings = [] @@ -2484,9 +2787,10 @@ class View(object): self.client_spec_path_cache = {} def append(self, view_line): - """Parse a view line, splitting it into depot and client - sides. Append to self.mappings, preserving order. This - is only needed for tag creation.""" + """Parse a view line, splitting it into depot and client sides. Append + to self.mappings, preserving order. This is only needed for tag + creation. + """ # Split the view line into exactly two words. P4 enforces # structure on these lines that simplifies this quite a bit. @@ -2535,7 +2839,7 @@ class View(object): return clientFile[len(self.client_prefix):] def update_client_spec_path_cache(self, files): - """ Caching file paths by "p4 where" batch query """ + """Caching file paths by "p4 where" batch query.""" # List depot file paths exclude that already cached fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache] @@ -2567,9 +2871,11 @@ class View(object): self.client_spec_path_cache[depotFile] = b'' def map_in_client(self, depot_path): - """Return the relative location in the client where this - depot file should live. Returns "" if the file should - not be mapped in the client.""" + """Return the relative location in the client where this depot file + should live. + + Returns "" if the file should not be mapped in the client. + """ if gitConfigBool("core.ignorecase"): depot_path = depot_path.lower() @@ -2577,14 +2883,16 @@ class View(object): if depot_path in self.client_spec_path_cache: return self.client_spec_path_cache[depot_path] - die( "Error: %s is not found in client spec path" % depot_path ) + die("Error: %s is not found in client spec path" % depot_path) return "" + def cloneExcludeCallback(option, opt_str, value, parser): # prepend "/" because the first "/" was consumed as part of the option itself. # ("-//depot/A/..." becomes "/depot/A/..." after option parsing) parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)] + class P4Sync(Command, P4UserMap): def __init__(self): @@ -2665,8 +2973,8 @@ class P4Sync(Command, P4UserMap): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) self.labels = {} - # Force a checkpoint in fast-import and wait for it to finish def checkpoint(self): + """Force a checkpoint in fast-import and wait for it to finish.""" self.gitStream.write("checkpoint\n\n") self.gitStream.write("progress checkpoint\n\n") self.gitStream.flush() @@ -2687,11 +2995,11 @@ class P4Sync(Command, P4UserMap): return True return False - def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0): + def extractFilesFromCommit(self, commit, shelved=False, shelved_cl=0): files = [] fnum = 0 while "depotFile%s" % fnum in commit: - path = commit["depotFile%s" % fnum] + path = commit["depotFile%s" % fnum] found = self.isPathWanted(decode_path(path)) if not found: fnum = fnum + 1 @@ -2718,10 +3026,10 @@ class P4Sync(Command, P4UserMap): return jobs def stripRepoPath(self, path, prefixes): - """When streaming files, this is called to map a p4 depot path - to where it should go in git. The prefixes are either - self.depotPaths, or self.branchPrefixes in the case of - branch detection.""" + """When streaming files, this is called to map a p4 depot path to where + it should go in git. The prefixes are either self.depotPaths, or + self.branchPrefixes in the case of branch detection. + """ if self.useClientSpec: # branch detection moves files up a level (the branch name) @@ -2750,8 +3058,9 @@ class P4Sync(Command, P4UserMap): return path def splitFilesIntoBranches(self, commit): - """Look at each depotFile in the commit to figure out to what - branch it belongs.""" + """Look at each depotFile in the commit to figure out to what branch it + belongs. + """ if self.clientSpecDirs: files = self.extractFilesFromCommit(commit) @@ -2811,10 +3120,12 @@ class P4Sync(Command, P4UserMap): print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)) return path - # output one file from the P4 stream - # - helper for streamP4Files - def streamOneP4File(self, file, contents): + """Output one file from the P4 stream. + + This is a helper for streamP4Files(). + """ + file_path = file['depotFile'] relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes) @@ -2822,12 +3133,13 @@ class P4Sync(Command, P4UserMap): if 'fileSize' in self.stream_file: size = int(self.stream_file['fileSize']) else: - size = 0 # deleted files don't get a fileSize apparently + # Deleted files don't get a fileSize apparently + size = 0 sys.stdout.write('\r%s --> %s (%s)\n' % ( file_path, relPath, format_size_human_readable(size))) sys.stdout.flush() - (type_base, type_mods) = split_p4_type(file["type"]) + type_base, type_mods = split_p4_type(file["type"]) git_mode = "100644" if "x" in type_mods: @@ -2869,8 +3181,8 @@ class P4Sync(Command, P4UserMap): raise e else: if p4_version_string().find('/NT') >= 0: - text = text.replace(b'\r\n', b'\n') - contents = [ text ] + text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00') + contents = [text] if type_base == "apple": # Apple filetype files will be streamed as a concatenation of @@ -2885,6 +3197,16 @@ class P4Sync(Command, P4UserMap): print("\nIgnoring apple filetype file %s" % file['depotFile']) return + if type_base == "utf8": + # The type utf8 explicitly means utf8 *with BOM*. These are + # streamed just like regular text files, however, without + # the BOM in the stream. + # Therefore, to accurately import these files into git, we + # need to explicitly re-add the BOM before writing. + # 'contents' is a set of bytes in this case, so create the + # BOM prefix as a b'' literal. + contents = [b'\xef\xbb\xbf' + contents[0]] + contents[1:] + # Note that we do not try to de-mangle keywords on utf16 files, # even though in theory somebody may want that. regexp = p4_keywords_regexp_for_type(type_base, type_mods) @@ -2892,7 +3214,7 @@ class P4Sync(Command, P4UserMap): contents = [regexp.sub(br'$\1$', c) for c in contents] if self.largeFileSystem: - (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents) + git_mode, contents = self.largeFileSystem.processContent(git_mode, relPath, contents) self.writeToGitStream(git_mode, relPath, contents) @@ -2906,8 +3228,8 @@ class P4Sync(Command, P4UserMap): if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath): self.largeFileSystem.removeLargeFile(relPath) - # handle another chunk of streaming data def streamP4FilesCb(self, marshalled): + """Handle another chunk of streaming data.""" # catch p4 errors and complain err = None @@ -2958,9 +3280,9 @@ class P4Sync(Command, P4UserMap): self.stream_file[k] = marshalled[k] if (verbose and - 'streamContentSize' in self.stream_file and - 'fileSize' in self.stream_file and - 'depotFile' in self.stream_file): + 'streamContentSize' in self.stream_file and + 'fileSize' in self.stream_file and + 'depotFile' in self.stream_file): size = int(self.stream_file["fileSize"]) if size > 0: progress = 100*self.stream_file['streamContentSize']/size @@ -2971,8 +3293,9 @@ class P4Sync(Command, P4UserMap): self.stream_have_file_info = True - # Stream directly from "p4 files" into "git fast-import" def streamP4Files(self, files): + """Stream directly from "p4 files" into "git fast-import.""" + filesForCommit = [] filesToRead = [] filesToDelete = [] @@ -3020,12 +3343,14 @@ class P4Sync(Command, P4UserMap): if userid in self.users: return self.users[userid] else: - return "%s <a@b>" % userid + userid_bytes = metadata_stream_to_writable_bytes(userid) + return b"%s <a@b>" % userid_bytes def streamTag(self, gitStream, labelName, labelDetails, commit, epoch): - """ Stream a p4 tag. - commit is either a git commit, or a fast-import mark, ":<p4commit>" - """ + """Stream a p4 tag. + + Commit is either a git commit, or a fast-import mark, ":<p4commit>". + """ if verbose: print("writing tag %s for commit %s" % (labelName, commit)) @@ -3043,11 +3368,12 @@ class P4Sync(Command, P4UserMap): email = self.make_email(owner) else: email = self.make_email(self.p4UserId()) - tagger = "%s %s %s" % (email, epoch, self.tz) - gitStream.write("tagger %s\n" % tagger) + gitStream.write("tagger ") + gitStream.write(email) + gitStream.write(" %s %s\n" % (epoch, self.tz)) - print("labelDetails=",labelDetails) + print("labelDetails=", labelDetails) if 'Description' in labelDetails: description = labelDetails['Description'] else: @@ -3075,15 +3401,18 @@ class P4Sync(Command, P4UserMap): return hasPrefix def findShadowedFiles(self, files, change): - # Perforce allows you commit files and directories with the same name, - # so you could have files //depot/foo and //depot/foo/bar both checked - # in. A p4 sync of a repository in this state fails. Deleting one of - # the files recovers the repository. - # - # Git will not allow the broken state to exist and only the most recent - # of the conflicting names is left in the repository. When one of the - # conflicting files is deleted we need to re-add the other one to make - # sure the git repository recovers in the same way as perforce. + """Perforce allows you commit files and directories with the same name, + so you could have files //depot/foo and //depot/foo/bar both checked + in. A p4 sync of a repository in this state fails. Deleting one of + the files recovers the repository. + + Git will not allow the broken state to exist and only the most + recent of the conflicting names is left in the repository. When one + of the conflicting files is deleted we need to re-add the other one + to make sure the git repository recovers in the same way as + perforce. + """ + deleted = [f for f in files if f['action'] in self.delete_actions] to_check = set() for f in deleted: @@ -3110,7 +3439,7 @@ class P4Sync(Command, P4UserMap): 'rev': record['headRev'], 'type': record['headType']}) - def commit(self, details, files, branch, parent = "", allow_empty=False): + def commit(self, details, files, branch, parent="", allow_empty=False): epoch = details["time"] author = details["user"] jobs = self.extractJobsFromCommit(details) @@ -3138,12 +3467,12 @@ class P4Sync(Command, P4UserMap): self.gitStream.write("commit %s\n" % branch) self.gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) - committer = "" if author not in self.users: self.getUserMapFromPerforceServer() - committer = "%s %s %s" % (self.make_email(author), epoch, self.tz) - self.gitStream.write("committer %s\n" % committer) + self.gitStream.write("committer ") + self.gitStream.write(self.make_email(author)) + self.gitStream.write(" %s %s\n" % (epoch, self.tz)) self.gitStream.write("data <<EOT\n") self.gitStream.write(details["desc"]) @@ -3200,8 +3529,11 @@ class P4Sync(Command, P4UserMap): print("Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)) - # Build a dictionary of changelists and labels, for "detect-labels" option. def getLabels(self): + """Build a dictionary of changelists and labels, for "detect-labels" + option. + """ + self.labels = {} l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths]) @@ -3227,11 +3559,12 @@ class P4Sync(Command, P4UserMap): if self.verbose: print("Label changes: %s" % self.labels.keys()) - # Import p4 labels as git tags. A direct mapping does not - # exist, so assume that if all the files are at the same revision - # then we can use that, or it's something more complicated we should - # just ignore. def importP4Labels(self, stream, p4Labels): + """Import p4 labels as git tags. A direct mapping does not exist, so + assume that if all the files are at the same revision then we can + use that, or it's something more complicated we should just ignore. + """ + if verbose: print("import p4 labels: " + ' '.join(p4Labels)) @@ -3246,7 +3579,7 @@ class P4Sync(Command, P4UserMap): if not m.match(name): if verbose: - print("label %s does not match regexp %s" % (name,validLabelRegexp)) + print("label %s does not match regexp %s" % (name, validLabelRegexp)) continue if name in ignoredP4Labels: @@ -3302,7 +3635,7 @@ class P4Sync(Command, P4UserMap): p = p[:-1] p = p[p.strip().rfind("/") + 1:] if not p.endswith("/"): - p += "/" + p += "/" return p def getBranchMapping(self): @@ -3322,7 +3655,7 @@ class P4Sync(Command, P4UserMap): continue source = paths[0] destination = paths[1] - ## HACK + # HACK if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]): source = source[len(self.depotPaths[0]):-4] destination = destination[len(self.depotPaths[0]):-4] @@ -3351,7 +3684,7 @@ class P4Sync(Command, P4UserMap): configBranches = gitConfigList("git-p4.branchList") for branch in configBranches: if branch: - (source, destination) = branch.split(":") + source, destination = branch.split(":") self.knownBranches[destination] = source lostAndFoundBranches.discard(destination) @@ -3359,7 +3692,6 @@ class P4Sync(Command, P4UserMap): if source not in self.knownBranches: lostAndFoundBranches.add(source) - for branch in lostAndFoundBranches: self.knownBranches[branch] = branch @@ -3431,27 +3763,22 @@ class P4Sync(Command, P4UserMap): def importNewBranch(self, branch, maxChange): # make fast-import flush all changes to disk and update the refs using the checkpoint # command so that we can try to find the branch parent in the git history - self.gitStream.write("checkpoint\n\n"); - self.gitStream.flush(); + self.gitStream.write("checkpoint\n\n") + self.gitStream.flush() branchPrefix = self.depotPaths[0] + branch + "/" range = "@1,%s" % maxChange - #print "prefix" + branchPrefix changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size) if len(changes) <= 0: return False firstChange = changes[0] - #print "first change in branch: %s" % firstChange sourceBranch = self.knownBranches[branch] sourceDepotPath = self.depotPaths[0] + sourceBranch sourceRef = self.gitRefForBranch(sourceBranch) - #print "source " + sourceBranch branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"]) - #print "branch parent: %s" % branchParentChange gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange) if len(gitParent) > 0: self.initialParents[self.gitRefForBranch(branch)] = gitParent - #print "parent git commit: %s" % gitParent self.importChanges(changes) return True @@ -3486,9 +3813,9 @@ class P4Sync(Command, P4UserMap): if self.detectBranches: branches = self.splitFilesIntoBranches(description) for branch in branches.keys(): - ## HACK --hwn + # HACK --hwn branchPrefix = self.depotPaths[0] + branch + "/" - self.branchPrefixes = [ branchPrefix ] + self.branchPrefixes = [branchPrefix] parent = "" @@ -3508,12 +3835,12 @@ class P4Sync(Command, P4UserMap): fullBranch = self.projectName + branch if fullBranch not in self.p4BranchesInGit: if not self.silent: - print("\n Importing new branch %s" % fullBranch); + print("\n Importing new branch %s" % fullBranch) if self.importNewBranch(branch, change - 1): parent = "" self.p4BranchesInGit.append(fullBranch) if not self.silent: - print("\n Resuming with change %s" % change); + print("\n Resuming with change %s" % change) if self.verbose: print("parent determined through known branches: %s" % parent) @@ -3572,7 +3899,7 @@ class P4Sync(Command, P4UserMap): newestRevision = 0 fileCnt = 0 - fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths] + fileArgs = ["%s...%s" % (p, revision) for p in self.depotPaths] for info in p4CmdList(["files"] + fileArgs): @@ -3581,24 +3908,21 @@ class P4Sync(Command, P4UserMap): % info['data']) if info['data'].find("must refer to client") >= 0: sys.stderr.write("This particular p4 error is misleading.\n") - sys.stderr.write("Perhaps the depot path was misspelled.\n"); + sys.stderr.write("Perhaps the depot path was misspelled.\n") sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths)) sys.exit(1) if 'p4ExitCode' in info: sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode']) sys.exit(1) - change = int(info["change"]) if change > newestRevision: newestRevision = change if info["action"] in self.delete_actions: - # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! - #fileCnt = fileCnt + 1 continue - for prop in ["depotFile", "rev", "action", "type" ]: + for prop in ["depotFile", "rev", "action", "type"]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 @@ -3618,7 +3942,6 @@ class P4Sync(Command, P4UserMap): print("IO error details: {}".format(err)) print(self.gitError.read()) - def importRevisions(self, args, branch_arg_given): changes = [] @@ -3690,7 +4013,7 @@ class P4Sync(Command, P4UserMap): self.importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE); + stderr=subprocess.PIPE) self.gitOutput = self.importProcess.stdout self.gitStream = self.importProcess.stdin self.gitError = self.importProcess.stderr @@ -3750,9 +4073,13 @@ class P4Sync(Command, P4UserMap): # restrict to just this one, disabling detect-branches if branch_arg_given: - short = self.branch.split("/")[-1] + short = shortP4Ref(self.branch, self.importIntoRemotes) if short in branches: - self.p4BranchesInGit = [ short ] + self.p4BranchesInGit = [short] + elif self.branch.startswith('refs/') and \ + branchExists(self.branch) and \ + '[git-p4:' in extractLogMessageFromGitCommit(self.branch): + self.p4BranchesInGit = [self.branch] else: self.p4BranchesInGit = branches.keys() @@ -3769,13 +4096,13 @@ class P4Sync(Command, P4UserMap): p4Change = 0 for branch in self.p4BranchesInGit: - logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) + logMsg = extractLogMessageFromGitCommit(fullP4Ref(branch, + self.importIntoRemotes)) settings = extractSettingsGitLog(logMsg) self.readOptions(settings) - if ('depot-paths' in settings - and 'change' in settings): + if 'depot-paths' in settings and 'change' in settings: change = int(settings['change']) + 1 p4Change = max(p4Change, change) @@ -3792,7 +4119,7 @@ class P4Sync(Command, P4UserMap): i = i - 1 break - paths.append ("/".join(cur_list[:i + 1])) + paths.append("/".join(cur_list[:i + 1])) self.previousDepotPaths = paths @@ -3802,18 +4129,7 @@ class P4Sync(Command, P4UserMap): if not self.silent and not self.detectBranches: print("Performing incremental import into %s git branch" % self.branch) - # accept multiple ref name abbreviations: - # refs/foo/bar/branch -> use it exactly - # p4/branch -> prepend refs/remotes/ or refs/heads/ - # branch -> prepend refs/remotes/p4/ or refs/heads/p4/ - if not self.branch.startswith("refs/"): - if self.importIntoRemotes: - prepend = "refs/remotes/" - else: - prepend = "refs/heads/" - if not self.branch.startswith("p4/"): - prepend += "p4/" - self.branch = prepend + self.branch + self.branch = fullP4Ref(self.branch, self.importIntoRemotes) if len(args) == 0 and self.depotPaths: if not self.silent: @@ -3821,8 +4137,8 @@ class P4Sync(Command, P4UserMap): else: if self.depotPaths and self.depotPaths != args: print("previous import used depot path %s and now %s was specified. " - "This doesn't work!" % (' '.join (self.depotPaths), - ' '.join (args))) + "This doesn't work!" % (' '.join(self.depotPaths), + ' '.join(args))) sys.exit(1) self.depotPaths = sorted(args) @@ -3862,7 +4178,7 @@ class P4Sync(Command, P4UserMap): if len(self.changesFile) == 0: revision = "#head" - p = re.sub ("\.\.\.$", "", p) + p = re.sub("\.\.\.$", "", p) if not p.endswith("/"): p += "/" @@ -3876,10 +4192,10 @@ class P4Sync(Command, P4UserMap): self.loadUserMapFromCache() self.labels = {} if self.detectLabels: - self.getLabels(); + self.getLabels() if self.detectBranches: - ## FIXME - what's a P4 projectName ? + # FIXME - what's a P4 projectName ? self.projectName = self.guessProjectName() if self.hasOrigin: @@ -3892,7 +4208,7 @@ class P4Sync(Command, P4UserMap): for b in self.p4BranchesInGit: if b != "master": - ## FIXME + # FIXME b = b[len(self.projectName):] self.createdBranches.add(b) @@ -3942,6 +4258,7 @@ class P4Sync(Command, P4UserMap): return True + class P4Rebase(Command): def __init__(self): Command.__init__(self) @@ -3961,11 +4278,11 @@ class P4Rebase(Command): def rebase(self): if os.system("git update-index --refresh") != 0: - die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash."); + die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.") if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0: - die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash."); + die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.") - [upstream, settings] = findUpstreamBranchPoint() + upstream, settings = findUpstreamBranchPoint() if len(upstream) == 0: die("Cannot find upstream branchpoint for rebase") @@ -3979,6 +4296,7 @@ class P4Rebase(Command): "HEAD", "--"]) return True + class P4Clone(P4Sync): def __init__(self): P4Sync.__init__(self) @@ -3996,7 +4314,7 @@ class P4Clone(P4Sync): self.cloneBare = False def defaultDestination(self, args): - ## TODO: use common prefix of args? + # TODO: use common prefix of args? depotPath = args[0] depotDir = re.sub("(@[^@]*)$", "", depotPath) depotDir = re.sub("(#[^#]*)$", "", depotDir) @@ -4032,7 +4350,7 @@ class P4Clone(P4Sync): os.makedirs(self.cloneDestination) chdir(self.cloneDestination) - init_cmd = [ "git", "init" ] + init_cmd = ["git", "init"] if self.cloneBare: init_cmd.append("--bare") retcode = subprocess.call(init_cmd) @@ -4044,19 +4362,28 @@ class P4Clone(P4Sync): # create a master branch and check out a work tree if gitBranchExists(self.branch): - system([ "git", "branch", currentGitBranch(), self.branch ]) + system(["git", "branch", currentGitBranch(), self.branch]) if not self.cloneBare: - system([ "git", "checkout", "-f" ]) + system(["git", "checkout", "-f"]) else: - print('Not checking out any branch, use ' \ + print('Not checking out any branch, use ' '"git checkout -q -b master <branch>"') # auto-set this variable if invoked with --use-client-spec if self.useClientSpec_from_options: system(["git", "config", "--bool", "git-p4.useclientspec", "true"]) + # persist any git-p4 encoding-handling config options passed in for clone: + if gitConfig('git-p4.metadataDecodingStrategy'): + system(["git", "config", "git-p4.metadataDecodingStrategy", gitConfig('git-p4.metadataDecodingStrategy')]) + if gitConfig('git-p4.metadataFallbackEncoding'): + system(["git", "config", "git-p4.metadataFallbackEncoding", gitConfig('git-p4.metadataFallbackEncoding')]) + if gitConfig('git-p4.pathEncoding'): + system(["git", "config", "git-p4.pathEncoding", gitConfig('git-p4.pathEncoding')]) + return True + class P4Unshelve(Command): def __init__(self): Command.__init__(self) @@ -4073,26 +4400,23 @@ class P4Unshelve(Command): self.destbranch = "refs/remotes/p4-unshelved" def renameBranch(self, branch_name): - """ Rename the existing branch to branch_name.N - """ + """Rename the existing branch to branch_name.N .""" - found = True - for i in range(0,1000): + for i in range(0, 1000): backup_branch_name = "{0}.{1}".format(branch_name, i) if not gitBranchExists(backup_branch_name): - gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup + # Copy ref to backup + gitUpdateRef(backup_branch_name, branch_name) gitDeleteRef(branch_name) - found = True print("renamed old unshelve branch to {0}".format(backup_branch_name)) break - - if not found: - sys.exit("gave up trying to rename existing branch {0}".format(sync.branch)) + else: + sys.exit("gave up trying to rename existing branch {0}".format(branch_name)) def findLastP4Revision(self, starting_point): - """ Look back from starting_point for the first commit created by git-p4 - to find the P4 commit we are based on, and the depot-paths. - """ + """Look back from starting_point for the first commit created by git-p4 + to find the P4 commit we are based on, and the depot-paths. + """ for parent in (range(65535)): log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent)) @@ -4103,8 +4427,9 @@ class P4Unshelve(Command): sys.exit("could not find git-p4 commits in {0}".format(self.origin)) def createShelveParent(self, change, branch_name, sync, origin): - """ Create a commit matching the parent of the shelved changelist 'change' - """ + """Create a commit matching the parent of the shelved changelist + 'change'. + """ parent_description = p4_describe(change, shelved=True) parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change) files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change) @@ -4172,10 +4497,11 @@ class P4Unshelve(Command): return True + class P4Branches(Command): def __init__(self): Command.__init__(self) - self.options = [ ] + self.options = [] self.description = ("Shows the git branches that hold imports and their " + "corresponding perforce depot paths") self.verbose = False @@ -4197,6 +4523,7 @@ class P4Branches(Command): print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])) return True + class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) @@ -4207,6 +4534,7 @@ class HelpFormatter(optparse.IndentedHelpFormatter): else: return "" + def printUsage(commands): print("usage: %s <command> [options]" % sys.argv[0]) print("") @@ -4215,16 +4543,18 @@ def printUsage(commands): print("Try %s <command> --help for command specific help." % sys.argv[0]) print("") + commands = { - "submit" : P4Submit, - "commit" : P4Submit, - "sync" : P4Sync, - "rebase" : P4Rebase, - "clone" : P4Clone, - "branches" : P4Branches, - "unshelve" : P4Unshelve, + "submit": P4Submit, + "commit": P4Submit, + "sync": P4Sync, + "rebase": P4Rebase, + "clone": P4Clone, + "branches": P4Branches, + "unshelve": P4Unshelve, } + def main(): if len(sys.argv[1:]) == 0: printUsage(commands.keys()) @@ -4251,11 +4581,11 @@ def main(): parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), options, - description = cmd.description, - formatter = HelpFormatter()) + description=cmd.description, + formatter=HelpFormatter()) try: - (cmd, args) = parser.parse_args(sys.argv[2:], cmd); + cmd, args = parser.parse_args(sys.argv[2:], cmd) except: parser.print_help() raise @@ -4263,7 +4593,7 @@ def main(): global verbose verbose = cmd.verbose if cmd.needsGit: - if cmd.gitdir == None: + if cmd.gitdir is None: cmd.gitdir = os.path.abspath(".git") if not isValidGitDir(cmd.gitdir): # "rev-parse --git-dir" without arguments will try $PWD/.git @@ -4271,7 +4601,7 @@ def main(): if os.path.exists(cmd.gitdir): cdup = read_pipe(["git", "rev-parse", "--show-cdup"]).strip() if len(cdup) > 0: - chdir(cdup); + chdir(cdup) if not isValidGitDir(cmd.gitdir): if isValidGitDir(cmd.gitdir + "/.git"): |