Merge branch 'fc/remote-bzr'

* fc/remote-bzr:
  remote-bzr: avoid bad refs
  remote-bzr: convert all unicode keys to str
  remote-bzr: access branches only when needed
  remote-bzr: delay peer branch usage
  remote-bzr: iterate revisions properly
  remote-bzr: improve progress reporting
  remote-bzr: add option to specify branches
  remote-bzr: add custom method to find branches
  remote-bzr: improve author sanitazion
  remote-bzr: add support for shared repo
  remote-bzr: fix branch names
  remote-bzr: add support for bzr repos
  remote-bzr: use branch variable when appropriate
  remote-bzr: fix partially pushed merge
  remote-bzr: fixes for branch diverge
  remote-bzr: add support to push merges
  remote-bzr: always try to update the worktree
  remote-bzr: fix order of locking in CustomTree
  remote-bzr: delay blob fetching until the very end
  remote-bzr: cleanup CustomTree

2 files changed, 298 insertions, 81 deletions

diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr
index c19ed0e..0ef30f8 100755
--- a/contrib/remote-helpers/git-remote-bzr
+++ b/contrib/remote-helpers/git-remote-bzr
@@ -13,6 +13,9 @@
 # or
 # % git clone bzr::lp:myrepo
 #
+# If you want to specify which branches you want track (per repo):
+# git config remote-bzr.branches 'trunk, devel, test'
+#
 
 import sys
 
@@ -27,6 +30,7 @@ import bzrlib.generate_ids
 import bzrlib.transport
 import bzrlib.errors
 import bzrlib.ui
+import bzrlib.urlutils
 
 import sys
 import os
@@ -37,6 +41,7 @@ import atexit, shutil, hashlib, urlparse, subprocess
 
 NAME_RE = re.compile('^([^<>]+)')
 AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
+EMAIL_RE = re.compile('^([^<>]+[^ \\\t<>])?\\b(?:[ \\t<>]*?)\\b([^ \\t<>]+@[^ \\t<>]+)')
 RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
 
 def die(msg, *args):
@@ -49,6 +54,12 @@ def warn(msg, *args):
 def gittz(tz):
     return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
 
+def get_config(config):
+    cmd = ['git', 'config', '--get', config]
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    output, _ = process.communicate()
+    return output
+
 class Marks:
 
     def __init__(self, path):
@@ -84,7 +95,7 @@ class Marks:
         return self.marks[rev]
 
     def to_rev(self, mark):
-        return self.rev_marks[mark]
+        return str(self.rev_marks[mark])
 
     def next_mark(self):
         self.last_mark += 1
@@ -174,9 +185,19 @@ def fixup_user(user):
         name = m.group(1)
         mail = m.group(2).strip()
     else:
-        m = NAME_RE.match(user)
+        m = EMAIL_RE.match(user)
         if m:
-            name = m.group(1).strip()
+            name = m.group(1)
+            mail = m.group(2)
+        else:
+            m = NAME_RE.match(user)
+            if m:
+                name = m.group(1).strip()
+
+    if not name:
+        name = 'unknown'
+    if not mail:
+        mail = 'Unknown'
 
     return '%s <%s>' % (name, mail)
 
@@ -250,22 +271,28 @@ def export_files(tree, files):
 
     return final
 
-def export_branch(branch, name):
+def export_branch(repo, name):
     global prefix
 
     ref = '%s/heads/%s' % (prefix, name)
     tip = marks.get_tip(name)
 
+    branch = bzrlib.branch.Branch.open(branches[name])
     repo = branch.repository
-    repo.lock_read()
+
+    branch.lock_read()
     revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
-    count = 0
+    tip_revno = branch.revision_id_to_revno(tip)
+    last_revno, _ = branch.last_revision_info()
+    total = last_revno - tip_revno
 
-    revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
+    for revid, _, seq, _ in revs:
 
-    for revid in revs:
+        if marks.is_marked(revid):
+            continue
 
         rev = repo.get_revision(revid)
+        revno = seq[0]
 
         parents = rev.parent_ids
         time = rev.timestamp
@@ -320,12 +347,15 @@ def export_branch(branch, name):
             print "M %s :%u %s" % f
         print
 
-        count += 1
-        if (count % 100 == 0):
-            print "progress revision %s (%d/%d)" % (revid, count, len(revs))
-            print "#############################################################"
+        if len(seq) > 1:
+            # let's skip branch revisions from the progress report
+            continue
 
-    repo.unlock()
+        progress = (revno - tip_revno)
+        if (progress % 100 == 0):
+            print "progress revision %d '%s' (%d/%d)" % (revno, name, progress, total)
+
+    branch.unlock()
 
     revid = branch.last_revision()
 
@@ -347,23 +377,24 @@ def export_tag(repo, name):
 def do_import(parser):
     global dirname
 
-    branch = parser.repo
+    repo = parser.repo
     path = os.path.join(dirname, 'marks-git')
 
     print "feature done"
     if os.path.exists(path):
         print "feature import-marks=%s" % path
     print "feature export-marks=%s" % path
+    print "feature force"
     sys.stdout.flush()
 
     while parser.check('import'):
         ref = parser[1]
         if ref.startswith('refs/heads/'):
             name = ref[len('refs/heads/'):]
-            export_branch(branch, name)
+            export_branch(repo, name)
         if ref.startswith('refs/tags/'):
             name = ref[len('refs/tags/'):]
-            export_tag(branch, name)
+            export_tag(repo, name)
         parser.next()
 
     print 'done'
@@ -382,23 +413,21 @@ def parse_blob(parser):
 
 class CustomTree():
 
-    def __init__(self, repo, revid, parents, files):
+    def __init__(self, branch, revid, parents, files):
         global files_cache
 
-        self.repo = repo
-        self.revid = revid
-        self.parents = parents
         self.updates = {}
+        self.branch = branch
 
         def copy_tree(revid):
             files = files_cache[revid] = {}
-            tree = repo.repository.revision_tree(revid)
-            repo.lock_read()
+            branch.lock_read()
+            tree = branch.repository.revision_tree(revid)
             try:
                 for path, entry in tree.iter_entries_by_dir():
-                    files[path] = entry.file_id
+                    files[path] = [entry.file_id, None]
             finally:
-                repo.unlock()
+                branch.unlock()
             return files
 
         if len(parents) == 0:
@@ -411,12 +440,18 @@ class CustomTree():
                 self.base_files = copy_tree(self.base_id)
 
         self.files = files_cache[revid] = self.base_files.copy()
+        self.rev_files = {}
+
+        for path, data in self.files.iteritems():
+            fid, mark = data
+            self.rev_files[fid] = [path, mark]
 
         for path, f in files.iteritems():
-            fid = self.files.get(path, None)
+            fid, mark = self.files.get(path, [None, None])
             if not fid:
                 fid = bzrlib.generate_ids.gen_file_id(path)
             f['path'] = path
+            self.rev_files[fid] = [path, mark]
             self.updates[fid] = f
 
     def last_revision(self):
@@ -426,16 +461,16 @@ class CustomTree():
         changes = []
 
         def get_parent(dirname, basename):
-            parent_fid = self.base_files.get(dirname, None)
+            parent_fid, mark = self.base_files.get(dirname, [None, None])
             if parent_fid:
                 return parent_fid
-            parent_fid = self.files.get(dirname, None)
+            parent_fid, mark = self.files.get(dirname, [None, None])
             if parent_fid:
                 return parent_fid
             if basename == '':
                 return None
             fid = bzrlib.generate_ids.gen_file_id(path)
-            d = add_entry(fid, dirname, 'directory')
+            add_entry(fid, dirname, 'directory')
             return fid
 
         def add_entry(fid, path, kind, mode = None):
@@ -456,9 +491,8 @@ class CustomTree():
                     (None, basename),
                     (None, kind),
                     (None, executable))
-            self.files[path] = change[0]
+            self.files[path] = [change[0], None]
             changes.append(change)
-            return change
 
         def update_entry(fid, path, kind, mode = None):
             dirname, basename = os.path.split(path)
@@ -478,9 +512,8 @@ class CustomTree():
                     (None, basename),
                     (None, kind),
                     (None, executable))
-            self.files[path] = change[0]
+            self.files[path] = [change[0], None]
             changes.append(change)
-            return change
 
         def remove_entry(fid, path, kind):
             dirname, basename = os.path.split(path)
@@ -495,7 +528,6 @@ class CustomTree():
                     (None, None))
             del self.files[path]
             changes.append(change)
-            return change
 
         for fid, f in self.updates.iteritems():
             path = f['path']
@@ -509,13 +541,30 @@ class CustomTree():
             else:
                 add_entry(fid, path, 'file', f['mode'])
 
+            self.files[path][1] = f['mark']
+            self.rev_files[fid][1] = f['mark']
+
         return changes
 
+    def get_content(self, file_id):
+        path, mark = self.rev_files[file_id]
+        if mark:
+            return blob_marks[mark]
+
+        # last resort
+        tree = self.branch.repository.revision_tree(self.base_id)
+        return tree.get_file_text(file_id)
+
     def get_file_with_stat(self, file_id, path=None):
-        return (StringIO.StringIO(self.updates[file_id]['data']), None)
+        content = self.get_content(file_id)
+        return (StringIO.StringIO(content), None)
 
     def get_symlink_target(self, file_id):
-        return self.updates[file_id]['data']
+        return self.get_content(file_id)
+
+    def id2path(self, file_id):
+        path, mark = self.rev_files[file_id]
+        return path
 
 def c_style_unescape(string):
     if string[0] == string[-1] == '"':
@@ -531,8 +580,11 @@ def parse_commit(parser):
     ref = parser[1]
     parser.next()
 
-    if ref != 'refs/heads/master':
-        die("bzr doesn't support multiple branches; use 'master'")
+    if ref.startswith('refs/heads/'):
+        name = ref[len('refs/heads/'):]
+        branch = bzrlib.branch.Branch.open(branches[name])
+    else:
+        die('unknown ref')
 
     commit_mark = parser.get_mark()
     parser.next()
@@ -559,7 +611,7 @@ def parse_commit(parser):
         if parser.check('M'):
             t, m, mark_ref, path = line.split(' ', 3)
             mark = int(mark_ref[1:])
-            f = { 'mode' : m, 'data' : blob_marks[mark] }
+            f = { 'mode' : m, 'mark' : mark }
         elif parser.check('D'):
             t, path = line.split(' ')
             f = { 'deleted' : True }
@@ -568,20 +620,18 @@ def parse_commit(parser):
         path = c_style_unescape(path).decode('utf-8')
         files[path] = f
 
-    repo = parser.repo
-
     committer, date, tz = committer
-    parents = [str(mark_to_rev(p)) for p in parents]
+    parents = [mark_to_rev(p) for p in parents]
     revid = bzrlib.generate_ids.gen_revision_id(committer, date)
     props = {}
-    props['branch-nick'] = repo.nick
+    props['branch-nick'] = branch.nick
 
-    mtree = CustomTree(repo, revid, parents, files)
+    mtree = CustomTree(branch, revid, parents, files)
     changes = mtree.iter_changes()
 
-    repo.lock_write()
+    branch.lock_write()
     try:
-        builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
+        builder = branch.get_commit_builder(parents, None, date, tz, committer, props, revid)
         try:
             list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
             builder.finish_inventory()
@@ -590,7 +640,7 @@ def parse_commit(parser):
             builder.abort()
             raise
     finally:
-        repo.unlock()
+        branch.unlock()
 
     parsed_refs[ref] = revid
     marks.new_mark(revid, commit_mark)
@@ -601,9 +651,6 @@ def parse_reset(parser):
     ref = parser[1]
     parser.next()
 
-    if ref != 'refs/heads/master':
-        die("bzr doesn't support multiple branches; use 'master'")
-
     # ugh
     if parser.check('commit'):
         parse_commit(parser)
@@ -616,7 +663,7 @@ def parse_reset(parser):
     parsed_refs[ref] = mark_to_rev(from_mark)
 
 def do_export(parser):
-    global parsed_refs, dirname, peer
+    global parsed_refs, dirname
 
     parser.next()
 
@@ -634,20 +681,33 @@ def do_export(parser):
         else:
             die('unhandled export command: %s' % line)
 
-    repo = parser.repo
-
     for ref, revid in parsed_refs.iteritems():
-        if ref == 'refs/heads/master':
-            repo.generate_revision_history(revid, marks.get_tip('master'))
-            if peer:
+        if ref.startswith('refs/heads/'):
+            name = ref[len('refs/heads/'):]
+            branch = bzrlib.branch.Branch.open(branches[name])
+            branch.generate_revision_history(revid, marks.get_tip(name))
+
+            if name in peers:
+                peer = bzrlib.branch.Branch.open(peers[name])
                 try:
-                    repo.push(peer, stop_revision=revid)
+                    peer.bzrdir.push_branch(branch, revision_id=revid)
                 except bzrlib.errors.DivergedBranches:
                     print "error %s non-fast forward" % ref
                     continue
-            else:
-                wt = repo.bzrdir.open_workingtree()
+
+            try:
+                wt = branch.bzrdir.open_workingtree()
                 wt.update()
+            except bzrlib.errors.NoWorkingTree:
+                pass
+        elif ref.startswith('refs/tags/'):
+            # TODO: implement tag push
+            print "error %s pushing tags not supported" % ref
+            continue
+        else:
+            # transport-helper/fast-export bugs
+            continue
+
         print "ok %s" % ref
 
     print
@@ -673,9 +733,15 @@ def ref_is_valid(name):
 
 def do_list(parser):
     global tags
-    print "? refs/heads/%s" % 'master'
 
-    branch = parser.repo
+    master_branch = None
+
+    for name in branches:
+        if not master_branch:
+            master_branch = name
+        print "? refs/heads/%s" % name
+
+    branch = bzrlib.branch.Branch.open(branches[master_branch])
     branch.lock_read()
     for tag, revid in branch.tags.get_tag_dict().items():
         try:
@@ -687,36 +753,112 @@ def do_list(parser):
         print "? refs/tags/%s" % tag
         tags[tag] = revid
     branch.unlock()
-    print "@refs/heads/%s HEAD" % 'master'
+
+    print "@refs/heads/%s HEAD" % master_branch
     print
 
+def get_remote_branch(origin, remote_branch, name):
+    global dirname, peers
+
+    branch_path = os.path.join(dirname, 'clone', name)
+    if os.path.exists(branch_path):
+        # pull
+        d = bzrlib.bzrdir.BzrDir.open(branch_path)
+        branch = d.open_branch()
+        try:
+            branch.pull(remote_branch, [], None, False)
+        except bzrlib.errors.DivergedBranches:
+            # use remote branch for now
+            return remote_branch
+    else:
+        # clone
+        d = origin.sprout(branch_path, None,
+                hardlink=True, create_tree_if_local=False,
+                force_new_repo=False,
+                source_branch=remote_branch)
+        branch = d.open_branch()
+
+    return branch
+
+def find_branches(repo, wanted):
+    transport = repo.user_transport
+
+    for fn in transport.iter_files_recursive():
+        if not fn.endswith('.bzr/branch-format'):
+            continue
+
+        name = subdir = fn[:-len('/.bzr/branch-format')]
+        name = name if name != '' else 'master'
+        name = name.replace('/', '+')
+
+        if wanted and not name in wanted:
+            continue
+
+        try:
+            cur = transport.clone(subdir)
+            branch = bzrlib.branch.Branch.open_from_transport(cur)
+        except bzrlib.errors.NotBranchError:
+            continue
+        else:
+            yield name, branch
+
 def get_repo(url, alias):
-    global dirname, peer
+    global dirname, peer, branches
 
+    normal_url = bzrlib.urlutils.normalize_url(url)
     origin = bzrlib.bzrdir.BzrDir.open(url)
-    branch = origin.open_branch()
+    is_local = isinstance(origin.transport, bzrlib.transport.local.LocalTransport)
+
+    shared_path = os.path.join(gitdir, 'bzr')
+    try:
+        shared_dir = bzrlib.bzrdir.BzrDir.open(shared_path)
+    except bzrlib.errors.NotBranchError:
+        shared_dir = bzrlib.bzrdir.BzrDir.create(shared_path)
+    try:
+        shared_repo = shared_dir.open_repository()
+    except bzrlib.errors.NoRepositoryPresent:
+        shared_repo = shared_dir.create_repository(shared=True)
 
-    if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
+    if not is_local:
         clone_path = os.path.join(dirname, 'clone')
-        remote_branch = branch
-        if os.path.exists(clone_path):
-            # pull
-            d = bzrlib.bzrdir.BzrDir.open(clone_path)
-            branch = d.open_branch()
-            result = branch.pull(remote_branch, [], None, False)
+        if not os.path.exists(clone_path):
+            os.mkdir(clone_path)
+
+    try:
+        repo = origin.open_repository()
+    except bzrlib.errors.NoRepositoryPresent:
+        # branch
+
+        name = 'master'
+        remote_branch = origin.open_branch()
+
+        if not is_local:
+            peers[name] = remote_branch.base
+            branch = get_remote_branch(origin, remote_branch, name)
         else:
-            # clone
-            d = origin.sprout(clone_path, None,
-                    hardlink=True, create_tree_if_local=False,
-                    source_branch=remote_branch)
-            branch = d.open_branch()
-            branch.bind(remote_branch)
-
-        peer = remote_branch
+            branch = remote_branch
+
+        branches[name] = branch.base
+
+        return branch.repository
     else:
-        peer = None
+        # repository
 
-    return branch
+        wanted = get_config('remote-bzr.branches').rstrip().split(', ')
+        # stupid python
+        wanted = [e for e in wanted if e]
+
+        for name, remote_branch in find_branches(repo, wanted):
+
+            if not is_local:
+                peers[name] = remote_branch.base
+                branch = get_remote_branch(origin, remote_branch, name)
+            else:
+                branch = remote_branch
+
+            branches[name] = branch.base
+
+        return repo
 
 def fix_path(alias, orig_url):
     url = urlparse.urlparse(orig_url, 'file')
@@ -727,12 +869,13 @@ def fix_path(alias, orig_url):
     subprocess.call(cmd)
 
 def main(args):
-    global marks, prefix, dirname
+    global marks, prefix, gitdir, dirname
     global tags, filenodes
     global blob_marks
     global parsed_refs
     global files_cache
     global is_tmp
+    global branches, peers
 
     alias = args[1]
     url = args[2]
@@ -743,6 +886,8 @@ def main(args):
     parsed_refs = {}
     files_cache = {}
     marks = None
+    branches = {}
+    peers = {}
 
     if alias[5:] == url:
         is_tmp = True
diff --git a/contrib/remote-helpers/test-bzr.sh b/contrib/remote-helpers/test-bzr.sh
index 34666e1..cec55f1 100755
--- a/contrib/remote-helpers/test-bzr.sh
+++ b/contrib/remote-helpers/test-bzr.sh
@@ -228,4 +228,76 @@ test_expect_success 'push utf-8 filenames' '
   test_cmp expected actual
 '
 
+test_expect_success 'pushing a merge' '
+  mkdir -p tmp && cd tmp &&
+  test_when_finished "cd .. && rm -rf tmp" &&
+
+  (
+  bzr init bzrrepo &&
+  cd bzrrepo &&
+  echo one > content &&
+  bzr add content &&
+  bzr commit -m one
+  ) &&
+
+  git clone "bzr::$PWD/bzrrepo" gitrepo &&
+
+  (
+  cd bzrrepo &&
+  echo two > content &&
+  bzr commit -m two
+  ) &&
+
+  (
+  cd gitrepo &&
+  echo three > content &&
+  git commit -a -m three &&
+  git fetch &&
+  git merge origin/master || true &&
+  echo three > content &&
+  git commit -a --no-edit &&
+  git push
+  ) &&
+
+  echo three > expected &&
+  cat bzrrepo/content > actual &&
+  test_cmp expected actual
+'
+
+cat > expected <<EOF
+origin/HEAD
+origin/branch
+origin/trunk
+EOF
+
+test_expect_success 'proper bzr repo' '
+  mkdir -p tmp && cd tmp &&
+  test_when_finished "cd .. && rm -rf tmp" &&
+
+  bzr init-repo bzrrepo &&
+
+  bzr init bzrrepo/trunk &&
+  (
+  cd bzrrepo/trunk &&
+  echo one >> content &&
+  bzr add content &&
+  bzr commit -m one
+  ) &&
+
+  bzr branch bzrrepo/trunk bzrrepo/branch &&
+  (
+  cd bzrrepo/branch &&
+  echo two >> content &&
+  bzr commit -m one
+  ) &&
+
+  git clone "bzr::$PWD/bzrrepo" gitrepo &&
+  (
+  cd gitrepo &&
+  git for-each-ref --format "%(refname:short)" refs/remotes/origin > ../actual
+  ) &&
+
+  test_cmp ../expected actual
+'
+
 test_done