Merge lp:~jelmer/brz-git/submodules into lp:brz-git

[Jelmer Vernooij]

Add basic support for submodules.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz-git-dev/183/

[lp:~jelmer/brz-git/submodules]

1918. By Jelmer Vernooij on 2018-05-05

Fix remaining test.

=== modified file 'branch.py'
--- branch.py	2018-04-02 15:43:48 +0000
+++ branch.py	2018-05-05 18:49:25 +0000
@@ -699,6 +699,20 @@
         from .memorytree import GitMemoryTree
         return GitMemoryTree(self, self.repository._git.object_store, self.head)
 
+    def reference_parent(self, path, file_id=None, possible_transports=None):
+        """Return the parent branch for a tree-reference file_id
+
+        :param path: The path of the file_id in the tree
+        :param file_id: Optional file_id of the tree reference
+        :return: A branch associated with the file_id
+        """
+        # FIXME should provide multiple branches, based on config
+        url = urlutils.join(self.user_url, path)
+        return branch.Branch.open(
+                url,
+                possible_transports=possible_transports)
+
+
 
 def _quick_lookup_revno(local_branch, remote_branch, revid):
     if type(revid) is not str:

=== modified file 'commit.py'
--- commit.py	2018-03-27 03:09:48 +0000
+++ commit.py	2018-05-05 18:49:25 +0000
@@ -45,6 +45,7 @@
     Blob,
     Commit,
     )
+from dulwich.index import read_submodule_head
 from dulwich.repo import Repo
 
 
@@ -85,8 +86,6 @@
         return self._any_changes
 
     def record_iter_changes(self, workingtree, basis_revid, iter_changes):
-        def treeref_sha1(path, file_id):
-            return Repo.open(os.path.join(workingtree.basedir, path)).head()
         seen_root = False
         for (file_id, path, changed_content, versioned, parent, name, kind,
              executable) in iter_changes:
@@ -129,7 +128,7 @@
                 entry.symlink_target = symlink_target
                 st = None
             elif kind[1] == "tree-reference":
-                sha = treeref_sha1(path[1], file_id)
+                sha = read_submodule_head(workingtree.abspath(path[1]))
                 reference_revision = workingtree.get_reference_revision(path[1], file_id)
                 entry.reference_revision = reference_revision
                 st = None

=== modified file 'mapping.py'
--- mapping.py	2018-04-07 15:40:32 +0000
+++ mapping.py	2018-05-05 18:49:25 +0000
@@ -599,7 +599,7 @@
 
 def entry_mode(entry):
     """Determine the git file mode for an inventory entry."""
-    return object_mode(entry.kind, entry.executable)
+    return object_mode(entry.kind, getattr(entry, 'executable', False))
 
 
 def extract_unusual_modes(rev):

=== modified file 'repository.py'
--- repository.py	2018-04-08 17:06:15 +0000
+++ repository.py	2018-05-05 18:49:25 +0000
@@ -642,7 +642,7 @@
     """Git repository format."""
 
     supports_versioned_directories = False
-    supports_tree_reference = False
+    supports_tree_reference = True
     rich_root_data = True
     supports_leaving_lock = False
     fast_deltas = True

=== modified file 'tree.py'
--- tree.py	2018-04-02 21:10:21 +0000
+++ tree.py	2018-05-05 18:49:25 +0000
@@ -34,11 +34,14 @@
     Blob,
     Tree,
     ZERO_SHA,
+    S_IFGITLINK,
+    S_ISGITLINK,
     )
 import stat
 import posixpath
 
 from ... import (
+    controldir as _mod_controldir,
     delta,
     errors,
     lock,
@@ -176,10 +179,43 @@
                 self.symlink_target)
 
 
+class GitTreeSubmodule(_mod_tree.TreeLink):
+
+    __slots__ = ['file_id', 'name', 'parent_id', 'reference_revision']
+
+    def __init__(self, file_id, name, parent_id, reference_revision=None):
+        self.file_id = file_id
+        self.name = name
+        self.parent_id = parent_id
+        self.reference_revision = reference_revision
+
+    @property
+    def kind(self):
+        return 'tree-reference'
+
+    def __repr__(self):
+        return "%s(file_id=%r, name=%r, parent_id=%r, reference_revision=%r)" % (
+            type(self).__name__, self.file_id, self.name, self.parent_id,
+            self.reference_revision)
+
+    def __eq__(self, other):
+        return (self.kind == other.kind and
+                self.file_id == other.file_id and
+                self.name == other.name and
+                self.parent_id == other.parent_id and
+                self.reference_revision == other.reference_revision)
+
+    def copy(self):
+        return self.__class__(
+                self.file_id, self.name, self.parent_id,
+                self.reference_revision)
+
+
 entry_factory = {
     'directory': GitTreeDirectory,
     'file': GitTreeFile,
     'symlink': GitTreeSymlink,
+    'tree-reference': GitTreeSubmodule,
     }
 
 
@@ -223,6 +259,11 @@
             self.tree = commit.tree
             self._fileid_map = self.mapping.get_fileid_map(self.store.__getitem__, self.tree)
 
+    def _get_nested_repository(self, path):
+        nested_repo_transport = self._repository.user_transport.clone(path)
+        nested_controldir = _mod_controldir.ControlDir.open_from_transport(nested_repo_transport)
+        return nested_controldir.find_repository()
+
     def supports_rename_tracking(self):
         return False
 
@@ -470,6 +511,15 @@
         else:
             return None
 
+    def get_reference_revision(self, path, file_id=None):
+        """See RevisionTree.get_symlink_target."""
+        (store, mode, hexsha) = self._lookup_path(path)
+        if S_ISGITLINK(mode):
+            nested_repo = self._get_nested_repository(path)
+            return nested_repo.lookup_foreign_revision_id(hexsha)
+        else:
+            return None
+
     def _comparison_data(self, entry, path):
         if entry is None:
             return None, False, None
@@ -488,6 +538,10 @@
             return (kind, len(contents), executable, osutils.sha_string(contents))
         elif kind == 'symlink':
             return (kind, None, None, store[hexsha].data)
+        elif kind == 'tree-reference':
+            nested_repo = self._get_nested_repository(path)
+            return (kind, None, None,
+                    nested_repo.lookup_foreign_revision_id(hexsha))
         else:
             return (kind, None, None, None)
 
@@ -746,7 +800,8 @@
     def is_versioned(self, path):
         with self.lock_read():
             path = path.rstrip('/').encode('utf-8')
-            return (path in self.index or self._has_dir(path))
+            (index, subpath) = self._lookup_index(path)
+            return (subpath in index or self._has_dir(path))
 
     def _has_dir(self, path):
         if path == "":
@@ -759,7 +814,8 @@
         if self._lock_mode is None:
             raise errors.ObjectNotLocked(self)
         self._versioned_dirs = set()
-        for p in self.index:
+        # TODO(jelmer): Browse over all indexes
+        for p, i in self._recurse_index_entries():
             self._ensure_versioned_dir(posixpath.dirname(p))
 
     def _ensure_versioned_dir(self, dirname):
@@ -814,11 +870,21 @@
                 raise errors.InvalidNormalization(path)
             self._index_add_entry(path, kind)
 
-    def _index_del_entry(self, path):
-        del self.index[path]
+    def _read_submodule_head(self, path):
+        raise NotImplementedError(self._read_submodule_head)
+
+    def _lookup_index(self, encoded_path):
+        if not isinstance(encoded_path, bytes):
+            raise TypeError(encoded_path)
+        # TODO(jelmer): Look in other indexes
+        return self.index, encoded_path
+
+    def _index_del_entry(self, index, path):
+        del index[path]
+        # TODO(jelmer): Keep track of dirty per index
         self._index_dirty = True
 
-    def _index_add_entry(self, path, kind, flags=0):
+    def _index_add_entry(self, path, kind, flags=0, reference_revision=None):
         if not isinstance(path, basestring):
             raise TypeError(path)
         if kind == "directory":
@@ -834,22 +900,40 @@
                 stat_val = os.stat_result(
                     (stat.S_IFREG | 0644, 0, 0, 0, 0, 0, 0, 0, 0, 0))
             blob.set_raw_string(file.read())
+            # Add object to the repository if it didn't exist yet
+            if not blob.id in self.store:
+                self.store.add_object(blob)
+            hexsha = blob.id
         elif kind == "symlink":
             blob = Blob()
             try:
                 stat_val = self._lstat(path)
-            except (errors.NoSuchFile, OSError):
+            except EnvironmentError:
                 # TODO: Rather than come up with something here, use the
                 # old index
                 stat_val = os.stat_result(
                     (stat.S_IFLNK, 0, 0, 0, 0, 0, 0, 0, 0, 0))
             blob.set_raw_string(
                 self.get_symlink_target(path).encode("utf-8"))
+            # Add object to the repository if it didn't exist yet
+            if not blob.id in self.store:
+                self.store.add_object(blob)
+            hexsha = blob.id
+        elif kind == "tree-reference":
+            if reference_revision is not None:
+                hexsha = self.branch.lookup_bzr_revision_id(reference_revision)[0]
+            else:
+                hexsha = self._read_submodule_head(path)
+                if hexsha is None:
+                    raise errors.NoCommits(path)
+            try:
+                stat_val = self._lstat(path)
+            except EnvironmentError:
+                stat_val = os.stat_result(
+                    (S_IFGITLINK, 0, 0, 0, 0, 0, 0, 0, 0, 0))
+            stat_val = os.stat_result((S_IFGITLINK, ) + stat_val[1:])
         else:
             raise AssertionError("unknown kind '%s'" % kind)
-        # Add object to the repository if it didn't exist yet
-        if not blob.id in self.store:
-            self.store.add_object(blob)
         # Add an entry to the index or update the existing entry
         ensure_normalized_path(path)
         encoded_path = path.encode("utf-8")
@@ -857,11 +941,23 @@
             # TODO(jelmer): Why do we need to do this?
             trace.mutter('ignoring path with invalid newline in it: %r', path)
             return
+        (index, index_path) = self._lookup_index(encoded_path)
+        index[index_path] = index_entry_from_stat(stat_val, hexsha, flags)
         self._index_dirty = True
-        self.index[encoded_path] = index_entry_from_stat(
-            stat_val, blob.id, flags)
         if self._versioned_dirs is not None:
-            self._ensure_versioned_dir(encoded_path)
+            self._ensure_versioned_dir(index_path)
+
+    def _recurse_index_entries(self, index=None, basepath=""):
+        # Iterate over all index entries
+        with self.lock_read():
+            if index is None:
+                index = self.index
+            for path, value in index.iteritems():
+                yield (posixpath.join(basepath, path), value)
+                (ctime, mtime, dev, ino, mode, uid, gid, size, sha, flags) = value
+                if S_ISGITLINK(mode):
+                    pass # TODO(jelmer): dive into submodule
+
 
     def iter_entries_by_dir(self, specific_files=None, yield_parents=False):
         if yield_parents:
@@ -876,7 +972,7 @@
             if specific_files is None or u"" in specific_files:
                 ret[(None, u"")] = root_ie
             dir_ids = {u"": root_ie.file_id}
-            for path, value in self.index.iteritems():
+            for path, value in self._recurse_index_entries():
                 if self.mapping.is_special_file(path):
                     continue
                 path = path.decode("utf-8")
@@ -895,6 +991,11 @@
                 ret[(posixpath.dirname(path), path)] = file_ie
             return ((path, ie) for ((_, path), ie) in sorted(ret.items()))
 
+    def iter_references(self):
+        # TODO(jelmer): Implement a more efficient version of this
+        for path, entry in self.iter_entries_by_dir():
+            if entry.kind == 'tree-reference':
+                yield path, self.mapping.generate_file_id(b'')
 
     def _get_dir_ie(self, path, parent_id):
         file_id = self.path2id(path)
@@ -916,6 +1017,8 @@
         ie = entry_factory[kind](file_id, name, parent_id)
         if kind == 'symlink':
             ie.symlink_target = self.get_symlink_target(path, file_id)
+        elif kind == 'tree-reference':
+            ie.reference_revision = self.get_reference_revision(path, file_id)
         else:
             try:
                 data = self.get_file_text(path, file_id)
@@ -953,14 +1056,16 @@
             raise errors.ObjectNotLocked(self)
         encoded_path = path.encode("utf-8")
         count = 0
+        (index, subpath) = self._lookup_index(encoded_path)
         try:
-            self._index_del_entry(encoded_path)
+            self._index_del_entry(index, encoded_path)
         except KeyError:
             # A directory, perhaps?
-            for p in list(self.index):
-                if p.startswith(encoded_path+b"/"):
+            # TODO(jelmer): Deletes that involve submodules?
+            for p in list(index):
+                if p.startswith(subpath+b"/"):
                     count += 1
-                    self._index_del_entry(p)
+                    self._index_del_entry(index, p)
         else:
             count = 1
         self._versioned_dirs = None
@@ -980,9 +1085,11 @@
     def update_basis_by_delta(self, revid, delta):
         # TODO(jelmer): This shouldn't be called, it's inventory specific.
         for (old_path, new_path, file_id, ie) in delta:
-            if old_path is not None and old_path.encode('utf-8') in self.index:
-                self._index_del_entry(old_path.encode('utf-8'))
-                self._versioned_dirs = None
+            if old_path is not None:
+                (index, old_subpath) = self._lookup_index(old_path.encode('utf-8'))
+                if old_subpath in index:
+                    self._index_del_entry(index, old_subpath)
+                    self._versioned_dirs = None
             if new_path is not None and ie.kind != 'directory':
                 self._index_add_entry(new_path, ie.kind)
         self.flush()
@@ -1050,10 +1157,12 @@
 
                 kind = self.kind(from_rel)
 
-            if not after and not from_path in self.index and kind != 'directory':
-                # It's not a file
-                raise errors.BzrMoveFailedError(from_rel, to_rel,
-                    errors.NotVersionedError(path=from_rel))
+            if not after and kind != 'directory':
+                (index, from_subpath) = self._lookup_index(from_path)
+                if from_subpath not in index:
+                    # It's not a file
+                    raise errors.BzrMoveFailedError(from_rel, to_rel,
+                        errors.NotVersionedError(path=from_rel))
 
             if not after:
                 try:
@@ -1064,17 +1173,21 @@
                             errors.NoSuchFile(to_rel))
                     raise
             if kind != 'directory':
+                (index, from_index_path) = self._lookup_index(from_path)
                 try:
-                    self._index_del_entry(from_path)
+                    self._index_del_entry(index, from_path)
                 except KeyError:
                     pass
                 self._index_add_entry(to_rel, kind)
             else:
-                todo = [p for p in self.index if p.startswith(from_path+'/')]
-                for p in todo:
+                todo = [(p, i) for (p, i) in self._recurse_index_entries() if p.startswith(from_path+'/')]
+                for child_path, child_value in todo:
+                    (child_to_index, child_to_index_path) = self._lookup_index(posixpath.join(to_path, posixpath.relpath(child_path, from_path)))
+                    child_to_index[child_to_index_path] = child_value
+                    # TODO(jelmer): Mark individual index as dirty
                     self._index_dirty = True
-                    self.index[posixpath.join(to_path, posixpath.relpath(p, from_path))] = self.index[p]
-                    self._index_del_entry(p)
+                    (child_from_index, child_from_index_path) = self._lookup_index(child_path)
+                    self._index_del_entry(child_from_index, child_from_index_path)
 
             self._versioned_dirs = None
             self.flush()
@@ -1097,3 +1210,43 @@
                 raise errors.PathsNotVersionedError(unversioned)
 
         return filter(self.is_versioned, paths)
+
+    def path_content_summary(self, path):
+        """See Tree.path_content_summary."""
+        try:
+            stat_result = self._lstat(path)
+        except OSError as e:
+            if getattr(e, 'errno', None) == errno.ENOENT:
+                # no file.
+                return ('missing', None, None, None)
+            # propagate other errors
+            raise
+        kind = mode_kind(stat_result.st_mode)
+        if kind == 'file':
+            return self._file_content_summary(path, stat_result)
+        elif kind == 'directory':
+            # perhaps it looks like a plain directory, but it's really a
+            # reference.
+            if self._directory_is_tree_reference(path):
+                kind = 'tree-reference'
+            return kind, None, None, None
+        elif kind == 'symlink':
+            target = osutils.readlink(self.abspath(path))
+            return ('symlink', None, None, target)
+        else:
+            return (kind, None, None, None)
+
+    def kind(self, relpath, file_id=None):
+        kind = osutils.file_kind(self.abspath(relpath))
+        if kind == 'directory':
+            (index, index_path) = self._lookup_index(relpath.encode('utf-8'))
+            try:
+                mode = index[index_path].mode
+            except KeyError:
+                return kind
+            else:
+                if S_ISGITLINK(mode):
+                    return 'tree-reference'
+                return 'directory'
+        else:
+            return kind

=== modified file 'workingtree.py'
--- workingtree.py	2018-05-05 16:39:36 +0000
+++ workingtree.py	2018-05-05 18:49:25 +0000
@@ -25,7 +25,6 @@
     )
 from collections import defaultdict
 import errno
-from dulwich.errors import NotGitRepository
 from dulwich.ignore import (
     IgnoreFilterManager,
     )
@@ -42,6 +41,7 @@
     iter_fresh_entries,
     blob_from_path_and_stat,
     FLAG_STAGEMASK,
+    read_submodule_head,
     validate_path,
     write_index_dict,
     )
@@ -55,7 +55,10 @@
     S_ISGITLINK,
     ZERO_SHA,
     )
-from dulwich.repo import Repo
+from dulwich.repo import (
+    NotGitRepository,
+    Repo as GitRepo,
+    )
 import os
 import posixpath
 import re
@@ -84,6 +87,7 @@
     inventory,
     )
 from ...mutabletree import (
+    BadReferenceTarget,
     MutableTree,
     )
 
@@ -127,7 +131,7 @@
         self._reset_data()
 
     def supports_tree_reference(self):
-        return False
+        return True
 
     def supports_rename_tracking(self):
         return False
@@ -455,7 +459,8 @@
                 abspath = self.abspath(filepath)
                 kind = osutils.file_kind(abspath)
                 if kind in ("file", "symlink"):
-                    if filepath in self.index:
+                    (index, subpath) = self._lookup_index(filepath.encode('utf-8'))
+                    if subpath in index:
                         # Already present
                         continue
                     call_action(filepath, kind)
@@ -463,7 +468,8 @@
                         self._index_add_entry(filepath, kind)
                     added.append(filepath)
                 elif kind == "directory":
-                    if filepath not in self.index:
+                    (index, subpath) = self._lookup_index(filepath.encode('utf-8'))
+                    if subpath not in index:
                         call_action(filepath, kind)
                     if recurse:
                         user_dirs.append(filepath)
@@ -546,10 +552,27 @@
         """Yield all unversioned files in this WorkingTree.
         """
         with self.lock_read():
-            for p in (set(self._iter_files_recursive(include_dirs=True)) - set([p.decode('utf-8') for p in self.index])):
+            index_paths = set([p.decode('utf-8') for p, i in self._recurse_index_entries()])
+            all_paths = set(self._iter_files_recursive(include_dirs=True))
+            for p in (all_paths - index_paths):
                 if not self._has_dir(p):
                     yield p
 
+    def _gather_kinds(self, files, kinds):
+        """See MutableTree._gather_kinds."""
+        with self.lock_tree_write():
+            for pos, f in enumerate(files):
+                if kinds[pos] is None:
+                    fullpath = osutils.normpath(self.abspath(f))
+                    try:
+                         kind = osutils.file_kind(fullpath)
+                    except OSError as e:
+                        if e.errno == errno.ENOENT:
+                            raise errors.NoSuchFile(fullpath)
+                    if kind == 'directory' and f != '' and os.path.exists(os.path.join(fullpath, '.git')):
+                        kind = 'tree-reference'
+                    kinds[pos] = kind
+
     def flush(self):
         if self._lock_mode != 'w':
             raise errors.NotWriteLocked(self)
@@ -665,8 +688,9 @@
 
     def get_file_verifier(self, path, file_id=None, stat_value=None):
         with self.lock_read():
+            (index, subpath) = self._lookup_index(path.encode('utf-8'))
             try:
-                return ("GIT", self.index[path.encode('utf-8')].sha)
+                return ("GIT", index[subpath].sha)
             except KeyError:
                 if self._has_dir(path):
                     return ("GIT", None)
@@ -699,8 +723,9 @@
 
     def stored_kind(self, path, file_id=None):
         with self.lock_read():
+            (index, subpath) = self._lookup_index(path.encode('utf-8'))
             try:
-                return mode_kind(self.index[path.encode("utf-8")].mode)
+                return mode_kind(index[subpath].mode)
             except KeyError:
                 # Maybe it's a directory?
                 if self._has_dir(path):
@@ -715,8 +740,9 @@
             if getattr(self, "_supports_executable", osutils.supports_executable)():
                 mode = self._lstat(path).st_mode
             else:
+                (index, subpath) = self._lookup_index(path.encode('utf-8'))
                 try:
-                    mode = self.index[path.encode('utf-8')].mode
+                    mode = index[subpath].mode
                 except KeyError:
                     mode = 0
             return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
@@ -733,7 +759,8 @@
         dir_ids = {}
         fk_entries = {'directory': tree.TreeDirectory,
                       'file': tree.TreeFile,
-                      'symlink': tree.TreeLink}
+                      'symlink': tree.TreeLink,
+                      'tree-reference': tree.TreeReference}
         with self.lock_read():
             root_ie = self._get_dir_ie(u"", None)
             if include_root and not from_dir:
@@ -747,19 +774,20 @@
                     and not self.mapping.is_special_file(name)])
             for path in path_iterator:
                 try:
-                    index_path = path.encode("utf-8")
+                    encoded_path = path.encode("utf-8")
                 except UnicodeEncodeError:
                     raise errors.BadFilenameEncoding(
                         path, osutils._fs_enc)
+                (index, index_path) = self._lookup_index(encoded_path)
                 try:
-                    value = self.index[index_path]
+                    value = index[index_path]
                 except KeyError:
                     value = None
-                kind = osutils.file_kind(self.abspath(path))
+                kind = self.kind(path)
                 parent, name = posixpath.split(path)
                 for dir_path, dir_ie in self._add_missing_parent_ids(parent, dir_ids):
                     pass
-                if kind == 'directory':
+                if kind in ('directory', 'tree-reference'):
                     if path != from_dir:
                         if self._has_dir(path):
                             ie = self._get_dir_ie(path, self.path2id(path))
@@ -810,10 +838,6 @@
                     paths.add(path)
             return paths
 
-    def _directory_is_tree_reference(self, path):
-        # FIXME: Check .gitsubmodules for path
-        return False
-
     def iter_child_entries(self, path, file_id=None):
         encoded_path = path.encode('utf-8')
         with self.lock_read():
@@ -1040,14 +1064,21 @@
     def apply_inventory_delta(self, changes):
         for (old_path, new_path, file_id, ie) in changes:
             if old_path is not None:
+                (index, old_subpath) = self._lookup_index(old_path.encode('utf-8'))
                 try:
-                    self._index_del_entry(old_path.encode('utf-8'))
+                    self._index_del_entry(index, old_subpath)
                 except KeyError:
                     pass
                 else:
                     self._versioned_dirs = None
             if new_path is not None and ie.kind != 'directory':
-                self._index_add_entry(new_path, ie.kind)
+                if ie.kind == 'tree-reference':
+                    self._index_add_entry(
+                            new_path, ie.kind,
+                            reference_revision=ie.reference_revision)
+                else:
+                    self._index_add_entry(new_path, ie.kind)
+        self.flush()
 
     def annotate_iter(self, path, file_id=None,
                       default_revision=_mod_revision.CURRENT_REVISION):
@@ -1142,7 +1173,8 @@
                             st = os.stat_result((entry.mode, 0, 0, 0,
                                   0, 0, len(obj.as_raw_string()), 0,
                                   0, 0))
-                    self.index[entry.path] = index_entry_from_stat(st, entry.sha, 0)
+                    (index, subpath) = self._lookup_index(entry.path)
+                    index[subpath] = index_entry_from_stat(st, entry.sha, 0)
 
     def pull(self, source, overwrite=False, stop_revision=None,
              change_reporter=None, possible_transports=None, local=False,
@@ -1166,6 +1198,77 @@
                                 show_base=show_base)
             return count
 
+    def add_reference(self, sub_tree):
+        """Add a TreeReference to the tree, pointing at sub_tree.
+
+        :param sub_tree: subtree to add.
+        """
+        with self.lock_tree_write():
+            try:
+                sub_tree_path = self.relpath(sub_tree.basedir)
+            except errors.PathNotChild:
+                raise BadReferenceTarget(
+                        self, sub_tree, 'Target not inside tree.')
+
+            self._add([sub_tree_path], [None], ['tree-reference'])
+
+    def _read_submodule_head(self, path):
+        return read_submodule_head(self.abspath(path))
+
+    def get_reference_revision(self, path, file_id=None):
+        hexsha = self._read_submodule_head(path)
+        if hexsha is None:
+            return _mod_revision.NULL_REVISION
+        return self.branch.lookup_foreign_revision_id(hexsha)
+
+    def get_nested_tree(self, path, file_id=None):
+        return workingtree.WorkingTree.open(self.abspath(path))
+
+    def _directory_is_tree_reference(self, relpath):
+        # as a special case, if a directory contains control files then
+        # it's a tree reference, except that the root of the tree is not
+        return relpath and osutils.lexists(self.abspath(relpath) + u"/.git")
+
+    def extract(self, sub_path, file_id=None, format=None):
+        """Extract a subtree from this tree.
+
+        A new branch will be created, relative to the path for this tree.
+        """
+        def mkdirs(path):
+            segments = osutils.splitpath(path)
+            transport = self.branch.controldir.root_transport
+            for name in segments:
+                transport = transport.clone(name)
+                transport.ensure_base()
+            return transport
+
+        with self.lock_tree_write():
+            self.flush()
+            branch_transport = mkdirs(sub_path)
+            if format is None:
+                format = self.controldir.cloning_metadir()
+            branch_transport.ensure_base()
+            branch_bzrdir = format.initialize_on_transport(branch_transport)
+            try:
+                repo = branch_bzrdir.find_repository()
+            except errors.NoRepositoryPresent:
+                repo = branch_bzrdir.create_repository()
+            if not repo.supports_rich_root():
+                raise errors.RootNotRich()
+            new_branch = branch_bzrdir.create_branch()
+            new_branch.pull(self.branch)
+            for parent_id in self.get_parent_ids():
+                new_branch.fetch(self.branch, parent_id)
+            tree_transport = self.controldir.root_transport.clone(sub_path)
+            if tree_transport.base != branch_transport.base:
+                tree_bzrdir = format.initialize_on_transport(tree_transport)
+                tree_bzrdir.set_branch_reference(new_branch)
+            else:
+                tree_bzrdir = branch_bzrdir
+            wt = tree_bzrdir.create_workingtree(_mod_revision.NULL_REVISION)
+            wt.set_parent_ids(self.get_parent_ids())
+            return wt
+
     def _get_check_refs(self):
         """Return the references needed to perform a check of this tree.
 
@@ -1284,7 +1387,7 @@
     # replaced with non-empty directories if they have contents.
     dirified = []
     target_root_path = target.abspath('.').encode(sys.getfilesystemencoding())
-    for path, index_entry in target.index.iteritems():
+    for path, index_entry in target._recurse_index_entries():
         try:
             live_entry = index_entry_from_path(
                     target.abspath(path.decode('utf-8')).encode(osutils._fs_enc))