Merge lp:~a-s-usov/python-fastimport/partial-streams-filtering into lp:~registry/python-fastimport/old

[Alex Usov]

Adds preserve_all_history flag to filter_processor.
 - prevents it from squashing empty commits
 - prevents it from rewriting/removing 'from' & 'merge' references

Primary usage -- being able to filter incremental stream.

[Jelmer Vernooij (jelmer)]

Hi,

Thanks for working on this.

I think "preserve_all_history" is a confusing name for this new behaviour. You're still changing history, just not removing commits.

Perhaps a better would be "squash_empty_commits", which could default to true?

Can you please also add a test that uses excludes/includes and filters? I suspect that if there are blobs that are removed, this will change the allocation of new ids, and might break the rewriting of from lines.

+ * preserve_all_history - if True filter processeor will be much more conservative
s/processeor/processor/

[Jelmer Vernooij (jelmer)]

review: Needs Fixing

[Alex Usov (a-s-usov)]

Will resubmit patch soon.

Regarding allocation of new id's, my understanding it that filter_processor doesn't generates mark id's but uses whatever is present in the stream.

[lp:~a-s-usov/python-fastimport/partial-streams-filtering]

304. By Oleksandr Usov <email address hidden> on 2011-10-21

Rename flag to squash_empty_commits and add few more tests with includes/excludes

Unmerged revisions

=== modified file 'fastimport/processors/filter_processor.py'
--- fastimport/processors/filter_processor.py	2011-10-06 00:24:50 +0000
+++ fastimport/processors/filter_processor.py	2011-10-21 09:38:25 +0000
@@ -36,16 +36,23 @@
 
     * exclude_paths - a list of paths that should not appear in the output
       stream
+
+    * squash_empty_commits - if set to False, filter processor will be much more
+      conservative w.r.t. history handling -- it will preserve all commits and
+      links between them, including those to unknown revisions. This is primarily
+      usefull for filtering incremental streams
     """
 
     known_params = [
         'include_paths',
         'exclude_paths',
+        'squash_empty_commits'
         ]
 
     def pre_process(self):
         self.includes = self.params.get('include_paths')
         self.excludes = self.params.get('exclude_paths')
+        self.squash_empty_commits = bool(self.params.get('squash_empty_commits', True))
         # What's the new root, if any
         self.new_root = helpers.common_directory(self.includes)
         # Buffer of blobs until we know we need them: mark -> cmd
@@ -91,7 +98,7 @@
         """Process a CommitCommand."""
         # These pass through if they meet the filtering conditions
         interesting_filecmds = self._filter_filecommands(cmd.iter_files)
-        if interesting_filecmds:
+        if interesting_filecmds or not self.squash_empty_commits:
             # If all we have is a single deleteall, skip this commit
             if len(interesting_filecmds) == 1 and isinstance(
                 interesting_filecmds[0], commands.FileDeleteAllCommand):
@@ -109,8 +116,9 @@
                             self.referenced_blobs.append(fc.dataref)
 
                 # Update from and merges to refer to commits in the output
-                cmd.from_ = self._find_interesting_from(cmd.from_)
-                cmd.merges = self._find_interesting_merges(cmd.merges)
+                if self.squash_empty_commits:
+                    cmd.from_ = self._find_interesting_from(cmd.from_)
+                    cmd.merges = self._find_interesting_merges(cmd.merges)
                 self.interesting_commits.add(cmd.id)
 
         # Keep track of the parents

=== modified file 'fastimport/tests/test_filter_processor.py'
--- fastimport/tests/test_filter_processor.py	2011-10-06 00:24:50 +0000
+++ fastimport/tests/test_filter_processor.py	2011-10-21 09:38:25 +0000
@@ -102,7 +102,6 @@
 M 644 :4 doc/index.txt
 """
 
-
 class TestCaseWithFiltering(TestCase):
 
     def assertFiltering(self, input, params, expected):
@@ -116,7 +115,6 @@
         out = outf.getvalue()
         self.assertEquals(expected, out)
 
-
 class TestNoFiltering(TestCaseWithFiltering):
 
     def test_params_not_given(self):
@@ -876,3 +874,232 @@
 reset refs/heads/bar
 from :101
 """)
+
+
+# A sample input stream containing empty commit
+_SAMPLE_EMPTY_COMMIT = \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :2
+committer Joe <[e-mail]> 1234567890 +1000
+data 14
+Initial import
+M 644 :1 COPYING
+commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+empty commit
+"""
+
+# A sample input stream containing unresolved from and merge references
+_SAMPLE_FROM_MERGE_COMMIT = \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+blob
+mark :2
+data 4
+bar
+commit refs/heads/master
+mark :4
+committer Joe <[e-mail]> 1234567890 +1000
+data 19
+unknown from commit
+from :999
+M 644 :2 data/DATA
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+merge :1001
+M 644 :99 data/DATA2
+"""
+
+class TestSquashEmptyCommitsFlag(TestCaseWithFiltering):
+    
+    def test_squash_empty_commit(self):
+        params = {'include_paths': None, 'exclude_paths': None}
+        self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :2
+committer Joe <[e-mail]> 1234567890 +1000
+data 14
+Initial import
+M 644 :1 COPYING
+""")
+        
+    def test_keep_empty_commit(self):
+        params = {'include_paths': None, 'exclude_paths': None, 'squash_empty_commits': False}
+        self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, _SAMPLE_EMPTY_COMMIT)
+
+    def test_squash_unresolved_references(self):
+        params = {'include_paths': None, 'exclude_paths': None}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+blob
+mark :2
+data 4
+bar
+commit refs/heads/master
+mark :4
+committer Joe <[e-mail]> 1234567890 +1000
+data 19
+unknown from commit
+M 644 :2 data/DATA
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+M 644 :99 data/DATA2
+""")
+        
+    def test_keep_unresolved_from_and_merge(self):
+        params = {'include_paths': None, 'exclude_paths': None, 'squash_empty_commits': False}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, _SAMPLE_FROM_MERGE_COMMIT)
+
+    def test_with_excludes(self):
+        params = {'include_paths': None,
+                  'exclude_paths': ['data/DATA'],
+                  'squash_empty_commits': False}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+commit refs/heads/master
+mark :4
+committer Joe <[e-mail]> 1234567890 +1000
+data 19
+unknown from commit
+from :999
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+merge :1001
+M 644 :99 data/DATA2
+""")
+
+    def test_with_file_includes(self):
+        params = {'include_paths': ['COPYING', 'data/DATA2'],
+                  'exclude_paths': None,
+                  'squash_empty_commits': False}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+commit refs/heads/master
+mark :4
+committer Joe <[e-mail]> 1234567890 +1000
+data 19
+unknown from commit
+from :999
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+merge :1001
+M 644 :99 data/DATA2
+"""
+)
+        
+    def test_with_directory_includes(self):
+        params = {'include_paths': ['data/'],
+                  'exclude_paths': None,
+                  'squash_empty_commits': False}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \
+"""commit refs/heads/master
+mark :3
+committer Joe <[e-mail]> 1234567890 +1000
+data 6
+import
+blob
+mark :2
+data 4
+bar
+commit refs/heads/master
+mark :4
+committer Joe <[e-mail]> 1234567890 +1000
+data 19
+unknown from commit
+from :999
+M 644 :2 DATA
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <[e-mail]> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+merge :1001
+M 644 :99 DATA2
+""")