Merge lp:~jelmer/brz/big-file-test into lp:brz

[Jelmer Vernooij]

Add some tests that use big files.

Add some tests with big files.

This uses rlimits to artificially restrict the available memory.

Adding files to a working tree works. Committing files currently does not and is marked as known failing.

[Martin Packman (gz)]

Looks good, thanks. We did discuss a while back some means of marking tests as big/slow so they could be skipped in some circumstances, but can look at that if CI has issues.

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/365/

=== modified file 'breezy/tests/blackbox/__init__.py'
--- breezy/tests/blackbox/__init__.py	2019-06-02 03:11:13 +0000
+++ breezy/tests/blackbox/__init__.py	2019-06-15 21:46:20 +0000
@@ -40,6 +40,7 @@
         'test_ancestry',
         'test_annotate',
         'test_bisect',
+        'test_big_file',
         'test_branch',
         'test_branches',
         'test_break_lock',

=== added file 'breezy/tests/blackbox/test_big_file.py'
--- breezy/tests/blackbox/test_big_file.py	1970-01-01 00:00:00 +0000
+++ breezy/tests/blackbox/test_big_file.py	2019-06-15 21:46:20 +0000
@@ -0,0 +1,95 @@
+# Copyright (C) 2009 Jelmer Vernooij <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+"""Tests with "big" files.
+
+These are meant to ensure that Breezy never keeps full copies of files in
+memory.
+"""
+
+import os
+import resource
+
+from breezy import (
+    osutils,
+    tests,
+    )
+from breezy.tests import (
+    features,
+    script,
+    )
+
+BIG_FILE_SIZE = 1024 * 1024 * 500
+BIG_FILE_CHUNK_SIZE = 1024 * 1024
+
+RESOURCE = resource.RLIMIT_DATA
+LIMIT = 1024 * 1024 * 200
+
+
+def make_big_file(path):
+    blob_1mb = BIG_FILE_CHUNK_SIZE * b'\x0c'
+    with open(path, 'wb') as f:
+        for i in range(BIG_FILE_SIZE // BIG_FILE_CHUNK_SIZE):
+            f.write(blob_1mb)
+
+
+class TestAdd(tests.TestCaseWithTransport):
+
+    def writeBigFile(self, path):
+        make_big_file(path)
+        self.addCleanup(os.unlink, path)
+
+    def setUp(self):
+        super(TestAdd, self).setUp()
+        previous = resource.getrlimit(RESOURCE)
+        self.addCleanup(resource.setrlimit, RESOURCE, previous)
+        resource.setrlimit(RESOURCE, (LIMIT, -1))
+
+    def test_allocate(self):
+        def allocate():
+            "." * BIG_FILE_SIZE
+        self.assertRaises(MemoryError, allocate)
+
+    def test_add(self):
+        tree = self.make_branch_and_tree('tree')
+        self.writeBigFile(os.path.join(tree.basedir, 'testfile'))
+        tree.add('testfile')
+
+    def test_make_file_big(self):
+        self.knownFailure('commit keeps entire files in memory')
+        tree = self.make_branch_and_tree('tree')
+        self.build_tree(['tree/testfile'])
+        tree.add('testfile')
+        tree.commit('add small file')
+        self.writeBigFile(os.path.join(tree.basedir, 'testfile'))
+        tree.commit('small files get big')
+        self.knownFailure('commit keeps entire files in memory')
+
+    def test_commit(self):
+        self.knownFailure('commit keeps entire files in memory')
+        tree = self.make_branch_and_tree('tree')
+        self.writeBigFile(os.path.join(tree.basedir, 'testfile'))
+        tree.add('testfile')
+        tree.commit('foo')
+
+    def test_clone(self):
+        self.knownFailure('commit keeps entire files in memory')
+        tree = self.make_branch_and_tree('tree')
+        self.writeBigFile(os.path.join(tree.basedir, 'testfile'))
+        tree.add('testfile')
+        tree.commit('foo')
+        tree.clone.sprout('newtree')