Merge lp:~blr/turnip/repack-api into lp:turnip

[Kit Randel]

Provides API for git repack, and per-repository configuration.

Provides API for git repack, and per-repository configuration.

Per-repository config defaults are read from git.config.yaml.

[Kit Randel (blr)]

Provides API for git repack.

Once support for per-repository configuration lands, this will potentially need to be refactored.

[William Grant (wgrant)]

git repack -l prevents inclusion of data from alternates. We also may want -f to force deltas to be recalculated.

[lp:~blr/turnip/repack-api]

158. By Kit Randel on 2015-05-21

Add git.config.yaml and fix imports.

[William Grant (wgrant)]

Repack can be a lengthy operation, so running it synchronously within an HTTP request is less than ideal, but it'll do until we have a job system of some kind. I'd like to see how it handles long requests, though -- if the connection dies, does the repack continue?

review: Approve (code)

[lp:~blr/turnip/repack-api]

159. By Kit Randel on 2015-05-21

Only extract json once is view.

160. By Kit Randel on 2015-05-21

Add -q to suppress output from repack.

161. By Kit Randel on 2015-05-21

* Add repack depth and window.
* ensure_config before repack.

162. By Kit Randel on 2015-05-26

* Add test to verify commits in separate packs exist in repacked pack.
* Set head/master to oid of commit in factory.add_commit().

163. By Kit Randel on 2015-05-26

Call pack-objects once.

164. By Kit Randel on 2015-05-26

Add inline comment on git gc.

165. By Kit Randel on 2015-05-26

Pass revlist to stdout.

166. By Kit Randel on 2015-05-26

Use subprocess.communicate rather than writing revlist.

167. By Kit Randel on 2015-05-26

Add test_helper factory.packs property.

168. By Kit Randel on 2015-05-26

Inline comment.

169. By Kit Randel on 2015-05-26

Assert repack results in 1 pack.

=== modified file '.bzrignore'
--- .bzrignore	2015-02-17 21:32:29 +0000
+++ .bzrignore	2015-05-26 07:47:04 +0000
@@ -1,13 +1,13 @@
 bin
+build
 develop-eggs
+dist
 download-cache
 eggs
+*.egg*
+*.egg-info
 .installed.cfg
+*.log
 parts
-*.egg-info
 tags
-TAGS
-build
-*.egg
-dist
-*.log
\ No newline at end of file
+TAGS
\ No newline at end of file

=== added file 'git.config.yaml'
--- git.config.yaml	1970-01-01 00:00:00 +0000
+++ git.config.yaml	2015-05-26 07:47:04 +0000
@@ -0,0 +1,8 @@
+# Per-repository configuration
+# See http://git-scm.com/docs/git-config.html for more info
+
+core.logallrefupdates: True
+pack.depth: 100
+pack.window: 100
+pack.windowMemory: 2g
+repack.writeBitmaps: True

=== modified file 'turnip/api/store.py'
--- turnip/api/store.py	2015-05-19 12:55:00 +0000
+++ turnip/api/store.py	2015-05-26 07:47:04 +0000
@@ -7,6 +7,7 @@
 import itertools
 import os
 import shutil
+import subprocess
 import uuid
 
 from pygit2 import (
@@ -22,6 +23,8 @@
     Repository,
     )
 
+from turnip.pack.helpers import ensure_config
+
 
 REF_TYPE_NAME = {
     GIT_OBJ_COMMIT: 'commit',
@@ -127,6 +130,7 @@
 
     if alternate_repo_paths:
         write_alternates(repo_path, alternate_repo_paths)
+    ensure_config(repo_path)  # set repository configuration defaults
     return repo_path
 
 
@@ -172,6 +176,36 @@
     shutil.rmtree(repo_path)
 
 
+def repack(repo_path, ignore_alternates=False, single=False,
+           prune=False, no_reuse_delta=False, window=None, depth=None):
+    """Repack a repository with git-repack.
+
+    :param ignore_alternates: Only repack local refs (git repack --local).
+    :param single: Create a single packfile (git repack -a).
+    :param prune: Remove redundant packs. (git repack -d)
+    :param no_reuse_delta: Force delta recalculation.
+    """
+    ensure_config(repo_path)
+
+    repack_args = ['git', 'repack', '-q']
+    if ignore_alternates:
+        repack_args.append('-l')
+    if no_reuse_delta:
+        repack_args.append('-f')
+    if prune:
+        repack_args.append('-d')
+    if single:
+        repack_args.append('-a')
+    if window:
+        repack_args.append('--window', window)
+    if depth:
+        repack_args.append('--depth', depth)
+
+    return subprocess.check_call(
+        repack_args, cwd=repo_path,
+        stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+
+
 def get_refs(repo_store, repo_name):
     """Return all refs for a git repository."""
     with open_repo(repo_store, repo_name) as repo:

=== modified file 'turnip/api/tests/test_api.py'
--- turnip/api/tests/test_api.py	2015-05-19 12:55:00 +0000
+++ turnip/api/tests/test_api.py	2015-05-26 07:47:04 +0000
@@ -3,6 +3,7 @@
 from __future__ import print_function
 
 import os
+import subprocess
 from textwrap import dedent
 import unittest
 import uuid
@@ -16,6 +17,7 @@
 
 from turnip import api
 from turnip.api.tests.test_helpers import (
+    chdir,
     get_revlist,
     open_repo,
     RepoFactory,
@@ -117,7 +119,6 @@
         """Merge diff can be requested across 2 repositories."""
         factory = RepoFactory(self.repo_store)
         c1 = factory.add_commit('foo', 'foobar.txt')
-        factory.set_head(c1)
 
         repo2_name = uuid.uuid4().hex
         factory2 = RepoFactory(
@@ -133,7 +134,6 @@
         """Diff can be requested across 2 repositories."""
         factory = RepoFactory(self.repo_store)
         c1 = factory.add_commit('foo', 'foobar.txt')
-        factory.set_head(c1)
 
         repo2_name = uuid.uuid4().hex
         factory2 = RepoFactory(
@@ -155,7 +155,6 @@
         """Cross repo diff with an invalid commit returns HTTP 404."""
         factory = RepoFactory(self.repo_store)
         c1 = factory.add_commit('foo', 'foobar.txt')
-        factory.set_head(c1)
 
         repo2_name = uuid.uuid4().hex
         RepoFactory(
@@ -201,7 +200,7 @@
 
         resp = self.app.get('/repo/{}/refs'.format(self.repo_path))
         refs = resp.json
-        self.assertEqual(0, len(refs.keys()))
+        self.assertEqual(1, len(refs.keys()))
 
     def test_allow_unicode_refs(self):
         """Ensure unicode refs are included in ref collection."""
@@ -213,7 +212,7 @@
 
         resp = self.app.get('/repo/{}/refs'.format(self.repo_path))
         refs = resp.json
-        self.assertEqual(1, len(refs.keys()))
+        self.assertEqual(2, len(refs.keys()))
 
     def test_repo_get_ref(self):
         RepoFactory(self.repo_store, num_commits=1).build()
@@ -510,6 +509,38 @@
         self.assertEqual(5, len(resp.json))
         self.assertNotIn(excluded_commit, resp.json)
 
+    def test_repo_repack_verify_pack(self):
+        """Ensure commit exists in pack."""
+        factory = RepoFactory(self.repo_store)
+        oid = factory.add_commit('foo', 'foobar.txt')
+        resp = self.app.post_json('/repo/{}/repack'.format(self.repo_path),
+                                  {'prune': True, 'single': True})
+        for filename in factory.packs:
+            pack = os.path.join(factory.pack_dir, filename)
+        out = subprocess.check_output(['git', 'verify-pack', pack, '-v'])
+        self.assertEqual(200, resp.status_code)
+        self.assertIn(oid.hex, out)
+
+    def test_repo_repack_verify_commits_to_pack(self):
+        """Ensure commits in different packs exist in merged pack."""
+        factory = RepoFactory(self.repo_store)
+        oid = factory.add_commit('foo', 'foobar.txt')
+        with chdir(factory.pack_dir):
+            subprocess.call(['git', 'gc', '-q'])  # pack first commit
+            oid2 = factory.add_commit('bar', 'foobar.txt', [oid])
+            p = subprocess.Popen(['git', 'pack-objects', '-q', 'pack2'],
+                                 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            p.communicate(input=oid2.hex)
+        self.assertEqual(2, len(factory.packs))  # ensure 2 packs exist
+        self.app.post_json('/repo/{}/repack'.format(self.repo_path),
+                           {'prune': True, 'single': True})
+        self.assertEqual(1, len(factory.packs))
+        repacked_pack = os.path.join(factory.pack_dir, factory.packs[0])
+        out = subprocess.check_output(['git', 'verify-pack',
+                                       repacked_pack, '-v'])
+        self.assertIn(oid.hex, out)
+        self.assertIn(oid2.hex, out)
+
 
 if __name__ == '__main__':
     unittest.main()

=== modified file 'turnip/api/tests/test_helpers.py'
--- turnip/api/tests/test_helpers.py	2015-04-22 21:41:44 +0000
+++ turnip/api/tests/test_helpers.py	2015-05-26 07:47:04 +0000
@@ -1,5 +1,7 @@
 # Copyright 2015 Canonical Ltd.  All rights reserved.
 
+import contextlib
+import fnmatch
 import itertools
 import os
 import urllib
@@ -28,6 +30,17 @@
     return Repository(repo_path)
 
 
[e-mail]
+def chdir(dirname=None):
+    curdir = os.getcwd()
+    try:
+        if dirname is not None:
+            os.chdir(dirname)
+        yield
+    finally:
+        os.chdir(curdir)
+
+
 class RepoFactory():
     """Builds a git repository in a user defined state."""
 
@@ -41,6 +54,7 @@
         self.num_commits = num_commits
         self.num_tags = num_tags
         self.repo_path = repo_path
+        self.pack_dir = os.path.join(repo_path, '.git', 'objects', 'pack')
         if clone_from:
             self.repo = self.clone_repo(clone_from)
         else:
@@ -52,6 +66,12 @@
         last = self.repo[self.repo.head.target]
         return list(self.repo.walk(last.id, GIT_SORT_TIME))
 
+    @property
+    def packs(self):
+        """Return list of pack files."""
+        return [filename for filename in fnmatch.filter(
+            os.listdir(self.pack_dir), '*.pack')]
+
     def add_commit(self, blob_content, file_path, parents=[],
                    ref=None, author=None, committer=None):
         """Create a commit from blob_content and file_path."""
@@ -66,10 +86,16 @@
         tree_id = repo.index.write_tree()
         oid = repo.create_commit(ref, author, committer,
                                  blob_content, tree_id, parents)
+        self.set_head(oid)  # set master
         return oid
 
     def set_head(self, oid):
-        self.repo.create_reference('refs/heads/master', oid)
+        try:
+            master_ref = self.repo.lookup_reference('refs/heads/master')
+        except KeyError:
+            master_ref = self.repo.create_reference('refs/heads/master', oid)
+        finally:
+            master_ref.set_target(oid)
 
     def add_branch(self, name, oid):
         commit = self.repo.get(oid)

=== modified file 'turnip/api/tests/test_store.py'
--- turnip/api/tests/test_store.py	2015-04-29 02:43:28 +0000
+++ turnip/api/tests/test_store.py	2015-05-26 07:47:04 +0000
@@ -16,6 +16,7 @@
     )
 import pygit2
 from testtools import TestCase
+import yaml
 
 from turnip.api import store
 from turnip.api.tests.test_helpers import (
@@ -74,6 +75,17 @@
         r = pygit2.Repository(path)
         self.assertEqual([], r.listall_references())
 
+    def test_repo_config(self):
+        """Assert repository is initialised with correct config defaults."""
+        repo_path = store.init_repo(self.repo_path)
+        repo_config = pygit2.Repository(repo_path).config
+        yaml_config = yaml.load(open('git.config.yaml'))
+
+        self.assertEqual(bool(yaml_config['core.logallrefupdates']),
+                         bool(repo_config['core.logallrefupdates']))
+        self.assertEqual(str(yaml_config['pack.depth']),
+                         repo_config['pack.depth'])
+
     def test_open_ephemeral_repo(self):
         """Opening a repo where a repo name contains ':' should return
         a new ephemeral repo.

=== modified file 'turnip/api/views.py'
--- turnip/api/views.py	2015-05-19 12:55:00 +0000
+++ turnip/api/views.py	2015-05-26 07:47:04 +0000
@@ -2,6 +2,7 @@
 
 import os
 import re
+from subprocess import CalledProcessError
 
 from cornice.resource import resource
 from cornice.util import extract_json_data
@@ -123,6 +124,36 @@
             return exc.HTTPNotFound()  # 404
 
 
+@resource(path='/repo/{name}/repack')
+class RepackAPI(BaseAPI):
+    """Provides HTTP API for repository repacking."""
+
+    def __init__(self, request):
+        super(RepackAPI, self).__init__()
+        self.request = request
+
+    @validate_path
+    def post(self, repo_store, repo_name):
+        repo_path = os.path.join(repo_store, repo_name)
+
+        data = extract_json_data(self.request)
+        ignore_alternates = data.get('ignore_alternates')
+        no_reuse_delta = data.get('no_reuse_delta')
+        prune = data.get('prune')
+        single = data.get('single')
+        window = data.get('window')
+        depth = data.get('depth')
+
+        try:
+            store.repack(repo_path, single=single, prune=prune,
+                         no_reuse_delta=no_reuse_delta,
+                         ignore_alternates=ignore_alternates,
+                         window=window, depth=depth)
+        except (CalledProcessError):
+            return exc.HTTPInternalServerError()
+        return
+
+
 @resource(collection_path='/repo/{name}/refs',
           path='/repo/{name}/refs/{ref:.*}')
 class RefAPI(BaseAPI):

=== modified file 'turnip/pack/helpers.py'
--- turnip/pack/helpers.py	2015-05-06 11:46:43 +0000
+++ turnip/pack/helpers.py	2015-05-26 07:47:04 +0000
@@ -14,6 +14,7 @@
     )
 
 from pygit2 import Repository
+import yaml
 
 import turnip.pack.hooks
 
@@ -104,10 +105,11 @@
     pygit2.Config handles locking itself, so we don't need to think too hard
     about concurrency.
     """
-
+    config_file = open('git.config.yaml')
+    git_config_defaults = yaml.load(config_file)
     config = Repository(repo_root).config
-    config['core.logallrefupdates'] = True
-    config['repack.writeBitmaps'] = True
+    for key, val in git_config_defaults.iteritems():
+        config[key] = val
 
 
 def ensure_hooks(repo_root):