Merge ~jugmac00/turnip:retrieve-launchpad-yaml-more-efficiently into turnip:master

[Jürgen Gmach]

commit API now can filter by blob paths

[Colin Watson (cjwatson)]

This basically looks good, thanks! Just a couple of suggestions.

review: Approve

[Jürgen Gmach (jugmac00)]

Thanks for the review! I applied the suggested changes.

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/347/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/348/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/349/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/350/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/351/

diff --git a/Makefile b/Makefile
index 98d40a1..035c607 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ test: $(ENV) bootstrap-test
 # this is a temporary solution to enable selecting single tests more easily
 # test setup will be redone in https://warthogs.atlassian.net/browse/LP-598
 # sample command:
-# make runpytest ARGS="-k filtered_by_blob_paths"
+# make runpytest ARGS="-k expression"
 runpytest: $(ENV) bootstrap-test
 	$(PYTHON) -m pip install pdbpp pytest
 	$(PYTHON) -m pytest $(ARGS)
diff --git a/turnip/api/tests/test_api.py b/turnip/api/tests/test_api.py
index ca47021..0e566b5 100644
--- a/turnip/api/tests/test_api.py
+++ b/turnip/api/tests/test_api.py
@@ -773,6 +773,58 @@ class ApiTestCase(TestCase, ApiRepoStoreMixin):
         self.assertEqual(1, len(resp.json))
         self.assertEqual(bulk_commits['commits'][0], resp.json[0]['sha1'])
 
+    def test_repo_get_launchpad_yaml_from_commit_collection(self):
+        factory = RepoFactory(self.repo_store, num_commits=10)
+        factory.build()
+
+        # standard configuration file name
+        c1 = factory.add_commit('bar', '.launchpad.yaml')
+        # commit which must not be returned
+        c2 = factory.commits[0]
+
+        payload = {
+            'filter_paths': [".launchpad.yaml", "debian/.launchpad.yaml"],
+            "commits": [c1.hex, c2.hex]
+        }
+
+        resp = self.app.post_json(
+            '/repo/{}/commits'.format(self.repo_path), payload
+        )
+
+        self.assertEqual(1, len(resp.json))
+        self.assertIn("blobs", resp.json[0])
+        self.assertEqual(
+            {'.launchpad.yaml': {'size': 3, 'data': 'YmFy'}},
+            resp.json[0]["blobs"]
+        )
+
+    def test_repo_get_launchpad_yaml_from_commit_collection_debian_edition(
+        self
+    ):
+        factory = RepoFactory(self.repo_store, num_commits=10)
+        factory.build()
+
+        # debian packaging: typical random files are added under `debian/...`
+        c1 = factory.add_commit('bar', 'debian/.launchpad.yaml')
+        # commit which must not be returned
+        c2 = factory.commits[0]
+
+        payload = {
+            'filter_paths': [".launchpad.yaml", "debian/.launchpad.yaml"],
+            "commits": [c1.hex, c2.hex]
+        }
+
+        resp = self.app.post_json(
+            '/repo/{}/commits'.format(self.repo_path), payload
+        )
+
+        self.assertEqual(1, len(resp.json))
+        self.assertIn("blobs", resp.json[0])
+        self.assertEqual(
+            {'debian/.launchpad.yaml': {'size': 3, 'data': 'YmFy'}},
+            resp.json[0]["blobs"]
+        )
+
     def test_repo_get_log_signatures(self):
         """Ensure signatures are correct."""
         factory = RepoFactory(self.repo_store)
diff --git a/turnip/api/views.py b/turnip/api/views.py
index d5cf3d7..2f199af 100644
--- a/turnip/api/views.py
+++ b/turnip/api/views.py
@@ -11,7 +11,10 @@ import pyramid.httpexceptions as exc
 from pyramid.response import Response
 
 from turnip.api import store
-from turnip.api.formatter import format_commit
+from turnip.api.formatter import (
+    format_blob,
+    format_commit,
+    )
 from turnip.config import config
 
 
@@ -357,12 +360,28 @@ class CommitAPI(BaseAPI):
     @validate_path
     def collection_post(self, repo_store, repo_name):
         """Get commits in bulk."""
-        commits = extract_cstruct(self.request)['body'].get('commits')
+        payload = extract_cstruct(self.request)["body"]
+        commits = payload.get("commits")
+        filter_paths = payload.get("filter_paths", [])
         try:
             commits = store.get_commits(repo_store, repo_name, commits)
         except GitError:
             return exc.HTTPNotFound()
-        return [format_commit(commit) for commit in commits]
+
+        # format commits and filter them if applicable
+        rv = []
+        for commit in commits:
+            d = dict()
+            # apply filter if given
+            for path in filter_paths:
+                if path in commit.tree:
+                    if not d.get("blobs"):
+                        d["blobs"] = dict()
+                    d["blobs"][path] = format_blob(commit.tree[path])
+            if not filter_paths or d:
+                d.update(format_commit(commit))
+                rv.append(d)
+        return rv
 
 
 @resource(path='/repo/{name}/log/{sha1}')