Merge ~pappacena/launchpad:aws-registry-push into launchpad:master

[Thiago F. Pappacena]

Adding OCI registry client for AWS' ECR

[Thiago F. Pappacena (pappacena)]

Dependencies MP: https://code.launchpad.net/~pappacena/lp-source-dependencies/+git/lp-source-dependencies/+merge/392640

[Colin Watson (cjwatson)]

review: Approve

9df07c9... by Thiago F. Pappacena on 2020-10-29

Fixing import formatting

diff --git a/constraints.txt b/constraints.txt
index 4c33491..116d95e 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -81,7 +81,7 @@ python-gettext==4.0
 pytz==2019.3
 # Handled in setup-requirements.txt instead.
 #setuptools==44.0.0
-six==1.14.0
+six==1.15.0
 transaction==3.0.0
 
 # zope.password needs these
@@ -135,7 +135,7 @@ lxml==4.5.0
 repoze.sphinx.autointerface==0.8
 requests==2.23.0
 certifi==2019.11.28
-urllib3==1.25.8
+urllib3==1.25.11
 idna==2.9
 chardet==3.0.4
 sphinxcontrib-applehelp==1.0.2
@@ -179,6 +179,8 @@ billiard==3.5.0.5
 bleach==3.1.0
 breezy==3.0.1
 bson==0.5.9
+boto3==1.16.2
+botocore==1.19.2
 # lp:~launchpad/bzr/lp
 bzr==2.6.0.lp.4
 celery==4.1.1
@@ -204,7 +206,7 @@ fastimport==0.9.8
 feedparser==5.2.1
 feedvalidator==0.0.0DEV-r1049
 FormEncode==1.3.1
-futures==3.2.0
+futures==3.3.0
 geoip2==2.9.0
 grokcore.component==3.1
 gunicorn==19.8.1
@@ -216,6 +218,7 @@ incremental==17.5.0
 ipaddress==1.0.18
 ipython==0.13.2
 iso8601==0.1.12
+jmespath==0.10.0
 jsautobuild==0.2
 keyring==0.6.2
 kombu==4.4.0
@@ -288,6 +291,7 @@ rabbitfixture==0.4.2
 requests-file==1.4.3
 requests-toolbelt==0.9.1
 responses==0.9.0
+s3transfer==0.3.3
 scandir==1.7
 service-identity==18.1.0
 setproctitle==1.1.7
diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
index 7c67b57..e3e095b 100644
--- a/lib/lp/oci/model/ociregistryclient.py
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -10,7 +10,7 @@ __all__ = [
     'OCIRegistryClient'
 ]
 
-
+import base64
 from functools import partial
 import hashlib
 from io import BytesIO
@@ -22,6 +22,7 @@ except ImportError:
 import logging
 import tarfile
 
+import boto3
 from requests.exceptions import (
     ConnectionError,
     HTTPError,
@@ -30,6 +31,7 @@ from six.moves.urllib.request import (
     parse_http_list,
     parse_keqv_list,
     )
+from six.moves.urllib.parse import urlparse
 from tenacity import (
     before_log,
     retry,
@@ -45,6 +47,7 @@ from lp.oci.interfaces.ociregistryclient import (
     MultipleOCIRegistryError,
     ManifestUploadFailed,
     )
+from lp.services.propertycache import cachedproperty
 from lp.services.timeout import urlfetch
 
 
@@ -426,6 +429,9 @@ class RegistryHTTPClient:
     def getInstance(cls, push_rule):
         """Returns an instance of RegistryHTTPClient adapted to the
         given push rule and registry's authentication flow."""
+        registry_domain = urlparse(push_rule.registry_url).netloc
+        if registry_domain.endswith(".amazonaws.com"):
+            return AWSRegistryHTTPClient(push_rule)
         try:
             proxy_urlfetch("{}/v2/".format(push_rule.registry_url))
             # No authorization error? Just return the basic RegistryHTTPClient.
@@ -516,3 +522,38 @@ class BearerTokenRegistryClient(RegistryHTTPClient):
                     url, auth_retry=False, headers=headers,
                     *args, **request_kwargs)
             raise
+
+
+class AWSRegistryHTTPClient(RegistryHTTPClient):
+
+    def _getRegion(self):
+        """Returns the region from the push URL domain."""
+        domain = urlparse(self.push_rule.registry_url).netloc
+        # The domain format should be something like
+        # 'xxx.dkr.ecr.sa-east-1.amazonaws.com'. 'sa-east-1' is the region.
+        return domain.split(".")[-3]
+
+    @cachedproperty
+    def credentials(self):
+        """Exchange aws_access_key_id and aws_secret_access_key with the
+        authentication token that should be used when talking to ECR."""
+        try:
+            auth = self.push_rule.registry_credentials.getCredentials()
+            username, password = auth['username'], auth.get('password')
+            region = self._getRegion()
+            log.info("Trying to authenticate with AWS in region %s" % region)
+            client = boto3.client('ecr', aws_access_key_id=username,
+                                  aws_secret_access_key=password,
+                                  region_name=region)
+            token = client.get_authorization_token()
+            auth_data = token["authorizationData"][0]
+            authorization_token = auth_data['authorizationToken']
+            username, password = base64.b64decode(
+                authorization_token).decode().split(':')
+            return username, password
+        except Exception as e:
+            log.error("Error trying to get authorization token for ECR "
+                      "registry: %s(%s)" % (e.__class__, e))
+            raise OCIRegistryAuthenticationError(
+                "It was not possible to get AWS credentials for %s: %s" %
+                (self.push_rule.registry_url, e))
diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py
index 8de88b4..93f33fb 100644
--- a/lib/lp/oci/tests/test_ociregistryclient.py
+++ b/lib/lp/oci/tests/test_ociregistryclient.py
@@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 __metaclass__ = type
 
+import base64
 from functools import partial
 import io
 import json
@@ -46,6 +47,7 @@ from lp.oci.interfaces.ociregistryclient import (
     )
 from lp.oci.model.ocirecipe import OCIRecipeBuildRequest
 from lp.oci.model.ociregistryclient import (
+    AWSRegistryHTTPClient,
     BearerTokenRegistryClient,
     OCIRegistryAuthenticationError,
     OCIRegistryClient,
@@ -668,6 +670,70 @@ class TestRegistryHTTPClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
         call = responses.calls[0]
         self.assertEqual("%s/v2/" % push_rule.registry_url, call.request.url)
 
+    @responses.activate
+    def test_get_aws_client_instance(self):
+        credentials = self.factory.makeOCIRegistryCredentials(
+            url="https://123456789.dkr.ecr.sa-east-1.amazonaws.com",
+            credentials={
+                'username': 'aws_access_key_id',
+                'password': "aws_secret_access_key"})
+        push_rule = removeSecurityProxy(self.factory.makeOCIPushRule(
+            registry_credentials=credentials,
+            image_name="ecr-test"))
+
+        instance = RegistryHTTPClient.getInstance(push_rule)
+        self.assertEqual(AWSRegistryHTTPClient, type(instance))
+        self.assertIsInstance(instance, RegistryHTTPClient)
+
+    @responses.activate
+    def test_aws_credentials(self):
+        boto_patch = self.useFixture(
+            MockPatch('lp.oci.model.ociregistryclient.boto3'))
+        boto = boto_patch.mock
+        get_authorization_token = (
+            boto.client.return_value.get_authorization_token)
+        get_authorization_token.return_value = {
+            "authorizationData": [{
+                "authorizationToken": base64.b64encode(
+                    b"the-username:the-token")
+            }]
+        }
+
+        credentials = self.factory.makeOCIRegistryCredentials(
+            url="https://123456789.dkr.ecr.sa-east-1.amazonaws.com",
+            credentials={
+                'username': 'my_aws_access_key_id',
+                'password': "my_aws_secret_access_key"})
+        push_rule = removeSecurityProxy(self.factory.makeOCIPushRule(
+            registry_credentials=credentials,
+            image_name="ecr-test"))
+
+        instance = RegistryHTTPClient.getInstance(push_rule)
+        # Check the credentials twice, to make sure they are cached.
+        for _ in range(2):
+            http_user, http_passwd = instance.credentials
+            self.assertEqual("the-username", http_user)
+            self.assertEqual("the-token", http_passwd)
+            self.assertEqual(1, boto.client.call_count)
+            self.assertEqual(mock.call(
+                'ecr', aws_access_key_id="my_aws_access_key_id",
+                aws_secret_access_key="my_aws_secret_access_key",
+                region_name="sa-east-1"),
+                boto.client.call_args)
+
+    @responses.activate
+    def test_aws_malformed_url_region(self):
+        credentials = self.factory.makeOCIRegistryCredentials(
+            url="https://.amazonaws.com",
+            credentials={'username': 'aa', 'password': "bb"})
+        push_rule = removeSecurityProxy(self.factory.makeOCIPushRule(
+            registry_credentials=credentials,
+            image_name="ecr-test"))
+
+        instance = RegistryHTTPClient.getInstance(push_rule)
+        self.assertRaises(
+            OCIRegistryAuthenticationError, getattr, instance, 'credentials')
+
 
 class TestBearerTokenRegistryClient(OCIConfigHelperMixin,
                                     SpyProxyCallsMixin, TestCaseWithFactory):
diff --git a/setup.py b/setup.py
index 4e76a25..f6d5b6b 100644
--- a/setup.py
+++ b/setup.py
@@ -153,6 +153,7 @@ setup(
         'ampoule',
         'backports.lzma; python_version < "3.3"',
         'beautifulsoup4[lxml]',
+        'boto3',
         'breezy',
         # XXX cjwatson 2020-08-07: This should eventually be removed
         # entirely, but we need to retain it until codeimport has been