Merge ~cjwatson/launchpad:access-token-auth into launchpad:master

[Colin Watson]

Support webservice authentication using AccessTokens

This only works with webservice methods that have appropriate `@scoped` decorators, of which there are none at present; but this lays some more of the groundwork.

[Ioana Lasc (ilasc)]

review: Approve

[William Grant (wgrant)]

review: Approve (code)

diff --git a/lib/lp/services/webapp/doc/webapp-publication.txt b/lib/lp/services/webapp/doc/webapp-publication.txt
index f0c2bcf..98f1fd0 100644
--- a/lib/lp/services/webapp/doc/webapp-publication.txt
+++ b/lib/lp/services/webapp/doc/webapp-publication.txt
@@ -1129,7 +1129,7 @@ The feeds implementation always returns the anonymous user.
     True
 
 The webservice implementation returns the principal for the person
-associated with the access token specified in the request.  The
+associated with the OAuth access token specified in the request.  The
 principal's access_level and scope will match what was specified in the
 token.
 
diff --git a/lib/lp/services/webapp/interaction.py b/lib/lp/services/webapp/interaction.py
index 8aec9d0..4b60242 100644
--- a/lib/lp/services/webapp/interaction.py
+++ b/lib/lp/services/webapp/interaction.py
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Methods dealing with interactions.
@@ -179,6 +179,7 @@ class InteractionExtras:
     """Extra data attached to all interactions.  See `IInteractionExtras`."""
 
     permit_timeout_from_features = False
+    access_token = None
 
 
 def get_interaction_extras():
diff --git a/lib/lp/services/webapp/interfaces.py b/lib/lp/services/webapp/interfaces.py
index 7c71623..a895808 100644
--- a/lib/lp/services/webapp/interfaces.py
+++ b/lib/lp/services/webapp/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 import logging
@@ -350,6 +350,9 @@ class IInteractionExtras(Interface):
         `lp.services.webapp.servers.set_permit_timeout_from_features`
         for more.""")
 
+    access_token = Attribute(
+        "The `IAccessToken` used to authenticate this interaction, if any.")
+
 
 #
 # Request
diff --git a/lib/lp/services/webapp/publication.py b/lib/lp/services/webapp/publication.py
index e7a4ec8..717cc65 100644
--- a/lib/lp/services/webapp/publication.py
+++ b/lib/lp/services/webapp/publication.py
@@ -70,6 +70,7 @@ from lp.registry.interfaces.person import (
     ITeam,
     )
 from lp.services import features
+from lp.services.auth.interfaces import IAccessTokenVerifiedRequest
 from lp.services.config import config
 from lp.services.database.interfaces import (
     IDatabasePolicy,
@@ -117,13 +118,13 @@ def maybe_block_offsite_form_post(request):
     if request.method != 'POST':
         return
     if (IOAuthSignedRequest.providedBy(request)
-        or not IBrowserRequest.providedBy(request)):
-        # We only want to check for the referrer header if we are
-        # in the middle of a request initiated by a web browser. A
-        # request to the web service (which is necessarily
-        # OAuth-signed) or a request that does not implement
-        # IBrowserRequest (such as an XML-RPC request) can do
-        # without a Referer.
+            or IAccessTokenVerifiedRequest.providedBy(request)
+            or not IBrowserRequest.providedBy(request)):
+        # We only want to check for the referrer header if we are in the
+        # middle of a request initiated by a web browser. A request to the
+        # web service (which is necessarily OAuth-signed or verified by an
+        # access token) or a request that does not implement IBrowserRequest
+        # (such as an XML-RPC request) can do without a Referer.
         return
     if request['PATH_INFO'] in OFFSITE_POST_WHITELIST:
         # XXX: jamesh 2007-11-23 bug=124421:
diff --git a/lib/lp/services/webapp/servers.py b/lib/lp/services/webapp/servers.py
index 8707437..31afcbf 100644
--- a/lib/lp/services/webapp/servers.py
+++ b/lib/lp/services/webapp/servers.py
@@ -57,6 +57,10 @@ from zope.session.interfaces import ISession
 from lp.app import versioninfo
 from lp.app.errors import UnexpectedFormData
 import lp.layers
+from lp.services.auth.interfaces import (
+    IAccessTokenSet,
+    IAccessTokenVerifiedRequest,
+    )
 from lp.services.config import config
 from lp.services.encoding import wsgi_native_string
 from lp.services.features import get_relevant_feature_controller
@@ -80,6 +84,7 @@ from lp.services.webapp.authorization import (
     LAUNCHPAD_SECURITY_POLICY_CACHE_UNAUTH_KEY,
     )
 from lp.services.webapp.errorlog import ErrorReportRequest
+from lp.services.webapp.interaction import get_interaction_extras
 from lp.services.webapp.interfaces import (
     IBasicLaunchpadRequest,
     IBrowserFormNG,
@@ -1171,26 +1176,32 @@ class WebServicePublication(WebServicePublicationMixin,
         else:
             return super(WebServicePublication, self).getResource(request, ob)
 
-    def getPrincipal(self, request):
-        """See `LaunchpadBrowserPublication`.
-
-        Web service requests are authenticated using OAuth, except for the
-        one made using (presumably) JavaScript on the /api override path.
-
-        Raises TokenException which has a webservice error status of
-        Unauthorized - 401.
-
-        Raises Unauthorized directly in the case where the consumer is None
-        for a non-anonymous request as it may represent a server error.
-        """
-        # Use the regular HTTP authentication, when the request is not
-        # on the API virtual host but comes through the path_override on
-        # the other regular virtual hosts.
-        request_path = request.get('PATH_INFO', '')
-        web_service_config = getUtility(IWebServiceConfiguration)
-        if request_path.startswith("/%s" % web_service_config.path_override):
-            return super(WebServicePublication, self).getPrincipal(request)
-
+    def _getPrincipalFromAccessToken(self, request):
+        """Authenticate a request using a personal access token."""
+        access_token = removeSecurityProxy(
+            getUtility(IAccessTokenSet).getBySecret(
+                request._auth[len("Token "):]))
+        if access_token is None:
+            raise TokenException("Unknown access token.")
+        elif access_token.is_expired:
+            raise TokenException("Expired access token.")
+        elif access_token.owner.account_status != AccountStatus.ACTIVE:
+            raise TokenException("Inactive account.")
+        access_token.updateLastUsed()
+        # GET requests will be rolled back, as will unsuccessful ones.
+        # Commit so that the last-used date is updated anyway.
+        transaction.commit()
+        logging_context.push(
+            access_token_id=access_token.id,
+            access_token_scopes=" ".join(
+                scope.title for scope in access_token.scopes))
+        alsoProvides(request, IAccessTokenVerifiedRequest)
+        get_interaction_extras().access_token = access_token
+        return getUtility(IPlacelessLoginSource).getPrincipal(
+            access_token.owner.accountID)
+
+    def _getPrincipalFromOAuth(self, request):
+        """Authenticate a request using OAuth."""
         # Fetch OAuth authorization information from the request.
         try:
             form = get_oauth_authorization(request)
@@ -1259,11 +1270,35 @@ class WebServicePublication(WebServicePublicationMixin,
             scope_url = canonical_url(token.context, force_local_path=True)
         else:
             scope_url = None
-        principal = getUtility(IPlacelessLoginSource).getPrincipal(
+        return getUtility(IPlacelessLoginSource).getPrincipal(
             token.person.account.id, access_level=token.permission,
             scope_url=scope_url)
 
-        return principal
+    def getPrincipal(self, request):
+        """See `LaunchpadBrowserPublication`.
+
+        Web service requests are authenticated using OAuth or personal
+        access tokens, except for the one made using (presumably) JavaScript
+        on the /api override path.
+
+        Raises TokenException which has a webservice error status of
+        Unauthorized - 401.
+
+        Raises Unauthorized directly in the case where the consumer is None
+        for a non-anonymous request as it may represent a server error.
+        """
+        # Use the regular HTTP authentication, when the request is not
+        # on the API virtual host but comes through the path_override on
+        # the other regular virtual hosts.
+        request_path = request.get('PATH_INFO', '')
+        web_service_config = getUtility(IWebServiceConfiguration)
+        if request_path.startswith("/%s" % web_service_config.path_override):
+            return super(WebServicePublication, self).getPrincipal(request)
+
+        if request._auth is not None and request._auth.startswith("Token "):
+            return self._getPrincipalFromAccessToken(request)
+        else:
+            return self._getPrincipalFromOAuth(request)
 
 
 @implementer(lp.layers.WebServiceLayer)
diff --git a/lib/lp/services/webapp/tests/test_publication.py b/lib/lp/services/webapp/tests/test_publication.py
index 332db55..57d63a9 100644
--- a/lib/lp/services/webapp/tests/test_publication.py
+++ b/lib/lp/services/webapp/tests/test_publication.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests publication.py"""
@@ -28,6 +28,7 @@ from zope.security.management import (
     thread_local as zope_security_thread_local,
     )
 
+from lp.services.auth.interfaces import IAccessTokenVerifiedRequest
 from lp.services.database.interfaces import IMasterStore
 from lp.services.identity.model.emailaddress import EmailAddress
 from lp.services.oauth.interfaces import (
@@ -124,7 +125,7 @@ class TestWebServicePublication(TestCaseWithFactory):
         person = self.factory.makePerson()
         self.assertNotEqual(person.id, person.account.id)
 
-        # Create an access token for our new person.
+        # Create an OAuth access token for our new person.
         consumer = getUtility(IOAuthConsumerSet).new(u'test-consumer')
         request_token, _ = consumer.newRequestToken()
         request_token.review(
@@ -244,6 +245,14 @@ class TestBlockingOffsitePosts(TestCase):
         # this call shouldn't raise an exception
         maybe_block_offsite_form_post(request)
 
+    def test_access_token_verified_requests(self):
+        # Requests that are verified with an access token are allowed.
+        request = LaunchpadTestRequest(
+            method='POST', environ=dict(PATH_INFO='/'))
+        directlyProvides(request, IAccessTokenVerifiedRequest)
+        # this call shouldn't raise an exception
+        maybe_block_offsite_form_post(request)
+
     def test_nonbrowser_requests(self):
         # Requests that are from non-browsers are allowed.
         class FakeNonBrowserRequest:
diff --git a/lib/lp/services/webapp/tests/test_servers.py b/lib/lp/services/webapp/tests/test_servers.py
index 50f48aa..a551f78 100644
--- a/lib/lp/services/webapp/tests/test_servers.py
+++ b/lib/lp/services/webapp/tests/test_servers.py
@@ -1,6 +1,10 @@
 # Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+from datetime import (
+    datetime,
+    timedelta,
+    )
 from doctest import (
     DocTestSuite,
     ELLIPSIS,
@@ -19,8 +23,14 @@ from lazr.restful.testing.webservice import (
     IGenericEntry,
     WebServiceTestCase,
     )
+import pytz
 from talisker.context import Context
 from talisker.logs import logging_context
+from testtools.matchers import (
+    ContainsDict,
+    Equals,
+    )
+import transaction
 from zope.component import (
     getGlobalSiteManager,
     getUtility,
@@ -29,8 +39,15 @@ from zope.interface import (
     implementer,
     Interface,
     )
+from zope.security.interfaces import Unauthorized
+from zope.security.management import newInteraction
+from zope.security.proxy import removeSecurityProxy
 
 from lp.app import versioninfo
+from lp.services.auth.enums import AccessTokenScope
+from lp.services.identity.interfaces.account import AccountStatus
+from lp.services.oauth.interfaces import TokenException
+from lp.services.webapp.interaction import get_interaction_extras
 from lp.services.webapp.interfaces import IFinishReadOnlyRequestEvent
 from lp.services.webapp.publication import LaunchpadBrowserPublication
 from lp.services.webapp.servers import (
@@ -50,9 +67,16 @@ from lp.services.webapp.servers import (
     )
 from lp.testing import (
     EventRecorder,
+    logout,
+    person_logged_in,
     TestCase,
+    TestCaseWithFactory,
     )
-from lp.testing.layers import FunctionalLayer
+from lp.testing.layers import (
+    DatabaseFunctionalLayer,
+    FunctionalLayer,
+    )
+from lp.testing.publication import get_request_and_publication
 
 
 class SetInWSGIEnvironmentTestCase(TestCase):
@@ -781,6 +805,138 @@ class TestFinishReadOnlyRequest(TestCase):
         self._test_publication(publication, ["ABORT"])
 
 
+class TestWebServiceAccessTokens(TestCaseWithFactory):
+    """Test personal access tokens for the webservice.
+
+    These are bearer tokens with an owner, a context, and some scopes.  We
+    can authenticate using one of these, and it will be recorded in the
+    interaction extras.
+    """
+
+    layer = DatabaseFunctionalLayer
+
+    def test_valid(self):
+        owner = self.factory.makePerson()
+        secret, token = self.factory.makeAccessToken(
+            owner=owner, scopes=[AccessTokenScope.REPOSITORY_BUILD_STATUS])
+        self.assertIsNone(removeSecurityProxy(token).date_last_used)
+        transaction.commit()
+        logout()
+
+        request, publication = get_request_and_publication(
+            "api.launchpad.test", "POST",
+            extra_environment={"HTTP_AUTHORIZATION": "Token %s" % secret})
+        newInteraction(request)
+        principal = publication.getPrincipal(request)
+        request.setPrincipal(principal)
+        self.assertEqual(owner, principal.person)
+        self.assertEqual(token, get_interaction_extras().access_token)
+        self.assertIsNotNone(token.date_last_used)
+        self.assertThat(logging_context.flat, ContainsDict({
+            "access_token_id": Equals(removeSecurityProxy(token).id),
+            "access_token_scopes": Equals("repository:build_status"),
+            }))
+
+        # token.date_last_used is still up to date even if the transaction
+        # is rolled back.
+        date_last_used = token.date_last_used
+        transaction.abort()
+        self.assertEqual(date_last_used, token.date_last_used)
+
+    def test_expired(self):
+        owner = self.factory.makePerson()
+        secret, token = self.factory.makeAccessToken(owner=owner)
+        with person_logged_in(owner):
+            token.date_expires = datetime.now(pytz.UTC) - timedelta(days=1)
+        transaction.commit()
+
+        request, publication = get_request_and_publication(
+            "api.launchpad.test", "POST",
+            extra_environment={"HTTP_AUTHORIZATION": "Token %s" % secret})
+        self.assertRaisesWithContent(
+            TokenException, "Expired access token.",
+            publication.getPrincipal, request)
+
+    def test_unknown(self):
+        request, publication = get_request_and_publication(
+            "api.launchpad.test", "POST",
+            extra_environment={"HTTP_AUTHORIZATION": "Token nonexistent"})
+        self.assertRaisesWithContent(
+            TokenException, "Unknown access token.",
+            publication.getPrincipal, request)
+
+    def test_inactive_account(self):
+        owner = self.factory.makePerson(account_status=AccountStatus.SUSPENDED)
+        secret, token = self.factory.makeAccessToken(owner=owner)
+        transaction.commit()
+
+        request, publication = get_request_and_publication(
+            "api.launchpad.test", "POST",
+            extra_environment={"HTTP_AUTHORIZATION": "Token %s" % secret})
+        self.assertRaisesWithContent(
+            TokenException, "Inactive account.",
+            publication.getPrincipal, request)
+
+    def _makeAccessTokenVerifiedRequest(self, **kwargs):
+        secret, token = self.factory.makeAccessToken(**kwargs)
+        transaction.commit()
+        logout()
+
+        request, publication = get_request_and_publication(
+            "api.launchpad.test", "POST",
+            extra_environment={"HTTP_AUTHORIZATION": "Token %s" % secret})
+        newInteraction(request)
+        principal = publication.getPrincipal(request)
+        request.setPrincipal(principal)
+
+    def test_checkRequest_valid(self):
+        repository = self.factory.makeGitRepository()
+        self._makeAccessTokenVerifiedRequest(
+            context=repository,
+            scopes=[AccessTokenScope.REPOSITORY_BUILD_STATUS])
+        getUtility(IWebServiceConfiguration).checkRequest(
+            repository,
+            ["repository:build_status", "repository:another_scope"])
+
+    def test_checkRequest_bad_context(self):
+        repository = self.factory.makeGitRepository()
+        self._makeAccessTokenVerifiedRequest(
+            context=repository,
+            scopes=[AccessTokenScope.REPOSITORY_BUILD_STATUS])
+        self.assertRaisesWithContent(
+            Unauthorized,
+            "Current authentication does not allow access to this object.",
+            getUtility(IWebServiceConfiguration).checkRequest,
+            self.factory.makeGitRepository(), ["repository:build_status"])
+
+    def test_checkRequest_unscoped_method(self):
+        repository = self.factory.makeGitRepository()
+        self._makeAccessTokenVerifiedRequest(
+            context=repository,
+            scopes=[AccessTokenScope.REPOSITORY_BUILD_STATUS])
+        self.assertRaisesWithContent(
+            Unauthorized,
+            "Current authentication only allows calling scoped methods.",
+            getUtility(IWebServiceConfiguration).checkRequest,
+            repository, None)
+
+    def test_checkRequest_wrong_scope(self):
+        repository = self.factory.makeGitRepository()
+        self._makeAccessTokenVerifiedRequest(
+            context=repository,
+            scopes=[
+                AccessTokenScope.REPOSITORY_BUILD_STATUS,
+                AccessTokenScope.REPOSITORY_PUSH,
+                ])
+        self.assertRaisesWithContent(
+            Unauthorized,
+            "Current authentication does not allow calling this method "
+            "(one of these scopes is required: "
+            "'repository:scope_1', 'repository:scope_2').",
+            getUtility(IWebServiceConfiguration).checkRequest,
+            repository, ["repository:scope_1", "repository:scope_2"])
+
+
 def test_suite():
     suite = unittest.TestSuite()
     suite.addTest(DocTestSuite(
diff --git a/lib/lp/services/webservice/configuration.py b/lib/lp/services/webservice/configuration.py
index 549e4fd..f52918a 100644
--- a/lib/lp/services/webservice/configuration.py
+++ b/lib/lp/services/webservice/configuration.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """A configuration class describing the Launchpad web service."""
@@ -10,9 +10,12 @@ __all__ = [
 from lazr.restful.simple import BaseWebServiceConfiguration
 import six
 from zope.component import getUtility
+from zope.security.interfaces import Unauthorized
 
 from lp.app import versioninfo
 from lp.services.config import config
+from lp.services.database.sqlbase import block_implicit_flushes
+from lp.services.webapp.interaction import get_interaction_extras
 from lp.services.webapp.interfaces import ILaunchBag
 from lp.services.webapp.servers import (
     WebServiceClientRequest,
@@ -92,3 +95,23 @@ class LaunchpadWebServiceConfiguration(BaseWebServiceConfiguration):
     def get_request_user(self):
         """See `IWebServiceConfiguration`."""
         return getUtility(ILaunchBag).user
+
+    @block_implicit_flushes
+    def checkRequest(self, context, required_scopes):
+        """See `IWebServiceConfiguration`."""
+        access_token = get_interaction_extras().access_token
+        if access_token is None:
+            return
+        if access_token.context != context:
+            raise Unauthorized(
+                "Current authentication does not allow access to this object.")
+        if not required_scopes:
+            raise Unauthorized(
+                "Current authentication only allows calling scoped methods.")
+        elif not any(
+                scope.title in required_scopes
+                for scope in access_token.scopes):
+            raise Unauthorized(
+                "Current authentication does not allow calling this method "
+                "(one of these scopes is required: %s)."
+                % ", ".join("'%s'" % scope for scope in required_scopes))