Merge ~cjwatson/lp-archive:translate-paths into lp-archive:main

[Colin Watson]

Add archive endpoints

If configured with a suitable Launchpad XML-RPC endpoint, this can serve apt archives based on publishing records and the librarian without needing access to the published archive on a local file system.

[Jürgen Gmach (jugmac00)]

review: Approve

diff --git a/.gitignore b/.gitignore
index af6421f..2c2f17b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 /.python-version
 /.tox
 /build
+/config.toml
 /env
 /htmlcov
 /tmp
diff --git a/.mypy.ini b/.mypy.ini
new file mode 100644
index 0000000..07a8171
--- /dev/null
+++ b/.mypy.ini
@@ -0,0 +1,5 @@
+[mypy]
+python_version = 3.10
+
+[mypy-tomllib.*]
+ignore_missing_imports = true
diff --git a/lp_archive/__init__.py b/lp_archive/__init__.py
index 63a22b1..83c12ec 100644
--- a/lp_archive/__init__.py
+++ b/lp_archive/__init__.py
@@ -3,16 +3,25 @@
 
 """The Launchpad archive service."""
 
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore
 from typing import Any
 
 from flask import Flask
 
-from lp_archive import root
+from lp_archive import archive, root, routing
 
 
 def create_app(test_config: dict[str, Any] | None = None) -> Flask:
     app = Flask(__name__)
-    if test_config is not None:
+    if test_config is None:
+        with open("config.toml", "rb") as f:
+            app.config.from_mapping(tomllib.load(f))
+    else:
         app.config.from_mapping(test_config)
+    app.url_map.converters["archive"] = routing.ArchiveConverter
     app.register_blueprint(root.bp)
+    app.register_blueprint(archive.bp)
     return app
diff --git a/lp_archive/archive.py b/lp_archive/archive.py
new file mode 100644
index 0000000..cf38f7a
--- /dev/null
+++ b/lp_archive/archive.py
@@ -0,0 +1,80 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""The main archive view."""
+
+from xmlrpc.client import Fault, ServerProxy
+
+from flask import Blueprint, current_app, g, request
+from werkzeug.datastructures import WWWAuthenticate
+from werkzeug.exceptions import Unauthorized
+from werkzeug.wrappers import Response
+
+bp = Blueprint("archive", __name__)
+
+
+def get_archive_proxy() -> ServerProxy:
+    archive_proxy = getattr(g, "archive_proxy", None)
+    if archive_proxy is None:
+        archive_proxy = ServerProxy(
+            current_app.config["ARCHIVE_ENDPOINT"], allow_none=True
+        )
+        g.archive_proxy = archive_proxy
+    return archive_proxy
+
+
+def check_auth(archive: str) -> None:
+    """Check whether the current request may access an archive.
+
+    If unauthorized, raises a suitable exception.
+    """
+    # Ideally we'd use a Flask extension for this rather than rolling most
+    # of the HTTP Basic Authentication logic for ourselves, but nothing
+    # seems to be quite suitable.  In particular, `flask-httpauth` doesn't
+    # currently support passing additional data from the route through to
+    # `verify_password`.
+    if request.authorization is None:
+        username = None
+        password = None
+        log_prefix = f"<anonymous>@{archive}"
+    else:
+        username = request.authorization.username
+        password = request.authorization.password
+        log_prefix = f"{username}@{archive}"
+    try:
+        # XXX cjwatson 2022-10-14: We should cache positive responses (maybe
+        # using `flask-caching`) for a while to reduce database load.
+        get_archive_proxy().checkArchiveAuthToken(archive, username, password)
+    except Fault as e:
+        if e.faultCode == 410:  # Unauthorized
+            current_app.logger.info("%s: Password does not match.", log_prefix)
+        else:
+            # Interpret any other fault as NotFound (320).
+            current_app.logger.info("%s: %s", log_prefix, e.faultString)
+        basic = WWWAuthenticate()
+        basic.set_basic()
+        raise Unauthorized(www_authenticate=basic)
+    else:
+        current_app.logger.info("%s: Authorized.", log_prefix)
+
+
+# The exact details of the URLs used here should be regarded as a proof of
+# concept for now.
[e-mail]("/<archive:archive>/<path:path>")
+def translate(archive: str, path: str) -> tuple[str, int, dict[str, str]]:
+    check_auth(archive)
+    try:
+        url = get_archive_proxy().translatePath(archive, path)
+    except Fault as f:
+        if f.faultCode == 320:  # NotFound
+            return "Not found", 404, {"Content-Type": "text/plain"}
+        else:
+            return "Internal server error", 500, {"Content-Type": "text/plain"}
+    assert isinstance(url, str)
+    return "", 307, {"Location": url}
+
+
[e-mail]_request
+def add_headers(response: Response) -> Response:
+    response.headers["Vary"] = "Authorization"
+    return response
diff --git a/lp_archive/routing.py b/lp_archive/routing.py
new file mode 100644
index 0000000..a3be94d
--- /dev/null
+++ b/lp_archive/routing.py
@@ -0,0 +1,23 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Routing helpers."""
+
+from werkzeug.routing import BaseConverter
+
+
+class ArchiveConverter(BaseConverter):
+    """Match an archive reference.
+
+    See `lp.soyuz.model.archive.Archive.reference` in Launchpad.
+    """
+
+    # This doesn't currently support the partner/copy archive reference
+    # syntax (distribution/archive), since it's hard to avoid that being
+    # ambiguous when parsing URLs (compare with the primary archive
+    # reference syntax).
+    #
+    # PPA: ~[^/]+/[^/]+/[^/]+ (~owner/distribution/archive)
+    # Primary: [^~+][^/]* (distribution)
+    regex = r"~[^/]+/[^/]+/[^/]+|[^~+][^/]*"
+    part_isolating = False
diff --git a/requirements.in b/requirements.in
index e3e9a71..da656c4 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1 +1,2 @@
 Flask
+tomli; python_version < "3.11"
diff --git a/requirements.txt b/requirements.txt
index 27acd36..73d76c5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,5 +16,7 @@ markupsafe==2.1.1
     # via
     #   jinja2
     #   werkzeug
+tomli==2.0.1 ; python_version < "3.11"
+    # via -r requirements.in
 werkzeug==2.2.2
     # via flask
diff --git a/setup.cfg b/setup.cfg
index 8410614..f54019b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,6 +18,7 @@ classifiers =
 packages = find:
 install_requires =
     Flask
+    tomli;python_version < "3.11"
 python_requires = >=3.10
 
 [options.extras_require]
diff --git a/tests/conftest.py b/tests/conftest.py
index d8c9780..2006629 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,8 +8,7 @@ from lp_archive import create_app
 
 @pytest.fixture()
 def app():
-    app = create_app()
-    app.config.update({"TESTING": True})
+    app = create_app({"TESTING": True})
     yield app
 
 
diff --git a/tests/test_archive.py b/tests/test_archive.py
new file mode 100644
index 0000000..22bd8e1
--- /dev/null
+++ b/tests/test_archive.py
@@ -0,0 +1,121 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from threading import Thread
+from typing import Any
+from xmlrpc.client import Fault
+from xmlrpc.server import SimpleXMLRPCServer
+
+import pytest
+
+
+class ArchiveXMLRPCServer(SimpleXMLRPCServer):
+
+    path_map = {"dists/focal/InRelease": "http://librarian.example.org/1"}
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.call_log = []
+        self.register_function(
+            self.check_archive_auth_token, name="checkArchiveAuthToken"
+        )
+        self.register_function(self.translate_path, name="translatePath")
+
+    def check_archive_auth_token(self, archive, username, password):
+        self.call_log.append(
+            ("checkArchiveAuthToken", archive, username, password)
+        )
+        if archive.endswith("/private"):
+            raise Fault(410, "Authorization required")
+        elif archive.endswith("/nonexistent"):
+            raise Fault(320, "Not found")
+        else:
+            return True
+
+    def translate_path(self, archive, path):
+        # See `lp.xmlrpc.faults` in Launchpad for fault codes.
+        self.call_log.append(("translatePath", archive, path))
+        if path == "oops":
+            raise Fault(380, "Oops")
+        elif path in self.path_map:
+            return self.path_map[path]
+        else:
+            raise Fault(320, "Not found")
+
+
[e-mail]
+def archive_proxy(app):
+    with ArchiveXMLRPCServer(("127.0.0.1", 0)) as server:
+        host, port = server.server_address
+        app.config.update({"ARCHIVE_ENDPOINT": f"http://{host}:{port}"})
+        thread = Thread(target=server.serve_forever)
+        thread.start()
+        yield server
+        server.shutdown()
+        thread.join()
+
+
+def test_auth_failed(client, archive_proxy):
+    response = client.get(
+        "/~user/ubuntu/private/dists/focal/InRelease", auth=("user", "secret")
+    )
+    assert response.status_code == 401
+    assert (
+        response.headers["WWW-Authenticate"]
+        == 'Basic realm="authentication required"'
+    )
+    assert response.headers["Vary"] == "Authorization"
+    assert archive_proxy.call_log == [
+        ("checkArchiveAuthToken", "~user/ubuntu/private", "user", "secret")
+    ]
+
+
+def test_auth_not_found(client, archive_proxy):
+    response = client.get(
+        "/~user/ubuntu/nonexistent/dists/focal/InRelease",
+        auth=("user", "secret"),
+    )
+    assert response.status_code == 401
+    assert (
+        response.headers["WWW-Authenticate"]
+        == 'Basic realm="authentication required"'
+    )
+    assert response.headers["Vary"] == "Authorization"
+    assert archive_proxy.call_log == [
+        ("checkArchiveAuthToken", "~user/ubuntu/nonexistent", "user", "secret")
+    ]
+
+
+def test_translate(client, archive_proxy):
+    response = client.get("/ubuntu/dists/focal/InRelease")
+    assert response.status_code == 307
+    assert response.headers["Location"] == "http://librarian.example.org/1"
+    assert response.headers["Vary"] == "Authorization"
+    assert archive_proxy.call_log == [
+        ("checkArchiveAuthToken", "ubuntu", None, None),
+        ("translatePath", "ubuntu", "dists/focal/InRelease"),
+    ]
+
+
+def test_translate_not_found(client, archive_proxy):
+    response = client.get("/ubuntu/nonexistent")
+    assert response.status_code == 404
+    assert response.headers["Content-Type"] == "text/plain"
+    assert response.headers["Vary"] == "Authorization"
+    assert response.data == b"Not found"
+    assert archive_proxy.call_log == [
+        ("checkArchiveAuthToken", "ubuntu", None, None),
+        ("translatePath", "ubuntu", "nonexistent"),
+    ]
+
+
+def test_translate_oops(client, archive_proxy):
+    response = client.get("/ubuntu/oops")
+    assert response.status_code == 500
+    assert response.headers["Content-Type"] == "text/plain"
+    assert response.headers["Vary"] == "Authorization"
+    assert response.data == b"Internal server error"
+    assert archive_proxy.call_log == [
+        ("checkArchiveAuthToken", "ubuntu", None, None),
+        ("translatePath", "ubuntu", "oops"),
+    ]
diff --git a/tests/test_factory.py b/tests/test_factory.py
index 85d71ca..54275b9 100644
--- a/tests/test_factory.py
+++ b/tests/test_factory.py
@@ -1,9 +1,23 @@
 # Copyright 2022 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+import os
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
 from lp_archive import create_app
 
 
-def test_config():
-    assert not create_app().testing
+def test_config_from_file():
+    with TemporaryDirectory() as empty:
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(empty)
+            Path("config.toml").touch()
+            assert not create_app().testing
+        finally:
+            os.chdir(old_cwd)
+
+
+def test_config_for_testing():
     assert create_app({"TESTING": True}).testing
diff --git a/tests/test_routing.py b/tests/test_routing.py
new file mode 100644
index 0000000..e11ae4e
--- /dev/null
+++ b/tests/test_routing.py
@@ -0,0 +1,41 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from flask import url_for
+
+
+def test_primary(app, client):
+    @app.route("/+test/<archive:archive>")
+    def index(archive):
+        return archive
+
+    response = client.get("/+test/ubuntu")
+    assert response.status_code == 200
+    assert response.data == b"ubuntu"
+
+    with app.test_request_context():
+        assert url_for("index", archive="ubuntu") == "/+test/ubuntu"
+
+
+def test_ppa(app, client):
+    @app.route("/+test/<archive:archive>")
+    def index(archive):
+        return archive
+
+    response = client.get("/+test/~owner/ubuntu/ppa")
+    assert response.status_code == 200
+    assert response.data == b"~owner/ubuntu/ppa"
+
+    with app.test_request_context():
+        assert (
+            url_for("index", archive="~owner/ubuntu/ppa")
+            == "/+test/~owner/ubuntu/ppa"
+        )
+
+
+def test_invalid_archive(app, client):
+    @app.route("/+test/<archive:archive>")
+    def index(archive):  # pragma: no cover
+        return archive
+
+    assert client.get("/+test/~owner/ubuntu").status_code == 404