Merge ~bowenfan/snapstore-client:SN2278-model_service_auth_and_cli_skeleton into snapstore-client:main

[Bowen Fan]

Add snapstore-client support for publishergw and model service

Upon first login to an airgapped proxy, snapstore-client will now
exchange the configured store admin macaroon for the offline publishergw
admin auth macaroon. The exchanged macaroon will be saved in the client
config and used automatically for subsequent publishergw requests.

Also add initial CLI for model service model creation and logic
webservices layers dependencies.

Solves: https://warthogs.atlassian.net/browse/SN-2337

This MP is part 1 of 5 for adding model service support to snapstore-client.

1: Add publishergw support and a skeleton for model service
2: Add remaining create CLI functions
3: List
4: Update
5: Delete

MPs 2-5 will be raised shortly.

CLI commands have been implemented, as much as possible, to follow this command structure guidance doc: https://discourse.ubuntu.com/t/command-structure/18556

On-prem model service spec: https://docs.google.com/document/d/1uxJ0Z1hCN_-6aJRwy9RhM3KblSuMgHqPwYEyP_88Ijk/edit

[Deep Fowdar (superalpaca)]

Thanks for this! I've left one comment in the diff

review: Approve

02f9201... by Bowen Fan on 2023-12-11

Hardcode UC series to 16 per MP feedback

Model series must be "16" for historical compatibility reasons, so we hardcode it to avoid confusing users.

Also improve model service model creation mock response (currently unused).

[Bowen Fan (bowenfan)]

> Thanks for this! I've left one comment in the diff

Thanks for the review! I've hardcoded 'series' so that users aren't asked for it in the CLI. Eventually as you said, we can hardcode it in publishergw or even snapmodels too.

diff --git a/conftest.py b/conftest.py
index 6ba7f19..48a6240 100644
--- a/conftest.py
+++ b/conftest.py
@@ -5,12 +5,11 @@ from pathlib import Path
 from unittest import mock
 from unittest.mock import patch
 
-from pymacaroons import Macaroon
 import pytest
+from pymacaroons import Macaroon
 
 from store_admin.config import Config
 
-
 root_logger = logging.getLogger(None)
 root_logger.setLevel(logging.DEBUG)
 
@@ -79,6 +78,27 @@ def mocked_config(mocked_xdgconfig):
 
 
 @pytest.fixture
+def mocked_offline_config(mocked_xdgconfig, exchanged_macaroon):
+    """Example expected config when using the client against an offline proxy"""
+    app_config_path = os.path.join(mocked_xdgconfig, "store-admin")
+    os.makedirs(app_config_path)
+
+    with open(os.path.join(app_config_path, "config.ini"), "w") as f:
+        f.write(
+            (
+                "[store:default]\n"
+                "gw_url = {gw_url}\n"
+                "pubgw_token = {pubgw_token}\n"
+            ).format(
+                gw_url="http://offline.local/",
+                pubgw_token=exchanged_macaroon,
+            )
+        )
+
+    yield Config()
+
+
[e-mail]
 def mocked_empty_config(mocked_xdgconfig):
     app_config_path = os.path.join(mocked_xdgconfig, "store-admin")
     os.makedirs(app_config_path)
diff --git a/store_admin/cli/model_service.py b/store_admin/cli/model_service.py
new file mode 100644
index 0000000..10b84d6
--- /dev/null
+++ b/store_admin/cli/model_service.py
@@ -0,0 +1,19 @@
+import click
+from store_admin.logic import model_service
+
+
[e-mail](name="model")
[e-mail]("model_name")
[e-mail](
+    "--api-key",
+    help="Model API key for validating serial requests",
+    prompt="API key (alphanumeric, `pwgen 40` for options)",
+)
+def create_model(model_name, api_key):
+    """Create a model in the model service.
+
+    MODEL_NAME must be lowercase alphanumeric, optionally with hyphens.
+    """
+    # UC series is hardcoded as 16 for historical reasons
+    series = "16"
+    model_service.create_model(model_name, api_key, series)
diff --git a/store_admin/cli/runner.py b/store_admin/cli/runner.py
index 03c172e..5ec6421 100644
--- a/store_admin/cli/runner.py
+++ b/store_admin/cli/runner.py
@@ -3,7 +3,7 @@ import logging
 import click
 
 from store_admin import defaults, log, version
-from store_admin.cli import common, overrides, snaps, stores
+from store_admin.cli import common, model_service, overrides, snaps, stores
 
 
 @click.group(
@@ -45,6 +45,11 @@ def run(debug, dashboard_root, devicegw_root, sso_url):
     log.configure(log_level=log_level)
 
 
[e-mail]()
+def create():
+    "Create various artifacts on the managed proxy."
+
+
 @run.group(name="list")
 def lst():
     "List snaps or overrides stored on the proxy."
@@ -69,6 +74,9 @@ def export():
 run.add_command(common.login)
 run.add_command(common.register)
 
+# Commands under create
+create.add_command(model_service.create_model)
+
 # Commands under list
 lst.add_command(overrides.list_overrides)
 
diff --git a/store_admin/logic/accounts.py b/store_admin/logic/accounts.py
index 5440af1..b8f0d6a 100644
--- a/store_admin/logic/accounts.py
+++ b/store_admin/logic/accounts.py
@@ -71,3 +71,20 @@ def export_token(ttl):
     logger.info(f"Exported token with permissions={permissions!r} and ttl={ttl!r}")
     logger.info("Usage:")
     logger.info(f"    export {STORE_ADMIN_ENV_AUTH_KEY}={auth_token}")
+
+
+def exchange_publishergw_macaroon(gw_url, admin_auth_token):
+    """Exchange a store admin macaroon for a publishergw gateway store admin token.
+
+    The exported store, which includes the store admin macaroon, must have already
+    been pushed to the proxy.
+    """
+    logger.info(
+        "Exchanging store admin macaroon for a publisher gateway admin macaroon..."
+    )
+    offline_exchange_url = urljoin(gw_url, "/publishergw/v1/tokens/offline/exchange")
+
+    resp = requests.post(offline_exchange_url, json={"macaroon": admin_auth_token})
+
+    exchanged_macaroon = resp.json()["macaroon"]
+    return exchanged_macaroon
diff --git a/store_admin/logic/login.py b/store_admin/logic/login.py
index 42cb43e..6b40ab0 100644
--- a/store_admin/logic/login.py
+++ b/store_admin/logic/login.py
@@ -2,6 +2,7 @@
 
 import getpass
 import logging
+import os
 from urllib.parse import urlparse
 
 from pymacaroons import Macaroon
@@ -11,7 +12,8 @@ from store_admin import (
     exceptions,
     webservices as ws,
 )
-
+from store_admin.logic.http_client import STORE_ADMIN_ENV_AUTH_KEY
+from store_admin.logic.accounts import exchange_publishergw_macaroon
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +36,18 @@ def login(store_url, email, sso_url, offline):
         cfg = config.Config()
         store = cfg.store_section("default")
         store.set("gw_url", gw_url)
+
+        # Logging in to an airgapped proxy should set the exchanged publishergw token
+        admin_token = os.getenv(STORE_ADMIN_ENV_AUTH_KEY)
+        if not admin_token:
+            logger.warning(
+                "Could not find store admin macaroon. Publisher gateway token not set."
+            )
+            return
+
+        pubgw_token = exchange_publishergw_macaroon(gw_url, admin_token)
+        store.set("pubgw_token", pubgw_token)
+
         cfg.save()
         return
 
diff --git a/store_admin/logic/model_service.py b/store_admin/logic/model_service.py
new file mode 100644
index 0000000..115d102
--- /dev/null
+++ b/store_admin/logic/model_service.py
@@ -0,0 +1,45 @@
+import logging
+import os
+from store_admin.utils import _check_default_store
+from store_admin import (
+    config,
+    webservices as ws,
+)
+
+logger = logging.getLogger(__name__)
+
+
+BRAND_ACCOUNT_ID_ENV_KEY = "BRAND_ACCOUNT_ID"
+
+
+def _get_or_set_brand_account_id(cfg):
+    brand_account_id = os.getenv(BRAND_ACCOUNT_ID_ENV_KEY)
+    store = cfg.store_section("default")
+
+    if brand_account_id:
+        store.set("brand_account_id", brand_account_id)
+        cfg.save()
+        return brand_account_id
+
+    saved = store.get("brand_account_id")
+    if not saved:
+        logger.error(
+            "No brand account ID saved.\n Re-run command after "
+            f"setting the {BRAND_ACCOUNT_ID_ENV_KEY} environment variable.\n"
+            "It will be saved for subsequent use."
+        )
+        return
+    return saved
+
+
+def create_model(model_name, api_key, series):
+    cfg = config.Config()
+    store = _check_default_store(cfg)
+    brand_account_id = _get_or_set_brand_account_id(cfg)
+    if not store or not brand_account_id:
+        return 1
+
+    json = {"name": model_name, "api-key": api_key, "series": series}
+    path = f"/publishergw/v1/brand/{brand_account_id}/model"
+    ws.model_service_create(store, path, json)
+    logger.info(f"Model {model_name} created.")
diff --git a/store_admin/logic/stores.py b/store_admin/logic/stores.py
index 7388ad0..46d2271 100644
--- a/store_admin/logic/stores.py
+++ b/store_admin/logic/stores.py
@@ -17,7 +17,7 @@ import requests
 
 from . import assertions, snaps
 from .accounts import get_account_details
-from .http_client import get_developer_session
+from .http_client import STORE_ADMIN_ENV_AUTH_KEY, get_developer_session
 from .. import defaults, exceptions
 
 
@@ -54,6 +54,8 @@ def export(store_id, channels, architectures, skip_snaps, account_key_ids=None):
     export_tarball_path.chmod(0o400)
     logger.info(f"Store data exported to: {export_tarball_path}")
     logger.info(f"Admin token exported to: {export_token_path}")
+    logger.info("Admin token usage:")
+    logger.info(f"    export {STORE_ADMIN_ENV_AUTH_KEY}=$(cat {export_token_path})")
 
 
 def export_files(
diff --git a/store_admin/logic/tests/conftest.py b/store_admin/logic/tests/conftest.py
index ac92aeb..914a994 100644
--- a/store_admin/logic/tests/conftest.py
+++ b/store_admin/logic/tests/conftest.py
@@ -2,6 +2,7 @@
 # Copyright 2022 Canonical Ltd.  This software is licensed under the
 # GNU General Public License version 3 (see the file LICENSE).
 
+from datetime import datetime
 import pytest
 import responses
 from pymacaroons import Macaroon, MACAROON_V2
@@ -197,6 +198,64 @@ def whoami_api(dashboard_root, requests_mock, whoami_data):
 
 
 @pytest.fixture
+def brand_account_id():
+    return "test-brand-account-id"
+
+
+def _create_model_service_user_response(display_name=None, id=None, username=None):
+    return {
+        "display-name": display_name or "User Display Name",
+        "id": id or "user_account_id",
+        "username": username or "username",
+    }
+
+
[e-mail]
+def model_service_model_apis(
+    requests_mock,
+    brand_account_id,
+):
+    def _make_model_response(brand_account_id, **kwargs):
+        model = {}
+
+        model["brand_account_id"] = brand_account_id
+        model["name"] = kwargs.get("name") or "test-model"
+        model["series"] = "16"
+        model["api-key"] = kwargs.get("api_key") or "test-api-key"
+
+        model["created-by"] = (
+            kwargs.get("created_by") or _create_model_service_user_response()
+        )
+        model["created-at"] = (
+            kwargs.get("created_at") or datetime.utcnow().isoformat() + "Z",
+        )
+        model["modified-by"] = (
+            _create_model_service_user_response()
+            if "modified_by" not in kwargs
+            else kwargs["modified_by"]
+        )
+        model["modified-at"] = (
+            datetime.utcnow().isoformat() + "Z"
+            if "modified_at" not in kwargs
+            else kwargs["modified_at"]
+        )
+
+        return model
+
+    def setup(gw_url):
+        create_response = _make_model_response(brand_account_id)
+        requests_mock.add(
+            "POST",
+            gw_url + f"publishergw/v1/brand/{brand_account_id}/model",
+            json=create_response,
+            status=201,
+        )
+        return (create_response, requests_mock)
+
+    return setup
+
+
[e-mail]
 def asserts_mock(devicegw_root, requests_mock):
     def add(assert_type, key, body=None, json=None):
         assert (json is None) != (body is None)
diff --git a/store_admin/logic/tests/test_login.py b/store_admin/logic/tests/test_login.py
index d4c6230..75b567d 100644
--- a/store_admin/logic/tests/test_login.py
+++ b/store_admin/logic/tests/test_login.py
@@ -2,12 +2,14 @@
 
 import logging
 import json
+import os
 from unittest import mock
 from urllib.parse import urljoin, urlparse
 
 import pytest
 import responses
 from pymacaroons import Macaroon
+from store_admin.logic import http_client
 from testtools.matchers import (
     ContainsDict,
     Equals,
@@ -241,6 +243,58 @@ class TestLogin:
             is None
         )
 
+    @mock.patch.dict(os.environ, {})
+    def test_login_to_offline_proxy_no_admin_macaroon(self, caplog):
+        gw_url = "http://offline.local:1234/"
+
+        login(**self.make_args(store_url=gw_url, offline=True))
+
+        assert len(caplog.records) == 1
+        assert (
+            caplog.records[-1].msg
+            == "Could not find store admin macaroon. Publisher gateway token not set."
+        )
+
+    @mock.patch.dict(
+        os.environ, {http_client.STORE_ADMIN_ENV_AUTH_KEY: "mock-credentials"}
+    )
+    def test_login_to_offline_proxy_success(
+        self, requests_mock, exchanged_macaroon, caplog
+    ):
+        gw_url = "http://offline.local:1234/"
+        exchanged = {"macaroon": exchanged_macaroon}
+        requests_mock.add(
+            "POST",
+            gw_url + "publishergw/v1/tokens/offline/exchange",
+            match=[
+                responses.matchers.json_params_matcher({"macaroon": "mock-credentials"})
+            ],
+            json=exchanged,
+            status=200,
+        )
+
+        login(**self.make_args(store_url=gw_url, offline=True))
+
+        assert len(requests_mock.calls) == 1
+        assert (
+            ContainsDict(
+                {
+                    "store:default": ContainsDict(
+                        {
+                            "gw_url": Equals(gw_url),
+                            "pubgw_token": Equals(exchanged_macaroon),
+                        }
+                    ),
+                }
+            ).match(config.Config().parser)
+            is None
+        )
+
+        assert len(caplog.messages) == 1
+        assert caplog.messages[-1] == (
+            "Exchanging store admin macaroon for a publisher gateway admin macaroon..."
+        )
+
     @responses.activate
     def test_store_url(self, mock_input, mock_getpass):
         gw_url = "http://otherstore.local:1234/"
diff --git a/store_admin/logic/tests/test_model_service.py b/store_admin/logic/tests/test_model_service.py
new file mode 100644
index 0000000..53b035b
--- /dev/null
+++ b/store_admin/logic/tests/test_model_service.py
@@ -0,0 +1,74 @@
+import json
+import os
+from unittest import mock
+import pytest
+from store_admin.logic.model_service import (
+    BRAND_ACCOUNT_ID_ENV_KEY,
+    create_model,
+)
+from testtools.matchers import (
+    ContainsDict,
+    Equals,
+)
+from store_admin import (
+    config,
+)
+import requests
+
+
[e-mail]("mocked_offline_config")
+class TestModelService:
+    gw_url = "http://offline.local/"
+
+    @mock.patch.dict(os.environ, {BRAND_ACCOUNT_ID_ENV_KEY: "test-brand-account-id"})
+    def test_set_brand_account_id(self, model_service_model_apis):
+        model_service_model_apis(self.gw_url)
+        create_model("test-name", "test-key", "16")
+
+        assert (
+            ContainsDict(
+                {
+                    "store:default": ContainsDict(
+                        {
+                            "brand_account_id": Equals("test-brand-account-id"),
+                        }
+                    ),
+                }
+            ).match(config.Config().parser)
+            is None
+        )
+
+    @mock.patch.dict(os.environ, {})
+    def test_no_brand_account_id_saved(self, caplog):
+        create_model("test-name", "test-key", "16")
+
+        assert caplog.messages[-1].startswith("No brand account ID saved.")
+
+    @mock.patch.dict(os.environ, {BRAND_ACCOUNT_ID_ENV_KEY: "test-brand-account-id"})
+    def test_authentication_failure(self, requests_mock, caplog):
+        requests_mock.add(
+            "POST",
+            self.gw_url + "publishergw/v1/brand/test-brand-account-id/model",
+            status=401,
+        )
+
+        with pytest.raises(requests.exceptions.HTTPError):
+            create_model("test-name", "test-key", "16")
+
+        assert (
+            caplog.messages[-1]
+            == "Authentication error. Please (re-)login to the offline store."
+        )
+
+    @mock.patch.dict(os.environ, {BRAND_ACCOUNT_ID_ENV_KEY: "test-brand-account-id"})
+    def test_create_model_success(self, model_service_model_apis, caplog):
+        _, api_mock = model_service_model_apis(self.gw_url)
+        create_model("test-name", "test-key", "16")
+
+        assert len(api_mock.calls) == 1
+        assert {
+            "name": "test-name",
+            "api-key": "test-key",
+            "series": "16",
+        } == json.loads(api_mock.calls[-1].request.body.decode())
+        assert caplog.messages[-1] == "Model test-name created."
diff --git a/store_admin/logic/tests/test_stores.py b/store_admin/logic/tests/test_stores.py
index 4793fe9..ee69b8d 100644
--- a/store_admin/logic/tests/test_stores.py
+++ b/store_admin/logic/tests/test_stores.py
@@ -13,7 +13,7 @@ import requests
 
 from store_admin import defaults, exceptions
 from store_admin.logic import stores
-
+from store_admin.logic.http_client import STORE_ADMIN_ENV_AUTH_KEY
 
 PATCH_PREFIX = "store_admin.logic.stores."
 
@@ -431,6 +431,8 @@ def test_export(caplog, tmpdir, snap_env, requests_mock, store_id):
         "Creating the export archive...",
         f"Store data exported to: {tarball_path}",
         f"Admin token exported to: {token_path}",
+        "Admin token usage:",
+        f"    export {STORE_ADMIN_ENV_AUTH_KEY}=$(cat {token_path})",
     ]
 
 
diff --git a/store_admin/webservices.py b/store_admin/webservices.py
index efd5484..a2196cb 100644
--- a/store_admin/webservices.py
+++ b/store_admin/webservices.py
@@ -164,6 +164,25 @@ def set_overrides(store, overrides, password=None):
     return resp.json()
 
 
+def model_service_create(store, create_path, json):
+    """Send a POST creation request for model service entities to the proxy URL."""
+    create_url = urllib.parse.urljoin(store.get("gw_url"), create_path)
+    pubgw_admin_token = store.get("pubgw_token")
+    headers = {"Authorization": f"Macaroon {pubgw_admin_token}"}
+
+    resp = requests.post(create_url, headers=headers, json=json)
+
+    if resp.status_code != 201:
+        if resp.status_code == 401:
+            logger.error(
+                "Authentication error. Please (re-)login to the offline store."
+            )
+        else:
+            _print_error_message("create model service entity", resp)
+        resp.raise_for_status()
+    return resp.json()
+
+
 def _print_error_message(action, response):
     """Print failure messages from other services in a standard way."""
     logger.error("Failed to %s:", action)