Merge ~orndorffgrant/ubuntu/+source/ubuntu-advantage-tools:upload-27.11.3-kinetic into ubuntu/+source/ubuntu-advantage-tools:ubuntu/devel

[Grant Orndorff]

This is release 27.11.3 of ubuntu-advantage-tools.

You can find the description of changes in the overall SRU bug here: https://bugs.launchpad.net/ubuntu/+source/ubuntu-advantage-tools/+bug/1993006

[Robie Basak (racb)]

SRU review.

I flagged the entire "apt news" concept as needing leadership approval in https://bugs.launchpad.net/ubuntu/+source/ubuntu-advantage-tools/+bug/1992026/comments/17, and I don't think I should accept this into proposed until that is resolved.

Apart from that this looks fine from an SRU perspective, from a quick review to look for significant changes that would impact the system and/or existing users (I've not reviewed every line). I made a couple of comments inline, but I don't think we need to block on them.

[Grant Orndorff (orndorffgrant)]

Thanks racb!

I responded to the inline comments - I agree with them and have added bugs to our github to follow up with them in a future release.

Ack to the "apt news" status and thanks for flagging that formally on the bug. I agree that we can't move forward until that is fully resolved.

[Paride Legovini (paride)]

Following-up to my previous review: the new changes LGTM. I'm proceeding with the uploads.

review: Approve

[Robie Basak (racb)]

"apt news" resolved by https://bugs.launchpad.net/ubuntu/+source/ubuntu-advantage-tools/+bug/1992026/comments/18. Nothing else outstanding, so SRU +1.

review: Approve (ubuntu-sru)

diff --git a/apt-hook/20apt-esm-hook.conf b/apt-hook/20apt-esm-hook.conf
index 0cc6823..07c3362 100644
--- a/apt-hook/20apt-esm-hook.conf
+++ b/apt-hook/20apt-esm-hook.conf
@@ -10,6 +10,6 @@ APT::Install::Pre-Invoke {
 	"[ ! -f /usr/lib/ubuntu-advantage/apt-esm-hook ] || /usr/lib/ubuntu-advantage/apt-esm-hook pre-invoke || true";
 };
 
-AptCli::Hooks::Upgrade {
+binary::apt::AptCli::Hooks::Upgrade {
 	"[ ! -f /usr/lib/ubuntu-advantage/apt-esm-json-hook ] || /usr/lib/ubuntu-advantage/apt-esm-json-hook || true";
 };
diff --git a/apt-hook/json-hook.cc b/apt-hook/json-hook.cc
index 0efaa81..16042f8 100644
--- a/apt-hook/json-hook.cc
+++ b/apt-hook/json-hook.cc
@@ -257,23 +257,33 @@ int run()
 
     json_object_put(hello_req.root_msg);
 
-    jsonrpc_request stats_req;
-    success = read_jsonrpc_request(socket_in, stats_req);
+    jsonrpc_request hook_req;
+    success = read_jsonrpc_request(socket_in, hook_req);
     if (!success) {
         std::cerr << "ua-hook: failed to read hook msg" << std::endl;
         return 0;
     }
-    if (stats_req.method == "org.debian.apt.hooks.install.statistics") {
+    if (hook_req.method == "org.debian.apt.hooks.install.statistics") {
         security_package_counts counts;
-        success = count_security_packages_from_apt_stats_json(stats_req.params, counts);
+        success = count_security_packages_from_apt_stats_json(hook_req.params, counts);
         if (success) {
             std::string message = create_count_message(counts);
             if (message != "") {
                 std::cout << message << std::endl;
             }
         }
+    } else if (hook_req.method == "org.debian.apt.hooks.install.pre-prompt") {
+        std::ifstream apt_news_flag_file("/var/lib/ubuntu-advantage/flags/show-apt-news");
+        if (apt_news_flag_file.is_open()) {
+            std::cout << "#" << std::endl;
+            std::cout << "# News about significant security updates, features and services will" << std::endl;
+            std::cout << "# appear here to raise awareness and perhaps tease /r/Linux ;)" << std::endl;
+            std::cout << "# Use 'pro config set apt_news=false' to hide this and future APT news." << std::endl;
+            std::cout << "#" << std::endl;
+            apt_news_flag_file.close();
+        }
     }
-    json_object_put(stats_req.root_msg);
+    json_object_put(hook_req.root_msg);
 
     jsonrpc_request bye_req;
     success = read_jsonrpc_request(socket_in, bye_req);
diff --git a/debian/changelog b/debian/changelog
index a5acfc4..e0d5868 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,16 @@
+ubuntu-advantage-tools (27.11.3~22.10.1) kinetic; urgency=medium
+
+  * New upstream release 27.11.3: (LP: #1993006)
+    - d/postinst: remove the Ubuntu Pro beta apt message and set up the
+      configurable flag for "APT news" instead
+    - collect-logs: do not fail if a file cannot be read (LP: #1991858)
+    - config: add a flag to disable "APT news" (LP: 1992026)
+    - messaging: add announcement of "APT news" to apt output
+    - messaging: only show "APT news" when using apt binary (GH: #2288)
+    - version: use /run instead of /tmp for version file (GH: #2294)
+
+ -- Grant Orndorff <[e-mail]>  Fri, 14 Oct 2022 16:55:27 -0400
+
 ubuntu-advantage-tools (27.11.2~22.10.1) kinetic; urgency=medium
 
   * New upstream release 27.11.2: (LP: #1991173)
diff --git a/debian/ubuntu-advantage-tools.postinst b/debian/ubuntu-advantage-tools.postinst
index 4a677fc..73b914b 100644
--- a/debian/ubuntu-advantage-tools.postinst
+++ b/debian/ubuntu-advantage-tools.postinst
@@ -57,6 +57,13 @@ OLD_LICENSE_CHECK_MARKER_FILE="/var/lib/ubuntu-advantage/marker-license-check"
 MACHINE_TOKEN_FILE="/var/lib/ubuntu-advantage/private/machine-token.json"
 PUBLIC_MACHINE_TOKEN_FILE="/var/lib/ubuntu-advantage/machine-token.json"
 
+UA_MESSAGES_DIR="/var/lib/ubuntu-advantage/messages"
+APT_ESM_MESSAGE_FILE="$UA_MESSAGES_DIR/apt-pre-invoke-esm-service-status"
+ESM_APPS_APT_MSG_FILES="$UA_MESSAGES_DIR/apt-pre-invoke-no-packages-apps.tmpl $UA_MESSAGES_DIR/apt-pre-invoke-no-packages-apps $UA_MESSAGES_DIR/apt-pre-invoke-packages-apps.tmpl $UA_MESSAGES_DIR/apt-pre-invoke-packages-apps"
+UA_FLAGS_DIR="/var/lib/ubuntu-advantage/flags"
+APT_NEWS_FLAG_FILE="$UA_FLAGS_DIR/show-apt-news"
+
+TMP_CANDIDATE_CACHE_PATH="/tmp/ubuntu-advantage/candidate-version"
 
 # Rename apt config files for ua services removing ubuntu release names
 redact_ubuntu_release_from_ua_apt_filenames() {
@@ -384,6 +391,29 @@ machine_token_file.write(content=content)
 "
 }
 
+migrate_ubuntu_pro_beta_banner() {
+    PREVIOUS_PKG_VER=$1
+    # This only shipped in 27.11.2~
+    if dpkg --compare-versions "$PREVIOUS_PKG_VER" ge "27.11.2~" \
+       && dpkg --compare-versions "$PREVIOUS_PKG_VER" lt "27.11.3~"; then
+        # if the banner was present, remove it and add the new flag file which is configurable
+        if [ -f $APT_ESM_MESSAGE_FILE ]; then
+            if cat $APT_ESM_MESSAGE_FILE | grep -q "Try Ubuntu Pro beta"; then
+                rm -f $APT_ESM_MESSAGE_FILE
+                rm -f $ESM_APPS_APT_MSG_FILES
+                mkdir -p $UA_FLAGS_DIR
+                touch $APT_NEWS_FLAG_FILE
+            fi
+        fi
+    fi
+}
+
+cleanup_candidate_version_stamp_permissions() {
+    if [ -f $TMP_CANDIDATE_CACHE_PATH ]; then
+        rm -f $TMP_CANDIDATE_CACHE_PATH
+    fi
+}
+
 case "$1" in
     configure)
       PREVIOUS_PKG_VER=$2
@@ -462,6 +492,8 @@ case "$1" in
       if [ -f $MACHINE_TOKEN_FILE ] && [ ! -f $PUBLIC_MACHINE_TOKEN_FILE ]; then
           create_public_machine_token_file
       fi
+      migrate_ubuntu_pro_beta_banner $PREVIOUS_PKG_VER
+      cleanup_candidate_version_stamp_permissions
       ;;
 esac
 
diff --git a/features/api_full_auto_attach.feature b/features/api_full_auto_attach.feature
index 240e06f..c65fca5 100644
--- a/features/api_full_auto_attach.feature
+++ b/features/api_full_auto_attach.feature
@@ -27,7 +27,7 @@ Feature: Full Auto-Attach Endpoint
         """
         Then stdout matches regexp:
         """
-        livepatch     +yes +disabled  +Canonical Livepatch service
+        livepatch     +yes +(disabled|n/a)  +Canonical Livepatch service
         """
         Examples:
            | release |
diff --git a/features/apt_messages.feature b/features/apt_messages.feature
index 27d67cb..b03524f 100644
--- a/features/apt_messages.feature
+++ b/features/apt_messages.feature
@@ -9,7 +9,7 @@ Feature: APT Messages
         When I run `apt-get upgrade -y` with sudo
 
         When I run `apt-get install -y --allow-downgrades <standard-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         2 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -18,7 +18,7 @@ Feature: APT Messages
         """
 
         When I run `apt-get install -y --allow-downgrades <infra-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         2 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -27,7 +27,7 @@ Feature: APT Messages
         """
 
         When I run `apt-get install -y --allow-downgrades <apps-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         1 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -37,7 +37,7 @@ Feature: APT Messages
 
         When I run `apt-get install -y --allow-downgrades <standard-pkg>` with sudo
         When I run `apt-get install -y --allow-downgrades <infra-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         4 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -47,7 +47,7 @@ Feature: APT Messages
 
         When I run `apt-get install -y --allow-downgrades <standard-pkg>` with sudo
         When I run `apt-get install -y --allow-downgrades <apps-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         3 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -57,7 +57,7 @@ Feature: APT Messages
 
         When I run `apt-get install -y --allow-downgrades <infra-pkg>` with sudo
         When I run `apt-get install -y --allow-downgrades <apps-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         3 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -68,7 +68,7 @@ Feature: APT Messages
         When I run `apt-get install -y --allow-downgrades <standard-pkg>` with sudo
         When I run `apt-get install -y --allow-downgrades <infra-pkg>` with sudo
         When I run `apt-get install -y --allow-downgrades <apps-pkg>` with sudo
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         5 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -76,7 +76,7 @@ Feature: APT Messages
 
         """
 
-        When I run `apt-get upgrade -y` with sudo
+        When I run `apt upgrade -y` with sudo
         Then stdout matches regexp:
         """
         0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
@@ -106,37 +106,52 @@ Feature: APT Messages
         When I run `apt-get -y upgrade` with sudo
         When I run `apt-get -y autoremove` with sudo
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade` with sudo
+        When I run `apt upgrade` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
         The following security updates require Ubuntu Pro with 'esm-infra' enabled:
           .*
         Learn more about Ubuntu Pro for 16\.04 at https:\/\/ubuntu\.com\/16-04
         0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded\.
         """
         When I attach `contract_token` with sudo
-        When I run `apt-get upgrade --dry-run` with sudo
+        When I run `apt upgrade --dry-run` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
         The following packages will be upgraded:
         """
         When I update contract to use `effectiveTo` as `days=+2`
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade --dry-run` with sudo
+        When I run `apt upgrade --dry-run` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
 
         CAUTION: Your Ubuntu Pro subscription will expire in 2 days.
         Renew your subscription at https:\/\/ubuntu.com\/pro to ensure continued security
@@ -146,13 +161,18 @@ Feature: APT Messages
         """
         When I update contract to use `effectiveTo` as `days=-3`
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade --dry-run` with sudo
+        When I run `apt upgrade --dry-run` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
 
         CAUTION: Your Ubuntu Pro subscription expired on \d+ \w+ \d+.
         Renew your subscription at https:\/\/ubuntu.com\/pro to ensure continued security
@@ -163,13 +183,18 @@ Feature: APT Messages
         """
         When I update contract to use `effectiveTo` as `days=-20`
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade --dry-run` with sudo
+        When I run `apt upgrade --dry-run` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
 
         \*Your Ubuntu Pro subscription has EXPIRED\*
         The following security updates require Ubuntu Pro with 'esm-infra' enabled:
@@ -179,13 +204,18 @@ Feature: APT Messages
         The following packages will be upgraded:
         """
         When I run `apt-get upgrade -y` with sudo
-        When I run `apt-get upgrade` with sudo
+        When I run `apt upgrade` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
 
         \*Your Ubuntu Pro subscription has EXPIRED\*
         Renew your service at https:\/\/ubuntu.com\/pro
@@ -194,13 +224,18 @@ Feature: APT Messages
         """
         When I run `pro detach --assume-yes` with sudo
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade` with sudo
+        When I run `apt upgrade` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
         Receive additional future security updates with Ubuntu Pro.
         Learn more about Ubuntu Pro for 16\.04 at https:\/\/ubuntu\.com\/16-04
         0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded\.
@@ -218,31 +253,48 @@ Feature: APT Messages
         When I run `apt-get -y autoremove` with sudo
         When I run `apt-get install hello` with sudo
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade` with sudo
+        When I run `apt upgrade` with sudo
         Then I will see the following on stdout:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
-        Try Ubuntu Pro beta with a free personal subscription on up to 5 machines.
-        Learn more at https://ubuntu.com/pro
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;)
+        # Use 'pro config set apt_news=false' to hide this and future APT news.
+        #
         0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
         """
+        When I run `apt-get upgrade` with sudo
+        Then stdout does not match regexp:
+        """
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
+        """
         When I attach `contract_token` with sudo
-        When I run `apt-get upgrade --dry-run` with sudo
+        When I run `apt upgrade --dry-run` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
         The following packages will be upgraded:
           hello
         """
 #        When I update contract to use `effectiveTo` as `days=-20`
 #        When I run `pro refresh messages` with sudo
-#        When I run `apt-get upgrade --dry-run` with sudo
+#        When I run `apt upgrade --dry-run` with sudo
 #        Then stdout matches regexp:
 #        """
 #        Reading package lists...
@@ -260,17 +312,30 @@ Feature: APT Messages
         When I run `apt-get upgrade -y` with sudo
         When I run `pro detach --assume-yes` with sudo
         When I run `pro refresh messages` with sudo
-        When I run `apt-get upgrade` with sudo
+        When I run `apt upgrade` with sudo
         Then stdout matches regexp:
         """
         Reading package lists...
         Building dependency tree...
         Reading state information...
         Calculating upgrade...
-        Try Ubuntu Pro beta with a free personal subscription on up to 5 machines.
-        Learn more at https:\/\/ubuntu.com\/pro
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
         0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded\.
         """
+        When I run `pro config set apt_news=false` with sudo
+        And I run `apt upgrade` with sudo
+        Then stdout does not match regexp:
+        """
+        #
+        # News about significant security updates, features and services will
+        # appear here to raise awareness and perhaps tease /r/Linux ;\)
+        # Use 'pro config set apt_news=false' to hide this and future APT news\.
+        #
+        """
         Examples: ubuntu release
           | release |
           | focal   |
diff --git a/features/cloud_pro_clone.feature b/features/cloud_pro_clone.feature
index f08bff0..48aa17a 100644
--- a/features/cloud_pro_clone.feature
+++ b/features/cloud_pro_clone.feature
@@ -2,7 +2,6 @@ Feature: Creating golden images based on Cloud Ubuntu Pro instances
 
     @series.lts
     @uses.config.machine_type.aws.pro
-    @uses.config.machine_type.azure.pro
     @uses.config.machine_type.gcp.pro
     Scenario Outline: Create a Pro fips-updates image and launch
         Given a `<release>` machine with ubuntu-advantage-tools installed
diff --git a/features/daemon.feature b/features/daemon.feature
index 593c86d..0b33c37 100644
--- a/features/daemon.feature
+++ b/features/daemon.feature
@@ -104,9 +104,10 @@ Feature: Pro Upgrade Daemon only runs in environments where necessary
         """
         Active: active \(running\)
         """
-        Then on `xenial`, systemd status output says memory usage is less than `14` MB
+        # TODO find out what caused memory to go up, try to lower it again
+        Then on `xenial`, systemd status output says memory usage is less than `15` MB
         Then on `bionic`, systemd status output says memory usage is less than `13` MB
-        Then on `focal`, systemd status output says memory usage is less than `11` MB
+        Then on `focal`, systemd status output says memory usage is less than `12` MB
         Then on `jammy`, systemd status output says memory usage is less than `12` MB
 
         When I run `cat /var/log/ubuntu-advantage-daemon.log` with sudo
diff --git a/features/docker.feature b/features/docker.feature
index e4ac590..2e82266 100644
--- a/features/docker.feature
+++ b/features/docker.feature
@@ -19,7 +19,9 @@ Feature: Build docker images with pro services
             apt-get update \
             && apt-get install --no-install-recommends -y ubuntu-advantage-tools ca-certificates \
 
-            && dpkg -i /ua.deb \
+            && ((dpkg -i /ua.deb || true)) \
+
+            && apt-get install -f \
 
             && pro attach --attach-config /run/secrets/ua-attach-config \
 
diff --git a/features/environment.py b/features/environment.py
index 39013d0..bfe0042 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -23,10 +23,6 @@ DEFAULT_UA_PPA_KEYID = "6E34E7116C0BC933"
 
 USERDATA_BLOCK_AUTO_ATTACH_IMG = """\
 #cloud-config
-[bootcmd, once]:
- - cp /usr/bin/ua /usr/bin/ua.orig
- - 'echo "#!/bin/sh\ndate >> /root/ua-calls\n" > /usr/bin/ua'
- - chmod 755 /usr/bin/ua
 """
 
 # we can't use write_files because it will clash with the
@@ -40,8 +36,10 @@ runcmd:
 USERDATA_RUNCMD_ENABLE_PROPOSED = """
 runcmd:
   - printf \"deb http://archive.ubuntu.com/ubuntu/ {series}-proposed main\" > /etc/apt/sources.list.d/uaclient-proposed.list
+  - printf \"deb http://archive.ubuntu.com/ubuntu/ {series}-proposed universe\" > /etc/apt/sources.list.d/uaclient-proposed-universe.list
   - "printf \\"Package: *\\nPin: release a={series}-proposed\\nPin-Priority: 400\\n\\" > /etc/apt/preferences.d/lower-proposed"
   - "printf \\"Package: ubuntu-advantage-tools\\nPin: release a={series}-proposed\\nPin-Priority: 1001\\n\\" > /etc/apt/preferences.d/uaclient-proposed"
+  - "printf \\"Package: ubuntu-advantage-pro\\nPin: release a={series}-proposed\\nPin-Priority: 1001\\n\\" > /etc/apt/preferences.d/uaclientpro-proposed"
 """  # noqa: E501
 
 USERDATA_APT_SOURCE_PPA = """\
@@ -333,6 +331,8 @@ class UAClientBehaveConfig:
 def before_all(context: Context) -> None:
     """behave will invoke this before anything else happens."""
     context.config.setup_logging()
+    logging.getLogger("botocore").setLevel(logging.INFO)
+    logging.getLogger("boto3").setLevel(logging.INFO)
     userdata = context.config.userdata
     if userdata:
         logging.debug("Userdata key / value pairs:")
diff --git a/features/ubuntu_pro.feature b/features/ubuntu_pro.feature
index f23dcde..c373a95 100644
--- a/features/ubuntu_pro.feature
+++ b/features/ubuntu_pro.feature
@@ -23,6 +23,7 @@ Feature: Command behaviour when auto-attached in an ubuntu PRO image
         """
         And I verify `/var/log/squid/access.log` is empty on `proxy` machine
         When I run `pro auto-attach` with sudo
+        When I run `pro status --all` with sudo
         Then stdout matches regexp:
             """
             SERVICE       +ENTITLED  STATUS    DESCRIPTION
@@ -94,6 +95,7 @@ Feature: Command behaviour when auto-attached in an ubuntu PRO image
         """
         And I verify `/var/log/squid/access.log` is empty on `proxy` machine
         When I run `pro auto-attach` with sudo
+        When I run `pro status --all` with sudo
         Then stdout matches regexp:
             """
             SERVICE       +ENTITLED  STATUS    DESCRIPTION
@@ -165,6 +167,7 @@ Feature: Command behaviour when auto-attached in an ubuntu PRO image
         """
         And I verify `/var/log/squid/access.log` is empty on `proxy` machine
         When I run `pro auto-attach` with sudo
+        When I run `pro status --all` with sudo
         Then stdout matches regexp:
             """
             SERVICE       +ENTITLED  STATUS    DESCRIPTION
diff --git a/features/unattached_commands.feature b/features/unattached_commands.feature
index 5f27952..d358c29 100644
--- a/features/unattached_commands.feature
+++ b/features/unattached_commands.feature
@@ -231,19 +231,18 @@ Feature: Command behaviour when unattached
         Given a `<release>` machine with ubuntu-advantage-tools installed
         #  Make sure we have a fresh, just rebooted, environment
         When I reboot the machine
-        Then I verify that no files exist matching `/tmp/ubuntu-advantage/candidate-version`
-        When I run `pro status` as non-root
+        Then I verify that no files exist matching `/run/ubuntu-advantage/candidate-version`
+        When I run `pro status` with sudo
         Then I will see the following on stderr
         """
         """
-        And I verify that files exist matching `/tmp/ubuntu-advantage/candidate-version`
+        And I verify that files exist matching `/run/ubuntu-advantage/candidate-version`
         # We forge a candidate to see results
-        When I delete the file `/tmp/ubuntu-advantage/candidate-version`
-        And I create the file `/tmp/ubuntu-advantage/candidate-version` with the following
+        When I delete the file `/run/ubuntu-advantage/candidate-version`
+        And I create the file `/run/ubuntu-advantage/candidate-version` with the following
         """
         99.9.9
         """
-        And I run `chmod 777 /tmp/ubuntu-advantage/candidate-version` with sudo
         And I run `pro status` as non-root
         Then stderr matches regexp:
         """
diff --git a/integration-requirements.txt b/integration-requirements.txt
index 9eaf14f..de2698d 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -2,7 +2,7 @@
 behave
 jsonschema
 PyHamcrest
-pycloudlib @ git+https://github.com/canonical/pycloudlib.git@25cca01b24d111e00e91a86eb1021b88592e069e
+pycloudlib @ git+https://github.com/canonical/pycloudlib.git@e7c4a42eb98a914b084b253c3f96c960de42e8fa
 toml==0.10
 
 
diff --git a/uaclient/actions.py b/uaclient/actions.py
index 9b1af00..3f602e8 100644
--- a/uaclient/actions.py
+++ b/uaclient/actions.py
@@ -160,6 +160,23 @@ def _write_command_output_to_file(
         system.write_file(filename, out)
 
 
+def _get_state_files(cfg: config.UAConfig):
+    # include cfg log files here because they could be set to non default
+    return [
+        cfg.cfg_path or DEFAULT_CONFIG_FILE,
+        cfg.log_file,
+        cfg.timer_log_file,
+        cfg.daemon_log_file,
+        cfg.data_path("jobs-status"),
+        CLOUD_BUILD_INFO,
+        *(
+            entitlement.repo_list_file_tmpl.format(name=entitlement.name)
+            for entitlement in entitlements.ENTITLEMENT_CLASSES
+            if issubclass(entitlement, entitlements.repo.RepoEntitlement)
+        ),
+    ]
+
+
 def collect_logs(cfg: config.UAConfig, output_dir: str):
     """
     Write all relevant Ubuntu Pro logs to the specified directory
@@ -199,28 +216,19 @@ def collect_logs(cfg: config.UAConfig, output_dir: str):
             return_codes=[0, 3],
         )
 
-    # include cfg log files here because they could be set to non default
-    state_files = [
-        cfg.cfg_path or DEFAULT_CONFIG_FILE,
-        cfg.log_file,
-        cfg.timer_log_file,
-        cfg.daemon_log_file,
-        cfg.data_path("jobs-status"),
-        CLOUD_BUILD_INFO,
-        *(
-            entitlement.repo_list_file_tmpl.format(name=entitlement.name)
-            for entitlement in entitlements.ENTITLEMENT_CLASSES
-            if issubclass(entitlement, entitlements.repo.RepoEntitlement)
-        ),
-    ]
+    state_files = _get_state_files(cfg)
 
     # also get default logrotated log files
     for f in state_files + glob.glob(DEFAULT_LOG_PREFIX + "*"):
         if os.path.isfile(f):
             try:
                 content = system.load_file(f)
-            except PermissionError as e:
-                logging.warning(e)
+            except Exception as e:
+                # If we fail to load that file for any reason we will
+                # not break the command, we will instead warn the user
+                # about the issue and try to process the other files
+                logging.warning("Failed to load file: %s\n%s", f, str(e))
+                continue
             content = util.redact_sensitive_logs(content)
             if os.getuid() == 0:
                 # if root, overwrite the original with redacted content
diff --git a/uaclient/cli.py b/uaclient/cli.py
index 3473414..55f8bec 100644
--- a/uaclient/cli.py
+++ b/uaclient/cli.py
@@ -49,7 +49,9 @@ from uaclient.entitlements.entitlement_status import (
     CanEnableFailureReason,
 )
 from uaclient.jobs.update_messaging import (
+    clear_apt_news_flag,
     refresh_motd,
+    set_apt_news_flag,
     update_apt_and_motd_messages,
 )
 
@@ -1022,6 +1024,13 @@ def action_config_set(args, *, cfg, **kwargs):
                     "<value> for interval must be a positive integer."
                 ).format(set_key, set_value)
             )
+    elif set_key == "apt_news":
+        set_value = set_value.lower() == "true"
+        if set_value:
+            set_apt_news_flag(cfg)
+        else:
+            clear_apt_news_flag(cfg)
+
     setattr(cfg, set_key, set_value)
 
 
diff --git a/uaclient/config.py b/uaclient/config.py
index 2b43d55..bb7bb06 100644
--- a/uaclient/config.py
+++ b/uaclient/config.py
@@ -49,6 +49,7 @@ UA_CONFIGURABLE_KEYS = (
     "update_messaging_timer",
     "update_status_timer",
     "metering_timer",
+    "apt_news",
 )
 
 # Basic schema validation top-level keys for parse_config handling
@@ -304,6 +305,17 @@ class UAConfig:
         self.cfg["ua_config"]["polling_error_retry_delay"] = value
         self.write_cfg()
 
+    @property
+    def apt_news(self) -> bool:
+        return self.cfg.get("ua_config", {}).get("apt_news", True)
+
+    @apt_news.setter
+    def apt_news(self, value: bool):
+        if "ua_config" not in self.cfg:
+            self.cfg["ua_config"] = {}
+        self.cfg["ua_config"]["apt_news"] = value
+        self.write_cfg()
+
     def check_lock_info(self) -> Tuple[int, str]:
         """Return lock info if config lock file is present the lock is active.
 
diff --git a/uaclient/defaults.py b/uaclient/defaults.py
index a2fa11f..7ffb140 100644
--- a/uaclient/defaults.py
+++ b/uaclient/defaults.py
@@ -6,14 +6,14 @@ any of our dependencies installed.
 """
 
 UAC_ETC_PATH = "/etc/ubuntu-advantage/"
-UAC_TMP_PATH = "/tmp/ubuntu-advantage/"
+UAC_RUN_PATH = "/run/ubuntu-advantage/"
 DEFAULT_DATA_DIR = "/var/lib/ubuntu-advantage"
 MACHINE_TOKEN_FILE = "machine-token.json"
 PRIVATE_SUBDIR = "/private"
 DEFAULT_PRIVATE_MACHINE_TOKEN_PATH = (
     DEFAULT_DATA_DIR + PRIVATE_SUBDIR + "/" + MACHINE_TOKEN_FILE
 )
-CANDIDATE_CACHE_PATH = UAC_TMP_PATH + "candidate-version"
+CANDIDATE_CACHE_PATH = UAC_RUN_PATH + "candidate-version"
 DEFAULT_CONFIG_FILE = UAC_ETC_PATH + "uaclient.conf"
 DEFAULT_HELP_FILE = UAC_ETC_PATH + "help_data.yaml"
 DEFAULT_UPGRADE_CONTRACT_FLAG_FILE = UAC_ETC_PATH + "request-update-contract"
diff --git a/uaclient/jobs/update_messaging.py b/uaclient/jobs/update_messaging.py
index 4a3893f..41a027b 100644
--- a/uaclient/jobs/update_messaging.py
+++ b/uaclient/jobs/update_messaging.py
@@ -28,7 +28,6 @@ from uaclient.messages import (
     CONTRACT_EXPIRED_MOTD_SOON_TMPL,
     DISABLED_APT_NO_PKGS_TMPL,
     DISABLED_APT_PKGS_TMPL,
-    TRY_UBUNTU_PRO_BETA,
 )
 
 XENIAL_ESM_URL = "https://ubuntu.com/16-04"
@@ -68,6 +67,17 @@ UPDATE_NOTIFIER_MOTD_SCRIPT = (
 )
 
 
+def set_apt_news_flag(cfg: config.UAConfig):
+    if system.is_current_series_lts():
+        system.create_file(
+            os.path.join(cfg.data_dir, "flags", "show-apt-news")
+        )
+
+
+def clear_apt_news_flag(cfg: config.UAConfig):
+    system.remove_file(os.path.join(cfg.data_dir, "flags", "show-apt-news"))
+
+
 def get_contract_expiry_status(
     cfg: config.UAConfig,
 ) -> Tuple[ContractExpiryStatus, int]:
@@ -311,32 +321,6 @@ def write_apt_and_motd_templates(cfg: config.UAConfig, series: str) -> None:
             ],
         )
 
-    if (
-        system.is_current_series_lts()
-        and not system.is_active_esm(series)
-        and not cfg.is_attached
-    ):
-        _write_template_or_remove(
-            TRY_UBUNTU_PRO_BETA, os.path.join(msg_dir, apps_no_pkg_file)
-        )
-        _write_template_or_remove(
-            TRY_UBUNTU_PRO_BETA, os.path.join(msg_dir, apps_pkg_file)
-        )
-        _write_template_or_remove("", os.path.join(msg_dir, infra_no_pkg_file))
-        _write_template_or_remove("", os.path.join(msg_dir, infra_pkg_file))
-        _write_template_or_remove(
-            "", os.path.join(msg_dir, motd_apps_no_pkg_file)
-        )
-        _write_template_or_remove(
-            "", os.path.join(msg_dir, motd_apps_pkg_file)
-        )
-        _write_template_or_remove(
-            "", os.path.join(msg_dir, motd_infra_no_pkg_file)
-        )
-        _write_template_or_remove(
-            "", os.path.join(msg_dir, motd_infra_pkg_file)
-        )
-
 
 def write_esm_announcement_message(cfg: config.UAConfig, series: str) -> None:
     """Write human-readable messages if ESM is offered on this LTS release.
@@ -396,6 +380,12 @@ def update_apt_and_motd_messages(cfg: config.UAConfig) -> bool:
 
     # Announce ESM availabilty on active ESM LTS releases
     # write_esm_announcement_message(cfg, series)
+
+    if cfg.apt_news:
+        set_apt_news_flag(cfg)
+    else:
+        clear_apt_news_flag(cfg)
+
     write_apt_and_motd_templates(cfg, series)
     # Now that we've setup/cleanedup templates render them with apt-hook
     system.subp(
diff --git a/uaclient/messages.py b/uaclient/messages.py
index 95f1605..e3061be 100644
--- a/uaclient/messages.py
+++ b/uaclient/messages.py
@@ -1034,7 +1034,3 @@ Package names in {bold}bold{end_bold} currently have an available update
 with '{{service}}' enabled""".format(
     bold=TxtColor.BOLD, end_bold=TxtColor.ENDC
 )
-
-TRY_UBUNTU_PRO_BETA = """\
-Try Ubuntu Pro beta with a free personal subscription on up to 5 machines.
-Learn more at https://ubuntu.com/pro"""
diff --git a/uaclient/system.py b/uaclient/system.py
index ef48d0c..3983e84 100644
--- a/uaclient/system.py
+++ b/uaclient/system.py
@@ -1,6 +1,7 @@
 import datetime
 import logging
 import os
+import pathlib
 import re
 import subprocess
 import time
@@ -205,6 +206,11 @@ def is_active_esm(series: str) -> bool:
 
 
 @lru_cache(maxsize=None)
+def is_current_series_active_esm() -> bool:
+    return is_active_esm(get_platform_info()["series"])
+
+
+@lru_cache(maxsize=None)
 def is_container(run_path: str = "/run") -> bool:
     """Checks to see if this code running in a container of some sort"""
 
@@ -350,6 +356,13 @@ def load_file(filename: str, decode: bool = True) -> str:
     return content.decode("utf-8")
 
 
+def create_file(filename: str, mode: int = 0o644) -> None:
+    logging.debug("Creating file: %s", filename)
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+    pathlib.Path(filename).touch()
+    os.chmod(filename, mode)
+
+
 def write_file(filename: str, content: str, mode: int = 0o644) -> None:
     """Write content to the provided filename encoding it if necessary.
 
diff --git a/uaclient/tests/test_actions.py b/uaclient/tests/test_actions.py
index 6f77d7f..12c05f1 100644
--- a/uaclient/tests/test_actions.py
+++ b/uaclient/tests/test_actions.py
@@ -1,8 +1,10 @@
+import logging
+
 import mock
 import pytest
 
 from uaclient import exceptions, messages
-from uaclient.actions import attach_with_token, auto_attach
+from uaclient.actions import attach_with_token, auto_attach, collect_logs
 from uaclient.exceptions import ContractAPIError, NonAutoAttachImageError
 from uaclient.tests.test_cli_auto_attach import fake_instance_factory
 
@@ -141,3 +143,36 @@ class TestAutoAttach:
             auto_attach(cfg, fake_instance_factory())
 
         assert unexpected_error == excinfo.value
+
+
[e-mail]("uaclient.actions._write_command_output_to_file")
+class TestCollectLogs:
+    @pytest.mark.parametrize("caplog_text", [logging.WARNING], indirect=True)
+    @mock.patch("os.getuid")
+    @mock.patch("uaclient.system.write_file")
+    @mock.patch("uaclient.system.load_file")
+    @mock.patch("uaclient.actions._get_state_files")
+    @mock.patch("glob.glob")
+    def test_collect_logs_invalid_file(
+        self,
+        m_glob,
+        m_get_state_files,
+        m_load_file,
+        m_write_file,
+        m_getuid,
+        m_write_cmd,
+        caplog_text,
+    ):
+        m_get_state_files.return_value = ["a", "b"]
+        m_load_file.side_effect = [UnicodeError("test"), "test"]
+        m_getuid.return_value = 1
+        m_glob.return_value = []
+
+        with mock.patch("os.path.isfile", return_value=True):
+            collect_logs(cfg=mock.MagicMock(), output_dir="test")
+
+        assert 2 == m_load_file.call_count
+        assert [mock.call("a"), mock.call("b")] == m_load_file.call_args_list
+        assert 1 == m_write_file.call_count
+        assert [mock.call("test/b", "test")] == m_write_file.call_args_list
+        assert "Failed to load file: a\n" in caplog_text()
diff --git a/uaclient/tests/test_cli_config_set.py b/uaclient/tests/test_cli_config_set.py
index 66aaf82..d8a7c49 100644
--- a/uaclient/tests/test_cli_config_set.py
+++ b/uaclient/tests/test_cli_config_set.py
@@ -16,7 +16,8 @@ positional arguments:
                   must be one of: http_proxy, https_proxy, apt_http_proxy,
                   apt_https_proxy, ua_apt_http_proxy, ua_apt_https_proxy,
                   global_apt_http_proxy, global_apt_https_proxy,
-                  update_messaging_timer, update_status_timer, metering_timer
+                  update_messaging_timer, update_status_timer, metering_timer,
+                  apt_news
 
 Flags:
   -h, --help      show this help message and exit
diff --git a/uaclient/tests/test_cli_config_show.py b/uaclient/tests/test_cli_config_show.py
index d4582fd..ebf760c 100644
--- a/uaclient/tests/test_cli_config_show.py
+++ b/uaclient/tests/test_cli_config_show.py
@@ -117,6 +117,7 @@ global_apt_https_proxy  http://global_apt_https_proxy
 update_messaging_timer  None
 update_status_timer     None
 metering_timer          None
+apt_news                True
 """
                 == out
             )
diff --git a/uaclient/tests/test_cli_config_unset.py b/uaclient/tests/test_cli_config_unset.py
index 115bc86..42c3625 100644
--- a/uaclient/tests/test_cli_config_unset.py
+++ b/uaclient/tests/test_cli_config_unset.py
@@ -15,7 +15,7 @@ positional arguments:
               http_proxy, https_proxy, apt_http_proxy, apt_https_proxy,
               ua_apt_http_proxy, ua_apt_https_proxy, global_apt_http_proxy,
               global_apt_https_proxy, update_messaging_timer,
-              update_status_timer, metering_timer
+              update_status_timer, metering_timer, apt_news
 
 Flags:
   -h, --help  show this help message and exit
diff --git a/uaclient/tests/test_config.py b/uaclient/tests/test_config.py
index bd8a49e..d3b493c 100644
--- a/uaclient/tests/test_config.py
+++ b/uaclient/tests/test_config.py
@@ -295,6 +295,7 @@ UA_CFG_DICT = {
     "ua_config": {
         "apt_http_proxy": None,
         "apt_https_proxy": None,
+        "apt_news": True,
         "global_apt_http_proxy": None,
         "global_apt_https_proxy": None,
         "ua_apt_http_proxy": None,
@@ -316,7 +317,9 @@ class TestUAConfigKeys:
     ):
         """Getters and settings are available fo UA_CONFIGURABLE_KEYS."""
         cfg = FakeConfig()
-        assert None is getattr(cfg, attr_name, None)
+        assert UA_CFG_DICT["ua_config"][attr_name] == getattr(
+            cfg, attr_name, None
+        )
         cfg_non_members = ("apt_http_proxy", "apt_https_proxy")
         if attr_name not in cfg_non_members:
             setattr(cfg, attr_name, attr_name + "value")
diff --git a/uaclient/tests/test_version.py b/uaclient/tests/test_version.py
index f6fd890..369dbbb 100644
--- a/uaclient/tests/test_version.py
+++ b/uaclient/tests/test_version.py
@@ -59,9 +59,12 @@ class TestGetLastKnownCandidate:
         m_open.side_effect = OSError()
         assert None is get_last_known_candidate()
 
+    @mock.patch("os.makedirs")
     @mock.patch("uaclient.version.get_apt_cache_policy_for_package")
     @mock.patch("os.path.exists", return_value=False)
-    def test_create_cache_before_returning(self, _m_exists, m_policy):
+    def test_create_cache_before_returning(
+        self, _m_exists, m_policy, _m_makedirs
+    ):
         m_policy.return_value = """
             Installed: 1.1.2
             Candidate: 1.2.3
@@ -83,7 +86,7 @@ class TestGetLastKnownCandidate:
             Version table:
         """
         m_open.side_effect = OSError()
-        assert None is get_last_known_candidate()
+        assert "1.2.3" == get_last_known_candidate()
 
 
 class TestCheckForNewVersion:
diff --git a/uaclient/version.py b/uaclient/version.py
index 54776f6..6232b30 100644
--- a/uaclient/version.py
+++ b/uaclient/version.py
@@ -14,11 +14,11 @@ from uaclient.apt import (
     get_apt_cache_policy_for_package,
     get_apt_cache_time,
 )
-from uaclient.defaults import CANDIDATE_CACHE_PATH, UAC_TMP_PATH
+from uaclient.defaults import CANDIDATE_CACHE_PATH, UAC_RUN_PATH
 from uaclient.exceptions import ProcessExecutionError
 from uaclient.system import subp
 
-__VERSION__ = "27.11.2"
+__VERSION__ = "27.11.3"
 PACKAGED_VERSION = "@@PACKAGED_VERSION@@"
 
 CANDIDATE_REGEX = r"Candidate: (?P<candidate>.*?)\n"
@@ -56,17 +56,19 @@ def get_last_known_candidate() -> Optional[str]:
         not os.path.exists(CANDIDATE_CACHE_PATH)
         or os.stat(CANDIDATE_CACHE_PATH).st_mtime < last_apt_cache_update
     ):
+        candidate_version = None
         try:
             policy = get_apt_cache_policy_for_package("ubuntu-advantage-tools")
             match = re.search(CANDIDATE_REGEX, policy)
             if match:
                 candidate_version = match.group("candidate")
-                os.makedirs(UAC_TMP_PATH, exist_ok=True)
+                os.makedirs(UAC_RUN_PATH, exist_ok=True)
                 with open(CANDIDATE_CACHE_PATH, "w") as f:
                     f.write(candidate_version)
-                os.chmod(CANDIDATE_CACHE_PATH, 0o777)
+                return candidate_version
         except Exception:
-            pass
+            if candidate_version is not None:
+                return candidate_version
 
     try:
         with open(CANDIDATE_CACHE_PATH, "r") as f: