Merge ~simpoir/ubuntu/+source/landscape-client:eoan_pulse into ubuntu/+source/landscape-client:ubuntu/eoan-devel

[Simon Poirier]

a2676a0... by Simon Poirier on 2019-10-22

d/p/apt-pulse-1758529.patch: Override the apt progress pulse (LP: #1758529)

49f65a5... by Simon Poirier on 2019-10-22

d/p/security_pocket_1848828.patch
d/p/reporter_logging.patch

660dc7f... by Simon Poirier on 2019-10-22

Updated changelog:

* d/p/apt-pulse-1758529.patch: Override the apt progress pulse (LP: #1758529)
* d/p/security_pocket_1848828.patch (LP: #1848828)
* d/p/reporter_logging.patch (LP: #1848828)<Paste>

[Eric Desrochers (slashd)]

lgmt

review: Approve

diff --git a/debian/changelog b/debian/changelog
index c8629f4..b72cdf0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+landscape-client (18.01-0ubuntu9.1) eoan; urgency=medium
+
+  * d/p/apt-pulse-1758529.patch: Override the apt progress pulse (LP: #1758529)
+  * d/p/security_pocket_1848828.patch (LP: #1848828)
+  * d/p/reporter_logging.patch (LP: #1848828)
+
+ -- Simon Poirier <[e-mail]>  Tue, 22 Oct 2019 18:54:21 -0400
+
 landscape-client (18.01-0ubuntu9) eoan; urgency=medium
 
   * d/p/product-name-vminfo-1828217.patch: Add product_name to things scanned
diff --git a/debian/patches/apt-pulse-1758529.patch b/debian/patches/apt-pulse-1758529.patch
new file mode 100644
index 0000000..14db379
--- /dev/null
+++ b/debian/patches/apt-pulse-1758529.patch
@@ -0,0 +1,27 @@
+Description: Override the apt progress pulse.
+ This should remove the tracebacks under python3 + bionic.
+Author: Simon Poirier <[e-mail]>
+Origin: upstream, https://github.com/CanonicalLtd/landscape-client/commit/8e52e1d3418b3b042a2f305d82336b9866914e43
+Bug-Ubuntu: https://bugs.launchpad.net/bugs/1758529
+Last-Update: 2019-09-30
+
+diff --git a/landscape/lib/apt/package/facade.py b/landscape/lib/apt/package/facade.py
+index 1c026beb..15c57ab5 100644
+--- a/landscape/lib/apt/package/facade.py
++++ b/landscape/lib/apt/package/facade.py
+@@ -57,6 +57,15 @@ class LandscapeAcquireProgress(AcquireProgress):
+         fcntl.ioctl API differences for different Python versions.
+         """
+ 
++    def pulse(self, owner):
++        """Override updating the acquire progress, which needs a tty.
++
++        Under Python3, StringIO.fileno() raises UnsupportedOperation instead
++        of an AttributeError. This would be uncaught by apt, thus we force a
++        NOOP here.
++        """
++        return True
++
+ 
+ class LandscapeInstallProgress(InstallProgress):
+ 
diff --git a/debian/patches/reporter_logging.patch b/debian/patches/reporter_logging.patch
new file mode 100644
index 0000000..42d8535
--- /dev/null
+++ b/debian/patches/reporter_logging.patch
@@ -0,0 +1,19 @@
+Description: Remove extra keyword which breaks log format
+Author: Simon Poirier <[e-mail]>
+Origin: upstream, https://github.com/CanonicalLtd/landscape-client/commit/ad37c28e80ebfb85f43bdabece9946d3a062c0e9
+Bug-Ubuntu: https://bugs.launchpad.net/bugs/1848828
+Last-Update: 2019-10-22
+
+diff --git a/landscape/client/package/reporter.py b/landscape/client/package/reporter.py
+index 9f34f2a1..39b67df3 100644
+--- a/landscape/client/package/reporter.py
++++ b/landscape/client/package/reporter.py
+@@ -755,7 +755,7 @@ class PackageReporter(PackageTaskHandler):
+             "%(not_locked)d not locked, "
+             "%(not_auto)d not autoremovable, "
+             "%(not_security)d not security.",
+-            extra=dict(
++            dict(
+                 installed=len(new_installed), available=len(new_available),
+                 upgrades=len(new_upgrades), locked=len(new_locked),
+                 auto=len(new_autoremovable), not_installed=len(not_installed),
diff --git a/debian/patches/security_pocket_1848828.patch b/debian/patches/security_pocket_1848828.patch
new file mode 100644
index 0000000..cbcd104
--- /dev/null
+++ b/debian/patches/security_pocket_1848828.patch
@@ -0,0 +1,255 @@
+Description: report packages from security-pocket
+Author: Simon Poirier <[e-mail]>
+Origin: upstream, https://github.com/CanonicalLtd/landscape-client/commit/77c54c2e9c01255f8ed43935b47b65abcd62d021
+Bug-Ubuntu: https://bugs.launchpad.net/bugs/1848828
+Last-Update: 2019-10-22
+
+diff --git a/Makefile.travis b/Makefile.travis
+index d82d473d..b532241a 100644
+--- a/Makefile.travis
++++ b/Makefile.travis
+@@ -11,6 +11,7 @@ pipinstallpythonapt: pipinstallpythonapt_deps
+ pipinstallpythonapt_deps:
+ 	pip install pyopenssl
+ 	pip install service_identity
++	sudo apt-get update
+ 	sudo apt-get -y build-dep python-apt python3-apt
+ 	sudo apt-get -y install libapt-pkg-dev
+ 
+diff --git a/landscape/client/package/reporter.py b/landscape/client/package/reporter.py
+index c48a4c04..6054ec3b 100644
+--- a/landscape/client/package/reporter.py
++++ b/landscape/client/package/reporter.py
+@@ -609,6 +609,9 @@ class PackageReporter(PackageTaskHandler):
+         - are now set, and were not;
+         - were previously set but are not anymore;
+ 
++        Also, packages coming from the security pocket will be
++        reported as such.
++
+         In all cases, the server is notified of the new situation
+         with a "packages" message.
+ 
+@@ -622,14 +625,17 @@ class PackageReporter(PackageTaskHandler):
+         old_upgrades = set(self._store.get_available_upgrades())
+         old_locked = set(self._store.get_locked())
+         old_autoremovable = set(self._store.get_autoremovable())
++        old_security = set(self._store.get_security())
+ 
+         current_installed = set()
+         current_available = set()
+         current_upgrades = set()
+         current_locked = set()
+         current_autoremovable = set()
++        current_security = set()
+         lsb = parse_lsb_release(LSB_RELEASE_FILENAME)
+         backports_archive = "{}-backports".format(lsb["code-name"])
++        security_archive = "{}-security".format(lsb["code-name"])
+ 
+         for package in self._facade.get_packages():
+             # Don't include package versions from the official backports
+@@ -664,6 +670,13 @@ class PackageReporter(PackageTaskHandler):
+                 if self._facade.is_package_upgrade(package):
+                     current_upgrades.add(id)
+ 
++                # Is this package present in the security pocket?
++                security_origins = any(
++                    origin for origin in package.origins
++                    if origin.archive == security_archive)
++                if security_origins:
++                    current_security.add(id)
++
+         for package in self._facade.get_locked_packages():
+             hash = self._facade.get_package_hash(package)
+             id = self._store.get_hash_id(hash)
+@@ -675,12 +688,14 @@ class PackageReporter(PackageTaskHandler):
+         new_upgrades = current_upgrades - old_upgrades
+         new_locked = current_locked - old_locked
+         new_autoremovable = current_autoremovable - old_autoremovable
++        new_security = current_security - old_security
+ 
+         not_installed = old_installed - current_installed
+         not_available = old_available - current_available
+         not_upgrades = old_upgrades - current_upgrades
+         not_locked = old_locked - current_locked
+         not_autoremovable = old_autoremovable - current_autoremovable
++        not_security = old_security - current_security
+ 
+         message = {}
+         if new_installed:
+@@ -703,6 +718,13 @@ class PackageReporter(PackageTaskHandler):
+             message["not-autoremovable"] = list(
+                 sequence_to_ranges(sorted(not_autoremovable)))
+ 
++        if new_security:
++            message["security"] = list(
++                sequence_to_ranges(sorted(new_security)))
++        if not_security:
++            message["not-security"] = list(
++                sequence_to_ranges(sorted(not_security)))
++
+         if not_installed:
+             message["not-installed"] = \
+                 list(sequence_to_ranges(sorted(not_installed)))
+@@ -722,17 +744,25 @@ class PackageReporter(PackageTaskHandler):
+         message["type"] = "packages"
+         result = self.send_message(message)
+ 
+-        logging.info("Queuing message with changes in known packages: "
+-                     "%d installed, %d available, %d available upgrades, "
+-                     "%d locked, %d autoremovable, %d not installed, "
+-                     "%d not available, %d not available upgrades, "
+-                     "%d not locked, %d not autoremovable. "
+-                     % (len(new_installed), len(new_available),
+-                        len(new_upgrades), len(new_locked),
+-                        len(new_autoremovable),
+-                        len(not_installed), len(not_available),
+-                        len(not_upgrades), len(not_locked),
+-                        len(not_autoremovable)))
++        logging.info(
++            "Queuing message with changes in known packages: "
++            "%(installed)d installed, %(available)d available, "
++            "%(upgrades)d available upgrades, %(locked)d locked, "
++            "%(auto)d autoremovable, %(security)d security, "
++            "%(not_installed)d not installed, "
++            "%(not_available)d not available, "
++            "%(not_upgrades)d not available upgrades, "
++            "%(not_locked)d not locked, "
++            "%(not_auto)d not autoremovable, "
++            "%(not_security)d not security.",
++            extra=dict(
++                installed=len(new_installed), available=len(new_available),
++                upgrades=len(new_upgrades), locked=len(new_locked),
++                auto=len(new_autoremovable), not_installed=len(not_installed),
++                not_available=len(not_available),
++                not_upgrades=len(not_upgrades), not_locked=len(not_locked),
++                not_auto=len(not_autoremovable), security=len(new_security),
++                not_security=len(not_security)))
+ 
+         def update_currently_known(result):
+             if new_installed:
+@@ -755,6 +785,10 @@ class PackageReporter(PackageTaskHandler):
+                 self._store.remove_locked(not_locked)
+             if not_autoremovable:
+                 self._store.remove_autoremovable(not_autoremovable)
++            if new_security:
++                self._store.add_security(new_security)
++            if not_security:
++                self._store.remove_security(not_security)
+             # Something has changed wrt the former run, let's update the
+             # timestamp and return True.
+             stamp_file = self._config.detect_package_changes_stamp
+diff --git a/landscape/client/package/tests/test_reporter.py b/landscape/client/package/tests/test_reporter.py
+index 851c8eb9..6500affb 100644
+--- a/landscape/client/package/tests/test_reporter.py
++++ b/landscape/client/package/tests/test_reporter.py
+@@ -999,6 +999,47 @@ class PackageReporterAptTest(LandscapeTest):
+         self.assertFalse(result)
+         self.assertEqual([1], self.store.get_autoremovable())
+ 
++    @inlineCallbacks
++    def test_detect_packages_from_security_pocket(self):
++        """Packages versions coming from security are reported as such."""
++        message_store = self.broker_service.message_store
++        message_store.set_accepted_types(["packages"])
++        lsb = parse_lsb_release(LSB_RELEASE_FILENAME)
++        release_path = os.path.join(self.repository_dir, "Release")
++        with open(release_path, "w") as release:
++            release.write("Suite: {}-security".format(lsb["code-name"]))
++
++        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
++
++        yield self.reporter.detect_packages_changes()
++
++        self.assertMessages(message_store.get_pending_messages(), [{
++            "type": "packages",
++            "available": [(1, 3)],
++            "security": [(1, 3)],
++        }])
++        self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])
++        self.assertEqual(sorted(self.store.get_security()), [1, 2, 3])
++
++    @inlineCallbacks
++    def test_detect_packages_not_from_security_pocket(self):
++        """Packages versions removed from security are reported as such."""
++        message_store = self.broker_service.message_store
++        message_store.set_accepted_types(["packages"])
++
++        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
++        self.store.add_available([1, 2, 3])
++        self.store.add_security([1, 2])
++
++        yield self.reporter.detect_packages_changes()
++
++        self.assertMessages(message_store.get_pending_messages(), [{
++            "type": "packages",
++            "not-security": [1, 2],
++        }])
++        self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])
++        self.assertEqual(self.store.get_security(), [])
++
+     def test_detect_packages_changes_with_backports(self):
+         """
+         Package versions coming from backports aren't considered to be
+diff --git a/landscape/lib/apt/package/store.py b/landscape/lib/apt/package/store.py
+index 2764f7cd..59d5ca4f 100644
+--- a/landscape/lib/apt/package/store.py
++++ b/landscape/lib/apt/package/store.py
+@@ -232,6 +232,25 @@ class PackageStore(HashIdStore):
+         cursor.execute("SELECT id FROM autoremovable")
+         return [row[0] for row in cursor.fetchall()]
+ 
++    @with_cursor
++    def add_security(self, cursor, ids):
++        for id in ids:
++            cursor.execute("REPLACE INTO security VALUES (?)", (id,))
++
++    @with_cursor
++    def remove_security(self, cursor, ids):
++        id_list = ",".join(str(int(id)) for id in ids)
++        cursor.execute("DELETE FROM security WHERE id IN (%s)" % id_list)
++
++    @with_cursor
++    def clear_security(self, cursor):
++        cursor.execute("DELETE FROM security")
++
++    @with_cursor
++    def get_security(self, cursor):
++        cursor.execute("SELECT id FROM security")
++        return [row[0] for row in cursor.fetchall()]
++
+     @with_cursor
+     def add_installed(self, cursor, ids):
+         for id in ids:
+@@ -450,6 +469,8 @@ def ensure_package_schema(db):
+     #       try block.
+     cursor = db.cursor()
+     try:
++        cursor.execute("CREATE TABLE security"
++                       " (id INTEGER PRIMARY KEY)")
+         cursor.execute("CREATE TABLE autoremovable"
+                        " (id INTEGER PRIMARY KEY)")
+         cursor.execute("CREATE TABLE locked"
+diff --git a/landscape/message_schemas/server_bound.py b/landscape/message_schemas/server_bound.py
+index be06c8d6..a03d76c3 100644
+--- a/landscape/message_schemas/server_bound.py
++++ b/landscape/message_schemas/server_bound.py
+@@ -367,13 +367,16 @@ PACKAGES = Message(
+      "locked": package_ids_or_ranges,
+      "autoremovable": package_ids_or_ranges,
+      "not-autoremovable": package_ids_or_ranges,
++     "security": package_ids_or_ranges,
+      "not-installed": package_ids_or_ranges,
+      "not-available": package_ids_or_ranges,
+      "not-available-upgrades": package_ids_or_ranges,
+-     "not-locked": package_ids_or_ranges},
++     "not-locked": package_ids_or_ranges,
++     "not-security": package_ids_or_ranges},
+     optional=["installed", "available", "available-upgrades", "locked",
+               "not-available", "not-installed", "not-available-upgrades",
+-              "not-locked", "autoremovable", "not-autoremovable"])
++              "not-locked", "autoremovable", "not-autoremovable", "security",
++              "not-security"])
+ 
+ package_locks = List(Tuple(Unicode(), Unicode(), Unicode()))
+ PACKAGE_LOCKS = Message(
diff --git a/debian/patches/series b/debian/patches/series
index ebfbf13..7dd8fac 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -9,3 +9,6 @@ unicode-tags-script.patch
 mem-usage-1685885.patch
 product-name-vminfo-1828217.patch
 stagger-launch-1788518.patch
+apt-pulse-1758529.patch
+security_pocket_1848828.patch
+reporter_logging.patch