Merge ubuntu-debuginfod:prepare-poller-for-ppa-support into ubuntu-debuginfod:master

[Sergio Durigan Junior]

This MP implements improvements/adjustments that are needed in the "poller" classes for the upcoming PPA support.

There's not much else to say; I'm following the same approach as in https://code.launchpad.net/~sergiodj/ubuntu-debuginfod/+git/ubuntu-debuginfod/+merge/438422.

[Bryce Harrington (bryce)]

Some random thoughts below, hope it helps!

review: Approve

[Sergio Durigan Junior (sergiodj)]

Thanks for the review, Bryce. I went through your comments and made the corresponding changes to the branch.

[Sergio Durigan Junior (sergiodj)]

For Lena and Athos: I will go ahead and merge the MP, but please feel free to review it and leave comments as you see fit; I can always make subsequential commits to address whatever is needed.

diff --git a/ddebpoller.py b/ddebpoller.py
index e4f586b..25b24ec 100644
--- a/ddebpoller.py
+++ b/ddebpoller.py
@@ -27,52 +27,62 @@ class DdebPoller(DebugPoller):
     """Perform the Launchpad polling and obtain a list of ddebs and source
     packages."""
 
-    # The delta we subtract from a timestamp in order to make sure we
-    # don't miss publications that may arrive out of order.
-    TIMESTAMP_DELTA = datetime.timedelta(hours=1)
-
-    def __init__(self, initial_interval=1, force_initial_interval=False, dry_run=False):
+    def __init__(
+        self,
+        module_name="ddeb",
+        initial_interval=1,
+        force_initial_interval=False,
+        fetch_all_on_first_run=False,
+        dry_run=False,
+        anonymous=True,
+    ):
         """Initialize the object using 'ddeb' as its name.
 
         Look at DebugPoller's docstring for an explanation about the arguments."""
         super().__init__(
-            "ddeb",
+            module_name=module_name,
             initial_interval=initial_interval,
             force_initial_interval=force_initial_interval,
+            fetch_all_on_first_run=fetch_all_on_first_run,
             dry_run=dry_run,
+            anonymous=anonymous,
         )
 
-    def _get_normal_and_real_timestamps(self):
-        """Get the previous timestamp and adjust it to account for
-        publications arriving out of order."""
-        timestamp = self._get_timestamp()
-        if timestamp is None:
-            raise RuntimeError("_get_timestamp returned None")
-        if not isinstance(timestamp, datetime.datetime):
-            raise RuntimeError(
-                f"_get_timestamp returned an invalid timestamp ({timestamp})"
-            )
-        # Allow a grace period to cope with publications arriving out of
-        # order during long transactions.
-        real_timestamp = timestamp - self.TIMESTAMP_DELTA
-        return timestamp, real_timestamp
-
-    def get_ddebs(self):
+    def get_ddebs(self, archive=None, ppainfo=None):
         """Get the list of ddebs that have been published since the last
         timestamp from Launchpad.
 
+        :param archive Launchpad archive: The Launchpad archive that
+            should be queried for new binaries.  Defaults to
+            self._main_archive.
+
+        :param ppainfo dict(str, str): A dictionary containing three
+            keys: "ppauser", "ppaname" and "isprivateppa".  It should
+            only be provided when dealing with a PPA; otherwise, the
+            default is None.
+
         :rtype: dict, datetime.datetime
 
         Return a dictionary containing all ddebs found, and also the
         new timestamp that should then be recorded by calling
-        record_timestamp."""
+        record_timestamp.
+
+        """
         timestamp, real_timestamp = self._get_normal_and_real_timestamps()
 
-        self._logger.info(f"Polling ddebs created since '{real_timestamp}'")
+        if archive is None:
+            archive = self._main_archive
+
+        if ppainfo is None:
+            archive_label = f"'{archive.displayname}' archive"
+        else:
+            archive_label = f"ppa:{ppainfo['ppauser']}/{ppainfo['ppaname']} (private: {ppainfo['isprivateppa']})"
+
+        self._logger.info(f"Polling ddebs created since '{real_timestamp}' from {archive_label}")
 
         result = []
         latest_timestamp_created = timestamp
-        for pkg in self._main_archive.getPublishedBinaries(
+        for pkg in archive.getPublishedBinaries(
             order_by_date=True, created_since_date=real_timestamp
         ):
             if pkg.status not in ("Pending", "Published"):
@@ -85,7 +95,10 @@ class DdebPoller(DebugPoller):
                 # Safety check.
                 continue
 
-            if pkg.date_created > latest_timestamp_created:
+            if (
+                latest_timestamp_created is None
+                or pkg.date_created > latest_timestamp_created
+            ):
                 latest_timestamp_created = pkg.date_created
 
             srcname = pkg.source_package_name
@@ -112,26 +125,48 @@ class DdebPoller(DebugPoller):
                     "ddeb_filename": ddeb,
                     "architecture": arch,
                 }
+                if ppainfo is not None:
+                    msg.update(ppainfo)
+
                 result.append(msg)
 
         return result, latest_timestamp_created
 
-    def get_sources(self):
+    def get_sources(self, archive=None, ppainfo=None):
         """Get the list of source packages that have been published since the
         last timestamp from Launchpad.
 
+        :param archive Launchpad archive: The Launchpad archive that
+            should be queried for new sources.  Defaults to
+            self._main_archive.
+
+        :param ppainfo dict(str, str): A dictionary containing three
+            keys: "ppauser", "ppaname" and "isprivateppa".  It should
+            only be provided when dealing with a PPA; otherwise, the
+            default is None.
+
         :rtype: dict, datetime.datetime
 
         Return a dictionary containing all source packages found, and
         also the new timestamp that should then be recorded by calling
-        record_timestamp."""
+        record_timestamp.
+
+        """
         timestamp, real_timestamp = self._get_normal_and_real_timestamps()
 
-        self._logger.info(f"Polling source packages created since '{real_timestamp}'")
+        if archive is None:
+            archive = self._main_archive
+
+        if ppainfo is None:
+            archive_label = f"'{archive.displayname}' archive"
+        else:
+            archive_label = f"ppa:{ppainfo['ppauser']}/{ppainfo['ppaname']} (private: {ppainfo['isprivateppa']})"
+
+        self._logger.info(f"Polling source packages created since '{real_timestamp}' from {archive_label}")
 
         result = []
         latest_timestamp_created = timestamp
-        for pkg in self._main_archive.getPublishedSources(
+        for pkg in archive.getPublishedSources(
             order_by_date=True, created_since_date=real_timestamp
         ):
             if pkg.status not in ("Pending", "Published"):
@@ -142,7 +177,10 @@ class DdebPoller(DebugPoller):
                 # Safety check.
                 continue
 
-            if pkg.date_created > latest_timestamp_created:
+            if (
+                latest_timestamp_created is None
+                or pkg.date_created > latest_timestamp_created
+            ):
                 latest_timestamp_created = pkg.date_created
 
             srcname = pkg.source_package_name
@@ -156,6 +194,9 @@ class DdebPoller(DebugPoller):
                 "source_urls": src_urls,
                 "architecture": "source",
             }
+            if ppainfo is not None:
+                msg.update(ppainfo)
+
             result.append(msg)
 
         return result, latest_timestamp_created
diff --git a/debugpoller.py b/debugpoller.py
index c29ea5d..4e4a657 100644
--- a/debugpoller.py
+++ b/debugpoller.py
@@ -31,19 +31,20 @@ class DebugPoller:
     # The timestamp file we will save our timestamp into.
     TIMESTAMP_FILE = os.path.expanduser("~/.config/ubuntu-debuginfod/timestamp")
 
+    # The delta we subtract from a timestamp in order to make sure we
+    # don't miss publications that may arrive out of order.
+    TIMESTAMP_DELTA = datetime.timedelta(hours=1)
+
     def __init__(
         self,
-        module_name,
         initial_interval=1,
         force_initial_interval=False,
+        fetch_all_on_first_run=False,
         dry_run=False,
+        anonymous=True,
     ):
         """Initalize the DebugPoller object.
 
-        :param str module_name: Name of the DebugPoller module.  This
-            is used to compose the timestamp filename so that it is
-            unique.
-
         :param int initial_interval: The initial interval (in hours),
             i.e., if no timestamp file has been found then we use this
             value to tell Launchpad that we want files created since
@@ -53,14 +54,31 @@ class DebugPoller:
             the initial interval to calculate the timestamp, instead
             of relying on the timestamp file.  Default is False.
 
+        :param bool fetch_all_on_first_run: Instruct the poller to
+            fetch everything from the archive when running for the
+            first time.  Useful to (re)build the entire pool of
+            artifacts.  In order to determine whether we're running
+            for the first time, we check if the timestamp file exists.
+            Default is False.
+
         :param bool dry_run: Tell the poller that it shouldn't record
             the timestamp in the file when the operation finishes.
-            Default is False."""
-        self._lp = Launchpad.login_anonymously(
-            "ubuntu-debuginfod poller", "production", version="devel"
-        )
+            Default is False.
+
+        :param bool anonymous: Whether we should login anonymously.
+            Default is True.
+
+        """
+        if anonymous:
+            self._lp = Launchpad.login_anonymously(
+                "ubuntu-debuginfod poller", "production", version="devel"
+            )
+        else:
+            self._lp = Launchpad.login_with(
+                "ubuntu-debuginfod poller", "production", version="devel"
+            )
+
         self._main_archive = self._lp.distributions["ubuntu"].main_archive
-        self.TIMESTAMP_FILE = self.TIMESTAMP_FILE + f"-{module_name}"
 
         self._logger = logging.getLogger(__name__)
         logging.basicConfig(
@@ -73,11 +91,20 @@ class DebugPoller:
 
         self._initial_interval = initial_interval
         self._force_initial_interval = force_initial_interval
+        self._fetch_all_on_first_run = fetch_all_on_first_run
         self._dry_run = dry_run
 
+    @property
+    def timestamp_filename(self):
+        """Get the filename that contains the timestamp for this module.
+
+        :rtype str: The timestamp filename."""
+        module_name = type(self).__name__.replace('Poller', '').lower()
+        return f"{self.TIMESTAMP_FILE}-{module_name}"
+
     def _generate_timestamp(self, interval=None):
         """Generate a timestamp that can be used when querying Launchpad
-        (via getPublishedSources).
+        (via getPublished{Sources,Binaries}).
 
         :param interval: Specify how long ago (in hours) the timestamp must
             refer to.  If not specified, the timestamp is generated for
@@ -97,19 +124,43 @@ class DebugPoller:
         new one.
 
         If a timestamp file exists, returns its value.  Otherwise,
-        generate a new one with self._initial_interval."""
-        if not os.path.exists(self.TIMESTAMP_FILE) or self._force_initial_interval:
-            self._logger.debug(f"Timestamp file '{self.TIMESTAMP_FILE}' doesn't exist")
+        generate a new one with self._initial_interval.
+
+        :raises RuntimeError: Invalid timestamp generated."""
+        tfile = self.timestamp_filename
+
+        if self._fetch_all_on_first_run and not os.path.exists(tfile):
+            self._logger.debug("Fetching everything on first run")
+            return None
+
+        if self._force_initial_interval or not os.path.exists(tfile):
+            self._logger.debug(f"Timestamp file '{tfile}' doesn't exist")
             self._logger.debug(
                 f"Or force_initial_interval = {self._force_initial_interval}"
             )
             return self._generate_timestamp(interval=self._initial_interval)
 
-        with open(self.TIMESTAMP_FILE, "r", encoding="UTF-8") as f:
+        with open(tfile, "r", encoding="UTF-8") as f:
             ts = int(f.readline().rstrip())
             d = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
+
+            if not isinstance(d, datetime.datetime):
+                raise RuntimeError(f"Invalid timestamp {d}")
             return d
 
+    def _get_normal_and_real_timestamps(self):
+        """Get the previous timestamp and adjust it to account for
+        publications arriving out of order."""
+        tfile = self.timestamp_filename
+        if not os.path.exists(tfile) and self._fetch_all_on_first_run:
+            return None, None
+
+        timestamp = self._get_timestamp()
+        # Allow a grace period to cope with publications arriving out of
+        # order during long transactions.
+        real_timestamp = timestamp - self.TIMESTAMP_DELTA
+        return timestamp, real_timestamp
+
     def record_timestamp(self, timestamp):
         """Save the timestamp into the timestamp file.
 
@@ -119,15 +170,10 @@ class DebugPoller:
             self._logger.debug("dry_run enabled, not recording timestamp file")
             return
 
-        dirname = os.path.dirname(self.TIMESTAMP_FILE)
+        tfile = self.timestamp_filename
+        dirname = os.path.dirname(tfile)
         os.makedirs(dirname, mode=0o755, exist_ok=True)
-        with open(self.TIMESTAMP_FILE, "w", encoding="UTF-8") as f:
+        with open(tfile, "w", encoding="UTF-8") as f:
             epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc)
             ts = int((timestamp - epoch).total_seconds())
             f.write("%d" % ts)
-
-    def set_initial_interval(self, initial_interval):
-        """Set the initial_interval value.
-
-        :param int initial_interval: The new initial_interval to be used."""
-        self._initial_interval = initial_interval