Merge ~lgp171188/launchpad:rsync-oval-data-when-publishing-a-distro into launchpad:master

[Guruprasad]

Perform OVAL data rsync when publishing a distro

[Jürgen Gmach (jugmac00)]

Thank you! This looks good to me.

Before you merge, please wait on a second review.

review: Approve

[Colin Watson (cjwatson)]

review: Approve

diff --git a/lib/lp/archivepublisher/scripts/publishdistro.py b/lib/lp/archivepublisher/scripts/publishdistro.py
index 3b91fa5..419b1ef 100644
--- a/lib/lp/archivepublisher/scripts/publishdistro.py
+++ b/lib/lp/archivepublisher/scripts/publishdistro.py
@@ -7,7 +7,9 @@ __all__ = [
     "PublishDistro",
 ]
 
+import os
 from optparse import OptionValueError
+from subprocess import CalledProcessError, check_call
 
 from storm.store import Store
 from zope.component import getUtility
@@ -19,6 +21,7 @@ from lp.archivepublisher.publishing import (
     getPublisher,
 )
 from lp.archivepublisher.scripts.base import PublisherScript
+from lp.services.config import config
 from lp.services.limitedlist import LimitedList
 from lp.services.scripts.base import LaunchpadScriptFailure
 from lp.services.webapp.adapter import (
@@ -504,11 +507,54 @@ class PublishDistro(PublisherScript):
                 # store and cause performance problems.
                 Store.of(archive).reset()
 
+    def rsync_oval_data(self):
+        if config.archivepublisher.oval_data_rsync_endpoint:
+            # Ensure that the rsync paths have a trailing slash.
+            rsync_src = os.path.join(
+                config.archivepublisher.oval_data_rsync_endpoint, ""
+            )
+            rsync_dest = os.path.join(
+                config.archivepublisher.oval_data_root, ""
+            )
+            rsync_command = [
+                "/usr/bin/rsync",
+                "-a",
+                "-q",
+                "--timeout={}".format(
+                    config.archivepublisher.oval_data_rsync_timeout
+                ),
+                "--delete",
+                "--delete-after",
+                rsync_src,
+                rsync_dest,
+            ]
+            try:
+                self.logger.info(
+                    "Attempting to rsync the OVAL data from '%s' to '%s'",
+                    rsync_src,
+                    rsync_dest,
+                )
+                check_call(rsync_command)
+            except CalledProcessError:
+                self.logger.exception(
+                    "Failed to rsync OVAL data from '%s' to '%s'",
+                    rsync_src,
+                    rsync_dest,
+                )
+                raise
+        else:
+            self.logger.info(
+                "Skipping the OVAL data sync as no rsync endpoint"
+                " has been configured."
+            )
+
     def main(self, reset_store_between_archives=True):
         """See `LaunchpadScript`."""
         self.validateOptions()
         self.logOptions()
 
+        self.rsync_oval_data()
+
         archive_ids = []
         for distribution in self.findDistros():
             for archive in self.getTargetArchives(distribution):
diff --git a/lib/lp/archivepublisher/tests/test_publishdistro.py b/lib/lp/archivepublisher/tests/test_publishdistro.py
index fd1d1e6..3d9e0cc 100644
--- a/lib/lp/archivepublisher/tests/test_publishdistro.py
+++ b/lib/lp/archivepublisher/tests/test_publishdistro.py
@@ -8,6 +8,7 @@ import shutil
 import subprocess
 from optparse import OptionValueError
 
+from fixtures import MockPatch
 from storm.store import Store
 from testtools.matchers import Not, PathExists
 from testtools.twistedsupport import AsynchronousDeferredRunTest
@@ -69,7 +70,8 @@ class TestPublishDistro(TestNativePublishingBase):
         if extra_args is not None:
             args.extend(extra_args)
         publish_distro = PublishDistro(test_args=args)
-        publish_distro.logger = BufferLogger()
+        self.logger = BufferLogger()
+        publish_distro.logger = self.logger
         publish_distro.txn = self.layer.txn
         switch_dbuser(config.archivepublisher.dbuser)
         publish_distro.main()
@@ -255,6 +257,80 @@ class TestPublishDistro(TestNativePublishingBase):
         pub_source = self.loadPubSource(pub_source_id)
         self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)
 
+    def setUpOVALDataRsync(self):
+        self.pushConfig(
+            "archivepublisher",
+            oval_data_rsync_endpoint="oval.internal::oval/",
+            oval_data_root="/tmp/oval-data",
+            oval_data_rsync_timeout=90,
+        )
+
+    def testPublishDistroOVALDataRsyncEndpointNotConfigured(self):
+        """
+        Test what happens when the OVAL data rsync endpoint is not configured.
+        """
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+        ).mock
+        self.runPublishDistro()
+        mock_subprocess_check_call.assert_not_called()
+        expected_log_line = (
+            "INFO Skipping the OVAL data sync as no rsync endpoint has been "
+            "configured."
+        )
+        self.assertTrue(expected_log_line in self.logger.getLogBuffer())
+
+    def testPublishDistroOVALDataRsyncEndpointConfigured(self):
+        """
+        Test the OVAL data rsync command.
+
+        When the endpoint is configured, verify that the command is run.
+        """
+        self.setUpOVALDataRsync()
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+        ).mock
+        self.runPublishDistro()
+        call_args = [
+            "/usr/bin/rsync",
+            "-a",
+            "-q",
+            "--timeout=90",
+            "--delete",
+            "--delete-after",
+            "oval.internal::oval/",
+            "/tmp/oval-data/",
+        ]
+        mock_subprocess_check_call.assert_called_once_with(call_args)
+
+    def testPublishDistroOVALDataRsyncErrorsOut(self):
+        self.setUpOVALDataRsync()
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch(
+                "lp.archivepublisher.scripts.publishdistro.check_call",
+                side_effect=subprocess.CalledProcessError(
+                    cmd="foo", returncode=5
+                ),
+            )
+        ).mock
+        self.assertRaises(subprocess.CalledProcessError, self.runPublishDistro)
+        call_args = [
+            "/usr/bin/rsync",
+            "-a",
+            "-q",
+            "--timeout=90",
+            "--delete",
+            "--delete-after",
+            "oval.internal::oval/",
+            "/tmp/oval-data/",
+        ]
+        mock_subprocess_check_call.assert_called_once_with(call_args)
+        expected_log_line = (
+            "ERROR Failed to rsync OVAL data from "
+            "'oval.internal::oval/' to '/tmp/oval-data/'"
+        )
+        self.assertTrue(expected_log_line in self.logger.getLogBuffer())
+
     @defer.inlineCallbacks
     def testForPPA(self):
         """Try to run publish-distro in PPA mode.
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index db9893c..28ce035 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -32,6 +32,15 @@ dbuser: archivepublisher
 # datatype: string
 run_parts_location: none
 
+# The rsync endpoint to use for synchronizing the OVAL data.
+oval_data_rsync_endpoint: none
+
+# The destination to rsync the OVAL data to.
+oval_data_root: /srv/launchpad.net/oval-data
+
+# Timeout (in seconds) to use when rsync'ing OVAL data.
+oval_data_rsync_timeout: 30
+
 
 [artifactory]
 # Base URL for publishing suitably-configured archives to Artifactory.