Merge ~cjwatson/launchpad:gina-reduce-transactions into launchpad:master

[Colin Watson]

gina: Don't hold transactions while unpacking source packages

Some source packages are large, and unpacking them can be very slow (over ten minutes in some extreme cases). Ensure that we don't hold a transaction open while doing so, since that sometimes causes monitoring scripts that kill idle transactions to ruin our day.

[Ines Almeida (ines-almeida)]

Makes sense.
Maybe add a comment in both places with a quick the explanation why adding TransactionFreeOperation is important there?

review: Approve

[Jürgen Gmach (jugmac00)]

Thanks, Colin. And good idea, Inês!

[Jürgen Gmach (jugmac00)]

review: Approve

[Colin Watson (cjwatson)]

Sure - done.

diff --git a/lib/lp/soyuz/scripts/gina/runner.py b/lib/lp/soyuz/scripts/gina/runner.py
index effe12e..f7909bc 100644
--- a/lib/lp/soyuz/scripts/gina/runner.py
+++ b/lib/lp/soyuz/scripts/gina/runner.py
@@ -13,6 +13,7 @@ import six
 from zope.component import getUtility
 
 from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.scripts.helpers import TransactionFreeOperation
 from lp.services.config import config
 from lp.services.features import getFeatureFlag
 from lp.services.scripts import log
@@ -192,8 +193,12 @@ def do_one_sourcepackage(distro, source, package_root, importer_handler):
         # already exists in the database
         log.info("%s already exists in the archive", source_data.package)
         return
-    source_data.process_package(distro, package_root)
-    source_data.ensure_complete()
+    importer_handler.commit()
+    # Unpacking source packages can be slow, so don't hold a transaction
+    # while doing so.
+    with TransactionFreeOperation():
+        source_data.process_package(distro, package_root)
+        source_data.ensure_complete()
     importer_handler.import_sourcepackage(source_data)
     importer_handler.commit()
 
diff --git a/lib/lp/soyuz/scripts/tests/test_gina.py b/lib/lp/soyuz/scripts/tests/test_gina.py
index edff14e..967d917 100644
--- a/lib/lp/soyuz/scripts/tests/test_gina.py
+++ b/lib/lp/soyuz/scripts/tests/test_gina.py
@@ -19,6 +19,7 @@ import lp.soyuz.scripts.gina.handlers
 from lp.archiveuploader.tagfiles import parse_tagfile
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.registry.interfaces.series import SeriesStatus
+from lp.scripts.helpers import TransactionFreeOperation
 from lp.services.database.constants import UTC_NOW
 from lp.services.features.testing import FeatureFixture
 from lp.services.log.logger import DevNullLogger
@@ -754,12 +755,15 @@ class TestRunner(TestCaseWithFactory):
         )
 
         def import_and_get_versions():
-            import_sourcepackages(
-                series.distribution.name,
-                packages_map,
-                archive_root,
-                importer_handler,
-            )
+            # Unpacking source packages can be slow, so ensure that we don't
+            # hold a transaction while doing so.
+            with TransactionFreeOperation.require():
+                import_sourcepackages(
+                    series.distribution.name,
+                    packages_map,
+                    archive_root,
+                    importer_handler,
+                )
             return [
                 p.source_package_version
                 for p in series.main_archive.getPublishedSources(