Merge lp:~stub/launchpad/staging into lp:launchpad/db-devel

[Stuart Bishop]

[r=allenap][bug=809123,824462] Wait for killed connections to terminate before proceeding to long transaction checks.

= Summary =

Wait for killed connections to terminate before proceeding to long transaction checks.

== Proposed fix ==

Loop until the connections are gone, repeatedly attempting to kill them. Report failure if they can't be killed after 10 seconds.

== Pre-implementation notes ==

== Implementation details ==

== Tests ==

== Demo and Q/A ==

= Launchpad lint =

Checking for conflicts and issues in changed files.

Linting changed files:
  database/schema/preflight.py
  database/schema/full-update.py

./database/schema/preflight.py
       7: '_pythonpath' imported but unused
./database/schema/full-update.py
       7: '_pythonpath' imported but unused

[Gavin Panella (allenap)]

review: Approve

=== modified file 'database/schema/full-update.py'
--- database/schema/full-update.py	2011-08-03 14:30:44 +0000
+++ database/schema/full-update.py	2011-08-11 11:42:24 +0000
@@ -105,17 +105,17 @@
     # work unattended.
     #
 
+    # Confirm we can invoke PGBOUNCER_INITD
+    log.debug("Confirming sudo access to pgbouncer startup script")
+    pgbouncer_rc = run_pgbouncer(log, 'status')
+    if pgbouncer_rc != 0:
+        return pgbouncer_rc
+
     # We initially ignore open connections, as they will shortly be
     # killed.
     if not NoConnectionCheckPreflight(log).check_all():
         return 99
 
-    # Confirm we can invoke PGBOUNCER_INITD
-    log.debug("Confirming sudo access to pgbouncer startup script")
-    pgbouncer_rc = run_pgbouncer(log, 'status')
-    if pgbouncer_rc != 0:
-        return pgbouncer_rc
-
     #
     # Start the actual upgrade. Failures beyond this point need to
     # generate informative messages to help with recovery.

=== modified file 'database/schema/preflight.py'
--- database/schema/preflight.py	2011-08-05 13:29:24 +0000
+++ database/schema/preflight.py	2011-08-11 11:42:24 +0000
@@ -15,7 +15,7 @@
 
 from datetime import timedelta
 from optparse import OptionParser
-import sys
+import time
 
 import psycopg2
 
@@ -282,21 +282,40 @@
 
         System users are defined by SYSTEM_USERS.
         """
-        for node in self.lpmain_nodes:
-            cur = node.con.cursor()
-            cur.execute("""
-                SELECT
-                    procpid, datname, usename, pg_terminate_backend(procpid)
-                FROM pg_stat_activity
-                WHERE
-                    datname=current_database()
-                    AND procpid <> pg_backend_pid()
-                    AND usename NOT IN %s
-                """ % sqlvalues(SYSTEM_USERS))
-            for procpid, datname, usename, ignored in cur.fetchall():
-                self.log.warning(
-                    "Killed %s [%s] on %s", usename, procpid, datname)
-        return True
+        # We keep trying to terminate connections every 0.5 seconds for
+        # up to 10 seconds.
+        num_tries = 20
+        seconds_to_pause = 0.5
+        for loop_count in range(num_tries):
+            all_clear = True
+            for node in self.lpmain_nodes:
+                cur = node.con.cursor()
+                cur.execute("""
+                    SELECT
+                        procpid, datname, usename,
+                        pg_terminate_backend(procpid)
+                    FROM pg_stat_activity
+                    WHERE
+                        datname=current_database()
+                        AND procpid <> pg_backend_pid()
+                        AND usename NOT IN %s
+                    """ % sqlvalues(SYSTEM_USERS))
+                for procpid, datname, usename, ignored in cur.fetchall():
+                    all_clear = False
+                    if loop_count == num_tries - 1:
+                        self.log.fatal(
+                            "Unable to kill %s [%s] on %s",
+                            usename, procpid, datname)
+                    else:
+                        self.log.warning(
+                            "Killed %s [%s] on %s", usename, procpid, datname)
+            if all_clear:
+                break
+
+            # Wait a little for any terminated connections to actually
+            # terminate.
+            time.sleep(seconds_to_pause)
+        return all_clear
 
 
 def main():
@@ -337,4 +356,4 @@
 
 
 if __name__ == '__main__':
-    sys.exit(main())
+    raise SystemExit(main())