Merge lp:~stub/charms/precise/postgresql/cleanups into lp:charms/postgresql

[Stuart Bishop]

Some minor cleanups.

The bulk of this MP is updating the backup script, pulling in a more recent copy from our internal branches.

[Marco Ceppi (marcoceppi)]

This LGTM, I found something else during review but it wasn't introduced by this merge, I filed a bug to track it lp:1313719

review: Approve

=== modified file 'config.yaml'
--- config.yaml	2014-02-13 12:16:30 +0000
+++ config.yaml	2014-03-31 13:35:53 +0000
@@ -347,13 +347,13 @@
         List of extra package sources, per charm-helpers standard.
         YAML format.
     type: string
-    default: ""
+    default: null
   install_keys:
     description: |
         List of signing keys for install_sources package sources, per
         charmhelpers standard. YAML format.
     type: string
-    default: ""
+    default: null
   extra_archives:
     default: ""
     type: string

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2014-02-13 13:38:07 +0000
+++ hooks/hooks.py	2014-03-31 13:35:53 +0000
@@ -408,6 +408,22 @@
             yield line.split()[s]
 
 
+def createcluster():
+    with switch_cwd('/tmp'):  # Ensure cwd is readable as the postgres user
+        create_cmd = [
+            "pg_createcluster",
+            "--locale", hookenv.config('locale'),
+            "-e", hookenv.config('encoding')]
+        if hookenv.config('listen_port'):
+            create_cmd.extend(["-p", str(hookenv.config('listen_port'))])
+        create_cmd.append(pg_version())
+        create_cmd.append(hookenv.config('cluster_name'))
+        run(create_cmd)
+        # Ensure SSL certificates exist, as we enable SSL by default.
+        create_ssl_cert(os.path.join(
+            postgresql_data_dir, pg_version(), hookenv.config('cluster_name')))
+
+
 def _get_system_ram():
     """ Return the system ram in Megabytes """
     import psutil
@@ -710,7 +726,7 @@
 
 
 def create_ssl_cert(cluster_dir):
-    # Debian by default expects SSL certificates in the datadir.
+    # PostgreSQL expects SSL certificates in the datadir.
     server_crt = os.path.join(cluster_dir, 'server.crt')
     server_key = os.path.join(cluster_dir, 'server.key')
     if not os.path.exists(server_crt):
@@ -1038,16 +1054,7 @@
             port_opt = "--port={}".format(config_data['listen_port'])
         else:
             port_opt = ''
-        with switch_cwd('/tmp'):
-            create_cmd = [
-                "pg_createcluster",
-                "--locale", config_data['locale'],
-                "-e", config_data['encoding']]
-            if listen_port:
-                create_cmd.extend(["-p", str(config_data['listen_port'])])
-            create_cmd.append(pg_version())
-            create_cmd.append(config_data['cluster_name'])
-            run(create_cmd)
+        createcluster()
         assert (
             not port_opt
             or get_service_port() == config_data['listen_port']), (
@@ -1070,15 +1077,13 @@
         'logs_dir': postgresql_logs_dir,
     }
     charm_dir = hookenv.charm_dir()
-    template_file = "{}/templates/dump-pg-db.tmpl".format(charm_dir)
-    dump_script = Template(open(template_file).read()).render(paths)
     template_file = "{}/templates/pg_backup_job.tmpl".format(charm_dir)
     backup_job = Template(open(template_file).read()).render(paths)
     host.write_file(
-        '{}/dump-pg-db'.format(postgresql_scripts_dir),
-        dump_script, perms=0755)
+        os.path.join(postgresql_scripts_dir, 'dump-pg-db'),
+        open('scripts/pgbackup.py', 'r').read(), perms=0o755)
     host.write_file(
-        '{}/pg_backup_job'.format(postgresql_scripts_dir),
+        os.path.join(postgresql_scripts_dir, 'pg_backup_job'),
         backup_job, perms=0755)
     install_postgresql_crontab(postgresql_crontab)
     hookenv.open_port(get_service_port())
@@ -2049,7 +2054,7 @@
                 # Clone the master with pg_basebackup.
                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
             log(output, DEBUG)
-            # Debian by default expects SSL certificates in the datadir.
+            # SSL certificates need to exist in the datadir.
             create_ssl_cert(postgresql_cluster_dir)
             create_recovery_conf(master_host, master_port)
         except subprocess.CalledProcessError as x:
@@ -2064,7 +2069,7 @@
                 shutil.rmtree(postgresql_cluster_dir)
             if os.path.exists(postgresql_config_dir):
                 shutil.rmtree(postgresql_config_dir)
-            run('pg_createcluster {} main'.format(version))
+            createcluster()
             config_changed()
             raise
         finally:

=== added file 'scripts/pgbackup.py'
--- scripts/pgbackup.py	1970-01-01 00:00:00 +0000
+++ scripts/pgbackup.py	2014-03-31 13:35:53 +0000
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+
+# Copyright 2008-2014 Canonical Ltd.  All rights reserved.
+
+"""
+Backup one or more PostgreSQL databases.
+
+Suitable for use in crontab for daily backups.
+"""
+
+__metaclass__ = type
+__all__ = []
+
+import sys
+import os
+import os.path
+import stat
+import logging
+import commands
+from datetime import datetime
+from optparse import OptionParser
+
+MB = float(1024 * 1024)
+
+
+def main(options, databases):
+    #Need longer file names if this is used more than daily
+    #today = datetime.now().strftime('%Y%m%d_%H:%M:%S')
+    today = datetime.now().strftime('%Y%m%d')
+
+    backup_dir = options.backup_dir
+    rv = 0
+
+    for database in databases:
+        dest = os.path.join(backup_dir, '%s.%s.dump' % (database, today))
+
+        # base cmd setup; to be modified per the compression desired
+        cmd = " ".join([
+            "/usr/bin/pg_dump",
+            "-U", "postgres",
+            "--format=c",
+            "--blobs",
+            ])
+
+        # alter the cmd to be used based on compression chosen
+        if options.compression_cmd == 'postgres':
+            cmd = " ".join([
+                cmd,
+                "--compress=%d" % options.compression_level
+                    if options.compression_level else "",
+                "--file=%s" % dest,
+                database])
+        elif options.compression_cmd == 'none':
+            cmd = " ".join([
+                cmd,
+                "--compress=0",
+                "--file=%s" % dest,
+                database])
+        else:
+            ext_map = dict(
+                gzip='.gz', pigz='.gz', bzip2='.bz2',
+                pixz='.xz', xz='.xz')
+            dest = dest + ext_map[options.compression_cmd]
+            compression_level_arg = ''
+            if options.compression_level:
+                compression_level_arg = '-%d' % options.compression_level
+            compression_procs_arg = ''
+            if options.processes:
+                compression_procs_arg = '-p %d' % options.processes
+
+            compression_cmd = options.compression_cmd
+            if options.compression_cmd != 'pixz':
+                compression_cmd = compression_cmd + ' -c'
+
+            cmd = " ".join([
+                cmd, "--compress=0", database, "|", compression_cmd,
+                compression_level_arg, compression_procs_arg,
+                ">", dest])
+
+        # If the file already exists, it is from an older dump today.
+        # We don't know if it was successful or not, so abort on this
+        # dump. Leave for operator intervention
+        if os.path.exists(dest):
+            log.error("%s already exists. Skipping." % dest)
+            continue
+
+        (rv, outtext) = commands.getstatusoutput(cmd)
+        if rv != 0:
+            log.critical("Failed to backup %s (%d)" % (database, rv))
+            log.critical(outtext)
+            continue
+
+        size = os.stat(dest)[stat.ST_SIZE]
+        log.info("Backed up %s (%0.2fMB)" % (database, size / MB))
+
+    return rv
+
+if __name__ == '__main__':
+    valid_compression_cmd = ['none'] + sorted([
+        "gzip", "bzip2", "postgres", "pigz", "xz", "pixz"])
+    multiproc_compression_cmd = ["pigz", "pixz"]
+
+    parser = OptionParser(
+        usage="usage: %prog [options] database [database ..]")
+    parser.add_option(
+        "-v", "--verbose", dest="verbose", default=0, action="count")
+    parser.add_option(
+        "-q", "--quiet", dest="quiet", default=0, action="count")
+    parser.add_option(
+        "-d", "--dir", dest="backup_dir",
+        default="/var/lib/postgresql/backups")
+    parser.add_option(
+        "-z", "--compression", dest="compression_cmd", metavar='COMP_CMD',
+        default="gzip",
+        help='Compression tool [{}]'.format(', '.join(valid_compression_cmd)))
+    parser.add_option(
+        "-l", "--compression-level", type=int, metavar='N',
+        dest="compression_level", default=None)
+    parser.add_option(
+        "-p", "--processes", type=int, dest="processes", default=None,
+        metavar="N",
+        help="Number of compression threads, if supported by COMP_CMD")
+    (options, databases) = parser.parse_args()
+    if len(databases) == 0:
+        parser.error("must specify at least one database")
+    if not os.path.isdir(options.backup_dir):
+        parser.error(
+            "Incorrect --dir. %s does not exist or is not a directory" % (
+                options.backup_dir))
+    if options.compression_cmd not in valid_compression_cmd:
+        parser.error(
+            "The compression command must be one of: " + ", ".join(
+                valid_compression_cmd))
+    if options.compression_level is not None and not (
+            1 <= options.compression_level <= 9):
+        parser.error(
+            "The compression level must be between 1 and 9: %s" %
+            options.compression_level)
+    if options.processes and (
+            options.compression_cmd not in multiproc_compression_cmd):
+        parser.error(
+            options.compression_cmd + " does not support multiple processes")
+
+    # Setup our log
+    log = logging.getLogger('pgbackup')
+    hdlr = logging.StreamHandler(sys.stderr)
+    hdlr.setFormatter(logging.Formatter(
+        fmt='%(asctime)s %(levelname)s %(message)s'))
+    log.addHandler(hdlr)
+    verbosity = options.verbose - options.quiet
+    if verbosity > 0:
+        log.setLevel(logging.DEBUG)
+    elif verbosity == 0:  # Default
+        log.setLevel(logging.INFO)
+    elif verbosity == -1:
+        log.setLevel(logging.WARN)
+    elif verbosity < -1:
+        log.setLevel(logging.ERROR)
+
+    sys.exit(main(options, databases))

=== removed file 'templates/dump-pg-db.tmpl'
--- templates/dump-pg-db.tmpl	2012-10-15 21:33:06 +0000
+++ templates/dump-pg-db.tmpl	1970-01-01 00:00:00 +0000
@@ -1,152 +0,0 @@
-#!/usr/bin/python
-
-# Copyright 2008-2011 Canonical Ltd.  All rights reserved.
-
-"""
-Backup one or more PostgreSQL databases.
-
-Suitable for use in crontab for daily backups.
-"""
-
-__metaclass__ = type
-__all__ = []
-
-import sys
-import os
-import os.path
-import stat
-import logging
-import commands
-from datetime import datetime
-from optparse import OptionParser
-
-MB = float(1024 * 1024)
-
-
-def main(options, databases):
-    #Need longer file names if this is used more than daily
-    #today = datetime.now().strftime('%Y%m%d_%H:%M:%S')
-    today = datetime.now().strftime('%Y%m%d')
-
-    backup_dir = options.backup_dir
-    rv = 0
-
-    for database in databases:
-        dest = os.path.join(backup_dir, '%s.%s.dump' % (database, today))
-
-        # base cmd setup; to be modified per the compression desired
-        cmd = " ".join([
-            "/usr/bin/pg_dump",
-            "-U", "postgres",
-            # Need to exclude slony schema from dumps, as read lock on
-            # event table blocks exclusive lock wanted by slony. And it
-            # normally only causes trouble anyway.
-            "--exclude-schema=_sl",
-            "--format=c",
-            "--blobs",
-            ])
-
-        # alter the cmd to be used based on compression chosen
-        if options.compression_cmd == 'gzip':
-            dest = dest + ".gz"
-            cmd = " ".join([
-                cmd,
-                "--compress=0",
-                database,
-                "| gzip -c%d" % options.compression_level,
-                ">", dest,
-                ])
-        elif options.compression_cmd == 'bzip2':
-            dest = dest + ".bz2"
-            cmd = " ".join([
-                cmd,
-                "--compress=0",
-                database,
-                "| bzip2 -c%d" % options.compression_level,
-                ">", dest,
-                ])
-        elif options.compression_cmd == 'postgres':
-            cmd = " ".join([
-                cmd,
-                "--compress=%d" % options.compression_level,
-                "--file=%s" % dest,
-                database,
-                ])
-        else:  # none
-            cmd = " ".join([
-                cmd,
-                "--compress=0",
-                "--file=%s" % dest,
-                database,
-                ])
-
-        # If the file already exists, it is from an older dump today.
-        # We don't know if it was successful or not, so abort on this
-        # dump. Leave for operator intervention
-        if os.path.exists(dest):
-            log.error("%s already exists. Skipping." % dest)
-            continue
-
-        (rv, outtext) = commands.getstatusoutput(cmd)
-        if rv != 0:
-            log.critical("Failed to backup %s (%d)" % (database, rv))
-            log.critical(outtext)
-            continue
-
-        size = os.stat(dest)[stat.ST_SIZE]
-        log.info("Backed up %s (%0.2fMB)" % (database, size / MB))
-
-    return rv
-
-if __name__ == '__main__':
-    parser = OptionParser(
-            usage="usage: %prog [options] database [database ..]"
-            )
-    parser.add_option("-v", "--verbose", dest="verbose", default=0,
-            action="count")
-    parser.add_option("-q", "--quiet", dest="quiet", default=0,
-            action="count")
-    parser.add_option("-d", "--dir", dest="backup_dir",
-            default="/var/lib/postgres/backups")
-    parser.add_option("-z", "--compression", dest="compression_cmd",
-            default="gzip")
-    parser.add_option("-l", "--compression-level", type=int,
-            dest="compression_level", default=6)
-    (options, databases) = parser.parse_args()
-    if len(databases) == 0:
-        parser.error("must specify at least one database")
-    if not os.path.isdir(options.backup_dir):
-        parser.error(
-                "Incorrect --dir. %s does not exist or is not a directory" % (
-                    options.backup_dir
-                    )
-                )
-    valid_compression_cmd = ["gzip", "bzip2", "postgres", "none"]
-    if options.compression_cmd not in valid_compression_cmd:
-        parser.error(
-                "The compression command must be one of: " + " ".join(valid_compression_cmd)
-                )
-    if options.compression_level < 1 or options.compression_level > 9:
-        parser.error(
-                "The compression level must be between 1 and 9: %s" %
-                options.compression_level
-        )
-
-    # Setup our log
-    log = logging.getLogger('pgbackup')
-    hdlr = logging.StreamHandler(sys.stderr)
-    hdlr.setFormatter(logging.Formatter(
-            fmt='%(asctime)s %(levelname)s %(message)s'
-            ))
-    log.addHandler(hdlr)
-    verbosity = options.verbose - options.quiet
-    if verbosity > 0:
-        log.setLevel(logging.DEBUG)
-    elif verbosity == 0:  # Default
-        log.setLevel(logging.INFO)
-    elif verbosity == -1:
-        log.setLevel(logging.WARN)
-    elif verbosity < -1:
-        log.setLevel(logging.ERROR)
-
-    sys.exit(main(options, databases))