Merge lp:~maas-maintainers/maas/backup into lp:maas/trunk

Proposed by Raphaël Badin on 2013-12-02
Status: Rejected
Rejected by: MAAS Lander on 2017-06-22
Proposed branch: lp:~maas-maintainers/maas/backup
Merge into: lp:maas/trunk
Diff against target: 499 lines (+473/-0)
5 files modified
backup/common.py (+142/-0)
backup/maas-backup (+134/-0)
backup/maas-restore (+186/-0)
backup/tests/maas_local_celeryconfig.py (+3/-0)
backup/tests/maas_local_settings.py (+8/-0)
To merge this branch: bzr merge lp:~maas-maintainers/maas/backup
Reviewer Review Type Date Requested Status
MAAS Maintainers 2013-12-02 Pending
Review via email: mp+197339@code.launchpad.net

Commit message

WIP

To post a comment you must log in.
lp:~maas-maintainers/maas/backup updated on 2013-12-06
1780. By Raphaël Badin on 2013-12-02

Couple of obvious fixes.

1781. By Raphaël Badin on 2013-12-02

One obvious fix.

1782. By Raphaël Badin on 2013-12-02

Define config variables.

1783. By Raphaël Badin on 2013-12-02

Fixes from actual testing.

1784. By Gavin Panella on 2013-12-02

No need for empty modules.

1785. By Gavin Panella on 2013-12-02

Extract code to name the database backup directory.

1786. By Gavin Panella on 2013-12-02

Fix typo.

1787. By Gavin Panella on 2013-12-02

Restore databases.

1788. By Gavin Panella on 2013-12-02

Fix mistake when deriving db dump dir.

1789. By Gavin Panella on 2013-12-02

Fix pg_restore command-line options.

1790. By Raphaël Badin on 2013-12-06

Fix pgdump/restore thanks jtv.

MAAS Lander (maas-lander) wrote :

Transitioned to Git.

lp:maas has now moved from Bzr to Git.
Please propose your branches with Launchpad using Git.

git clone https://git.launchpad.net/maas

Unmerged revisions

1790. By Raphaël Badin on 2013-12-06

Fix pgdump/restore thanks jtv.

1789. By Gavin Panella on 2013-12-02

Fix pg_restore command-line options.

1788. By Gavin Panella on 2013-12-02

Fix mistake when deriving db dump dir.

1787. By Gavin Panella on 2013-12-02

Restore databases.

1786. By Gavin Panella on 2013-12-02

Fix typo.

1785. By Gavin Panella on 2013-12-02

Extract code to name the database backup directory.

1784. By Gavin Panella on 2013-12-02

No need for empty modules.

1783. By Raphaël Badin on 2013-12-02

Fixes from actual testing.

1782. By Raphaël Badin on 2013-12-02

Define config variables.

1781. By Raphaël Badin on 2013-12-02

One obvious fix.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added directory 'backup'
2=== added file 'backup/common.py'
3--- backup/common.py 1970-01-01 00:00:00 +0000
4+++ backup/common.py 2013-12-06 10:37:36 +0000
5@@ -0,0 +1,142 @@
6+#!/usr/bin/env python2.7
7+# -*- mode: python -*-
8+# Copyright 2013 Canonical Ltd. This software is licensed under the
9+# GNU Affero General Public License version 3 (see the file LICENSE).
10+
11+"""Common utilities."""
12+
13+from __future__ import (
14+ absolute_import,
15+ print_function,
16+ unicode_literals,
17+ )
18+
19+str = None
20+
21+__metaclass__ = type
22+__all__ = [
23+ "get_database_backup_directory",
24+ "import_settings",
25+ "logger",
26+ "MAAS_CONFIG_DIR",
27+ "MAAS_CONFIG_TARBALL",
28+ "makedirs",
29+ "run_command",
30+ "System",
31+]
32+
33+import errno
34+import logging
35+import os
36+from os import path
37+from pipes import quote
38+from subprocess import (
39+ PIPE,
40+ Popen,
41+ )
42+import sys
43+
44+import apt
45+
46+
47+logging.basicConfig(
48+ level=logging.DEBUG,
49+ format='%(asctime)s %(levelname)s %(message)s')
50+
51+logger = logging.getLogger("maas-backup")
52+
53+# MAAS variables.
54+MAAS_CONFIG_DIR = '/etc/maas'
55+MAAS_CONFIG_TARBALL = 'etc-maas.tgz'
56+
57+# Do *NOT* "from ... import ..." these names; they are modified by
58+# import_settings().
59+django_settings = None
60+celery_settings = None
61+
62+
63+def import_settings(test=False):
64+ if test:
65+ config_dir = path.join(path.dirname(__file__), 'tests')
66+ sys.path[:0] = [config_dir]
67+ os.environ['DJANGO_SETTINGS_MODULE'] = 'maas_local_settings'
68+ os.environ['CELERY_CONFIG_MODULE'] = 'maas_local_celeryconfig'
69+ else:
70+ sys.path[:0] = [MAAS_CONFIG_DIR, '/usr/share/maas/']
71+ os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'
72+ os.environ['CELERY_CONFIG_MODULE'] = 'celeryconfig_cluster'
73+ # Import Django settings.
74+ from django.conf import settings
75+ global django_settings
76+ django_settings = settings
77+ # Import Celery settings.
78+ from celery.app import app_or_default
79+ global celery_settings
80+ celery_settings = app_or_default().conf
81+
82+
83+def run_command(args, input=None, check_call=True, dry_run=False, env=None):
84+ """A wrapper to Popen to run commands in the command-line."""
85+ if dry_run:
86+ cmd = " ".join(quote(arg) for arg in args)
87+ logger.info("Running command: %s" % cmd)
88+ return
89+ process = Popen(
90+ args, stdout=PIPE, stderr=PIPE, stdin=PIPE,
91+ shell=False, env=env)
92+ stdout, stderr = process.communicate(input)
93+ retcode = process.returncode
94+ if check_call and retcode != 0:
95+ cmd = " ".join(quote(arg) for arg in args)
96+ raise Exception(
97+ "Command '%s' failed (%d):\n%s\n%s" % (
98+ cmd, retcode,
99+ stdout.decode('ascii'),
100+ stderr.decode('ascii')))
101+ return retcode, stdout, stderr
102+
103+
104+def makedirs(path):
105+ """Similar to os.makedirs but does not error if the directory exists."""
106+ try:
107+ os.makedirs(path)
108+ except OSError as error:
109+ if error.errno != errno.EEXIST:
110+ raise
111+ # Directory already exists.
112+
113+
114+def get_database_backup_directory(database):
115+ """Return a relative directory in which a db will or has been backed up.
116+
117+ :param database: A Django `DATABASES` entry.
118+ :type database: `dict`
119+ """
120+ return "db.%(NAME)s.user=%(USER)s" % database
121+
122+
123+class System:
124+ """Encapsulates information about the system.
125+
126+ For example, what packages are installed, what is the purpose of
127+ this system.
128+
129+ """
130+
131+ def __init__(self):
132+ super(System, self).__init__()
133+ self.cache = apt.Cache()
134+
135+ def is_package_installed(self, package_name):
136+ try:
137+ package = self.cache[package_name]
138+ except KeyError:
139+ return False
140+ else:
141+ return package.is_installed
142+
143+ def is_region_controller(self):
144+ return self.is_package_installed("maas-region-controller")
145+
146+ def is_cluster_controller(self):
147+ return self.is_package_installed("maas-cluster-controller")
148
149=== added file 'backup/maas-backup'
150--- backup/maas-backup 1970-01-01 00:00:00 +0000
151+++ backup/maas-backup 2013-12-06 10:37:36 +0000
152@@ -0,0 +1,134 @@
153+#!/usr/bin/env python2.7
154+# -*- mode: python -*-
155+# Copyright 2013 Canonical Ltd. This software is licensed under the
156+# GNU Affero General Public License version 3 (see the file LICENSE).
157+
158+"""Backup MAAS."""
159+
160+from __future__ import (
161+ absolute_import,
162+ print_function,
163+ unicode_literals,
164+ )
165+
166+str = None
167+
168+__metaclass__ = type
169+
170+import argparse
171+import os
172+from os import path
173+import sys
174+
175+import common
176+from common import (
177+ get_database_backup_directory,
178+ logger,
179+ System,
180+ MAAS_CONFIG_TARBALL,
181+ MAAS_CONFIG_DIR,
182+ run_command,
183+ makedirs,
184+ )
185+from six import text_type
186+
187+# For development, add the script's directory to sys.path.
188+sys.path[:0] = [path.dirname(__file__)]
189+
190+
191+def backup_config_files(target_directory, dry_run=False):
192+ dest_file = path.join(target_directory, MAAS_CONFIG_TARBALL)
193+ run_command(
194+ ['tar', 'zcpf', dest_file, '-C', MAAS_CONFIG_DIR, '.'],
195+ dry_run=dry_run)
196+
197+
198+def backup_databases(target_directory, dry_run=False):
199+ databases = [
200+ database
201+ for database in common.django_settings.DATABASES.values()
202+ if database["ENGINE"] == "django.db.backends.postgresql_psycopg2"
203+ ]
204+ for database in databases:
205+ backup_to = path.join(
206+ target_directory, get_database_backup_directory(database))
207+ command = (
208+ "pg_dump",
209+ "--file", backup_to,
210+ "--host", database["HOST"],
211+ "--username", database["USER"],
212+ database["NAME"],
213+ )
214+ # The password cannot be passed on the command-line, bit libpq
215+ # recognises PGPASSWORD in the environment.
216+ env = dict(os.environ, PGPASSWORD=database["PASSWORD"])
217+ run_command(command, env=env, dry_run=dry_run)
218+
219+
220+def backup_region(target_directory, dry_run=False):
221+ """Backup region controller to `target_directory`."""
222+ logger.info("Backing-up region controller to %s.", target_directory)
223+ if dry_run:
224+ logger.info("Dry-run.")
225+ backup_config_files(target_directory, dry_run=dry_run)
226+ backup_databases(target_directory, dry_run=dry_run)
227+
228+
229+def backup_leases_file(target_directory, dry_run=False):
230+ leases_file = common.celery_settings.DHCP_LEASES_FILE
231+ run_command(
232+ ['cp', '-p', '--', leases_file, target_directory],
233+ check_call=False,
234+ dry_run=dry_run)
235+
236+
237+def backup_cluster(target_directory, dry_run=False):
238+ """Backup cluster controller to `target_directory`."""
239+ logger.info("Backing-up cluster controller to %s.", target_directory)
240+ if dry_run:
241+ logger.info("Dry-run.")
242+ backup_config_files(target_directory, dry_run=dry_run)
243+ backup_leases_file(target_directory, dry_run=dry_run)
244+
245+
246+def backup(target_directory, dry_run=False, test=False):
247+ """Backup cluster and/or region controller to `target_directory`."""
248+ system = System()
249+ if test or system.is_region_controller():
250+ target_dir = path.join(target_directory, "region")
251+ makedirs(target_dir)
252+ backup_region(target_dir, dry_run or test)
253+ if test or system.is_cluster_controller():
254+ target_dir = path.join(target_directory, "cluster")
255+ makedirs(target_dir)
256+ backup_cluster(target_dir, dry_run or test)
257+
258+
259+# See http://docs.python.org/release/2.7/library/argparse.html.
260+argument_parser = argparse.ArgumentParser(description=__doc__)
261+argument_parser.add_argument(
262+ 'target_directory', metavar="target-directory", type=text_type,
263+ help="The directory where the files will be put.")
264+argument_parser.add_argument(
265+ '--dry-run', action='store_true', default=False, help=(
266+ "No action; perform a simulation of events that would occur but do "
267+ "not actually do anything."))
268+argument_parser.add_argument(
269+ '--test', action='store_true', default=False, help=(
270+ "Run a test run. Equivalent to --dry-run plus skipping all the "
271+ "checks."))
272+
273+
274+def main():
275+ args = argument_parser.parse_args()
276+ if not args.test and os.geteuid() != 0:
277+ logger.error(
278+ "This script must be run with root privileges ("
279+ "i.e. use 'sudo').")
280+ return
281+ common.import_settings(args.test)
282+ backup(args.target_directory, args.dry_run, args.test)
283+
284+
285+if __name__ == "__main__":
286+ main()
287
288=== added file 'backup/maas-restore'
289--- backup/maas-restore 1970-01-01 00:00:00 +0000
290+++ backup/maas-restore 2013-12-06 10:37:36 +0000
291@@ -0,0 +1,186 @@
292+#!/usr/bin/env python2.7
293+# -*- mode: python -*-
294+# Copyright 2013 Canonical Ltd. This software is licensed under the
295+# GNU Affero General Public License version 3 (see the file LICENSE).
296+
297+"""Restore MAAS from a backup."""
298+
299+from __future__ import (
300+ absolute_import,
301+ print_function,
302+ unicode_literals,
303+ )
304+
305+str = None
306+
307+__metaclass__ = type
308+
309+import argparse
310+import os
311+from os import path
312+import sys
313+from urlparse import urlparse
314+
315+import common
316+from common import (
317+ get_database_backup_directory,
318+ logger,
319+ System,
320+ MAAS_CONFIG_TARBALL,
321+ MAAS_CONFIG_DIR,
322+ run_command,
323+ )
324+from six import text_type
325+
326+# For development, add the script's directory to sys.path.
327+sys.path[:0] = [path.dirname(__file__)]
328+
329+
330+def restore_config_files(source_directory, dry_run=False):
331+ src_file = path.join(source_directory, MAAS_CONFIG_TARBALL)
332+ run_command(
333+ ['tar', 'zxpf', src_file, '-C', MAAS_CONFIG_DIR],
334+ dry_run=dry_run)
335+
336+
337+def restore_databases(source_directory, dry_run=False):
338+ databases = [
339+ database
340+ for database in common.django_settings.DATABASES.values()
341+ if database["ENGINE"] == "django.db.backends.postgresql_psycopg2"
342+ ]
343+ for database in databases:
344+ restore_from = path.join(
345+ source_directory, get_database_backup_directory(database))
346+ command = (
347+ "psql",
348+ "--host", database["HOST"],
349+ "--dbname", database["NAME"],
350+ "--username", database["USER"],
351+ "--single-transaction",
352+ "--file", restore_from,
353+ )
354+ # The password cannot be passed on the command-line, bit libpq
355+ # recognises PGPASSWORD in the environment.
356+ env = dict(os.environ, PGPASSWORD=database["PASSWORD"])
357+ run_command(command, env=env, dry_run=dry_run)
358+
359+
360+def update_rabbitmq_passwords(dry_run=False, test=False):
361+ # Update password for maas_longpoll.
362+ run_command(
363+ ['rabbitmqctl', 'change_password',
364+ common.django_settings.RABBITMQ_USERID,
365+ common.django_settings.RABBITMQ_PASSWORD],
366+ dry_run=dry_run)
367+ # Update password for maas_workers.
368+ broker_url = common.celery_settings.BROKER_URL
369+ parsed_url = urlparse(broker_url)
370+ run_command(
371+ ['rabbitmqctl', 'change_password',
372+ parsed_url.username, parsed_url.password],
373+ dry_run=dry_run)
374+
375+
376+def run_db_migrations(dry_run=False):
377+ run_command(
378+ ['maas', 'migrate', 'maasserver', '--noinput'],
379+ dry_run=dry_run)
380+ run_command(
381+ ['maas', 'migrate', 'metadataserver', '--noinput'],
382+ dry_run=dry_run)
383+
384+
385+def restart_region_services(dry_run=False):
386+ run_command(['service', 'apache2', 'restart'], dry_run=dry_run)
387+ run_command(['service', 'maas-pserv', 'restart'], dry_run=dry_run)
388+ run_command(
389+ ['service', 'maas-region-celery', 'restart'], dry_run=dry_run)
390+ run_command(['service', 'maas-txlongpoll', 'restart'], dry_run=dry_run)
391+ # Do not error if this fails because this MAAS instance might not be
392+ # using maas-dns.
393+ run_command(
394+ ['service', 'bind9', 'restart'], check_call=False, dry_run=dry_run)
395+
396+
397+def restore_region(source_directory, dry_run=False):
398+ """Restore region controller from `source_directory`."""
399+ logger.info("Restoring region controller from %s.", source_directory)
400+ if dry_run:
401+ logger.info("Dry-run.")
402+ restore_config_files(source_directory, dry_run=dry_run)
403+ restore_databases(source_directory, dry_run=dry_run)
404+ update_rabbitmq_passwords(dry_run=dry_run)
405+ restart_region_services(dry_run=dry_run)
406+ run_db_migrations(dry_run=dry_run)
407+
408+
409+def restore_leases_file(source_directory, dry_run=False):
410+ leases_file = common.celery_settings.DHCP_LEASES_FILE
411+ source_file = path.join(
412+ source_directory, path.basename(leases_file))
413+ run_command(
414+ ['cp', '-p', '--', source_file, leases_file],
415+ check_call=False,
416+ dry_run=dry_run)
417+
418+
419+def restart_cluser_services(dry_run=False):
420+ run_command(
421+ ['service', 'maas-cluster-celery', 'restart'], dry_run=dry_run)
422+ # Do not error if this fails because this MAAS instance might not be
423+ # using maas-dhcp.
424+ run_command(
425+ ['service', 'maas-dhcp-server', 'restart'], check_call=False,
426+ dry_run=dry_run)
427+
428+
429+def restore_cluster(source_directory, dry_run=False):
430+ """Restore cluster controller from `source_directory`."""
431+ logger.info("Restoring cluster controller from %s.", source_directory)
432+ if dry_run:
433+ logger.info("Dry-run.")
434+ restore_config_files(source_directory, dry_run=dry_run)
435+ restore_leases_file(source_directory, dry_run=dry_run)
436+ restart_cluser_services(dry_run=dry_run)
437+
438+
439+def restore(source_directory, dry_run=False, test=False):
440+ """Restore cluster and/or region controller from `source_directory`."""
441+ system = System()
442+ if test or system.is_region_controller():
443+ source_dir = path.join(source_directory, "region")
444+ restore_region(source_dir, dry_run or test)
445+ if test or system.is_cluster_controller():
446+ source_dir = path.join(source_directory, "cluster")
447+ restore_cluster(source_dir, dry_run or test)
448+
449+
450+# See http://docs.python.org/release/2.7/library/argparse.html.
451+argument_parser = argparse.ArgumentParser(description=__doc__)
452+argument_parser.add_argument(
453+ 'source_directory', metavar="source-directory", type=text_type,
454+ help="The directory where the files from a backup can be found.")
455+argument_parser.add_argument(
456+ '--dry-run', action='store_true', default=False, help=(
457+ "No action; perform a simulation of events that would occur but do "
458+ "not actually do anything."))
459+argument_parser.add_argument(
460+ '--test', action='store_true', default=False, help=(
461+ "Run a test run. Equivalent to --dry-run plus skipping all the "
462+ "checks."))
463+
464+
465+def main():
466+ args = argument_parser.parse_args()
467+ if not args.test and os.geteuid() != 0:
468+ logger.error(
469+ "This script must be run with root privileges ("
470+ "i.e. use 'sudo').")
471+ return
472+ common.import_settings(args.test)
473+ restore(args.source_directory, args.dry_run, args.test)
474+
475+
476+if __name__ == "__main__":
477+ main()
478
479=== added directory 'backup/tests'
480=== added file 'backup/tests/maas_local_celeryconfig.py'
481--- backup/tests/maas_local_celeryconfig.py 1970-01-01 00:00:00 +0000
482+++ backup/tests/maas_local_celeryconfig.py 2013-12-06 10:37:36 +0000
483@@ -0,0 +1,3 @@
484+BROKER_URL = 'amqp://maas_workers:TqgnPD6jXe7GT2nVFDJc@10.55.61.76:5672//maas_workers'
485+
486+DHCP_LEASES_FILE = '/var/lib/maas/dhcp/dhcpd.leases'
487
488=== added file 'backup/tests/maas_local_settings.py'
489--- backup/tests/maas_local_settings.py 1970-01-01 00:00:00 +0000
490+++ backup/tests/maas_local_settings.py 2013-12-06 10:37:36 +0000
491@@ -0,0 +1,8 @@
492+# RabbitMQ settings.
493+RABBITMQ_HOST = 'localhost'
494+RABBITMQ_USERID = 'maas_longpoll'
495+RABBITMQ_PASSWORD = 'gQaFNcMkzoSloOxHY7ZN'
496+RABBITMQ_VIRTUAL_HOST = '/maas_longpoll'
497+
498+# Settings set so that Django accepts this file as a valid config file.
499+SECRET_KEY = '546908346890'