Merge lp:~longbow/kewpie/xb_manager into lp:kewpie

Proposed by Valentine Gostev on 2012-06-11
Status: Merged
Merged at revision: 126
Proposed branch: lp:~longbow/kewpie/xb_manager
Merge into: lp:kewpie
Diff against target: 335 lines (+272/-2)
5 files modified
kewpie.py (+4/-1)
lib/test_mgmt/execution_management.py (+2/-1)
lib/test_mgmt/test_execution.py (+1/-0)
lib/util/xb_manager.py (+203/-0)
percona_tests/xbm/xb_incremental_test.py (+62/-0)
To merge this branch: bzr merge lp:~longbow/kewpie/xb_manager
Reviewer Review Type Date Requested Status
Valentine Gostev Approve on 2012-06-11
Review via email: mp+109727@code.launchpad.net

Description of the change

Added xtrabackup management library and a sample test in 'xmb' suite

To post a comment you must log in.
Valentine Gostev (longbow) :
review: Approve
Patrick Crews (patrick-crews) wrote :

You rock! Glad to see you are still finding use for the tools : )
I'm currently working on some updates myself...just been locked in a basement for a few weeks ; )

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'kewpie.py'
2--- kewpie.py 2012-02-20 20:49:50 +0000
3+++ kewpie.py 2012-06-11 20:30:31 +0000
4@@ -44,6 +44,7 @@
5 from lib.sys_mgmt.system_management import systemManager
6 from lib.test_mgmt.execution_management import executionManager
7 from lib.opts.matrix_manager import matrixManager
8+from lib.util.xb_manager import xtrabackupManager
9
10 # functions
11 def handle_sys_config(input_args, defaults):
12@@ -93,6 +94,8 @@
13 # Create our server_manager
14 server_manager = serverManager(system_manager, variables)
15
16+ # Create XtraBackup manager
17+ xtrabackup_manager = xtrabackupManager(system_manager, server_manager, variables)
18 # Get our mode-specific test_manager and test_executor
19 (test_manager,test_executor) = handle_mode(variables, system_manager)
20
21@@ -102,7 +105,7 @@
22 # Initialize test execution manager
23 execution_manager = executionManager(server_manager, system_manager
24 , test_manager, test_executor
25- , variables, matrix_manager)
26+ , variables, matrix_manager, xtrabackup_manager)
27
28 # Execute our tests!
29 execution_manager.execute_tests()
30
31=== modified file 'lib/test_mgmt/execution_management.py'
32--- lib/test_mgmt/execution_management.py 2012-02-20 20:49:50 +0000
33+++ lib/test_mgmt/execution_management.py 2012-06-11 20:30:31 +0000
34@@ -39,11 +39,12 @@
35 """
36
37 def __init__(self, server_manager, system_manager, test_manager
38- , executor_type, variables, matrix_manager):
39+ , executor_type, variables, matrix_manager, xtrabackup_manager):
40
41 self.server_manager = server_manager
42 self.system_manager = system_manager
43 self.matrix_manager = matrix_manager
44+ self.xtrabackup_manager = xtrabackup_manager
45 self.logging = system_manager.logging
46 self.test_manager = test_manager
47 if variables['verbose']:
48
49=== modified file 'lib/test_mgmt/test_execution.py'
50--- lib/test_mgmt/test_execution.py 2012-02-20 20:49:50 +0000
51+++ lib/test_mgmt/test_execution.py 2012-06-11 20:30:31 +0000
52@@ -58,6 +58,7 @@
53 self.server_manager = self.execution_manager.server_manager
54 self.time_manager = self.system_manager.time_manager
55 self.matrix_manager = self.execution_manager.matrix_manager
56+ self.xtrabackup_manager = self.execution_manager.xtrabackup_manager
57 self.name = name
58 self.working_environment = {} # we pass env dict to define what we need
59 self.dirset = { self.name : { 'log': None } }
60
61=== added file 'lib/util/xb_manager.py'
62--- lib/util/xb_manager.py 1970-01-01 00:00:00 +0000
63+++ lib/util/xb_manager.py 2012-06-11 20:30:31 +0000
64@@ -0,0 +1,203 @@
65+#! /usr/bin/env python
66+# -*- mode: python; indent-tabs-mode: nil; -*-
67+# vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
68+#
69+# Copyright (C) 2011-2012 Patrick Crews, Valentine Gostev
70+#
71+#
72+# This program is free software; you can redistribute it and/or modify
73+# it under the terms of the GNU General Public License as published by
74+# the Free Software Foundation; either version 2 of the License, or
75+# (at your option) any later version.
76+#
77+# This program is distributed in the hope that it will be useful,
78+# but WITHOUT ANY WARRANTY; without even the implied warranty of
79+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
80+# GNU General Public License for more details.
81+#
82+# You should have received a copy of the GNU General Public License
83+# along with this program; if not, write to the Free Software
84+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
85+
86+import os
87+import shutil
88+import re
89+import subprocess
90+
91+""" Xtrabackup manager class handles all mysql backup operations
92+ it creates a backup object from server objects
93+"""
94+
95+class xtrabackupManager:
96+ def __init__(self, server_manager, system_manager, variables):
97+ self.system_manager = system_manager
98+ self.code_manager = system_manager.code_manager
99+ self.server_manager = server_manager
100+ self.env_manager = system_manager.env_manager
101+ self.logging = system_manager.logging
102+ self.xb_bin_path = variables['xtrabackuppath']
103+ self.ib_bin_path = variables['innobackupexpath']
104+ self.backup_dir = os.path.join(variables['workdir'],'backups')
105+ if not os.path.isdir(self.backup_dir):
106+ os.makedirs(self.backup_dir)
107+
108+ """ clean_dir method is used to wipe out data from server object's
109+ data directory in order to restore a prepared backup
110+ Usage clean_dir()
111+ """
112+ def clean_dir(self,path):
113+ for top,dirs,files in os.walk(path):
114+ for file in files:
115+ os.unlink(os.path.join(top,file))
116+ for dir in dirs:
117+ shutil.rmtree(os.path.join(top,dir))
118+
119+ """ alloc_dir returns a directory name for next backup
120+ """
121+ def alloc_dir(self, topdir, dir_pattern='backup'):
122+ dir_pattern_obj = '%s(\\d+)' %dir_pattern
123+ dir_pattern_obj = re.compile(dir_pattern_obj)
124+ dir_list = [list(dirs)[1] for dirs in os.walk(topdir) if list(dirs)[0] == topdir][0]
125+ dir_list = [b for b in dir_list if dir_pattern_obj.match(b)]
126+ if not dir_list:
127+ dir_suffix = '0'
128+ else:
129+ dir_suffix = str(max([int(re.split(dir_pattern,b)[1]) for b in dir_list])+1)
130+
131+ return '%s%s' %(dir_pattern,dir_suffix)
132+
133+ """ execute_cmd executes backup program to create backup object
134+ """
135+ def execute_cmd(self, cmd, exec_path, outfile_path):
136+ outfile = open(outfile_path,'w')
137+ cmd_subproc = subprocess.Popen( cmd
138+ , cwd = exec_path
139+ , shell=True
140+ , stdout = outfile
141+ , stderr = subprocess.STDOUT
142+ )
143+ cmd_subproc.wait()
144+ retcode = cmd_subproc.returncode
145+ outfile.close
146+ in_file = open(outfile_path,'r')
147+ output = ''.join(in_file.readlines())
148+ return retcode,output
149+
150+ """ method creates full backup from server object
151+ new_backup_object = backup_full(server_object)
152+ """
153+ def backup_full(self,server_object):
154+ self.datadir = server_object.datadir
155+ self.ib_bin = self.ib_bin_path
156+ self.xb_bin = self.xb_bin_path
157+ self.b_root_dir = self.backup_dir
158+ allocated_dir = self.alloc_dir(self.b_root_dir)
159+ self.b_path = os.path.join(self.b_root_dir, allocated_dir)
160+ temp_log = os.path.join(self.b_root_dir, '%s.log' %allocated_dir)
161+ self.xb_log = os.path.join(self.b_path, '%s.log' %allocated_dir)
162+ cmd = [self.ib_bin
163+ , "--defaults-file=%s" %server_object.cnf_file
164+ , "--no-timestamp"
165+ , "--user=root"
166+ , "--port=%d" %server_object.master_port
167+ , "--host=127.0.0.1"
168+ , "--ibbackup=%s" %self.xb_bin
169+ , self.b_path
170+ ]
171+ cmd = " ".join(cmd)
172+ self.retcode, self.output = self.execute_cmd(cmd, self.b_root_dir, temp_log)
173+ shutil.move(temp_log, self.xb_log)
174+ self.status = 'full-backup'
175+ return self
176+
177+ """ Create incremental backup from full backup and server objects
178+ Usage: incremental_backup_object = backup_inc(server_obj,previous_backup_object)
179+ """
180+ def backup_inc(self,server_object,backup_object):
181+ self.ib_bin = self.ib_bin_path
182+ self.xb_bin = self.xb_bin_path
183+ self.b_root_dir = self.backup_dir
184+ allocated_dir = self.alloc_dir(self.b_root_dir)
185+ self.b_path = os.path.join(self.b_root_dir, allocated_dir)
186+ temp_log = os.path.join(self.b_root_dir, '%s.log' %allocated_dir)
187+ self.xb_log = os.path.join(self.b_path, '%s.log' %allocated_dir)
188+ cmd = [self.ib_bin
189+ , '--defualts-file=%s' %server_object.cnf_file
190+ , '--no-timestamp'
191+ , '--user=root'
192+ , '--port=%d' %server_object.master_port
193+ , '--host=127.0.0.1'
194+ , '--ibbackup=%s' %self.xb_bin
195+ , '--incremental'
196+ , '--incremental-basedir=%s' %backup_object.b_path
197+ , self.b_path
198+ ]
199+ cmd = " ".join(cmd)
200+ self.retcode, self.output = self.execute_cmd(cmd, self.b_root_dir, temp_log)
201+ shutil.move(temp_log, self.xb_log)
202+ self.status = 'inc-backup'
203+ return self
204+
205+ """ Method to prepare a backup
206+ Usage prepare(unprepared_backup_object)
207+ """
208+ def prepare(self,backup_object,rollback=True):
209+ if rollback:
210+ rollback = ''
211+ else:
212+ rollback = '--redo-only'
213+
214+ cmd = [backup_object.ib_bin
215+ , "--apply-log"
216+ , " %s" %rollback
217+ , "--ibbackup=%s" %backup_object.xb_bin
218+ , backup_object.b_path
219+ ]
220+ cmd = " ".join(cmd)
221+ temp_log = os.path.join(backup_object.b_path, self.alloc_dir(backup_object.b_path,dir_pattern='prepare'))
222+ retcode, output = self.execute_cmd(cmd, backup_object.b_path, temp_log)
223+ if rollback and retcode==0:
224+ backup_object.status = 'prepared-redo-only'
225+ elif not rollback and retcode==0:
226+ backup_object.status = 'prepared'
227+ else:
228+ backup_object.status = 'prepare-failed'
229+
230+ return retcode, output
231+
232+ """ Method restores server_object's data from backup
233+ If backup is not ready for restore an appropriate error
234+ message will be returned.
235+ Usage: restore(prepared_backup_object,server_object)
236+ """
237+ def restore(self,backup_object,server_object):
238+ if backup_object.status=='prepared-redo-only' or backup_object.status=='prepared':
239+ pass
240+ elif backup_object.status=='full-backup':
241+ retcode = 1
242+ output = 'Backup has to be prepared before restore!\n'
243+ return retcode, output
244+ elif backup_object.status=='inc-backup':
245+ retcode = 1
246+ output = 'You have to apply incrementals to full backup to restore it!\n'
247+ elif backup_object.status=='prepare-failed':
248+ retcode = 1
249+ output = 'Backup failed to prepare, see prepare log for details.\n'
250+ else:
251+ retcode = 1
252+ output = 'CRITICAL ERROR: Unknown backup status!\n'
253+
254+ cmd = [backup_object.ib_bin
255+ , "--defaults-file=%s" %server_object.cnf_file
256+ , "--copy-back"
257+ , "--ibbackup=%s" %backup_object.xb_bin
258+ , backup_object.b_path
259+ ]
260+ cmd = " ".join(cmd)
261+ server_object.stop()
262+ self.clean_dir(server_object.datadir)
263+ temp_log = os.path.join(backup_object.b_path, self.alloc_dir(backup_object.b_path,dir_pattern='restore'))
264+ retcode, output = self.execute_cmd(cmd, backup_object.b_path, temp_log)
265+ server_object.start()
266+ return retcode, output
267+
268
269=== added directory 'percona_tests/xbm'
270=== added file 'percona_tests/xbm/xb_incremental_test.py'
271--- percona_tests/xbm/xb_incremental_test.py 1970-01-01 00:00:00 +0000
272+++ percona_tests/xbm/xb_incremental_test.py 2012-06-11 20:30:31 +0000
273@@ -0,0 +1,62 @@
274+#! /usr/bin/env python
275+# -*- mode: python; indent-tabs-mode: nil; -*-
276+# vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
277+#
278+# Copyright (C) 2011 Patrick Crews
279+#
280+#
281+# This program is free software; you can redistribute it and/or modify
282+# it under the terms of the GNU General Public License as published by
283+# the Free Software Foundation; either version 2 of the License, or
284+# (at your option) any later version.
285+#
286+# This program is distributed in the hope that it will be useful,
287+# but WITHOUT ANY WARRANTY; without even the implied warranty of
288+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
289+# GNU General Public License for more details.
290+#
291+# You should have received a copy of the GNU General Public License
292+# along with this program; if not, write to the Free Software
293+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
294+
295+import os
296+import shutil
297+
298+from lib.util.mysqlBaseTestCase import mysqlBaseTestCase
299+
300+server_requirements = [['--innodb-file-per-table'],['--innodb-file-per-table']]
301+servers = []
302+server_manager = None
303+test_executor = None
304+# we explicitly use the --no-timestamp option
305+# here. We will be using a generic / vanilla backup dir
306+backup_path = None
307+
308+class basicTest(mysqlBaseTestCase):
309+
310+
311+ def test_ib_incremental(self):
312+ master_server = servers[0]
313+ slave_server = servers[1]
314+ self.servers = servers
315+ logging = test_executor.logging
316+ xb_manager = test_executor.xtrabackup_manager
317+ # backup0 = xb_manager.backup_full(master_server)
318+ # xb_manager.prepare(backup0)
319+ # xb_manager.restore(backup0,master_server)
320+ randgen_manager = test_executor.randgen_manager
321+ randgen_manager.create_test_bed(master_server, optional_parameters)
322+ backup0 = xb_manager.backup_full(master_server)
323+ load0 = randgen_manager.create_load(master_server, runtime_options)
324+ load0.start()
325+ backup1 = xb_manager.backup_inc(master_server, backup0)
326+ xb_manager.prepare(backup0,rollback=False)
327+ xb_manager.prepare_inc(backup0,backup1)
328+ xb_manager.restore(slave_server)
329+ slave_server.start_repl(master_server)
330+ load0.wait()
331+ compare_checksums(master_server,slave_server,db1="test",db2="test2")
332+
333+
334+
335+

Subscribers

People subscribed via source and target branches