Merge ~afreiberger/charm-glance-sync:blacken-20.08 into charm-glance-sync:master
- Git
- lp:~afreiberger/charm-glance-sync
- blacken-20.08
- Merge into master
Proposed by
Drew Freiberger
Status: | Merged |
---|---|
Merged at revision: | b37ab82942fd8266a0998a1ce591aef755fc70de |
Proposed branch: | ~afreiberger/charm-glance-sync:blacken-20.08 |
Merge into: | charm-glance-sync:master |
Prerequisite: | ~afreiberger/charm-glance-sync:makefile-20.08 |
Diff against target: |
2500 lines (+756/-712) 10 files modified
src/files/check_stale_lockfile_master.py (+18/-13) src/files/check_stale_lockfile_slave.py (+18/-13) src/files/db_purge_deleted_master/db_purge_deleted_glance_images.py (+20/-19) src/files/db_purge_deleted_slave/db_purge_deleted_glance_images.py (+11/-11) src/files/glance_sync_master.py (+94/-81) src/files/glance_sync_slave.py (+280/-231) src/reactive/glance_sync.py (+282/-314) src/tests/functional/tests/test_glance_sync.py (+29/-26) src/tests/unit/__init__.py (+2/-1) src/tox.ini (+2/-3) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Xav Paice (community) | Approve | ||
Review via email: mp+388630@code.launchpad.net |
Commit message
Blackened repository to 88 lines and fixed up lint
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/src/files/check_stale_lockfile_master.py b/src/files/check_stale_lockfile_master.py | |||
2 | index bd6aa3d..341441d 100644 | |||
3 | --- a/src/files/check_stale_lockfile_master.py | |||
4 | +++ b/src/files/check_stale_lockfile_master.py | |||
5 | @@ -5,20 +5,25 @@ import os.path | |||
6 | 5 | import sys | 5 | import sys |
7 | 6 | import time | 6 | import time |
8 | 7 | 7 | ||
11 | 8 | ('Check the status of lock file to be sure it is not stale, ' | 8 | ( |
12 | 9 | 'warn at 7200 seconds crit at 14400 seconds') | 9 | "Check the status of lock file to be sure it is not stale, " |
13 | 10 | "warn at 7200 seconds crit at 14400 seconds" | ||
14 | 11 | ) | ||
15 | 10 | 12 | ||
16 | 11 | parser = optparse.OptionParser() | 13 | parser = optparse.OptionParser() |
17 | 12 | parser.add_option( | 14 | parser.add_option( |
19 | 13 | '-w', action='store', default='7200', help='seconds to warn', type='int') | 15 | "-w", action="store", default="7200", help="seconds to warn", type="int" |
20 | 16 | ) | ||
21 | 14 | parser.add_option( | 17 | parser.add_option( |
23 | 15 | '-c', action='store', default='14400', help='seconds to crit', type='int') | 18 | "-c", action="store", default="14400", help="seconds to crit", type="int" |
24 | 19 | ) | ||
25 | 16 | parser.add_option( | 20 | parser.add_option( |
31 | 17 | '-f', | 21 | "-f", |
32 | 18 | action='store', | 22 | action="store", |
33 | 19 | default='/tmp/glance_sync_master.lock', | 23 | default="/tmp/glance_sync_master.lock", |
34 | 20 | help='file to check', | 24 | help="file to check", |
35 | 21 | type='string') | 25 | type="string", |
36 | 26 | ) | ||
37 | 22 | 27 | ||
38 | 23 | options, args = parser.parse_args() | 28 | options, args = parser.parse_args() |
39 | 24 | 29 | ||
40 | @@ -29,7 +34,7 @@ nagCrit = 2 | |||
41 | 29 | try: | 34 | try: |
42 | 30 | statInfo = os.stat(options.f) | 35 | statInfo = os.stat(options.f) |
43 | 31 | except OSError: | 36 | except OSError: |
45 | 32 | print('OK: lockfile {} not present'.format(options.f)) | 37 | print("OK: lockfile {} not present".format(options.f)) |
46 | 33 | sys.exit(nagOk) | 38 | sys.exit(nagOk) |
47 | 34 | 39 | ||
48 | 35 | now = int(time.time()) | 40 | now = int(time.time()) |
49 | @@ -37,11 +42,11 @@ statInfoSlice = statInfo[8] | |||
50 | 37 | timeDiff = now - statInfoSlice | 42 | timeDiff = now - statInfoSlice |
51 | 38 | 43 | ||
52 | 39 | if timeDiff > options.c: | 44 | if timeDiff > options.c: |
54 | 40 | print('CRIT: lock file is older than {} seconds'.format(options.c)) | 45 | print("CRIT: lock file is older than {} seconds".format(options.c)) |
55 | 41 | sys.exit(nagCrit) | 46 | sys.exit(nagCrit) |
56 | 42 | elif timeDiff > options.w: | 47 | elif timeDiff > options.w: |
58 | 43 | print('WARN: lock file is older than {} seconds'.format(options.w)) | 48 | print("WARN: lock file is older than {} seconds".format(options.w)) |
59 | 44 | sys.exit(nagWarn) | 49 | sys.exit(nagWarn) |
60 | 45 | else: | 50 | else: |
62 | 46 | print('OK: lock file is under 3 hours') | 51 | print("OK: lock file is under 3 hours") |
63 | 47 | sys.exit(nagOk) | 52 | sys.exit(nagOk) |
64 | diff --git a/src/files/check_stale_lockfile_slave.py b/src/files/check_stale_lockfile_slave.py | |||
65 | index 1be8b9b..08412be 100644 | |||
66 | --- a/src/files/check_stale_lockfile_slave.py | |||
67 | +++ b/src/files/check_stale_lockfile_slave.py | |||
68 | @@ -5,20 +5,25 @@ import os.path | |||
69 | 5 | import sys | 5 | import sys |
70 | 6 | import time | 6 | import time |
71 | 7 | 7 | ||
74 | 8 | ('Check the status of lock file to be sure it is not stale, ' | 8 | ( |
75 | 9 | 'warn at 7200 seconds crit at 14400 seconds') | 9 | "Check the status of lock file to be sure it is not stale, " |
76 | 10 | "warn at 7200 seconds crit at 14400 seconds" | ||
77 | 11 | ) | ||
78 | 10 | 12 | ||
79 | 11 | parser = optparse.OptionParser() | 13 | parser = optparse.OptionParser() |
80 | 12 | parser.add_option( | 14 | parser.add_option( |
82 | 13 | '-w', action='store', default='7200', help='seconds to warn', type='int') | 15 | "-w", action="store", default="7200", help="seconds to warn", type="int" |
83 | 16 | ) | ||
84 | 14 | parser.add_option( | 17 | parser.add_option( |
86 | 15 | '-c', action='store', default='14400', help='seconds to crit', type='int') | 18 | "-c", action="store", default="14400", help="seconds to crit", type="int" |
87 | 19 | ) | ||
88 | 16 | parser.add_option( | 20 | parser.add_option( |
94 | 17 | '-f', | 21 | "-f", |
95 | 18 | action='store', | 22 | action="store", |
96 | 19 | default='/tmp/glance_sync_slave.lock', | 23 | default="/tmp/glance_sync_slave.lock", |
97 | 20 | help='file to check', | 24 | help="file to check", |
98 | 21 | type='string') | 25 | type="string", |
99 | 26 | ) | ||
100 | 22 | 27 | ||
101 | 23 | options, args = parser.parse_args() | 28 | options, args = parser.parse_args() |
102 | 24 | 29 | ||
103 | @@ -29,7 +34,7 @@ nagCrit = 2 | |||
104 | 29 | try: | 34 | try: |
105 | 30 | statInfo = os.stat(options.f) | 35 | statInfo = os.stat(options.f) |
106 | 31 | except OSError: | 36 | except OSError: |
108 | 32 | print('OK: lockfile {} not present'.format(options.f)) | 37 | print("OK: lockfile {} not present".format(options.f)) |
109 | 33 | sys.exit(nagOk) | 38 | sys.exit(nagOk) |
110 | 34 | 39 | ||
111 | 35 | now = int(time.time()) | 40 | now = int(time.time()) |
112 | @@ -37,11 +42,11 @@ statInfoSlice = statInfo[8] | |||
113 | 37 | timeDiff = now - statInfoSlice | 42 | timeDiff = now - statInfoSlice |
114 | 38 | 43 | ||
115 | 39 | if timeDiff > options.c: | 44 | if timeDiff > options.c: |
117 | 40 | print('CRIT: lock file is older than {} seconds'.format(options.c)) | 45 | print("CRIT: lock file is older than {} seconds".format(options.c)) |
118 | 41 | sys.exit(nagCrit) | 46 | sys.exit(nagCrit) |
119 | 42 | elif timeDiff > options.w: | 47 | elif timeDiff > options.w: |
121 | 43 | print('WARN: lock file is older than {} seconds'.format(options.w)) | 48 | print("WARN: lock file is older than {} seconds".format(options.w)) |
122 | 44 | sys.exit(nagWarn) | 49 | sys.exit(nagWarn) |
123 | 45 | else: | 50 | else: |
125 | 46 | print('OK: lock file is under 3 hours') | 51 | print("OK: lock file is under 3 hours") |
126 | 47 | sys.exit(nagOk) | 52 | sys.exit(nagOk) |
127 | diff --git a/src/files/db_purge_deleted_master/db_purge_deleted_glance_images.py b/src/files/db_purge_deleted_master/db_purge_deleted_glance_images.py | |||
128 | index 144383f..bbc97c6 100644 | |||
129 | --- a/src/files/db_purge_deleted_master/db_purge_deleted_glance_images.py | |||
130 | +++ b/src/files/db_purge_deleted_master/db_purge_deleted_glance_images.py | |||
131 | @@ -30,27 +30,28 @@ config = {} | |||
132 | 30 | 30 | ||
133 | 31 | # pull connection information from glance-api.conf | 31 | # pull connection information from glance-api.conf |
134 | 32 | try: | 32 | try: |
136 | 33 | with open('/etc/glance/glance-api.conf', 'r') as conf_file: | 33 | with open("/etc/glance/glance-api.conf", "r") as conf_file: |
137 | 34 | for line in conf_file.readlines(): | 34 | for line in conf_file.readlines(): |
139 | 35 | if line.startswith('connection ='): | 35 | if line.startswith("connection ="): |
140 | 36 | # connection = mysql://glance:<password>@<host>/glance | 36 | # connection = mysql://glance:<password>@<host>/glance |
146 | 37 | connection = urlparse.urlparse(line.split('=')[1].strip()) | 37 | connection = urlparse.urlparse(line.split("=")[1].strip()) |
147 | 38 | config['host'] = connection.hostname | 38 | config["host"] = connection.hostname |
148 | 39 | config['user'] = connection.username | 39 | config["user"] = connection.username |
149 | 40 | config['password'] = connection.password | 40 | config["password"] = connection.password |
150 | 41 | config['database'] = connection.path[1:].strip() | 41 | config["database"] = connection.path[1:].strip() |
151 | 42 | break | 42 | break |
152 | 43 | except IOError as e: | 43 | except IOError as e: |
153 | 44 | sys.exit(e) | 44 | sys.exit(e) |
154 | 45 | 45 | ||
155 | 46 | try: | 46 | try: |
156 | 47 | connection = MySQLdb.connect( | 47 | connection = MySQLdb.connect( |
161 | 48 | host=config['host'], | 48 | host=config["host"], |
162 | 49 | user=config['user'], | 49 | user=config["user"], |
163 | 50 | passwd=config['password'], | 50 | passwd=config["password"], |
164 | 51 | db=config['database']) | 51 | db=config["database"], |
165 | 52 | ) | ||
166 | 52 | except Exception as e: | 53 | except Exception as e: |
168 | 53 | print('ERROR: unable to connect to mysql database') | 54 | print("ERROR: unable to connect to mysql database") |
169 | 54 | sys.exit(e) | 55 | sys.exit(e) |
170 | 55 | 56 | ||
171 | 56 | cursor = connection.cursor() | 57 | cursor = connection.cursor() |
172 | @@ -60,14 +61,14 @@ cursor.execute("SELECT id FROM glance.images WHERE status='deleted';") | |||
173 | 60 | image_ids = cursor.fetchall() | 61 | image_ids = cursor.fetchall() |
174 | 61 | 62 | ||
175 | 62 | for image_id in image_ids: | 63 | for image_id in image_ids: |
178 | 63 | print('purging {}'.format(image_id[0])) | 64 | print("purging {}".format(image_id[0])) |
179 | 64 | args = (image_id[0]) | 65 | args = image_id[0] |
180 | 65 | commands = [ | 66 | commands = [ |
186 | 66 | 'DELETE FROM glance.image_properties WHERE image_id=%s;', | 67 | "DELETE FROM glance.image_properties WHERE image_id=%s;", |
187 | 67 | 'DELETE FROM glance.image_members WHERE image_id=%s;', | 68 | "DELETE FROM glance.image_members WHERE image_id=%s;", |
188 | 68 | 'DELETE FROM glance.image_tags WHERE image_id=%s;', | 69 | "DELETE FROM glance.image_tags WHERE image_id=%s;", |
189 | 69 | 'DELETE FROM glance.image_locations WHERE image_id=%s;', | 70 | "DELETE FROM glance.image_locations WHERE image_id=%s;", |
190 | 70 | 'DELETE FROM glance.images WHERE id=%s;' | 71 | "DELETE FROM glance.images WHERE id=%s;", |
191 | 71 | ] | 72 | ] |
192 | 72 | 73 | ||
193 | 73 | for command in commands: | 74 | for command in commands: |
194 | diff --git a/src/files/db_purge_deleted_slave/db_purge_deleted_glance_images.py b/src/files/db_purge_deleted_slave/db_purge_deleted_glance_images.py | |||
195 | index c0acc47..74f57be 100644 | |||
196 | --- a/src/files/db_purge_deleted_slave/db_purge_deleted_glance_images.py | |||
197 | +++ b/src/files/db_purge_deleted_slave/db_purge_deleted_glance_images.py | |||
198 | @@ -27,10 +27,11 @@ from contextlib import closing | |||
199 | 27 | import mysql.connector as mysql | 27 | import mysql.connector as mysql |
200 | 28 | 28 | ||
201 | 29 | con = mysql.connect( | 29 | con = mysql.connect( |
206 | 30 | host=os.environ['OS_MYSQL_HOST'], | 30 | host=os.environ["OS_MYSQL_HOST"], |
207 | 31 | user=os.environ['OS_MYSQL_USER'], | 31 | user=os.environ["OS_MYSQL_USER"], |
208 | 32 | password=os.environ['OS_MYSQL_PASS'], | 32 | password=os.environ["OS_MYSQL_PASS"], |
209 | 33 | database=os.environ['OS_MYSQL_DB']) | 33 | database=os.environ["OS_MYSQL_DB"], |
210 | 34 | ) | ||
211 | 34 | 35 | ||
212 | 35 | # delete from images where status = 'deleted' | 36 | # delete from images where status = 'deleted' |
213 | 36 | with closing(con.cursor()) as cur: | 37 | with closing(con.cursor()) as cur: |
214 | @@ -38,15 +39,14 @@ with closing(con.cursor()) as cur: | |||
215 | 38 | cur.execute(sql) | 39 | cur.execute(sql) |
216 | 39 | image_ids = cur.fetchall() | 40 | image_ids = cur.fetchall() |
217 | 40 | for image_id in image_ids: | 41 | for image_id in image_ids: |
220 | 41 | print('purging {}'.format(image_id[0])) | 42 | print("purging {}".format(image_id[0])) |
221 | 42 | args = (image_id[0]) | 43 | args = image_id[0] |
222 | 43 | commands = [ | 44 | commands = [ |
226 | 44 | "DELETE FROM glance.image_properties WHERE image_id='{}';".format( | 45 | "DELETE FROM glance.image_properties WHERE image_id='{}';".format(args), |
227 | 45 | args), "DELETE FROM glance.image_members WHERE image_id='{}';". | 46 | "DELETE FROM glance.image_members WHERE image_id='{}';".format(args), |
225 | 46 | format(args), | ||
228 | 47 | "DELETE FROM glance.image_tags WHERE image_id='{}';".format(args), | 47 | "DELETE FROM glance.image_tags WHERE image_id='{}';".format(args), |
231 | 48 | "DELETE FROM glance.image_locations WHERE image_id='{}';".format( | 48 | "DELETE FROM glance.image_locations WHERE image_id='{}';".format(args), |
232 | 49 | args), "DELETE FROM glance.images WHERE id='{}';".format(args) | 49 | "DELETE FROM glance.images WHERE id='{}';".format(args), |
233 | 50 | ] | 50 | ] |
234 | 51 | 51 | ||
235 | 52 | for command in commands: | 52 | for command in commands: |
236 | diff --git a/src/files/glance_sync_master.py b/src/files/glance_sync_master.py | |||
237 | index 57139b3..5cd53b5 100755 | |||
238 | --- a/src/files/glance_sync_master.py | |||
239 | +++ b/src/files/glance_sync_master.py | |||
240 | @@ -20,31 +20,33 @@ from keystoneclient.v3 import client as keystone_v3_client | |||
241 | 20 | 20 | ||
242 | 21 | def get_keystone_client(): | 21 | def get_keystone_client(): |
243 | 22 | # We know that we set OS_AUTH_VERSION, so use it (and cast to int) | 22 | # We know that we set OS_AUTH_VERSION, so use it (and cast to int) |
245 | 23 | if int(os.environ['OS_AUTH_VERSION']) == 3: | 23 | if int(os.environ["OS_AUTH_VERSION"]) == 3: |
246 | 24 | ksc = keystone_v3_client.Client( | 24 | ksc = keystone_v3_client.Client( |
253 | 25 | auth_url=os.environ['OS_AUTH_URL'], | 25 | auth_url=os.environ["OS_AUTH_URL"], |
254 | 26 | username=os.environ['OS_USERNAME'], | 26 | username=os.environ["OS_USERNAME"], |
255 | 27 | password=os.environ['OS_PASSWORD'], | 27 | password=os.environ["OS_PASSWORD"], |
256 | 28 | user_domain_name=os.environ['OS_USER_DOMAIN_NAME'], | 28 | user_domain_name=os.environ["OS_USER_DOMAIN_NAME"], |
257 | 29 | project_domain_name=os.environ['OS_PROJECT_DOMAIN_NAME'], | 29 | project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"], |
258 | 30 | project_name=os.environ['OS_PROJECT_NAME']) | 30 | project_name=os.environ["OS_PROJECT_NAME"], |
259 | 31 | ) | ||
260 | 31 | else: | 32 | else: |
261 | 32 | ksc = keystone_v2_client.Client( | 33 | ksc = keystone_v2_client.Client( |
266 | 33 | username=os.environ['OS_USERNAME'], | 34 | username=os.environ["OS_USERNAME"], |
267 | 34 | password=os.environ['OS_PASSWORD'], | 35 | password=os.environ["OS_PASSWORD"], |
268 | 35 | tenant_name=os.environ['OS_TENANT_NAME'], | 36 | tenant_name=os.environ["OS_TENANT_NAME"], |
269 | 36 | auth_url=os.environ['OS_AUTH_URL']) | 37 | auth_url=os.environ["OS_AUTH_URL"], |
270 | 38 | ) | ||
271 | 37 | return ksc | 39 | return ksc |
272 | 38 | 40 | ||
273 | 39 | 41 | ||
274 | 40 | def get_glance_client(ksc): | 42 | def get_glance_client(ksc): |
275 | 41 | # create a glance client, using the provided keystone client for details | 43 | # create a glance client, using the provided keystone client for details |
276 | 42 | token = ksc.auth_token | 44 | token = ksc.auth_token |
279 | 43 | service = ksc.services.find(name='glance') | 45 | service = ksc.services.find(name="glance") |
280 | 44 | endpoint = ksc.endpoints.find(service_id=service.id, interface='internal') | 46 | endpoint = ksc.endpoints.find(service_id=service.id, interface="internal") |
281 | 45 | glance_url = endpoint.url | 47 | glance_url = endpoint.url |
282 | 46 | 48 | ||
284 | 47 | return GlanceClient('2', endpoint=glance_url, token=token) | 49 | return GlanceClient("2", endpoint=glance_url, token=token) |
285 | 48 | 50 | ||
286 | 49 | 51 | ||
287 | 50 | class BootStackMetadataError(Exception): | 52 | class BootStackMetadataError(Exception): |
288 | @@ -54,14 +56,15 @@ class BootStackMetadataError(Exception): | |||
289 | 54 | helps map different tenants between regions (with | 56 | helps map different tenants between regions (with |
290 | 55 | different tenant-id, but same/similar tenant-name | 57 | different tenant-id, but same/similar tenant-name |
291 | 56 | """ | 58 | """ |
292 | 59 | |||
293 | 57 | pass | 60 | pass |
294 | 58 | 61 | ||
295 | 59 | 62 | ||
296 | 60 | class ImageSyncMaster: | 63 | class ImageSyncMaster: |
298 | 61 | def __init__(self, data_dir='/srv/glance_master_sync/data'): | 64 | def __init__(self, data_dir="/srv/glance_master_sync/data"): |
299 | 62 | self.tenants = {} | 65 | self.tenants = {} |
300 | 63 | self.DATA_DIR = data_dir | 66 | self.DATA_DIR = data_dir |
302 | 64 | tmp_dir = '/tmp/glance_master_sync' | 67 | tmp_dir = "/tmp/glance_master_sync" |
303 | 65 | if not os.path.isdir(tmp_dir): | 68 | if not os.path.isdir(tmp_dir): |
304 | 66 | os.makedirs(tmp_dir) | 69 | os.makedirs(tmp_dir) |
305 | 67 | 70 | ||
306 | @@ -71,76 +74,81 @@ class ImageSyncMaster: | |||
307 | 71 | def glance_connect(self): | 74 | def glance_connect(self): |
308 | 72 | try: | 75 | try: |
309 | 73 | 76 | ||
311 | 74 | self.log('connecting to Keystone') | 77 | self.log("connecting to Keystone") |
312 | 75 | self.keystone = get_keystone_client() | 78 | self.keystone = get_keystone_client() |
313 | 76 | except Exception as e: | 79 | except Exception as e: |
315 | 77 | self.log('EXCEPTION: {0}'.format(e)) | 80 | self.log("EXCEPTION: {0}".format(e)) |
316 | 78 | sys.exit(2) | 81 | sys.exit(2) |
317 | 79 | if not self.tenants: | 82 | if not self.tenants: |
318 | 80 | # In the call to keystone we know that we get a list response. | 83 | # In the call to keystone we know that we get a list response. |
319 | 81 | # 1st element is the response code, 2nd is the data in dicts form | 84 | # 1st element is the response code, 2nd is the data in dicts form |
320 | 82 | self.tenants = dict( | 85 | self.tenants = dict( |
324 | 83 | [(tenant['id'], tenant['name']) | 86 | [ |
325 | 84 | for tenant in self.keystone.get('/projects')[1]['projects'] | 87 | (tenant["id"], tenant["name"]) |
326 | 85 | if tenant['enabled']]) | 88 | for tenant in self.keystone.get("/projects")[1]["projects"] |
327 | 89 | if tenant["enabled"] | ||
328 | 90 | ] | ||
329 | 91 | ) | ||
330 | 86 | self.glance = get_glance_client(self.keystone) | 92 | self.glance = get_glance_client(self.keystone) |
331 | 87 | return self.glance | 93 | return self.glance |
332 | 88 | 94 | ||
333 | 89 | def timestamp_now(self): | 95 | def timestamp_now(self): |
335 | 90 | return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') | 96 | return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
336 | 91 | 97 | ||
337 | 92 | def log(self, msg): | 98 | def log(self, msg): |
339 | 93 | print('{0} {1}'.format(self.timestamp_now(), msg)) | 99 | print("{0} {1}".format(self.timestamp_now(), msg)) |
340 | 94 | 100 | ||
341 | 95 | def delete_files(self, existing_images_ids): | 101 | def delete_files(self, existing_images_ids): |
342 | 96 | if not existing_images_ids: | 102 | if not existing_images_ids: |
345 | 97 | self.log('WARNING: precautionary halt. ' | 103 | self.log("WARNING: precautionary halt. No glance images found. noop.") |
344 | 98 | 'No glance images found. noop.') | ||
346 | 99 | return | 104 | return |
347 | 100 | 105 | ||
348 | 101 | for dirpath, dirnames, filenames in os.walk(self.DATA_DIR): | 106 | for dirpath, dirnames, filenames in os.walk(self.DATA_DIR): |
352 | 102 | if dirpath != self.DATA_DIR and \ | 107 | if dirpath != self.DATA_DIR and len(dirnames) == 0 and len(filenames) == 0: |
350 | 103 | len(dirnames) == 0 and \ | ||
351 | 104 | len(filenames) == 0: | ||
353 | 105 | os.rmdir(dirpath) | 108 | os.rmdir(dirpath) |
354 | 106 | continue | 109 | continue |
355 | 107 | 110 | ||
356 | 108 | for filename in filenames: | 111 | for filename in filenames: |
357 | 109 | full_path = os.path.join(dirpath, filename) | 112 | full_path = os.path.join(dirpath, filename) |
361 | 110 | if filename.endswith('.json.tmp'): | 113 | if filename.endswith(".json.tmp"): |
362 | 111 | self.log('WARNING: temporary file skipped. Please check ' | 114 | self.log( |
363 | 112 | '{0}'.format(full_path)) | 115 | "WARNING: temporary file skipped. Please check " |
364 | 116 | "{0}".format(full_path) | ||
365 | 117 | ) | ||
366 | 113 | continue | 118 | continue |
369 | 114 | elif filename.endswith('.json') and \ | 119 | elif ( |
370 | 115 | filename[:-5] in existing_images_ids: | 120 | filename.endswith(".json") and filename[:-5] in existing_images_ids |
371 | 121 | ): | ||
372 | 116 | continue | 122 | continue |
373 | 117 | else: | 123 | else: |
376 | 118 | self.log('INFO: image not found in glance - deleting ' | 124 | self.log( |
377 | 119 | '{0}'.format(full_path)) | 125 | "INFO: image not found in glance - deleting " |
378 | 126 | "{0}".format(full_path) | ||
379 | 127 | ) | ||
380 | 120 | os.remove(full_path) | 128 | os.remove(full_path) |
381 | 121 | 129 | ||
382 | 122 | def create_lock(self, lockfile): | 130 | def create_lock(self, lockfile): |
383 | 123 | try: | 131 | try: |
384 | 124 | os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR) | 132 | os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR) |
385 | 125 | except OSError: | 133 | except OSError: |
387 | 126 | self.log('ERROR: could not create lockfile {}'.format(lockfile)) | 134 | self.log("ERROR: could not create lockfile {}".format(lockfile)) |
388 | 127 | 135 | ||
390 | 128 | def file_locked(self, lockfile='/tmp/glance_sync_master.lock'): | 136 | def file_locked(self, lockfile="/tmp/glance_sync_master.lock"): |
391 | 129 | if os.path.isfile(lockfile): | 137 | if os.path.isfile(lockfile): |
392 | 130 | return True | 138 | return True |
393 | 131 | else: | 139 | else: |
394 | 132 | return False | 140 | return False |
395 | 133 | 141 | ||
397 | 134 | def release_lock(self, lockfile='/tmp/glance_sync_master.lock'): | 142 | def release_lock(self, lockfile="/tmp/glance_sync_master.lock"): |
398 | 135 | if os.path.isfile(lockfile): | 143 | if os.path.isfile(lockfile): |
399 | 136 | try: | 144 | try: |
400 | 137 | os.remove(lockfile) | 145 | os.remove(lockfile) |
401 | 138 | except OSError as e: | 146 | except OSError as e: |
402 | 139 | self.log(e) | 147 | self.log(e) |
403 | 140 | 148 | ||
405 | 141 | def set_filelock(self, lockfile='/tmp/glance_sync_master.lock'): | 149 | def set_filelock(self, lockfile="/tmp/glance_sync_master.lock"): |
406 | 142 | if self.file_locked(lockfile): | 150 | if self.file_locked(lockfile): |
408 | 143 | self.log('WARNING: sync already in progress, exiting') | 151 | self.log("WARNING: sync already in progress, exiting") |
409 | 144 | sys.exit(2) | 152 | sys.exit(2) |
410 | 145 | 153 | ||
411 | 146 | self.create_lock(lockfile) | 154 | self.create_lock(lockfile) |
412 | @@ -152,21 +160,20 @@ class ImageSyncMaster: | |||
413 | 152 | len(image.id) < 2 : DATA_DIR/<image-id>.json | 160 | len(image.id) < 2 : DATA_DIR/<image-id>.json |
414 | 153 | len(image.id) >= 2: DATA_DIR/XX/XXZZZ.json | 161 | len(image.id) >= 2: DATA_DIR/XX/XXZZZ.json |
415 | 154 | """ | 162 | """ |
417 | 155 | self.log('getting image from database.') | 163 | self.log("getting image from database.") |
418 | 156 | existing_images = self.get_community_images_from_database() | 164 | existing_images = self.get_community_images_from_database() |
420 | 157 | self.log('Extending with data from api.') | 165 | self.log("Extending with data from api.") |
421 | 158 | for image in self.glance.images.list(): | 166 | for image in self.glance.images.list(): |
422 | 159 | existing_images.add(image.id) | 167 | existing_images.add(image.id) |
423 | 160 | 168 | ||
424 | 161 | for image_helper in existing_images: | 169 | for image_helper in existing_images: |
425 | 162 | image = self.glance.images.get(image_helper) | 170 | image = self.glance.images.get(image_helper) |
428 | 163 | if len(image['id']) < 2: | 171 | if len(image["id"]) < 2: |
429 | 164 | basename = '{0}.json'.format(image['id']) | 172 | basename = "{0}.json".format(image["id"]) |
430 | 165 | else: | 173 | else: |
433 | 166 | basename = '{0}/{1}.json'.format( | 174 | basename = "{0}/{1}.json".format(str(image["id"])[:2], image["id"]) |
432 | 167 | str(image['id'])[:2], image['id']) | ||
434 | 168 | filename = os.path.join(self.DATA_DIR, basename) | 175 | filename = os.path.join(self.DATA_DIR, basename) |
436 | 169 | if not self.is_latest_metadata(filename, image['updated_at']): | 176 | if not self.is_latest_metadata(filename, image["updated_at"]): |
437 | 170 | self.update_metadata(filename, image) | 177 | self.update_metadata(filename, image) |
438 | 171 | 178 | ||
439 | 172 | return existing_images | 179 | return existing_images |
440 | @@ -181,31 +188,32 @@ class ImageSyncMaster: | |||
441 | 181 | with open(filename) as meta_file: | 188 | with open(filename) as meta_file: |
442 | 182 | try: | 189 | try: |
443 | 183 | data = json.load(meta_file) | 190 | data = json.load(meta_file) |
446 | 184 | local_updated_at = data['updated_at'] | 191 | local_updated_at = data["updated_at"] |
447 | 185 | imageid = data['id'] | 192 | imageid = data["id"] |
448 | 186 | except Exception as e: | 193 | except Exception as e: |
450 | 187 | self.log('EXCEPTION: {0}'.format(e)) | 194 | self.log("EXCEPTION: {0}".format(e)) |
451 | 188 | return False | 195 | return False |
452 | 189 | 196 | ||
453 | 190 | local_dup = dateutil.parser.parse(local_updated_at) | 197 | local_dup = dateutil.parser.parse(local_updated_at) |
454 | 191 | glance_dup = dateutil.parser.parse(glance_updated_at) | 198 | glance_dup = dateutil.parser.parse(glance_updated_at) |
455 | 192 | 199 | ||
456 | 193 | if local_dup >= glance_dup: | 200 | if local_dup >= glance_dup: |
458 | 194 | self.log('INFO: {0} up to date'.format(imageid)) | 201 | self.log("INFO: {0} up to date".format(imageid)) |
459 | 195 | return True | 202 | return True |
460 | 196 | else: | 203 | else: |
463 | 197 | self.log('INFO: {0} outdated. Re-creating local ' | 204 | self.log( |
464 | 198 | 'copy of the metadata'.format(imageid)) | 205 | "INFO: {0} outdated. Re-creating local " |
465 | 206 | "copy of the metadata".format(imageid) | ||
466 | 207 | ) | ||
467 | 199 | else: | 208 | else: |
470 | 200 | self.log('INFO: {0} not found. Creating a local ' | 209 | self.log("INFO: {0} not found. Creating a local copy".format(filename)) |
469 | 201 | 'copy'.format(filename)) | ||
471 | 202 | return False | 210 | return False |
472 | 203 | 211 | ||
473 | 204 | def update_metadata(self, filename, glance_metadata): | 212 | def update_metadata(self, filename, glance_metadata): |
474 | 205 | """creates or replaces file with image metadata | 213 | """creates or replaces file with image metadata |
475 | 206 | creates subdirectory if it doesn't exist | 214 | creates subdirectory if it doesn't exist |
476 | 207 | """ | 215 | """ |
478 | 208 | tmp_file = '{0}.tmp'.format(filename) | 216 | tmp_file = "{0}.tmp".format(filename) |
479 | 209 | 217 | ||
480 | 210 | if not os.path.exists(os.path.dirname(filename)): | 218 | if not os.path.exists(os.path.dirname(filename)): |
481 | 211 | os.mkdir(os.path.dirname(filename), 0o750) | 219 | os.mkdir(os.path.dirname(filename), 0o750) |
482 | @@ -216,32 +224,34 @@ class ImageSyncMaster: | |||
483 | 216 | self.log(e) | 224 | self.log(e) |
484 | 217 | return False | 225 | return False |
485 | 218 | 226 | ||
487 | 219 | with open(tmp_file, 'w') as f: | 227 | with open(tmp_file, "w") as f: |
488 | 220 | json.dump(glance_metadata, f, indent=4, ensure_ascii=False) | 228 | json.dump(glance_metadata, f, indent=4, ensure_ascii=False) |
489 | 221 | 229 | ||
490 | 222 | os.rename(tmp_file, filename) | 230 | os.rename(tmp_file, filename) |
492 | 223 | self.log('INFO: update_metadata :: {0}'.format(filename)) | 231 | self.log("INFO: update_metadata :: {0}".format(filename)) |
493 | 224 | return True | 232 | return True |
494 | 225 | 233 | ||
495 | 226 | def add_bs_metadata_keys(self, glance_metadata): | 234 | def add_bs_metadata_keys(self, glance_metadata): |
501 | 227 | keys = [k for k in glance_metadata if k.startswith('bs_')] | 235 | keys = [k for k in glance_metadata if k.startswith("bs_")] |
502 | 228 | if 'bs_owner' in keys: | 236 | if "bs_owner" in keys: |
503 | 229 | msg = 'WARNING: bs_owner metadata should not exist (image: ' \ | 237 | msg = ( |
504 | 230 | '{0}; bs_owner: {1})'.format(glance_metadata.id, | 238 | "WARNING: bs_owner metadata should not exist (image: " |
505 | 231 | glance_metadata.bs_owner) | 239 | "{0}; bs_owner: {1})".format( |
506 | 240 | glance_metadata.id, glance_metadata.bs_owner | ||
507 | 241 | ) | ||
508 | 242 | ) | ||
509 | 232 | raise BootStackMetadataError(msg) | 243 | raise BootStackMetadataError(msg) |
513 | 233 | elif glance_metadata['owner'] in self.tenants: | 244 | elif glance_metadata["owner"] in self.tenants: |
514 | 234 | glance_metadata['bs_owner'] = self.tenants[ | 245 | glance_metadata["bs_owner"] = self.tenants[glance_metadata["owner"]] |
512 | 235 | glance_metadata['owner']] | ||
515 | 236 | return glance_metadata | 246 | return glance_metadata |
516 | 237 | 247 | ||
517 | 238 | def main(self): | 248 | def main(self): |
519 | 239 | self.log('starting glance sync') | 249 | self.log("starting glance sync") |
520 | 240 | # updates local metadata files if outdated | 250 | # updates local metadata files if outdated |
521 | 241 | existing_images = self.parse_glance_images() | 251 | existing_images = self.parse_glance_images() |
522 | 242 | # removes local metadata files from deleted images | 252 | # removes local metadata files from deleted images |
523 | 243 | self.delete_files(existing_images) | 253 | self.delete_files(existing_images) |
525 | 244 | self.log('ending glance sync') | 254 | self.log("ending glance sync") |
526 | 245 | self.release_lock() | 255 | self.release_lock() |
527 | 246 | 256 | ||
528 | 247 | def get_community_images_from_database(self): | 257 | def get_community_images_from_database(self): |
529 | @@ -249,34 +259,37 @@ class ImageSyncMaster: | |||
530 | 249 | 259 | ||
531 | 250 | db_img_list = list() | 260 | db_img_list = list() |
532 | 251 | con = mysql.connect( | 261 | con = mysql.connect( |
537 | 252 | host=os.environ['OS_MYSQL_HOST'], | 262 | host=os.environ["OS_MYSQL_HOST"], |
538 | 253 | user=os.environ['OS_MYSQL_USER'], | 263 | user=os.environ["OS_MYSQL_USER"], |
539 | 254 | password=os.environ['OS_MYSQL_PASS'], | 264 | password=os.environ["OS_MYSQL_PASS"], |
540 | 255 | database=os.environ['OS_MYSQL_DB']) | 265 | database=os.environ["OS_MYSQL_DB"], |
541 | 266 | ) | ||
542 | 256 | with closing(con.cursor()) as cur: | 267 | with closing(con.cursor()) as cur: |
545 | 257 | sql = 'SELECT id, name FROM images WHERE deleted = 0 AND ' \ | 268 | sql = ( |
546 | 258 | "visibility = 'community'" | 269 | "SELECT id, name FROM images WHERE deleted = 0 AND " |
547 | 270 | "visibility = 'community'" | ||
548 | 271 | ) | ||
549 | 259 | cur.execute(sql) | 272 | cur.execute(sql) |
550 | 260 | for (id, name) in cur.fetchall(): | 273 | for (id, name) in cur.fetchall(): |
551 | 261 | self.log( | 274 | self.log( |
554 | 262 | 'Retrieved community image with id [{}] and name [{}] ' | 275 | "Retrieved community image with id [{}] and name [{}] " |
555 | 263 | 'from database'.format(id, name)) | 276 | "from database".format(id, name) |
556 | 277 | ) | ||
557 | 264 | db_img_list.append(id) | 278 | db_img_list.append(id) |
558 | 265 | 279 | ||
559 | 266 | return set(db_img_list) | 280 | return set(db_img_list) |
560 | 267 | 281 | ||
561 | 268 | 282 | ||
566 | 269 | if __name__ == '__main__': | 283 | if __name__ == "__main__": |
567 | 270 | parser = argparse.ArgumentParser(description='Synchronize glance images ' | 284 | parser = argparse.ArgumentParser(description="Synchronize glance images to disk ") |
568 | 271 | 'to disk ') | 285 | parser.add_argument("-d", "--datadir", help="directory to write images to") |
565 | 272 | parser.add_argument('-d', '--datadir', help='directory to write images to') | ||
569 | 273 | args = parser.parse_args() | 286 | args = parser.parse_args() |
570 | 274 | 287 | ||
571 | 275 | if args.datadir: | 288 | if args.datadir: |
572 | 276 | data_dir = args.datadir | 289 | data_dir = args.datadir |
573 | 277 | else: | 290 | else: |
574 | 278 | parser.print_help() | 291 | parser.print_help() |
576 | 279 | sys.exit('ERROR: please specify an output directory for images') | 292 | sys.exit("ERROR: please specify an output directory for images") |
577 | 280 | 293 | ||
578 | 281 | master = ImageSyncMaster(data_dir) | 294 | master = ImageSyncMaster(data_dir) |
579 | 282 | master.main() | 295 | master.main() |
580 | diff --git a/src/files/glance_sync_slave.py b/src/files/glance_sync_slave.py | |||
581 | index 702b362..daf6dc6 100755 | |||
582 | --- a/src/files/glance_sync_slave.py | |||
583 | +++ b/src/files/glance_sync_slave.py | |||
584 | @@ -10,6 +10,7 @@ import json | |||
585 | 10 | import datetime | 10 | import datetime |
586 | 11 | import dateutil.parser | 11 | import dateutil.parser |
587 | 12 | import atexit | 12 | import atexit |
588 | 13 | |||
589 | 13 | # import re | 14 | # import re |
590 | 14 | import shlex | 15 | import shlex |
591 | 15 | import os_client_config | 16 | import os_client_config |
592 | @@ -22,61 +23,70 @@ class OSProjectNotFound(Exception): | |||
593 | 22 | """This indicates no sync is possible | 23 | """This indicates no sync is possible |
594 | 23 | (not defaulting to admin project) | 24 | (not defaulting to admin project) |
595 | 24 | """ | 25 | """ |
596 | 26 | |||
597 | 25 | pass | 27 | pass |
598 | 26 | 28 | ||
599 | 27 | 29 | ||
600 | 28 | class ImageSyncSlave: | 30 | class ImageSyncSlave: |
631 | 29 | extra_properties = set(['bs_owner']) | 31 | extra_properties = set(["bs_owner"]) |
632 | 30 | glance_properties = set(["architecture", | 32 | glance_properties = set( |
633 | 31 | "checksum", | 33 | [ |
634 | 32 | "container_format", | 34 | "architecture", |
635 | 33 | "created_at", | 35 | "checksum", |
636 | 34 | "deleted", | 36 | "container_format", |
637 | 35 | "deleted_at", | 37 | "created_at", |
638 | 36 | "direct_url", | 38 | "deleted", |
639 | 37 | "disk_format", | 39 | "deleted_at", |
640 | 38 | "file", | 40 | "direct_url", |
641 | 39 | "id", | 41 | "disk_format", |
642 | 40 | "instance_uuid", | 42 | "file", |
643 | 41 | "kernel_id", | 43 | "id", |
644 | 42 | "locations", | 44 | "instance_uuid", |
645 | 43 | "min_disk", | 45 | "kernel_id", |
646 | 44 | "min_ram", | 46 | "locations", |
647 | 45 | "name", | 47 | "min_disk", |
648 | 46 | "os_distro", | 48 | "min_ram", |
649 | 47 | "os_version", | 49 | "name", |
650 | 48 | "owner", | 50 | "os_distro", |
651 | 49 | "protected", | 51 | "os_version", |
652 | 50 | "ramdisk_id", | 52 | "owner", |
653 | 51 | "schema", | 53 | "protected", |
654 | 52 | "self", | 54 | "ramdisk_id", |
655 | 53 | "size", | 55 | "schema", |
656 | 54 | "status", | 56 | "self", |
657 | 55 | "tags", | 57 | "size", |
658 | 56 | "updated_at", | 58 | "status", |
659 | 57 | "virtual_size", | 59 | "tags", |
660 | 58 | "visibility"]) | 60 | "updated_at", |
661 | 61 | "virtual_size", | ||
662 | 62 | "visibility", | ||
663 | 63 | ] | ||
664 | 64 | ) | ||
665 | 59 | # egrep -B2 readOnly glanceclient/v2/image_schema.py | \ | 65 | # egrep -B2 readOnly glanceclient/v2/image_schema.py | \ |
666 | 60 | # awk '/\{/ {print $1}' | tr -d \": | 66 | # awk '/\{/ {print $1}' | tr -d \": |
678 | 61 | readonly_properties = set(['file', | 67 | readonly_properties = set( |
679 | 62 | 'size', | 68 | [ |
680 | 63 | 'status', | 69 | "file", |
681 | 64 | 'self', | 70 | "size", |
682 | 65 | 'direct_url', | 71 | "status", |
683 | 66 | 'schema', | 72 | "self", |
684 | 67 | 'updated_at', | 73 | "direct_url", |
685 | 68 | 'locations', | 74 | "schema", |
686 | 69 | 'virtual_size', | 75 | "updated_at", |
687 | 70 | 'checksum', | 76 | "locations", |
688 | 71 | 'created_at']) | 77 | "virtual_size", |
689 | 78 | "checksum", | ||
690 | 79 | "created_at", | ||
691 | 80 | ] | ||
692 | 81 | ) | ||
693 | 72 | 82 | ||
694 | 73 | def __init__(self, data_dir, source): | 83 | def __init__(self, data_dir, source): |
695 | 74 | self.projects_slave = {} | 84 | self.projects_slave = {} |
696 | 75 | self.DATA_DIR = data_dir | 85 | self.DATA_DIR = data_dir |
697 | 76 | self.SOURCE = source | 86 | self.SOURCE = source |
698 | 77 | self.valid_properties = self.glance_properties.difference( | 87 | self.valid_properties = self.glance_properties.difference( |
701 | 78 | self.readonly_properties.union( | 88 | self.readonly_properties.union(self.extra_properties) |
702 | 79 | self.extra_properties)) | 89 | ) |
703 | 80 | self.set_filelock() | 90 | self.set_filelock() |
704 | 81 | self.glance_connect_slave() | 91 | self.glance_connect_slave() |
705 | 82 | self.glance_connect_master() | 92 | self.glance_connect_master() |
706 | @@ -84,22 +94,25 @@ class ImageSyncSlave: | |||
707 | 84 | def download_metadata_from_master(self): | 94 | def download_metadata_from_master(self): |
708 | 85 | """rsync metadata files from source to data_dir""" | 95 | """rsync metadata files from source to data_dir""" |
709 | 86 | 96 | ||
721 | 87 | if not self.SOURCE.endswith('/'): | 97 | if not self.SOURCE.endswith("/"): |
722 | 88 | self.SOURCE += '/' | 98 | self.SOURCE += "/" |
723 | 89 | if not self.DATA_DIR.endswith('/'): | 99 | if not self.DATA_DIR.endswith("/"): |
724 | 90 | self.DATA_DIR += '/' | 100 | self.DATA_DIR += "/" |
725 | 91 | 101 | ||
726 | 92 | command = '/usr/bin/rsync -az --delete -e ' \ | 102 | command = ( |
727 | 93 | "'ssh -o StrictHostKeyChecking=no' " \ | 103 | "/usr/bin/rsync -az --delete -e " |
728 | 94 | '{0} {1}'.format(self.SOURCE, self.DATA_DIR) | 104 | "'ssh -o StrictHostKeyChecking=no' " |
729 | 95 | proc = subprocess.Popen(shlex.split(command), | 105 | "{0} {1}".format(self.SOURCE, self.DATA_DIR) |
730 | 96 | stdout=subprocess.PIPE, | 106 | ) |
731 | 97 | stderr=subprocess.PIPE) | 107 | proc = subprocess.Popen( |
732 | 108 | shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE | ||
733 | 109 | ) | ||
734 | 98 | (stdout, stderr) = proc.communicate() | 110 | (stdout, stderr) = proc.communicate() |
735 | 99 | if proc.returncode: | 111 | if proc.returncode: |
739 | 100 | self.log('ERROR: problem while getting data from master ' | 112 | self.log( |
740 | 101 | '({0})'.format(command)) | 113 | "ERROR: problem while getting data from master ({0})".format(command) |
741 | 102 | self.log('ERROR: {0}'.format(stderr)) | 114 | ) |
742 | 115 | self.log("ERROR: {0}".format(stderr)) | ||
743 | 103 | self.release_lock() | 116 | self.release_lock() |
744 | 104 | sys.exit(2) | 117 | sys.exit(2) |
745 | 105 | else: | 118 | else: |
746 | @@ -111,17 +124,22 @@ class ImageSyncSlave: | |||
747 | 111 | 124 | ||
748 | 112 | db_img_list = list() | 125 | db_img_list = list() |
749 | 113 | con = mysql.connect( | 126 | con = mysql.connect( |
754 | 114 | host=os.environ['OS_MYSQL_HOST'], | 127 | host=os.environ["OS_MYSQL_HOST"], |
755 | 115 | user=os.environ['OS_MYSQL_USER'], | 128 | user=os.environ["OS_MYSQL_USER"], |
756 | 116 | password=os.environ['OS_MYSQL_PASS'], | 129 | password=os.environ["OS_MYSQL_PASS"], |
757 | 117 | database=os.environ['OS_MYSQL_DB']) | 130 | database=os.environ["OS_MYSQL_DB"], |
758 | 131 | ) | ||
759 | 118 | with closing(con.cursor()) as cur: | 132 | with closing(con.cursor()) as cur: |
762 | 119 | sql = "SELECT id, name FROM images WHERE deleted = 0 AND " \ | 133 | sql = ( |
763 | 120 | "visibility = 'community'" | 134 | "SELECT id, name FROM images WHERE deleted = 0 AND " |
764 | 135 | "visibility = 'community'" | ||
765 | 136 | ) | ||
766 | 121 | cur.execute(sql) | 137 | cur.execute(sql) |
767 | 122 | for (id, name) in cur.fetchall(): | 138 | for (id, name) in cur.fetchall(): |
770 | 123 | self.log('Retrieved community image with id [{}] and name ' | 139 | self.log( |
771 | 124 | '[{}] from database'.format(id, name)) | 140 | "Retrieved community image with id [{}] and name " |
772 | 141 | "[{}] from database".format(id, name) | ||
773 | 142 | ) | ||
774 | 125 | db_img_list.append(id) | 143 | db_img_list.append(id) |
775 | 126 | 144 | ||
776 | 127 | return set(db_img_list) | 145 | return set(db_img_list) |
777 | @@ -134,7 +152,7 @@ class ImageSyncSlave: | |||
778 | 134 | 152 | ||
779 | 135 | @returns processed (aka. parsed) images | 153 | @returns processed (aka. parsed) images |
780 | 136 | """ | 154 | """ |
782 | 137 | self.log('getting image list from slave') | 155 | self.log("getting image list from slave") |
783 | 138 | processed_images_ids = set() | 156 | processed_images_ids = set() |
784 | 139 | to_delete_images_ids = set() | 157 | to_delete_images_ids = set() |
785 | 140 | existing_images = self.get_community_images_from_database() | 158 | existing_images = self.get_community_images_from_database() |
786 | @@ -143,10 +161,9 @@ class ImageSyncSlave: | |||
787 | 143 | for image_helper in existing_images: | 161 | for image_helper in existing_images: |
788 | 144 | image = self.glance_slave.images.get(image_helper) | 162 | image = self.glance_slave.images.get(image_helper) |
789 | 145 | if len(image.id) < 2: | 163 | if len(image.id) < 2: |
791 | 146 | basename = '{0}.json'.format(image.id) | 164 | basename = "{0}.json".format(image.id) |
792 | 147 | else: | 165 | else: |
795 | 148 | basename = '{0}/{1}.json'.format(str(image.id)[:2], | 166 | basename = "{0}/{1}.json".format(str(image.id)[:2], image.id) |
794 | 149 | image.id) | ||
796 | 150 | filename = os.path.join(self.DATA_DIR, basename) | 167 | filename = os.path.join(self.DATA_DIR, basename) |
797 | 151 | if not os.path.isfile(filename): | 168 | if not os.path.isfile(filename): |
798 | 152 | to_delete_images_ids.add(image.id) | 169 | to_delete_images_ids.add(image.id) |
799 | @@ -154,14 +171,16 @@ class ImageSyncSlave: | |||
800 | 154 | 171 | ||
801 | 155 | metadata_local = self.read_metadata(filename) | 172 | metadata_local = self.read_metadata(filename) |
802 | 156 | if not metadata_local: | 173 | if not metadata_local: |
805 | 157 | self.log('ERROR: read_metadata did not retrieve anything ' | 174 | self.log( |
806 | 158 | '({0})'.format(filename)) | 175 | "ERROR: read_metadata did not retrieve anything " |
807 | 176 | "({0})".format(filename) | ||
808 | 177 | ) | ||
809 | 159 | continue | 178 | continue |
810 | 160 | 179 | ||
815 | 161 | if metadata_local['checksum'] == image.checksum: | 180 | if metadata_local["checksum"] == image.checksum: |
816 | 162 | if not self.is_latest_metadata(metadata_local['id'], | 181 | if not self.is_latest_metadata( |
817 | 163 | metadata_local['updated_at'], | 182 | metadata_local["id"], metadata_local["updated_at"], image.updated_at |
818 | 164 | image.updated_at): | 183 | ): |
819 | 165 | # checksum ok, metadata outdated | 184 | # checksum ok, metadata outdated |
820 | 166 | self.update_metadata(metadata_local, image) | 185 | self.update_metadata(metadata_local, image) |
821 | 167 | processed_images_ids.add(image.id) | 186 | processed_images_ids.add(image.id) |
822 | @@ -170,16 +189,17 @@ class ImageSyncSlave: | |||
823 | 170 | self.upload_to_slave(metadata_local) | 189 | self.upload_to_slave(metadata_local) |
824 | 171 | processed_images_ids.add(image.id) | 190 | processed_images_ids.add(image.id) |
825 | 172 | 191 | ||
828 | 173 | self.log('DEBUG: images pending to be deleted: ' | 192 | self.log( |
829 | 174 | '{0}'.format(to_delete_images_ids)) | 193 | "DEBUG: images pending to be deleted: {0}".format(to_delete_images_ids) |
830 | 194 | ) | ||
831 | 175 | self.delete_images_from_slave(to_delete_images_ids) | 195 | self.delete_images_from_slave(to_delete_images_ids) |
834 | 176 | self.log('DEBUG: processed images (to skip while parsing metadata ' | 196 | self.log( |
835 | 177 | 'files): {0}'.format(processed_images_ids)) | 197 | "DEBUG: processed images (to skip while parsing metadata " |
836 | 198 | "files): {0}".format(processed_images_ids) | ||
837 | 199 | ) | ||
838 | 178 | return processed_images_ids | 200 | return processed_images_ids |
839 | 179 | 201 | ||
843 | 180 | def is_latest_metadata(self, image_id, | 202 | def is_latest_metadata(self, image_id, master_updated_at, slave_updated_at): |
841 | 181 | master_updated_at, | ||
842 | 182 | slave_updated_at): | ||
844 | 183 | """Compares filename content (JSON metadata) and glance service info | 203 | """Compares filename content (JSON metadata) and glance service info |
845 | 184 | @return | 204 | @return |
846 | 185 | True: no need to update | 205 | True: no need to update |
847 | @@ -189,70 +209,78 @@ class ImageSyncSlave: | |||
848 | 189 | slave_dup = dateutil.parser.parse(slave_updated_at) | 209 | slave_dup = dateutil.parser.parse(slave_updated_at) |
849 | 190 | 210 | ||
850 | 191 | if master_dup <= slave_dup: | 211 | if master_dup <= slave_dup: |
853 | 192 | self.log('INFO: is_latest_metadata :: {0} up to ' | 212 | self.log("INFO: is_latest_metadata :: {0} up to date".format(image_id)) |
852 | 193 | 'date'.format(image_id)) | ||
854 | 194 | return True | 213 | return True |
855 | 195 | else: | 214 | else: |
858 | 196 | self.log('INFO: is_latest_metadata :: {0} outdated. Needs ' | 215 | self.log( |
859 | 197 | 'update_metadata.'.format(image_id)) | 216 | "INFO: is_latest_metadata :: {0} outdated. Needs " |
860 | 217 | "update_metadata.".format(image_id) | ||
861 | 218 | ) | ||
862 | 198 | return False | 219 | return False |
863 | 199 | 220 | ||
864 | 200 | def upload_to_slave(self, metadata_local): # noqa: C901 is too complex (12) | 221 | def upload_to_slave(self, metadata_local): # noqa: C901 is too complex (12) |
865 | 201 | """upload image to glance slave service | 222 | """upload image to glance slave service |
866 | 202 | """ | 223 | """ |
868 | 203 | tmp_image_basename = '{0}.img'.format(metadata_local['id']) | 224 | tmp_image_basename = "{0}.img".format(metadata_local["id"]) |
869 | 204 | tmp_image = os.path.join(self.DATA_DIR, tmp_image_basename) | 225 | tmp_image = os.path.join(self.DATA_DIR, tmp_image_basename) |
870 | 205 | try: | 226 | try: |
871 | 206 | clean_metadata, removed_props = self.mangle_metadata(metadata_local) | 227 | clean_metadata, removed_props = self.mangle_metadata(metadata_local) |
872 | 207 | except OSProjectNotFound as e: | 228 | except OSProjectNotFound as e: |
877 | 208 | self.log('EXCEPTION: upload_to_slave :: image-id {0} :: ' | 229 | self.log( |
878 | 209 | 'problem uploading data to glance ' | 230 | "EXCEPTION: upload_to_slave :: image-id {0} :: " |
879 | 210 | 'slave (image could not be removed) :: ' | 231 | "problem uploading data to glance " |
880 | 211 | '{1}'.format(metadata_local['id'], e)) | 232 | "slave (image could not be removed) :: " |
881 | 233 | "{1}".format(metadata_local["id"], e) | ||
882 | 234 | ) | ||
883 | 212 | return False | 235 | return False |
884 | 213 | 236 | ||
885 | 214 | for k in removed_props: | 237 | for k in removed_props: |
886 | 215 | if k in clean_metadata: | 238 | if k in clean_metadata: |
887 | 216 | del clean_metadata[k] | 239 | del clean_metadata[k] |
888 | 217 | 240 | ||
890 | 218 | self.log('INFO: creating image {0}'.format(clean_metadata['id'])) | 241 | self.log("INFO: creating image {0}".format(clean_metadata["id"])) |
891 | 219 | try: | 242 | try: |
892 | 220 | self.glance_slave.images.create(**clean_metadata) | 243 | self.glance_slave.images.create(**clean_metadata) |
894 | 221 | self.log('DEBUG: create image: {0}'.format(clean_metadata)) | 244 | self.log("DEBUG: create image: {0}".format(clean_metadata)) |
895 | 222 | except Exception as e: # TODO narrow this exception down | 245 | except Exception as e: # TODO narrow this exception down |
897 | 223 | self.log('EXCEPTION: upload_to_slave :: {0}'.format(e)) | 246 | self.log("EXCEPTION: upload_to_slave :: {0}".format(e)) |
898 | 224 | try: | 247 | try: |
899 | 225 | # update metadata | 248 | # update metadata |
905 | 226 | self.glance_slave.images.update(clean_metadata['id'], | 249 | self.glance_slave.images.update( |
906 | 227 | remove_props=removed_props, | 250 | clean_metadata["id"], remove_props=removed_props, **clean_metadata |
907 | 228 | **clean_metadata) | 251 | ) |
908 | 229 | self.log('DEBUG: update_to_slave :: update metadata ' | 252 | self.log( |
909 | 230 | '{0}'.format(clean_metadata)) | 253 | "DEBUG: update_to_slave :: update metadata " |
910 | 254 | "{0}".format(clean_metadata) | ||
911 | 255 | ) | ||
912 | 231 | except Exception as e: | 256 | except Exception as e: |
913 | 232 | if "HTTPNotFound" not in e: | 257 | if "HTTPNotFound" not in e: |
920 | 233 | self.log('ERROR: update_to_slave (both image ' | 258 | self.log( |
921 | 234 | 'create/update failed :: {0} - this can ' | 259 | "ERROR: update_to_slave (both image " |
922 | 235 | 'happen if the image was deleted through ' | 260 | "create/update failed :: {0} - this can " |
923 | 236 | 'the API but still exists in the glance ' | 261 | "happen if the image was deleted through " |
924 | 237 | 'database :: {1}' | 262 | "the API but still exists in the glance " |
925 | 238 | .format(clean_metadata['id'], e)) | 263 | "database :: {1}".format(clean_metadata["id"], e) |
926 | 264 | ) | ||
927 | 239 | return False | 265 | return False |
928 | 240 | 266 | ||
931 | 241 | self.log('ERROR: {0} {1} is likely deleted'.format( | 267 | self.log( |
932 | 242 | clean_metadata['id'], e)) | 268 | "ERROR: {0} {1} is likely deleted".format(clean_metadata["id"], e) |
933 | 269 | ) | ||
934 | 243 | 270 | ||
935 | 244 | try: | 271 | try: |
936 | 245 | # Upload. | 272 | # Upload. |
939 | 246 | self.glance_slave.images.upload(clean_metadata['id'], | 273 | self.glance_slave.images.upload(clean_metadata["id"], open(tmp_image, "rb")) |
938 | 247 | open(tmp_image, 'rb')) | ||
940 | 248 | os.remove(tmp_image) | 274 | os.remove(tmp_image) |
942 | 249 | self.log('DEBUG: update_to_slave :: upload {0}'.format(tmp_image)) | 275 | self.log("DEBUG: update_to_slave :: upload {0}".format(tmp_image)) |
943 | 250 | except Exception as e: | 276 | except Exception as e: |
944 | 251 | os.remove(tmp_image) | 277 | os.remove(tmp_image) |
949 | 252 | self.log('ERROR: upload_to_slave :: image-id {0} :: ' | 278 | self.log( |
950 | 253 | 'problem uploading data to glance ' | 279 | "ERROR: upload_to_slave :: image-id {0} :: " |
951 | 254 | 'slave (image could not be removed) :: ' | 280 | "problem uploading data to glance " |
952 | 255 | '{1}'.format(clean_metadata['id'], e)) | 281 | "slave (image could not be removed) :: " |
953 | 282 | "{1}".format(clean_metadata["id"], e) | ||
954 | 283 | ) | ||
955 | 256 | return False | 284 | return False |
956 | 257 | 285 | ||
957 | 258 | def download_from_master(self, metadata_local): # noqa: C901 is too complex (12) | 286 | def download_from_master(self, metadata_local): # noqa: C901 is too complex (12) |
958 | @@ -261,45 +289,54 @@ class ImageSyncSlave: | |||
959 | 261 | @return True: downloaded or already on local storage | 289 | @return True: downloaded or already on local storage |
960 | 262 | @return False: error | 290 | @return False: error |
961 | 263 | """ | 291 | """ |
963 | 264 | tmp_image_basename = '{0}.img'.format(metadata_local['id']) | 292 | tmp_image_basename = "{0}.img".format(metadata_local["id"]) |
964 | 265 | tmp_image = os.path.join(self.DATA_DIR, tmp_image_basename) | 293 | tmp_image = os.path.join(self.DATA_DIR, tmp_image_basename) |
965 | 266 | if os.path.isfile(tmp_image): | 294 | if os.path.isfile(tmp_image): |
967 | 267 | if self.check_md5(metadata_local['checksum'], tmp_image): | 295 | if self.check_md5(metadata_local["checksum"], tmp_image): |
968 | 268 | return True | 296 | return True |
969 | 269 | 297 | ||
970 | 270 | try: | 298 | try: |
971 | 271 | os.remove(tmp_image) | 299 | os.remove(tmp_image) |
972 | 272 | except Exception as e: | 300 | except Exception as e: |
974 | 273 | self.log('ERROR: download_from_master :: {0}'.format(e)) | 301 | self.log("ERROR: download_from_master :: {0}".format(e)) |
975 | 274 | return False | 302 | return False |
976 | 275 | downloaded = False | 303 | downloaded = False |
977 | 276 | retries = 3 | 304 | retries = 3 |
978 | 277 | for i in range(0, retries): | 305 | for i in range(0, retries): |
979 | 278 | try: | 306 | try: |
980 | 279 | bin_image = self.glance_master.images.data( | 307 | bin_image = self.glance_master.images.data( |
982 | 280 | image_id=metadata_local['id']) | 308 | image_id=metadata_local["id"] |
983 | 309 | ) | ||
984 | 281 | 310 | ||
985 | 282 | hash_md5 = hashlib.md5() | 311 | hash_md5 = hashlib.md5() |
987 | 283 | with open(tmp_image, 'wb') as fd: | 312 | with open(tmp_image, "wb") as fd: |
988 | 284 | for chunk in bin_image: | 313 | for chunk in bin_image: |
989 | 285 | fd.write(chunk) | 314 | fd.write(chunk) |
990 | 286 | hash_md5.update(chunk) | 315 | hash_md5.update(chunk) |
991 | 287 | bin_image_checksum = hash_md5.hexdigest() | 316 | bin_image_checksum = hash_md5.hexdigest() |
993 | 288 | if metadata_local['checksum'] == bin_image_checksum: | 317 | if metadata_local["checksum"] == bin_image_checksum: |
994 | 289 | downloaded = True | 318 | downloaded = True |
998 | 290 | self.log('INFO: download_from_master ({0} - {1}):: ' | 319 | self.log( |
999 | 291 | 'checksum OK'.format(metadata_local['id'], | 320 | "INFO: download_from_master ({0} - {1}):: " |
1000 | 292 | metadata_local['checksum'])) | 321 | "checksum OK".format( |
1001 | 322 | metadata_local["id"], metadata_local["checksum"] | ||
1002 | 323 | ) | ||
1003 | 324 | ) | ||
1004 | 293 | break | 325 | break |
1005 | 294 | elif os.path.exists(tmp_image): | 326 | elif os.path.exists(tmp_image): |
1010 | 295 | self.log('INFO: download_from_master ({0}/{1}; {2}):: ' | 327 | self.log( |
1011 | 296 | 'invalid checksum ' | 328 | "INFO: download_from_master ({0}/{1}; {2}):: " |
1012 | 297 | '{3}'.format(metadata_local['id'], i, retries, | 329 | "invalid checksum " |
1013 | 298 | bin_image_checksum)) | 330 | "{3}".format( |
1014 | 331 | metadata_local["id"], i, retries, bin_image_checksum | ||
1015 | 332 | ) | ||
1016 | 333 | ) | ||
1017 | 299 | os.remove(tmp_image) | 334 | os.remove(tmp_image) |
1018 | 300 | except Exception as e: | 335 | except Exception as e: |
1021 | 301 | self.log('EXCEPTION: download_from_master ({0}/{1}; {2}):: ' | 336 | self.log( |
1022 | 302 | '{3}'.format(i, retries, metadata_local['id'], e)) | 337 | "EXCEPTION: download_from_master ({0}/{1}; {2}):: " |
1023 | 338 | "{3}".format(i, retries, metadata_local["id"], e) | ||
1024 | 339 | ) | ||
1025 | 303 | if os.path.exists(tmp_image): | 340 | if os.path.exists(tmp_image): |
1026 | 304 | os.remove(tmp_image) | 341 | os.remove(tmp_image) |
1027 | 305 | 342 | ||
1028 | @@ -310,86 +347,90 @@ class ImageSyncSlave: | |||
1029 | 310 | deletes images not found in local storage | 347 | deletes images not found in local storage |
1030 | 311 | """ | 348 | """ |
1031 | 312 | if not to_delete_images_ids: | 349 | if not to_delete_images_ids: |
1034 | 313 | self.log('WARNING: precautionary halt. No glance images found ' | 350 | self.log( |
1035 | 314 | 'to be deleted. noop.') | 351 | "WARNING: precautionary halt. No glance images found " |
1036 | 352 | "to be deleted. noop." | ||
1037 | 353 | ) | ||
1038 | 315 | return | 354 | return |
1039 | 316 | 355 | ||
1040 | 317 | for image_id in to_delete_images_ids: | 356 | for image_id in to_delete_images_ids: |
1042 | 318 | self.log('INFO: removing image {0}'.format(image_id)) | 357 | self.log("INFO: removing image {0}".format(image_id)) |
1043 | 319 | try: | 358 | try: |
1044 | 320 | self.glance_slave.images.delete(image_id) | 359 | self.glance_slave.images.delete(image_id) |
1046 | 321 | self.log('DEBUG: image {0} removed'.format(image_id)) | 360 | self.log("DEBUG: image {0} removed".format(image_id)) |
1047 | 322 | except Exception as e: # TODO narrow the exception down | 361 | except Exception as e: # TODO narrow the exception down |
1050 | 323 | self.log('ERROR: could not delete {0} :: ' | 362 | self.log("ERROR: could not delete {0} :: {1}".format(image_id, e)) |
1049 | 324 | '{1}'.format(image_id, e)) | ||
1051 | 325 | 363 | ||
1052 | 326 | def create_missing_slave_images(self, processed_images_ids): | 364 | def create_missing_slave_images(self, processed_images_ids): |
1053 | 327 | 365 | ||
1054 | 328 | for dirpath, dirnames, filenames in os.walk(self.DATA_DIR): | 366 | for dirpath, dirnames, filenames in os.walk(self.DATA_DIR): |
1058 | 329 | if dirpath != self.DATA_DIR and \ | 367 | if dirpath != self.DATA_DIR and len(dirnames) == 0 and len(filenames) == 0: |
1056 | 330 | len(dirnames) == 0 and \ | ||
1057 | 331 | len(filenames) == 0: | ||
1059 | 332 | os.rmdir(dirpath) | 368 | os.rmdir(dirpath) |
1060 | 333 | continue | 369 | continue |
1061 | 334 | 370 | ||
1062 | 335 | for filename in filenames: | 371 | for filename in filenames: |
1063 | 336 | full_path = os.path.join(dirpath, filename) | 372 | full_path = os.path.join(dirpath, filename) |
1065 | 337 | if filename.endswith('.json'): | 373 | if filename.endswith(".json"): |
1066 | 338 | image_id = filename[:-5] | 374 | image_id = filename[:-5] |
1067 | 339 | if image_id in processed_images_ids: | 375 | if image_id in processed_images_ids: |
1068 | 340 | continue | 376 | continue |
1069 | 341 | 377 | ||
1070 | 342 | metadata_local = self.read_metadata(full_path) | 378 | metadata_local = self.read_metadata(full_path) |
1071 | 343 | if not metadata_local: | 379 | if not metadata_local: |
1075 | 344 | self.log('ERROR: read_metadata did not ' | 380 | self.log( |
1076 | 345 | 'retrieve anything ' | 381 | "ERROR: read_metadata did not " |
1077 | 346 | '({0})'.format(full_path)) | 382 | "retrieve anything " |
1078 | 383 | "({0})".format(full_path) | ||
1079 | 384 | ) | ||
1080 | 347 | continue | 385 | continue |
1081 | 348 | 386 | ||
1082 | 349 | slave_project_id = self.project_mapping(metadata_local) | 387 | slave_project_id = self.project_mapping(metadata_local) |
1083 | 350 | if not slave_project_id: | 388 | if not slave_project_id: |
1086 | 351 | self.log('DEBUG: could not map image into any ' | 389 | self.log( |
1087 | 352 | 'slave project :: {0}'.format(metadata_local)) | 390 | "DEBUG: could not map image into any " |
1088 | 391 | "slave project :: {0}".format(metadata_local) | ||
1089 | 392 | ) | ||
1090 | 353 | continue | 393 | continue |
1091 | 354 | 394 | ||
1093 | 355 | metadata_local['owner'] = slave_project_id | 395 | metadata_local["owner"] = slave_project_id |
1094 | 356 | if self.download_from_master(metadata_local): | 396 | if self.download_from_master(metadata_local): |
1095 | 357 | self.upload_to_slave(metadata_local) | 397 | self.upload_to_slave(metadata_local) |
1096 | 358 | else: | 398 | else: |
1099 | 359 | self.log('ERROR: image {0} could not be downloaded ' | 399 | self.log( |
1100 | 360 | 'from master'.format(metadata_local['id'])) | 400 | "ERROR: image {0} could not be downloaded " |
1101 | 401 | "from master".format(metadata_local["id"]) | ||
1102 | 402 | ) | ||
1103 | 361 | 403 | ||
1104 | 362 | def project_mapping(self, metadata_local): | 404 | def project_mapping(self, metadata_local): |
1105 | 363 | """can master/slave projects be mapped, no matter project_ids are not | 405 | """can master/slave projects be mapped, no matter project_ids are not |
1106 | 364 | the same? | 406 | the same? |
1107 | 365 | """ | 407 | """ |
1108 | 366 | # master/slave match can't be done (no project_id) | 408 | # master/slave match can't be done (no project_id) |
1110 | 367 | if 'owner' not in metadata_local: | 409 | if "owner" not in metadata_local: |
1111 | 368 | return False | 410 | return False |
1112 | 369 | 411 | ||
1113 | 370 | # master/slave project_ids match | 412 | # master/slave project_ids match |
1116 | 371 | if metadata_local['owner'] in self.projects_slave: | 413 | if metadata_local["owner"] in self.projects_slave: |
1117 | 372 | return metadata_local['owner'] | 414 | return metadata_local["owner"] |
1118 | 373 | 415 | ||
1119 | 374 | # no extra project_name passed -- can't check match | 416 | # no extra project_name passed -- can't check match |
1121 | 375 | if 'bs_owner' not in metadata_local: | 417 | if "bs_owner" not in metadata_local: |
1122 | 376 | return False | 418 | return False |
1123 | 377 | 419 | ||
1125 | 378 | master_project_name = metadata_local['bs_owner'] | 420 | master_project_name = metadata_local["bs_owner"] |
1126 | 379 | 421 | ||
1127 | 380 | # XXX(aluria): no image on slave service | 422 | # XXX(aluria): no image on slave service |
1128 | 381 | # XXX(aluria): look for similar project on slave | 423 | # XXX(aluria): look for similar project on slave |
1133 | 382 | for slave_project_id, slave_project_name in ( | 424 | for slave_project_id, slave_project_name in self.projects_slave.items(): |
1134 | 383 | self.projects_slave.items()): | 425 | slave_to_master = slave_project_name.replace( |
1135 | 384 | slave_to_master = slave_project_name.replace(self.REGION_SLAVE, | 426 | self.REGION_SLAVE, self.REGION_MASTER |
1136 | 385 | self.REGION_MASTER) | 427 | ) |
1137 | 386 | # XXX(aluria): pitfall, if on master service there are | 428 | # XXX(aluria): pitfall, if on master service there are |
1138 | 387 | # XXX(aluria): 2 projects: | 429 | # XXX(aluria): 2 projects: |
1139 | 388 | # XXX(auria): REGION_SLAVE-restofprojectname | 430 | # XXX(auria): REGION_SLAVE-restofprojectname |
1140 | 389 | # XXX(auria): REGION_MASTER-restofprojectname | 431 | # XXX(auria): REGION_MASTER-restofprojectname |
1141 | 390 | # XXX(auria): first found gets image assigned | 432 | # XXX(auria): first found gets image assigned |
1144 | 391 | if master_project_name in (slave_project_name, | 433 | if master_project_name in (slave_project_name, slave_to_master): |
1143 | 392 | slave_to_master): | ||
1145 | 393 | return slave_project_id | 434 | return slave_project_id |
1146 | 394 | return False | 435 | return False |
1147 | 395 | 436 | ||
1148 | @@ -413,49 +454,47 @@ class ImageSyncSlave: | |||
1149 | 413 | data = json.load(meta_file) | 454 | data = json.load(meta_file) |
1150 | 414 | return data | 455 | return data |
1151 | 415 | except Exception as e: | 456 | except Exception as e: |
1153 | 416 | self.log('EXCEPTION: {0}'.format(e)) | 457 | self.log("EXCEPTION: {0}".format(e)) |
1154 | 417 | return False | 458 | return False |
1155 | 418 | else: | 459 | else: |
1157 | 419 | self.log('INFO: {0} not found.'.format(metadata_file)) | 460 | self.log("INFO: {0} not found.".format(metadata_file)) |
1158 | 420 | return False | 461 | return False |
1159 | 421 | 462 | ||
1160 | 422 | def glance_connect_slave(self): | 463 | def glance_connect_slave(self): |
1161 | 423 | try: | 464 | try: |
1162 | 424 | self.keystone = os_client_config.session_client( | 465 | self.keystone = os_client_config.session_client( |
1169 | 425 | 'identity', | 466 | "identity", cloud="envvars", |
1164 | 426 | cloud='envvars', | ||
1165 | 427 | ) | ||
1166 | 428 | self.glance_slave = os_client_config.make_client( | ||
1167 | 429 | 'image', | ||
1168 | 430 | cloud='envvars', | ||
1170 | 431 | ) | 467 | ) |
1171 | 468 | self.glance_slave = os_client_config.make_client("image", cloud="envvars") | ||
1172 | 432 | except Exception as e: | 469 | except Exception as e: |
1176 | 433 | self.log('EXCEPTION: {0}'.format(e)) | 470 | self.log("EXCEPTION: {0}".format(e)) |
1177 | 434 | self.log('ERROR: unable to load environment variables, please ' | 471 | self.log( |
1178 | 435 | 'source novarc') | 472 | "ERROR: unable to load environment variables, please source novarc" |
1179 | 473 | ) | ||
1180 | 436 | self.release_lock() | 474 | self.release_lock() |
1181 | 437 | sys.exit(2) | 475 | sys.exit(2) |
1182 | 438 | if not self.projects_slave: | 476 | if not self.projects_slave: |
1183 | 439 | self.projects_slave = dict( | 477 | self.projects_slave = dict( |
1187 | 440 | [(tenant['id'], tenant['name']) | 478 | [ |
1188 | 441 | for tenant in self.keystone.get('/projects').json()['projects'] | 479 | (tenant["id"], tenant["name"]) |
1189 | 442 | if tenant['enabled']] | 480 | for tenant in self.keystone.get("/projects").json()["projects"] |
1190 | 481 | if tenant["enabled"] | ||
1191 | 482 | ] | ||
1192 | 443 | ) | 483 | ) |
1194 | 444 | self.REGION_SLAVE = os.environ['OS_REGION_NAME'].upper() | 484 | self.REGION_SLAVE = os.environ["OS_REGION_NAME"].upper() |
1195 | 445 | return self.glance_slave | 485 | return self.glance_slave |
1196 | 446 | 486 | ||
1197 | 447 | def glance_connect_master(self): | 487 | def glance_connect_master(self): |
1198 | 448 | try: | 488 | try: |
1204 | 449 | self.glance_master = os_client_config.make_client( | 489 | self.glance_master = os_client_config.make_client("image", cloud="master") |
1205 | 450 | 'image', | 490 | self.REGION_MASTER = os.environ["OS_MASTER_REGION"] |
1201 | 451 | cloud='master', | ||
1202 | 452 | ) | ||
1203 | 453 | self.REGION_MASTER = os.environ['OS_MASTER_REGION'] | ||
1206 | 454 | except Exception as e: | 491 | except Exception as e: |
1211 | 455 | self.log('EXCEPTION: {0}'.format(e)) | 492 | self.log("EXCEPTION: {0}".format(e)) |
1212 | 456 | self.log('ERROR: unable to load master cloud environment, ' | 493 | self.log( |
1213 | 457 | 'please check master_creds settings and ' | 494 | "ERROR: unable to load master cloud environment, " |
1214 | 458 | '/etc/openstack/clouds.yaml') | 495 | "please check master_creds settings and " |
1215 | 496 | "/etc/openstack/clouds.yaml" | ||
1216 | 497 | ) | ||
1217 | 459 | self.release_lock() | 498 | self.release_lock() |
1218 | 460 | sys.exit(2) | 499 | sys.exit(2) |
1219 | 461 | return self.glance_master | 500 | return self.glance_master |
1220 | @@ -464,7 +503,7 @@ class ImageSyncSlave: | |||
1221 | 464 | return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") | 503 | return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
1222 | 465 | 504 | ||
1223 | 466 | def log(self, msg): | 505 | def log(self, msg): |
1225 | 467 | print('{0} {1}'.format(self.timestamp_now(), msg)) | 506 | print("{0} {1}".format(self.timestamp_now(), msg)) |
1226 | 468 | 507 | ||
1227 | 469 | def mangle_metadata(self, metadata_local, metadata_slave=None): | 508 | def mangle_metadata(self, metadata_local, metadata_slave=None): |
1228 | 470 | """Maps projects in MASTER region with projects in SLAVE region | 509 | """Maps projects in MASTER region with projects in SLAVE region |
1229 | @@ -519,122 +558,132 @@ class ImageSyncSlave: | |||
1230 | 519 | 'base_image_ref', | 558 | 'base_image_ref', |
1231 | 520 | 'owner_id'] | 559 | 'owner_id'] |
1232 | 521 | """ | 560 | """ |
1235 | 522 | if 'owner' not in metadata_local: | 561 | if "owner" not in metadata_local: |
1236 | 523 | raise (OSProjectNotFound, 'no owner :: {0}'.format(metadata_local)) | 562 | raise (OSProjectNotFound, "no owner :: {0}".format(metadata_local)) |
1237 | 524 | 563 | ||
1241 | 525 | if metadata_local['owner'] not in self.projects_slave: | 564 | if metadata_local["owner"] not in self.projects_slave: |
1242 | 526 | if 'bs_owner' in metadata_local: | 565 | if "bs_owner" in metadata_local: |
1243 | 527 | master_project_name = metadata_local['bs_owner'] | 566 | master_project_name = metadata_local["bs_owner"] |
1244 | 528 | else: | 567 | else: |
1247 | 529 | raise (OSProjectNotFound, 'no bs_owner :: ' | 568 | raise ( |
1248 | 530 | '{0}'.format(metadata_local)) | 569 | OSProjectNotFound, |
1249 | 570 | "no bs_owner :: {0}".format(metadata_local), | ||
1250 | 571 | ) | ||
1251 | 531 | 572 | ||
1252 | 532 | # XXX(aluria): image does not exist on slave service | 573 | # XXX(aluria): image does not exist on slave service |
1253 | 533 | if not metadata_slave: | 574 | if not metadata_slave: |
1254 | 534 | slave_project_id = self.project_mapping(metadata_local) | 575 | slave_project_id = self.project_mapping(metadata_local) |
1255 | 535 | if not slave_project_id: | 576 | if not slave_project_id: |
1260 | 536 | raise (OSProjectNotFound, 'no project_mapping :: ' | 577 | raise ( |
1261 | 537 | '{0}'.format(metadata_local)) | 578 | OSProjectNotFound, |
1262 | 538 | elif metadata_local['owner'] != slave_project_id: | 579 | "no project_mapping :: {0}".format(metadata_local), |
1263 | 539 | metadata_local['owner'] = slave_project_id | 580 | ) |
1264 | 581 | elif metadata_local["owner"] != slave_project_id: | ||
1265 | 582 | metadata_local["owner"] = slave_project_id | ||
1266 | 540 | # XXX(aluria): image exists on slave service | 583 | # XXX(aluria): image exists on slave service |
1267 | 541 | # XXX(aluria): keep all metadata and mangle project_id (owner) | 584 | # XXX(aluria): keep all metadata and mangle project_id (owner) |
1268 | 542 | else: | 585 | else: |
1269 | 543 | # ie. admin, services, SLAVE-CENTRAL | 586 | # ie. admin, services, SLAVE-CENTRAL |
1271 | 544 | slave_project_name = self.projects_slave[metadata_slave['owner']] | 587 | slave_project_name = self.projects_slave[metadata_slave["owner"]] |
1272 | 545 | # ie. admin, services, MASTER-CENTRAL | 588 | # ie. admin, services, MASTER-CENTRAL |
1276 | 546 | slave_to_master = \ | 589 | slave_to_master = slave_project_name.replace( |
1277 | 547 | slave_project_name.replace(self.REGION_SLAVE, | 590 | self.REGION_SLAVE, self.REGION_MASTER |
1278 | 548 | self.REGION_MASTER) | 591 | ) |
1279 | 549 | # ie. admin, services, MASTER-CENTRAL | 592 | # ie. admin, services, MASTER-CENTRAL |
1283 | 550 | if master_project_name in (slave_project_name, | 593 | if master_project_name in (slave_project_name, slave_to_master): |
1284 | 551 | slave_to_master): | 594 | metadata_local["owner"] = metadata_slave["owner"] |
1282 | 552 | metadata_local['owner'] = metadata_slave['owner'] | ||
1285 | 553 | else: | 595 | else: |
1288 | 554 | raise (OSProjectNotFound, 'project not found: ' | 596 | raise ( |
1289 | 555 | '{0}'.format(metadata_local)) | 597 | OSProjectNotFound, |
1290 | 598 | "project not found: {0}".format(metadata_local), | ||
1291 | 599 | ) | ||
1292 | 556 | 600 | ||
1295 | 557 | removed_props = [k for k in metadata_local.keys() if k not in | 601 | removed_props = [ |
1296 | 558 | self.valid_properties] | 602 | k for k in metadata_local.keys() if k not in self.valid_properties |
1297 | 603 | ] | ||
1298 | 559 | return (metadata_local, removed_props) | 604 | return (metadata_local, removed_props) |
1299 | 560 | 605 | ||
1300 | 561 | def update_metadata(self, metadata_local, metadata_slave): | 606 | def update_metadata(self, metadata_local, metadata_slave): |
1303 | 562 | self.log('INFO: image-id {0}: updating ' | 607 | self.log("INFO: image-id {0}: updating metadata".format(metadata_local["id"])) |
1302 | 563 | 'metadata'.format(metadata_local['id'])) | ||
1304 | 564 | 608 | ||
1307 | 565 | metadata, removed_props = self.mangle_metadata(metadata_local, | 609 | metadata, removed_props = self.mangle_metadata(metadata_local, metadata_slave) |
1306 | 566 | metadata_slave) | ||
1308 | 567 | for k in removed_props: | 610 | for k in removed_props: |
1309 | 568 | if k in metadata: | 611 | if k in metadata: |
1310 | 569 | del metadata[k] | 612 | del metadata[k] |
1311 | 570 | 613 | ||
1312 | 571 | try: | 614 | try: |
1315 | 572 | self.glance_slave.images.update(metadata['id'], | 615 | self.glance_slave.images.update(metadata["id"], **metadata) |
1314 | 573 | **metadata) | ||
1316 | 574 | except Exception as e: | 616 | except Exception as e: |
1319 | 575 | self.log('EXCEPTION: update_metadata :: {0} - ' | 617 | self.log( |
1320 | 576 | '{1}'.format(metadata['id'], e)) | 618 | "EXCEPTION: update_metadata :: {0} - {1}".format(metadata["id"], e) |
1321 | 619 | ) | ||
1322 | 577 | raise e | 620 | raise e |
1323 | 578 | 621 | ||
1324 | 579 | def create_lock(self, lockfile): | 622 | def create_lock(self, lockfile): |
1325 | 580 | try: | 623 | try: |
1327 | 581 | with open(lockfile, 'w') as lock: | 624 | with open(lockfile, "w") as lock: |
1328 | 582 | lock.write(str(os.getpid())) | 625 | lock.write(str(os.getpid())) |
1329 | 583 | except OSError: | 626 | except OSError: |
1331 | 584 | self.log('ERROR: could not create lockfile {0}'.format(lockfile)) | 627 | self.log("ERROR: could not create lockfile {0}".format(lockfile)) |
1332 | 585 | 628 | ||
1334 | 586 | def file_locked(self, lockfile='/tmp/glance_sync_slave.lock'): | 629 | def file_locked(self, lockfile="/tmp/glance_sync_slave.lock"): |
1335 | 587 | if os.path.isfile(lockfile): | 630 | if os.path.isfile(lockfile): |
1336 | 588 | return True | 631 | return True |
1337 | 589 | else: | 632 | else: |
1338 | 590 | return False | 633 | return False |
1339 | 591 | 634 | ||
1341 | 592 | def release_lock(self, lockfile='/tmp/glance_sync_slave.lock'): | 635 | def release_lock(self, lockfile="/tmp/glance_sync_slave.lock"): |
1342 | 593 | if os.path.isfile(lockfile): | 636 | if os.path.isfile(lockfile): |
1343 | 594 | try: | 637 | try: |
1344 | 595 | os.remove(lockfile) | 638 | os.remove(lockfile) |
1345 | 596 | except OSError as e: | 639 | except OSError as e: |
1346 | 597 | self.log(e) | 640 | self.log(e) |
1347 | 598 | 641 | ||
1349 | 599 | def set_filelock(self, lockfile='/tmp/glance_sync_slave.lock'): | 642 | def set_filelock(self, lockfile="/tmp/glance_sync_slave.lock"): |
1350 | 600 | if self.file_locked(lockfile): | 643 | if self.file_locked(lockfile): |
1352 | 601 | self.log('WARNING: sync already in progress, exiting') | 644 | self.log("WARNING: sync already in progress, exiting") |
1353 | 602 | sys.exit(2) | 645 | sys.exit(2) |
1354 | 603 | 646 | ||
1355 | 604 | self.create_lock(lockfile) | 647 | self.create_lock(lockfile) |
1356 | 605 | atexit.register(self.release_lock) | 648 | atexit.register(self.release_lock) |
1357 | 606 | 649 | ||
1358 | 607 | def main(self): | 650 | def main(self): |
1361 | 608 | self.log('starting glance sync') | 651 | self.log("starting glance sync") |
1362 | 609 | self.log('getting metadata from master') | 652 | self.log("getting metadata from master") |
1363 | 610 | self.download_metadata_from_master() | 653 | self.download_metadata_from_master() |
1364 | 611 | processed_images_ids = self.parse_glance_slave_images() | 654 | processed_images_ids = self.parse_glance_slave_images() |
1365 | 612 | self.create_missing_slave_images(processed_images_ids) | 655 | self.create_missing_slave_images(processed_images_ids) |
1367 | 613 | self.log('ending glance image sync slave run') | 656 | self.log("ending glance image sync slave run") |
1368 | 614 | self.release_lock() | 657 | self.release_lock() |
1369 | 615 | 658 | ||
1370 | 616 | 659 | ||
1374 | 617 | if __name__ == '__main__': | 660 | if __name__ == "__main__": |
1375 | 618 | parser = argparse.ArgumentParser(description='Synchronize remote images ' | 661 | parser = argparse.ArgumentParser( |
1376 | 619 | 'metadata to disk and import into glance') | 662 | description="Synchronize remote images " |
1377 | 663 | "metadata to disk and import into glance" | ||
1378 | 664 | ) | ||
1379 | 620 | parser.add_argument("-d", "--datadir", help="directory to write images to") | 665 | parser.add_argument("-d", "--datadir", help="directory to write images to") |
1384 | 621 | parser.add_argument("-s", "--source", help="full path to master rsync " | 666 | parser.add_argument( |
1385 | 622 | "source. Format: " | 667 | "-s", |
1386 | 623 | "<user>@<hostname>:<port>/" | 668 | "--source", |
1387 | 624 | "<directory>") | 669 | help="full path to master rsync " |
1388 | 670 | "source. Format: " | ||
1389 | 671 | "<user>@<hostname>:<port>/" | ||
1390 | 672 | "<directory>", | ||
1391 | 673 | ) | ||
1392 | 625 | args = parser.parse_args() | 674 | args = parser.parse_args() |
1393 | 626 | 675 | ||
1394 | 627 | if args.datadir: | 676 | if args.datadir: |
1395 | 628 | data_dir = args.datadir | 677 | data_dir = args.datadir |
1396 | 629 | else: | 678 | else: |
1397 | 630 | parser.print_help() | 679 | parser.print_help() |
1399 | 631 | sys.exit('ERROR: please specify an output directory for images') | 680 | sys.exit("ERROR: please specify an output directory for images") |
1400 | 632 | 681 | ||
1401 | 633 | if args.source: | 682 | if args.source: |
1402 | 634 | source = args.source | 683 | source = args.source |
1403 | 635 | else: | 684 | else: |
1404 | 636 | parser.print_help() | 685 | parser.print_help() |
1406 | 637 | sys.exit('ERROR: please specify an image source to sync from') | 686 | sys.exit("ERROR: please specify an image source to sync from") |
1407 | 638 | 687 | ||
1408 | 639 | slave = ImageSyncSlave(data_dir, source) | 688 | slave = ImageSyncSlave(data_dir, source) |
1409 | 640 | slave.main() | 689 | slave.main() |
1410 | diff --git a/src/reactive/glance_sync.py b/src/reactive/glance_sync.py | |||
1411 | index bd80f43..dff83c6 100644 | |||
1412 | --- a/src/reactive/glance_sync.py | |||
1413 | +++ b/src/reactive/glance_sync.py | |||
1414 | @@ -15,59 +15,53 @@ from charmhelpers.contrib.openstack.utils import config_flags_parser | |||
1415 | 15 | from charms.reactive import hook, clear_flag, when, when_any | 15 | from charms.reactive import hook, clear_flag, when, when_any |
1416 | 16 | 16 | ||
1417 | 17 | 17 | ||
1419 | 18 | @hook('install') | 18 | @hook("install") |
1420 | 19 | def install_glance_sync(): | 19 | def install_glance_sync(): |
1421 | 20 | """Install glance-sync charm.""" | 20 | """Install glance-sync charm.""" |
1423 | 21 | hookenv.status_set('maintenance', 'Installing') | 21 | hookenv.status_set("maintenance", "Installing") |
1424 | 22 | configure_config_dir() | 22 | configure_config_dir() |
1425 | 23 | configure_log_dir() | 23 | configure_log_dir() |
1426 | 24 | configure_script_dir() | 24 | configure_script_dir() |
1427 | 25 | configure_sync_mode() | 25 | configure_sync_mode() |
1428 | 26 | 26 | ||
1431 | 27 | homedir = os.path.expanduser('~ubuntu') | 27 | homedir = os.path.expanduser("~ubuntu") |
1432 | 28 | ssh_identity = '{}/.ssh/id_rsa'.format(homedir) | 28 | ssh_identity = "{}/.ssh/id_rsa".format(homedir) |
1433 | 29 | if not os.path.exists(ssh_identity): | 29 | if not os.path.exists(ssh_identity): |
1439 | 30 | command = ['ssh-keygen', '-t', 'rsa', '-N', '', | 30 | command = ["ssh-keygen", "-t", "rsa", "-N", "", "-f", ssh_identity] |
1440 | 31 | '-f', ssh_identity] | 31 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
1436 | 32 | proc = subprocess.Popen(command, | ||
1437 | 33 | stdout=subprocess.PIPE, | ||
1438 | 34 | stderr=subprocess.PIPE) | ||
1441 | 35 | (stdout, stderr) = proc.communicate() | 32 | (stdout, stderr) = proc.communicate() |
1442 | 36 | if proc.returncode: | 33 | if proc.returncode: |
1445 | 37 | print("ERROR: problem generating ssh key '{}':" | 34 | print("ERROR: problem generating ssh key '{}':".format(command)) |
1444 | 38 | .format(command)) | ||
1446 | 39 | print(stderr) | 35 | print(stderr) |
1450 | 40 | os.chown(ssh_identity, | 36 | os.chown( |
1451 | 41 | pwd.getpwnam('ubuntu').pw_uid, | 37 | ssh_identity, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid |
1452 | 42 | grp.getgrnam('ubuntu').gr_gid) | 38 | ) |
1453 | 43 | 39 | ||
1455 | 44 | hookenv.status_set('active', 'Unit is ready') | 40 | hookenv.status_set("active", "Unit is ready") |
1456 | 45 | 41 | ||
1457 | 46 | 42 | ||
1459 | 47 | @when('config.changed.master_mode') | 43 | @when("config.changed.master_mode") |
1460 | 48 | def configure_sync_mode(): | 44 | def configure_sync_mode(): |
1461 | 49 | """Configure glance-sync charm to be either master or slave.""" | 45 | """Configure glance-sync charm to be either master or slave.""" |
1464 | 50 | master_enabled = hookenv.config('master_mode') | 46 | master_enabled = hookenv.config("master_mode") |
1465 | 51 | hookenv.log('configuring mode') | 47 | hookenv.log("configuring mode") |
1466 | 52 | if master_enabled: | 48 | if master_enabled: |
1470 | 53 | hookenv.status_set('maintenance', | 49 | hookenv.status_set("maintenance", "Configuring master") |
1471 | 54 | 'Configuring master') | 50 | hookenv.log("configuring unit as master") |
1469 | 55 | hookenv.log('configuring unit as master') | ||
1472 | 56 | perform_slave_cleanup() | 51 | perform_slave_cleanup() |
1473 | 57 | install_master_sync_script() | 52 | install_master_sync_script() |
1476 | 58 | hookenv.log('opening TCP port 22') | 53 | hookenv.log("opening TCP port 22") |
1477 | 59 | open_port(22, protocol='TCP') | 54 | open_port(22, protocol="TCP") |
1478 | 60 | else: | 55 | else: |
1482 | 61 | hookenv.status_set('maintenance', | 56 | hookenv.status_set("maintenance", "Configuring slave") |
1483 | 62 | 'Configuring slave') | 57 | hookenv.log("configuring unit as slave") |
1481 | 63 | hookenv.log('configuring unit as slave') | ||
1484 | 64 | perform_master_cleanup() | 58 | perform_master_cleanup() |
1485 | 65 | install_slave_sync_script() | 59 | install_slave_sync_script() |
1486 | 66 | 60 | ||
1487 | 67 | install_db_cleanup_script() | 61 | install_db_cleanup_script() |
1488 | 68 | configure_cron() | 62 | configure_cron() |
1489 | 69 | configure_data_dir() | 63 | configure_data_dir() |
1491 | 70 | hookenv.status_set('active', 'Unit is ready') | 64 | hookenv.status_set("active", "Unit is ready") |
1492 | 71 | 65 | ||
1493 | 72 | 66 | ||
1494 | 73 | def perform_slave_cleanup(): | 67 | def perform_slave_cleanup(): |
1495 | @@ -75,29 +69,29 @@ def perform_slave_cleanup(): | |||
1496 | 75 | Cleanup glance-sync slave files, once the charm is set | 69 | Cleanup glance-sync slave files, once the charm is set |
1497 | 76 | to be the master, after being a slave. | 70 | to be the master, after being a slave. |
1498 | 77 | """ | 71 | """ |
1500 | 78 | data_dir = hookenv.config('data_dir') | 72 | data_dir = hookenv.config("data_dir") |
1501 | 79 | if os.path.exists(data_dir): | 73 | if os.path.exists(data_dir): |
1502 | 80 | shutil.rmtree(data_dir, ignore_errors=True) | 74 | shutil.rmtree(data_dir, ignore_errors=True) |
1503 | 81 | 75 | ||
1505 | 82 | script_dir = hookenv.config('script_dir') | 76 | script_dir = hookenv.config("script_dir") |
1506 | 83 | if os.path.exists(script_dir): | 77 | if os.path.exists(script_dir): |
1507 | 84 | shutil.rmtree(script_dir, ignore_errors=True) | 78 | shutil.rmtree(script_dir, ignore_errors=True) |
1508 | 85 | 79 | ||
1511 | 86 | config_dir = hookenv.config('config_dir') | 80 | config_dir = hookenv.config("config_dir") |
1512 | 87 | novarc_file = os.path.join(config_dir, 'novarc') | 81 | novarc_file = os.path.join(config_dir, "novarc") |
1513 | 88 | if os.path.isfile(novarc_file): | 82 | if os.path.isfile(novarc_file): |
1514 | 89 | os.remove(novarc_file) | 83 | os.remove(novarc_file) |
1515 | 90 | 84 | ||
1517 | 91 | clouds_yaml_dir = '/etc/openstack' | 85 | clouds_yaml_dir = "/etc/openstack" |
1518 | 92 | if os.path.exists(clouds_yaml_dir): | 86 | if os.path.exists(clouds_yaml_dir): |
1519 | 93 | shutil.rmtree(clouds_yaml_dir, ignore_errors=True) | 87 | shutil.rmtree(clouds_yaml_dir, ignore_errors=True) |
1520 | 94 | 88 | ||
1522 | 95 | cron_file = '/etc/cron.d/glance_sync_slave' | 89 | cron_file = "/etc/cron.d/glance_sync_slave" |
1523 | 96 | if os.path.isfile(cron_file): | 90 | if os.path.isfile(cron_file): |
1524 | 97 | os.remove(cron_file) | 91 | os.remove(cron_file) |
1526 | 98 | clear_flag('cron.configured') | 92 | clear_flag("cron.configured") |
1527 | 99 | 93 | ||
1529 | 100 | clear_flag('slave.configured') | 94 | clear_flag("slave.configured") |
1530 | 101 | 95 | ||
1531 | 102 | 96 | ||
1532 | 103 | def perform_master_cleanup(): | 97 | def perform_master_cleanup(): |
1533 | @@ -105,168 +99,162 @@ def perform_master_cleanup(): | |||
1534 | 105 | Cleanup glance-sync master files, once the charm is set | 99 | Cleanup glance-sync master files, once the charm is set |
1535 | 106 | to be a slave, after being the master. | 100 | to be a slave, after being the master. |
1536 | 107 | """ | 101 | """ |
1538 | 108 | data_dir = hookenv.config('data_dir') | 102 | data_dir = hookenv.config("data_dir") |
1539 | 109 | if os.path.exists(data_dir): | 103 | if os.path.exists(data_dir): |
1540 | 110 | shutil.rmtree(data_dir, ignore_errors=True) | 104 | shutil.rmtree(data_dir, ignore_errors=True) |
1541 | 111 | 105 | ||
1543 | 112 | script_dir = hookenv.config('script_dir') | 106 | script_dir = hookenv.config("script_dir") |
1544 | 113 | if os.path.exists(script_dir): | 107 | if os.path.exists(script_dir): |
1545 | 114 | shutil.rmtree(script_dir, ignore_errors=True) | 108 | shutil.rmtree(script_dir, ignore_errors=True) |
1546 | 115 | 109 | ||
1549 | 116 | config_dir = hookenv.config('config_dir') | 110 | config_dir = hookenv.config("config_dir") |
1550 | 117 | novarc_file = os.path.join(config_dir, 'novarc') | 111 | novarc_file = os.path.join(config_dir, "novarc") |
1551 | 118 | if os.path.isfile(novarc_file): | 112 | if os.path.isfile(novarc_file): |
1552 | 119 | os.remove(novarc_file) | 113 | os.remove(novarc_file) |
1553 | 120 | 114 | ||
1555 | 121 | cron_file = '/etc/cron.d/glance_sync_master' | 115 | cron_file = "/etc/cron.d/glance_sync_master" |
1556 | 122 | if os.path.isfile(cron_file): | 116 | if os.path.isfile(cron_file): |
1557 | 123 | os.remove(cron_file) | 117 | os.remove(cron_file) |
1559 | 124 | clear_flag('cron.configured') | 118 | clear_flag("cron.configured") |
1560 | 125 | 119 | ||
1562 | 126 | clear_flag('master.configured') | 120 | clear_flag("master.configured") |
1563 | 127 | 121 | ||
1564 | 128 | 122 | ||
1566 | 129 | @hook('upgrade-charm') | 123 | @hook("upgrade-charm") |
1567 | 130 | def upgrade_glance_sync(): | 124 | def upgrade_glance_sync(): |
1568 | 131 | """Perform charm upgrade.""" | 125 | """Perform charm upgrade.""" |
1569 | 132 | install_glance_sync() | 126 | install_glance_sync() |
1570 | 133 | 127 | ||
1571 | 134 | 128 | ||
1573 | 135 | @when('config.changed.config_dir') | 129 | @when("config.changed.config_dir") |
1574 | 136 | def configure_config_dir(): | 130 | def configure_config_dir(): |
1575 | 137 | """Configure 'config_dir' directory, and configure cron.""" | 131 | """Configure 'config_dir' directory, and configure cron.""" |
1578 | 138 | hookenv.status_set('maintenance', 'Configuring') | 132 | hookenv.status_set("maintenance", "Configuring") |
1579 | 139 | config_dir = hookenv.config('config_dir') | 133 | config_dir = hookenv.config("config_dir") |
1580 | 140 | if not os.path.exists(config_dir): | 134 | if not os.path.exists(config_dir): |
1581 | 141 | os.makedirs(config_dir) | 135 | os.makedirs(config_dir) |
1585 | 142 | os.chown(config_dir, | 136 | os.chown( |
1586 | 143 | pwd.getpwnam('ubuntu').pw_uid, | 137 | config_dir, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid |
1587 | 144 | grp.getgrnam('ubuntu').gr_gid) | 138 | ) |
1588 | 145 | 139 | ||
1589 | 146 | configure_cron() | 140 | configure_cron() |
1591 | 147 | hookenv.status_set('active', 'Unit is ready') | 141 | hookenv.status_set("active", "Unit is ready") |
1592 | 148 | 142 | ||
1593 | 149 | 143 | ||
1595 | 150 | @when('config.changed.data_dir') | 144 | @when("config.changed.data_dir") |
1596 | 151 | def configure_data_dir(): | 145 | def configure_data_dir(): |
1597 | 152 | """Configure 'data_dir' directory, and configure cron.""" | 146 | """Configure 'data_dir' directory, and configure cron.""" |
1600 | 153 | hookenv.status_set('maintenance', 'Configuring') | 147 | hookenv.status_set("maintenance", "Configuring") |
1601 | 154 | data_dir = hookenv.config('data_dir') | 148 | data_dir = hookenv.config("data_dir") |
1602 | 155 | if not os.path.exists(data_dir): | 149 | if not os.path.exists(data_dir): |
1603 | 156 | os.makedirs(data_dir) | 150 | os.makedirs(data_dir) |
1607 | 157 | os.chown(data_dir, | 151 | os.chown(data_dir, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
1605 | 158 | pwd.getpwnam('ubuntu').pw_uid, | ||
1606 | 159 | grp.getgrnam('ubuntu').gr_gid) | ||
1608 | 160 | 152 | ||
1609 | 161 | configure_cron() | 153 | configure_cron() |
1611 | 162 | hookenv.status_set('active', 'Unit is ready') | 154 | hookenv.status_set("active", "Unit is ready") |
1612 | 163 | 155 | ||
1613 | 164 | 156 | ||
1615 | 165 | @when('config.changed.log_dir') | 157 | @when("config.changed.log_dir") |
1616 | 166 | def configure_log_dir(): | 158 | def configure_log_dir(): |
1617 | 167 | """ | 159 | """ |
1618 | 168 | Configure 'log_dir' directory, setup lograte for glance-sync | 160 | Configure 'log_dir' directory, setup lograte for glance-sync |
1619 | 169 | log files, and configure cron. | 161 | log files, and configure cron. |
1620 | 170 | """ | 162 | """ |
1623 | 171 | hookenv.status_set('maintenance', 'Configuring') | 163 | hookenv.status_set("maintenance", "Configuring") |
1624 | 172 | log_dir = hookenv.config('log_dir') | 164 | log_dir = hookenv.config("log_dir") |
1625 | 173 | if not os.path.exists(log_dir): | 165 | if not os.path.exists(log_dir): |
1626 | 174 | os.makedirs(log_dir) | 166 | os.makedirs(log_dir) |
1630 | 175 | os.chown(log_dir, | 167 | os.chown(log_dir, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
1628 | 176 | pwd.getpwnam('ubuntu').pw_uid, | ||
1629 | 177 | grp.getgrnam('ubuntu').gr_gid) | ||
1631 | 178 | templating.render( | 168 | templating.render( |
1636 | 179 | source='logrotate.d.j2', | 169 | source="logrotate.d.j2", |
1637 | 180 | target='/etc/logrotate.d/glance_sync', | 170 | target="/etc/logrotate.d/glance_sync", |
1638 | 181 | owner='root', | 171 | owner="root", |
1639 | 182 | group='root', | 172 | group="root", |
1640 | 183 | perms=0o644, | 173 | perms=0o644, |
1641 | 184 | context=hookenv.config(), | 174 | context=hookenv.config(), |
1642 | 185 | ) | 175 | ) |
1643 | 186 | 176 | ||
1644 | 187 | configure_cron() | 177 | configure_cron() |
1646 | 188 | hookenv.status_set('active', 'Unit is ready') | 178 | hookenv.status_set("active", "Unit is ready") |
1647 | 189 | 179 | ||
1648 | 190 | 180 | ||
1650 | 191 | @when('config.changed.script_dir') | 181 | @when("config.changed.script_dir") |
1651 | 192 | def configure_script_dir(): | 182 | def configure_script_dir(): |
1652 | 193 | """Configure 'script_dir' directory, and configure cron.""" | 183 | """Configure 'script_dir' directory, and configure cron.""" |
1655 | 194 | hookenv.status_set('maintenance', 'Configuring') | 184 | hookenv.status_set("maintenance", "Configuring") |
1656 | 195 | script_dir = hookenv.config('script_dir') | 185 | script_dir = hookenv.config("script_dir") |
1657 | 196 | if not os.path.exists(script_dir): | 186 | if not os.path.exists(script_dir): |
1658 | 197 | os.makedirs(script_dir) | 187 | os.makedirs(script_dir) |
1662 | 198 | os.chown(script_dir, | 188 | os.chown( |
1663 | 199 | pwd.getpwnam('ubuntu').pw_uid, | 189 | script_dir, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid |
1664 | 200 | grp.getgrnam('ubuntu').gr_gid) | 190 | ) |
1665 | 201 | 191 | ||
1666 | 202 | configure_cron() | 192 | configure_cron() |
1668 | 203 | hookenv.status_set('active', 'Unit is ready') | 193 | hookenv.status_set("active", "Unit is ready") |
1669 | 204 | 194 | ||
1670 | 205 | 195 | ||
1672 | 206 | @when('config.changed.authorized_keys') | 196 | @when("config.changed.authorized_keys") |
1673 | 207 | def configure_authorized_keys(): | 197 | def configure_authorized_keys(): |
1674 | 208 | """ | 198 | """ |
1675 | 209 | Configure authorized_keys file containing command-limited | 199 | Configure authorized_keys file containing command-limited |
1676 | 210 | public keys for slave(s). | 200 | public keys for slave(s). |
1677 | 211 | """ | 201 | """ |
1683 | 212 | hookenv.status_set('maintenance', 'Configuring') | 202 | hookenv.status_set("maintenance", "Configuring") |
1684 | 213 | ssh_conf_dir = '/etc/ssh/authorized-keys' | 203 | ssh_conf_dir = "/etc/ssh/authorized-keys" |
1685 | 214 | auth_keys_file = os.path.join(ssh_conf_dir, | 204 | auth_keys_file = os.path.join(ssh_conf_dir, "glance-sync-slaves") |
1686 | 215 | 'glance-sync-slaves') | 205 | authorized_keys = hookenv.config("authorized_keys") |
1682 | 216 | authorized_keys = hookenv.config('authorized_keys') | ||
1687 | 217 | if authorized_keys: | 206 | if authorized_keys: |
1688 | 218 | keys_bytestring = base64.b64decode(authorized_keys) | 207 | keys_bytestring = base64.b64decode(authorized_keys) |
1690 | 219 | keys = str(keys_bytestring, 'utf-8') | 208 | keys = str(keys_bytestring, "utf-8") |
1691 | 220 | if not os.path.exists(ssh_conf_dir): | 209 | if not os.path.exists(ssh_conf_dir): |
1692 | 221 | os.makedirs(ssh_conf_dir) | 210 | os.makedirs(ssh_conf_dir) |
1696 | 222 | os.chown(ssh_conf_dir, | 211 | os.chown( |
1697 | 223 | pwd.getpwnam('root').pw_uid, | 212 | ssh_conf_dir, pwd.getpwnam("root").pw_uid, grp.getgrnam("root").gr_gid |
1698 | 224 | grp.getgrnam('root').gr_gid) | 213 | ) |
1699 | 225 | os.chmod(ssh_conf_dir, 0o755) | 214 | os.chmod(ssh_conf_dir, 0o755) |
1701 | 226 | with open(auth_keys_file, 'w') as f: | 215 | with open(auth_keys_file, "w") as f: |
1702 | 227 | f.write(keys) | 216 | f.write(keys) |
1703 | 228 | os.chmod(auth_keys_file, 0o644) | 217 | os.chmod(auth_keys_file, 0o644) |
1704 | 229 | configure_sshd(auth_keys_file) | 218 | configure_sshd(auth_keys_file) |
1706 | 230 | hookenv.status_set('active', 'Unit is ready') | 219 | hookenv.status_set("active", "Unit is ready") |
1707 | 231 | else: | 220 | else: |
1708 | 232 | if os.path.isfile(auth_keys_file): | 221 | if os.path.isfile(auth_keys_file): |
1709 | 233 | os.remove(auth_keys_file) | 222 | os.remove(auth_keys_file) |
1711 | 234 | hookenv.status_set('active', 'Unit is ready') | 223 | hookenv.status_set("active", "Unit is ready") |
1712 | 235 | 224 | ||
1713 | 236 | 225 | ||
1714 | 237 | def configure_sshd(keys_file): | 226 | def configure_sshd(keys_file): |
1715 | 238 | """Add 'AuthorizedKeysFile' line to '/etc/ssh/sshd_config'.""" | 227 | """Add 'AuthorizedKeysFile' line to '/etc/ssh/sshd_config'.""" |
1718 | 239 | original = '/etc/ssh/sshd_config' | 228 | original = "/etc/ssh/sshd_config" |
1719 | 240 | temp_config = '/tmp/sshd_config' | 229 | temp_config = "/tmp/sshd_config" |
1720 | 241 | old_lines = [] | 230 | old_lines = [] |
1721 | 242 | auth_keys_file_added = False | 231 | auth_keys_file_added = False |
1722 | 243 | auth_keys_file_existed = False | 232 | auth_keys_file_existed = False |
1725 | 244 | homedirs = '%h/.ssh/authorized_keys' | 233 | homedirs = "%h/.ssh/authorized_keys" |
1726 | 245 | with open(original, 'r') as old_file: | 234 | with open(original, "r") as old_file: |
1727 | 246 | for line in old_file: | 235 | for line in old_file: |
1728 | 247 | old_lines.append(line) | 236 | old_lines.append(line) |
1730 | 248 | with open(temp_config, 'w') as new_file: | 237 | with open(temp_config, "w") as new_file: |
1731 | 249 | for line in old_lines: | 238 | for line in old_lines: |
1734 | 250 | if re.search(r'^AuthorizedKeysFile.*{}.*' | 239 | if re.search(r"^AuthorizedKeysFile.*{}.*".format(keys_file), line): |
1733 | 251 | .format(keys_file), line): | ||
1735 | 252 | auth_keys_file_existed = True | 240 | auth_keys_file_existed = True |
1736 | 253 | new_file.write(line) | 241 | new_file.write(line) |
1737 | 254 | continue | 242 | continue |
1738 | 255 | else: | 243 | else: |
1743 | 256 | replaced = re.sub(r'^AuthorizedKeysFile\s(.*)$', | 244 | replaced = re.sub( |
1744 | 257 | r'AuthorizedKeysFile \1 {} {}' | 245 | r"^AuthorizedKeysFile\s(.*)$", |
1745 | 258 | .format(homedirs, keys_file), | 246 | r"AuthorizedKeysFile \1 {} {}".format(homedirs, keys_file), |
1746 | 259 | line) | 247 | line, |
1747 | 248 | ) | ||
1748 | 260 | if replaced is not line: | 249 | if replaced is not line: |
1749 | 261 | auth_keys_file_added = True | 250 | auth_keys_file_added = True |
1750 | 262 | new_file.write(replaced) | 251 | new_file.write(replaced) |
1751 | 263 | 252 | ||
1752 | 264 | if not auth_keys_file_existed and not auth_keys_file_added: | 253 | if not auth_keys_file_existed and not auth_keys_file_added: |
1755 | 265 | new_file.write('AuthorizedKeysFile {} {}\n' | 254 | new_file.write("AuthorizedKeysFile {} {}\n".format(homedirs, keys_file)) |
1754 | 266 | .format(homedirs, keys_file)) | ||
1756 | 267 | 255 | ||
1757 | 268 | shutil.copy(temp_config, original) | 256 | shutil.copy(temp_config, original) |
1759 | 269 | os.system('sudo sshd -t && sudo service ssh reload') | 257 | os.system("sudo sshd -t && sudo service ssh reload") |
1760 | 270 | 258 | ||
1761 | 271 | 259 | ||
1762 | 272 | def install_slave_sync_script(): | 260 | def install_slave_sync_script(): |
1763 | @@ -274,22 +262,19 @@ def install_slave_sync_script(): | |||
1764 | 274 | Install slave files, and corresponding directory | 262 | Install slave files, and corresponding directory |
1765 | 275 | structure. | 263 | structure. |
1766 | 276 | """ | 264 | """ |
1771 | 277 | hookenv.status_set('maintenance', 'Installing') | 265 | hookenv.status_set("maintenance", "Installing") |
1772 | 278 | hookenv.log('installing slave sync script') | 266 | hookenv.log("installing slave sync script") |
1773 | 279 | script_dir = hookenv.config('script_dir') | 267 | script_dir = hookenv.config("script_dir") |
1774 | 280 | files = ['glance_sync_slave.py'] | 268 | files = ["glance_sync_slave.py"] |
1775 | 281 | for file in files: | 269 | for file in files: |
1778 | 282 | source = os.path.join(hookenv.charm_dir(), | 270 | source = os.path.join(hookenv.charm_dir(), "files", file) |
1777 | 283 | 'files', file) | ||
1779 | 284 | dest = os.path.join(script_dir, file) | 271 | dest = os.path.join(script_dir, file) |
1780 | 285 | if not os.path.exists(os.path.dirname(dest)): | 272 | if not os.path.exists(os.path.dirname(dest)): |
1781 | 286 | os.makedirs(os.path.dirname(dest)) | 273 | os.makedirs(os.path.dirname(dest)) |
1782 | 287 | shutil.copy(source, dest) | 274 | shutil.copy(source, dest) |
1786 | 288 | os.chown(dest, | 275 | os.chown(dest, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
1784 | 289 | pwd.getpwnam('ubuntu').pw_uid, | ||
1785 | 290 | grp.getgrnam('ubuntu').gr_gid) | ||
1787 | 291 | os.chmod(dest, 0o750) | 276 | os.chmod(dest, 0o750) |
1789 | 292 | hookenv.status_set('active', 'Unit is ready') | 277 | hookenv.status_set("active", "Unit is ready") |
1790 | 293 | 278 | ||
1791 | 294 | 279 | ||
1792 | 295 | def install_master_sync_script(): | 280 | def install_master_sync_script(): |
1793 | @@ -297,22 +282,19 @@ def install_master_sync_script(): | |||
1794 | 297 | Install master files, and corresponding directory | 282 | Install master files, and corresponding directory |
1795 | 298 | structure. | 283 | structure. |
1796 | 299 | """ | 284 | """ |
1801 | 300 | hookenv.status_set('maintenance', 'Installing') | 285 | hookenv.status_set("maintenance", "Installing") |
1802 | 301 | hookenv.log('installing master sync script') | 286 | hookenv.log("installing master sync script") |
1803 | 302 | script_dir = hookenv.config('script_dir') | 287 | script_dir = hookenv.config("script_dir") |
1804 | 303 | files = ['glance_sync_master.py'] | 288 | files = ["glance_sync_master.py"] |
1805 | 304 | for file in files: | 289 | for file in files: |
1808 | 305 | source = os.path.join(hookenv.charm_dir(), | 290 | source = os.path.join(hookenv.charm_dir(), "files", file) |
1807 | 306 | 'files', file) | ||
1809 | 307 | dest = os.path.join(script_dir, file) | 291 | dest = os.path.join(script_dir, file) |
1810 | 308 | if not os.path.exists(os.path.dirname(dest)): | 292 | if not os.path.exists(os.path.dirname(dest)): |
1811 | 309 | os.makedirs(os.path.dirname(dest)) | 293 | os.makedirs(os.path.dirname(dest)) |
1812 | 310 | shutil.copy(source, dest) | 294 | shutil.copy(source, dest) |
1816 | 311 | os.chown(dest, | 295 | os.chown(dest, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
1814 | 312 | pwd.getpwnam('ubuntu').pw_uid, | ||
1815 | 313 | grp.getgrnam('ubuntu').gr_gid) | ||
1817 | 314 | os.chmod(dest, 0o750) | 296 | os.chmod(dest, 0o750) |
1819 | 315 | hookenv.status_set('active', 'Unit is ready') | 297 | hookenv.status_set("active", "Unit is ready") |
1820 | 316 | 298 | ||
1821 | 317 | 299 | ||
1822 | 318 | def install_db_cleanup_script(): | 300 | def install_db_cleanup_script(): |
1823 | @@ -320,102 +302,102 @@ def install_db_cleanup_script(): | |||
1824 | 320 | Install 'db_purge_deleted_glance_images.py' for | 302 | Install 'db_purge_deleted_glance_images.py' for |
1825 | 321 | master or slave. | 303 | master or slave. |
1826 | 322 | """ | 304 | """ |
1830 | 323 | hookenv.status_set('maintenance', 'Installing') | 305 | hookenv.status_set("maintenance", "Installing") |
1831 | 324 | script_dir = hookenv.config('script_dir') | 306 | script_dir = hookenv.config("script_dir") |
1832 | 325 | master_enabled = hookenv.config('master_mode') | 307 | master_enabled = hookenv.config("master_mode") |
1833 | 326 | if master_enabled: | 308 | if master_enabled: |
1835 | 327 | mode = 'master' | 309 | mode = "master" |
1836 | 328 | else: | 310 | else: |
1843 | 329 | mode = 'slave' | 311 | mode = "slave" |
1844 | 330 | source = os.path.join(hookenv.charm_dir(), 'files', | 312 | source = os.path.join( |
1845 | 331 | 'db_purge_deleted_' + mode, | 313 | hookenv.charm_dir(), |
1846 | 332 | 'db_purge_deleted_glance_images.py') | 314 | "files", |
1847 | 333 | dest = os.path.join(script_dir, | 315 | "db_purge_deleted_" + mode, |
1848 | 334 | 'db_purge_deleted_glance_images.py') | 316 | "db_purge_deleted_glance_images.py", |
1849 | 317 | ) | ||
1850 | 318 | dest = os.path.join(script_dir, "db_purge_deleted_glance_images.py") | ||
1851 | 335 | shutil.copy(source, dest) | 319 | shutil.copy(source, dest) |
1855 | 336 | os.chown(dest, | 320 | os.chown(dest, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
1853 | 337 | pwd.getpwnam('ubuntu').pw_uid, | ||
1854 | 338 | grp.getgrnam('ubuntu').gr_gid) | ||
1856 | 339 | os.chmod(dest, 0o750) | 321 | os.chmod(dest, 0o750) |
1858 | 340 | hookenv.status_set('active', 'Unit is ready') | 322 | hookenv.status_set("active", "Unit is ready") |
1859 | 341 | 323 | ||
1860 | 342 | 324 | ||
1862 | 343 | @when('config.changed.master_creds') | 325 | @when("config.changed.master_creds") |
1863 | 344 | def configure_master_novarc(): | 326 | def configure_master_novarc(): |
1864 | 345 | """ | 327 | """ |
1865 | 346 | Get context from 'master_creds'. Put the info into 'clouds.yaml' | 328 | Get context from 'master_creds'. Put the info into 'clouds.yaml' |
1866 | 347 | for the openstacksdk. | 329 | for the openstacksdk. |
1867 | 348 | """ | 330 | """ |
1870 | 349 | keystone_creds = config_flags_parser(hookenv.config('master_creds')) | 331 | keystone_creds = config_flags_parser(hookenv.config("master_creds")) |
1871 | 350 | clouds_yaml_dir = '/etc/openstack' | 332 | clouds_yaml_dir = "/etc/openstack" |
1872 | 351 | if not keystone_creds: | 333 | if not keystone_creds: |
1874 | 352 | hookenv.status_set('blocked', 'Please add master_creds') | 334 | hookenv.status_set("blocked", "Please add master_creds") |
1875 | 353 | return | 335 | return |
1876 | 354 | elif not os.path.exists(clouds_yaml_dir): | 336 | elif not os.path.exists(clouds_yaml_dir): |
1877 | 355 | creds = {} | 337 | creds = {} |
1880 | 356 | keystone_creds = config_flags_parser(hookenv.config('master_creds')) | 338 | keystone_creds = config_flags_parser(hookenv.config("master_creds")) |
1881 | 357 | if '/v3' in keystone_creds['auth_url']: | 339 | if "/v3" in keystone_creds["auth_url"]: |
1882 | 358 | creds = { | 340 | creds = { |
1891 | 359 | 'master_username': keystone_creds['username'], | 341 | "master_username": keystone_creds["username"], |
1892 | 360 | 'master_password': keystone_creds['password'], | 342 | "master_password": keystone_creds["password"], |
1893 | 361 | 'master_project': keystone_creds['project'], | 343 | "master_project": keystone_creds["project"], |
1894 | 362 | 'master_region': keystone_creds['region'], | 344 | "master_region": keystone_creds["region"], |
1895 | 363 | 'master_auth_url': keystone_creds['auth_url'], | 345 | "master_auth_url": keystone_creds["auth_url"], |
1896 | 364 | 'master_auth_version': '3', | 346 | "master_auth_version": "3", |
1897 | 365 | 'master_user_domain': keystone_creds['domain'], | 347 | "master_user_domain": keystone_creds["domain"], |
1898 | 366 | 'master_project_domain': keystone_creds['domain'], | 348 | "master_project_domain": keystone_creds["domain"], |
1899 | 367 | } | 349 | } |
1900 | 368 | else: | 350 | else: |
1901 | 369 | creds = { | 351 | creds = { |
1907 | 370 | 'master_username': keystone_creds['username'], | 352 | "master_username": keystone_creds["username"], |
1908 | 371 | 'master_password': keystone_creds['password'], | 353 | "master_password": keystone_creds["password"], |
1909 | 372 | 'master_project': keystone_creds['project'], | 354 | "master_project": keystone_creds["project"], |
1910 | 373 | 'master_region': keystone_creds['region'], | 355 | "master_region": keystone_creds["region"], |
1911 | 374 | 'master_auth_url': keystone_creds['auth_url'], | 356 | "master_auth_url": keystone_creds["auth_url"], |
1912 | 375 | } | 357 | } |
1913 | 376 | elif os.path.exists(clouds_yaml_dir): | 358 | elif os.path.exists(clouds_yaml_dir): |
1914 | 377 | shutil.rmtree(clouds_yaml_dir, ignore_errors=True) | 359 | shutil.rmtree(clouds_yaml_dir, ignore_errors=True) |
1915 | 378 | creds = {} | 360 | creds = {} |
1918 | 379 | keystone_creds = config_flags_parser(hookenv.config('master_creds')) | 361 | keystone_creds = config_flags_parser(hookenv.config("master_creds")) |
1919 | 380 | if '/v3' in keystone_creds['auth_url']: | 362 | if "/v3" in keystone_creds["auth_url"]: |
1920 | 381 | creds = { | 363 | creds = { |
1929 | 382 | 'master_username': keystone_creds['username'], | 364 | "master_username": keystone_creds["username"], |
1930 | 383 | 'master_password': keystone_creds['password'], | 365 | "master_password": keystone_creds["password"], |
1931 | 384 | 'master_project': keystone_creds['project'], | 366 | "master_project": keystone_creds["project"], |
1932 | 385 | 'master_region': keystone_creds['region'], | 367 | "master_region": keystone_creds["region"], |
1933 | 386 | 'master_auth_url': keystone_creds['auth_url'], | 368 | "master_auth_url": keystone_creds["auth_url"], |
1934 | 387 | 'master_auth_version': '3', | 369 | "master_auth_version": "3", |
1935 | 388 | 'master_user_domain': keystone_creds['domain'], | 370 | "master_user_domain": keystone_creds["domain"], |
1936 | 389 | 'master_project_domain': keystone_creds['domain'], | 371 | "master_project_domain": keystone_creds["domain"], |
1937 | 390 | } | 372 | } |
1938 | 391 | else: | 373 | else: |
1939 | 392 | creds = { | 374 | creds = { |
1945 | 393 | 'master_username': keystone_creds['username'], | 375 | "master_username": keystone_creds["username"], |
1946 | 394 | 'master_password': keystone_creds['password'], | 376 | "master_password": keystone_creds["password"], |
1947 | 395 | 'master_project': keystone_creds['project'], | 377 | "master_project": keystone_creds["project"], |
1948 | 396 | 'master_region': keystone_creds['region'], | 378 | "master_region": keystone_creds["region"], |
1949 | 397 | 'master_auth_url': keystone_creds['auth_url'], | 379 | "master_auth_url": keystone_creds["auth_url"], |
1950 | 398 | } | 380 | } |
1952 | 399 | clouds_yaml = os.path.join(clouds_yaml_dir, 'clouds.yaml') | 381 | clouds_yaml = os.path.join(clouds_yaml_dir, "clouds.yaml") |
1953 | 400 | templating.render( | 382 | templating.render( |
1955 | 401 | source='clouds.yaml.j2', | 383 | source="clouds.yaml.j2", |
1956 | 402 | target=clouds_yaml, | 384 | target=clouds_yaml, |
1959 | 403 | owner='ubuntu', | 385 | owner="ubuntu", |
1960 | 404 | group='ubuntu', | 386 | group="ubuntu", |
1961 | 405 | perms=0o600, | 387 | perms=0o600, |
1962 | 406 | context=creds, | 388 | context=creds, |
1963 | 407 | ) | 389 | ) |
1964 | 408 | configure_novarc() | 390 | configure_novarc() |
1966 | 409 | hookenv.status_set('active', 'Unit is ready') | 391 | hookenv.status_set("active", "Unit is ready") |
1967 | 410 | 392 | ||
1968 | 411 | 393 | ||
1970 | 412 | @when('config.changed.novarc') | 394 | @when("config.changed.novarc") |
1971 | 413 | def configure_custom_novarc(): | 395 | def configure_custom_novarc(): |
1972 | 414 | """Configure 'novarc' file after config change.""" | 396 | """Configure 'novarc' file after config change.""" |
1973 | 415 | configure_novarc() | 397 | configure_novarc() |
1974 | 416 | 398 | ||
1975 | 417 | 399 | ||
1977 | 418 | @hook('keystone-admin-relation-{joined,changed}') | 400 | @hook("keystone-admin-relation-{joined,changed}") |
1978 | 419 | def configure_relation_novarc(relation=None): | 401 | def configure_relation_novarc(relation=None): |
1979 | 420 | """ | 402 | """ |
1980 | 421 | Configure 'novarc' file after adding keystone | 403 | Configure 'novarc' file after adding keystone |
1981 | @@ -424,7 +406,7 @@ def configure_relation_novarc(relation=None): | |||
1982 | 424 | configure_novarc() | 406 | configure_novarc() |
1983 | 425 | 407 | ||
1984 | 426 | 408 | ||
1986 | 427 | @hook('database-relation-{joined,changed}') | 409 | @hook("database-relation-{joined,changed}") |
1987 | 428 | def configure_relation_glancedb(relation=None): | 410 | def configure_relation_glancedb(relation=None): |
1988 | 429 | """ | 411 | """ |
1989 | 430 | Configure 'novarc' file after adding database | 412 | Configure 'novarc' file after adding database |
1990 | @@ -438,94 +420,88 @@ def configure_novarc(): | |||
1991 | 438 | Configure 'novarc' file from user supplied custom novarc | 420 | Configure 'novarc' file from user supplied custom novarc |
1992 | 439 | file, or using keystone and mysql relations. | 421 | file, or using keystone and mysql relations. |
1993 | 440 | """ | 422 | """ |
2000 | 441 | hookenv.status_set('maintenance', 'Configuring') | 423 | hookenv.status_set("maintenance", "Configuring") |
2001 | 442 | keystone_relations = hookenv.relations_of_type('keystone-admin') | 424 | keystone_relations = hookenv.relations_of_type("keystone-admin") |
2002 | 443 | db_relations = hookenv.relations_of_type('database') | 425 | db_relations = hookenv.relations_of_type("database") |
2003 | 444 | config_dir = hookenv.config('config_dir') | 426 | config_dir = hookenv.config("config_dir") |
2004 | 445 | novarc_file = os.path.join(config_dir, 'novarc') | 427 | novarc_file = os.path.join(config_dir, "novarc") |
2005 | 446 | custom_novarc = hookenv.config('novarc') | 428 | custom_novarc = hookenv.config("novarc") |
2006 | 447 | if len(keystone_relations) > 0 and len(db_relations) > 0: | 429 | if len(keystone_relations) > 0 and len(db_relations) > 0: |
2007 | 448 | write_relation_novarc(novarc_file) | 430 | write_relation_novarc(novarc_file) |
2009 | 449 | elif not custom_novarc == '': | 431 | elif not custom_novarc == "": |
2010 | 450 | write_custom_novarc(novarc_file) | 432 | write_custom_novarc(novarc_file) |
2011 | 451 | else: | 433 | else: |
2014 | 452 | hookenv.log('ERROR: set novarc config or add keystone and ' | 434 | hookenv.log("ERROR: set novarc config or add keystone and database relations") |
2013 | 453 | 'database relations') | ||
2015 | 454 | if os.path.isfile(novarc_file): | 435 | if os.path.isfile(novarc_file): |
2016 | 455 | os.remove(novarc_file) | 436 | os.remove(novarc_file) |
2020 | 456 | clear_flag('novarc.configured') | 437 | clear_flag("novarc.configured") |
2021 | 457 | hookenv.status_set('blocked', 'Set novarc config or add keystone ' | 438 | hookenv.status_set( |
2022 | 458 | 'and database relations') | 439 | "blocked", "Set novarc config or add keystone and database relations" |
2023 | 440 | ) | ||
2024 | 459 | 441 | ||
2025 | 460 | 442 | ||
2026 | 461 | def write_relation_novarc(path): # noqa: C901 is too complex (14) | 443 | def write_relation_novarc(path): # noqa: C901 is too complex (14) |
2027 | 462 | """Write 'novarc' file.""" | 444 | """Write 'novarc' file.""" |
2030 | 463 | hookenv.status_set('maintenance', 'Configuring novarc') | 445 | hookenv.status_set("maintenance", "Configuring novarc") |
2031 | 464 | hookenv.log('configuring novarc based on keystone relation') | 446 | hookenv.log("configuring novarc based on keystone relation") |
2032 | 465 | # TODO: replace this with some better way to get the master region | 447 | # TODO: replace this with some better way to get the master region |
2033 | 466 | # name available. Query from the client doesn't have permissions, | 448 | # name available. Query from the client doesn't have permissions, |
2034 | 467 | # it is in 'clouds.yaml'. Possible alternative is to use a cross model | 449 | # it is in 'clouds.yaml'. Possible alternative is to use a cross model |
2035 | 468 | # relation. | 450 | # relation. |
2036 | 469 | context = {} | 451 | context = {} |
2040 | 470 | clouds_yaml_dir = '/etc/openstack' | 452 | clouds_yaml_dir = "/etc/openstack" |
2041 | 471 | master_creds_set = hookenv.config('master_creds') | 453 | master_creds_set = hookenv.config("master_creds") |
2042 | 472 | master_enabled = hookenv.config('master_mode') | 454 | master_enabled = hookenv.config("master_mode") |
2043 | 473 | if master_enabled: | 455 | if master_enabled: |
2045 | 474 | keystone = hookenv.relations_of_type('keystone-admin') | 456 | keystone = hookenv.relations_of_type("keystone-admin") |
2046 | 475 | if len(keystone) > 0: | 457 | if len(keystone) > 0: |
2047 | 476 | relation = keystone[0] | 458 | relation = keystone[0] |
2051 | 477 | if 'service_password' in relation and relation['service_password']: | 459 | if "service_password" in relation and relation["service_password"]: |
2052 | 478 | context['keystone'] = copy(relation) | 460 | context["keystone"] = copy(relation) |
2053 | 479 | elif not master_enabled and master_creds_set != '': | 461 | elif not master_enabled and master_creds_set != "": |
2054 | 480 | if not os.path.exists(clouds_yaml_dir): | 462 | if not os.path.exists(clouds_yaml_dir): |
2055 | 481 | configure_master_novarc() | 463 | configure_master_novarc() |
2060 | 482 | keystone_creds = config_flags_parser(hookenv | 464 | keystone_creds = config_flags_parser(hookenv.config("master_creds")) |
2061 | 483 | .config('master_creds')) | 465 | context["keystone_master_region"] = keystone_creds["region"] |
2062 | 484 | context['keystone_master_region'] = keystone_creds['region'] | 466 | keystone = hookenv.relations_of_type("keystone-admin") |
2059 | 485 | keystone = hookenv.relations_of_type('keystone-admin') | ||
2063 | 486 | if len(keystone) > 0: | 467 | if len(keystone) > 0: |
2066 | 487 | filtered_keystone = ([x for x in keystone | 468 | filtered_keystone = [x for x in keystone if "service_password" in x] |
2065 | 488 | if 'service_password' in x]) | ||
2067 | 489 | if len(filtered_keystone) > 0: | 469 | if len(filtered_keystone) > 0: |
2069 | 490 | context['keystone'] = copy(filtered_keystone[0]) | 470 | context["keystone"] = copy(filtered_keystone[0]) |
2070 | 491 | else: | 471 | else: |
2076 | 492 | if master_creds_set == '': | 472 | if master_creds_set == "": |
2077 | 493 | hookenv.log('master_creds missing') | 473 | hookenv.log("master_creds missing") |
2078 | 494 | hookenv.status_set('blocked', | 474 | hookenv.status_set("blocked", "Please add master_creds") |
2079 | 495 | 'Please add master_creds') | 475 | mysql = hookenv.relations_of_type("database") |
2075 | 496 | mysql = hookenv.relations_of_type('database') | ||
2080 | 497 | if len(mysql) > 0: | 476 | if len(mysql) > 0: |
2081 | 498 | relation = mysql[0] | 477 | relation = mysql[0] |
2083 | 499 | context['db'] = copy(relation) | 478 | context["db"] = copy(relation) |
2084 | 500 | if len(context.keys()) == 2: | 479 | if len(context.keys()) == 2: |
2085 | 501 | templating.render( | 480 | templating.render( |
2087 | 502 | source='novarc_master.j2', | 481 | source="novarc_master.j2", |
2088 | 503 | target=path, | 482 | target=path, |
2091 | 504 | owner='ubuntu', | 483 | owner="ubuntu", |
2092 | 505 | group='ubuntu', | 484 | group="ubuntu", |
2093 | 506 | perms=0o600, | 485 | perms=0o600, |
2094 | 507 | context=context, | 486 | context=context, |
2095 | 508 | ) | 487 | ) |
2097 | 509 | hookenv.status_set('active', 'Unit is ready') | 488 | hookenv.status_set("active", "Unit is ready") |
2098 | 510 | elif len(context.keys()) == 3: | 489 | elif len(context.keys()) == 3: |
2099 | 511 | templating.render( | 490 | templating.render( |
2101 | 512 | source='novarc_slave.j2', | 491 | source="novarc_slave.j2", |
2102 | 513 | target=path, | 492 | target=path, |
2105 | 514 | owner='ubuntu', | 493 | owner="ubuntu", |
2106 | 515 | group='ubuntu', | 494 | group="ubuntu", |
2107 | 516 | perms=0o600, | 495 | perms=0o600, |
2108 | 517 | context=context, | 496 | context=context, |
2109 | 518 | ) | 497 | ) |
2117 | 519 | hookenv.status_set('active', 'Unit is ready') | 498 | hookenv.status_set("active", "Unit is ready") |
2118 | 520 | elif 'db' not in context.keys(): | 499 | elif "db" not in context.keys(): |
2119 | 521 | hookenv.status_set('maintenance', | 500 | hookenv.status_set("maintenance", "mysql relation incomplete") |
2120 | 522 | 'mysql relation incomplete') | 501 | elif "keystone" not in context.keys(): |
2121 | 523 | elif 'keystone' not in context.keys(): | 502 | hookenv.status_set("maintenance", "keystone relation incomplete") |
2115 | 524 | hookenv.status_set('maintenance', | ||
2116 | 525 | 'keystone relation incomplete') | ||
2122 | 526 | else: | 503 | else: |
2125 | 527 | hookenv.status_set('maintenance', 'keystone and ' | 504 | hookenv.status_set("maintenance", "keystone and mysql relation incomplete") |
2124 | 528 | 'mysql relation incomplete') | ||
2126 | 529 | 505 | ||
2127 | 530 | 506 | ||
2128 | 531 | def write_custom_novarc(path): | 507 | def write_custom_novarc(path): |
2129 | @@ -536,172 +512,164 @@ def write_custom_novarc(path): | |||
2130 | 536 | # To add a custom novarc file, run: | 512 | # To add a custom novarc file, run: |
2131 | 537 | # `juju config <glance-sync-application> | 513 | # `juju config <glance-sync-application> |
2132 | 538 | # novarc=$(base64 -w 0 /path/to/novarc)` | 514 | # novarc=$(base64 -w 0 /path/to/novarc)` |
2137 | 539 | hookenv.status_set('maintenance', | 515 | hookenv.status_set("maintenance", "Configuring custom novarc") |
2138 | 540 | 'Configuring custom novarc') | 516 | hookenv.log("configuring custom novarc") |
2139 | 541 | hookenv.log('configuring custom novarc') | 517 | novarc = hookenv.config("novarc") |
2136 | 542 | novarc = hookenv.config('novarc') | ||
2140 | 543 | novarc_bytestring = base64.b64decode(novarc) | 518 | novarc_bytestring = base64.b64decode(novarc) |
2142 | 544 | with open(path, 'wb') as f: | 519 | with open(path, "wb") as f: |
2143 | 545 | f.write(novarc_bytestring) | 520 | f.write(novarc_bytestring) |
2147 | 546 | os.chown(path, | 521 | os.chown(path, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
2145 | 547 | pwd.getpwnam('ubuntu').pw_uid, | ||
2146 | 548 | grp.getgrnam('ubuntu').gr_gid) | ||
2148 | 549 | os.chmod(path, 0o600) | 522 | os.chmod(path, 0o600) |
2152 | 550 | clouds_yaml_dir = '/etc/openstack' | 523 | clouds_yaml_dir = "/etc/openstack" |
2153 | 551 | master_creds_set = hookenv.config('master_creds') | 524 | master_creds_set = hookenv.config("master_creds") |
2154 | 552 | master_enabled = hookenv.config('master_mode') | 525 | master_enabled = hookenv.config("master_mode") |
2155 | 553 | if master_enabled: | 526 | if master_enabled: |
2157 | 554 | hookenv.status_set('active', 'Unit is ready') | 527 | hookenv.status_set("active", "Unit is ready") |
2158 | 555 | return | 528 | return |
2160 | 556 | elif not master_enabled and master_creds_set != '': | 529 | elif not master_enabled and master_creds_set != "": |
2161 | 557 | if not os.path.exists(clouds_yaml_dir): | 530 | if not os.path.exists(clouds_yaml_dir): |
2162 | 558 | configure_master_novarc() | 531 | configure_master_novarc() |
2164 | 559 | hookenv.status_set('active', 'Unit is ready') | 532 | hookenv.status_set("active", "Unit is ready") |
2165 | 560 | else: | 533 | else: |
2170 | 561 | if master_creds_set == '': | 534 | if master_creds_set == "": |
2171 | 562 | hookenv.log('ERROR: master_creds missing') | 535 | hookenv.log("ERROR: master_creds missing") |
2172 | 563 | hookenv.status_set('blocked', | 536 | hookenv.status_set("blocked", "Please add master_creds") |
2169 | 564 | 'Please add master_creds') | ||
2173 | 565 | 537 | ||
2174 | 566 | 538 | ||
2176 | 567 | @when('config.changed.sync_source') | 539 | @when("config.changed.sync_source") |
2177 | 568 | def configure_sync_source(): | 540 | def configure_sync_source(): |
2178 | 569 | """Configure cron after 'sync_source' config change.""" | 541 | """Configure cron after 'sync_source' config change.""" |
2179 | 570 | configure_cron() | 542 | configure_cron() |
2180 | 571 | 543 | ||
2181 | 572 | 544 | ||
2184 | 573 | @when_any('config.changed.sync_enabled', | 545 | @when_any("config.changed.sync_enabled", "config.changed.cron_frequency") |
2183 | 574 | 'config.changed.cron_frequency') | ||
2185 | 575 | def configure_cron(): | 546 | def configure_cron(): |
2186 | 576 | """ | 547 | """ |
2187 | 577 | Configure cron after 'sync_enabled' or | 548 | Configure cron after 'sync_enabled' or |
2188 | 578 | 'cron_frequency' config change. | 549 | 'cron_frequency' config change. |
2189 | 579 | """ | 550 | """ |
2194 | 580 | hookenv.status_set('maintenance', 'Configuring') | 551 | hookenv.status_set("maintenance", "Configuring") |
2195 | 581 | sync_enabled = hookenv.config('sync_enabled') | 552 | sync_enabled = hookenv.config("sync_enabled") |
2196 | 582 | hookenv.log('configuring sync cronjob') | 553 | hookenv.log("configuring sync cronjob") |
2197 | 583 | master_enabled = hookenv.config('master_mode') | 554 | master_enabled = hookenv.config("master_mode") |
2198 | 584 | if master_enabled: | 555 | if master_enabled: |
2200 | 585 | cron_file = '/etc/cron.d/glance_sync_master' | 556 | cron_file = "/etc/cron.d/glance_sync_master" |
2201 | 586 | else: | 557 | else: |
2207 | 587 | cron_file = '/etc/cron.d/glance_sync_slave' | 558 | cron_file = "/etc/cron.d/glance_sync_slave" |
2208 | 588 | if not hookenv.config('sync_source'): | 559 | if not hookenv.config("sync_source"): |
2209 | 589 | hookenv.log('ERROR: sync_source not set') | 560 | hookenv.log("ERROR: sync_source not set") |
2210 | 590 | hookenv.status_set('blocked', 'Please set a ' | 561 | hookenv.status_set( |
2211 | 591 | 'sync_source to configure crontab') | 562 | "blocked", "Please set a sync_source to configure crontab" |
2212 | 563 | ) | ||
2213 | 592 | return | 564 | return |
2214 | 593 | 565 | ||
2215 | 594 | if sync_enabled and master_enabled: | 566 | if sync_enabled and master_enabled: |
2217 | 595 | hookenv.log('adding cronjob') | 567 | hookenv.log("adding cronjob") |
2218 | 596 | templating.render( | 568 | templating.render( |
2220 | 597 | source='glance_sync_master_cron.j2', | 569 | source="glance_sync_master_cron.j2", |
2221 | 598 | target=cron_file, | 570 | target=cron_file, |
2224 | 599 | owner='root', | 571 | owner="root", |
2225 | 600 | group='root', | 572 | group="root", |
2226 | 601 | perms=0o640, | 573 | perms=0o640, |
2227 | 602 | context=hookenv.config(), | 574 | context=hookenv.config(), |
2228 | 603 | ) | 575 | ) |
2229 | 604 | elif sync_enabled and not master_enabled: | 576 | elif sync_enabled and not master_enabled: |
2230 | 605 | # Just an alias to make sure that the paths are as expected. | 577 | # Just an alias to make sure that the paths are as expected. |
2231 | 606 | context = hookenv.config() | 578 | context = hookenv.config() |
2237 | 607 | if context['sync_source'][-1] != '/': | 579 | if context["sync_source"][-1] != "/": |
2238 | 608 | context['sync_source'] += '/' | 580 | context["sync_source"] += "/" |
2239 | 609 | if context['data_dir'][-1] != '/': | 581 | if context["data_dir"][-1] != "/": |
2240 | 610 | context['data_dir'] += '/' | 582 | context["data_dir"] += "/" |
2241 | 611 | hookenv.log('adding cronjob') | 583 | hookenv.log("adding cronjob") |
2242 | 612 | templating.render( | 584 | templating.render( |
2244 | 613 | source='glance_sync_slave_cron.j2', | 585 | source="glance_sync_slave_cron.j2", |
2245 | 614 | target=cron_file, | 586 | target=cron_file, |
2248 | 615 | owner='root', | 587 | owner="root", |
2249 | 616 | group='root', | 588 | group="root", |
2250 | 617 | perms=0o640, | 589 | perms=0o640, |
2251 | 618 | context=hookenv.config(), | 590 | context=hookenv.config(), |
2252 | 619 | ) | 591 | ) |
2253 | 620 | else: | 592 | else: |
2255 | 621 | hookenv.log('removing cronjob') | 593 | hookenv.log("removing cronjob") |
2256 | 622 | if os.path.isfile(cron_file): | 594 | if os.path.isfile(cron_file): |
2257 | 623 | os.remove(cron_file) | 595 | os.remove(cron_file) |
2259 | 624 | clear_flag('cron.configured') | 596 | clear_flag("cron.configured") |
2260 | 625 | configure_novarc() | 597 | configure_novarc() |
2262 | 626 | hookenv.status_set('active', 'Unit is ready') | 598 | hookenv.status_set("active", "Unit is ready") |
2263 | 627 | 599 | ||
2264 | 628 | 600 | ||
2266 | 629 | @hook('nrpe-external-master-relation-changed') | 601 | @hook("nrpe-external-master-relation-changed") |
2267 | 630 | def setup_nrpe_checks(nagios): | 602 | def setup_nrpe_checks(nagios): |
2268 | 631 | """Configure NRPE checks.""" | 603 | """Configure NRPE checks.""" |
2270 | 632 | hookenv.status_set('maintenance', 'Configuring nrpe checks') | 604 | hookenv.status_set("maintenance", "Configuring nrpe checks") |
2271 | 633 | config = hookenv.config() | 605 | config = hookenv.config() |
2273 | 634 | modes = ['slave', 'master'] | 606 | modes = ["slave", "master"] |
2274 | 635 | 607 | ||
2275 | 636 | for mode in modes: | 608 | for mode in modes: |
2276 | 637 | nagios.add_check( | 609 | nagios.add_check( |
2277 | 638 | [ | 610 | [ |
2284 | 639 | '/usr/lib/nagios/plugins/check_file_age', | 611 | "/usr/lib/nagios/plugins/check_file_age", |
2285 | 640 | '-w 14400', '-c 25200', '-i', '-f', | 612 | "-w 14400", |
2286 | 641 | os.path.join( | 613 | "-c 25200", |
2287 | 642 | config['log_dir'], | 614 | "-i", |
2288 | 643 | 'glance_sync_' + mode + '.log' | 615 | "-f", |
2289 | 644 | ), | 616 | os.path.join(config["log_dir"], "glance_sync_" + mode + ".log"), |
2290 | 645 | ], | 617 | ], |
2297 | 646 | name='glance_sync_' + mode + '_log', | 618 | name="glance_sync_" + mode + "_log", |
2298 | 647 | description=( | 619 | description=("Verify age of last image sync from glance to disk"), |
2299 | 648 | 'Verify age of last image sync ' | 620 | context=config["nagios_context"], |
2294 | 649 | 'from glance to disk' | ||
2295 | 650 | ), | ||
2296 | 651 | context=config['nagios_context'], | ||
2300 | 652 | unit=hookenv.local_unit(), | 621 | unit=hookenv.local_unit(), |
2301 | 653 | ) | 622 | ) |
2302 | 654 | 623 | ||
2303 | 655 | # Copy nrpe check plugin for stale lockfiles in place. | 624 | # Copy nrpe check plugin for stale lockfiles in place. |
2308 | 656 | script_dir = hookenv.config('script_dir') | 625 | script_dir = hookenv.config("script_dir") |
2309 | 657 | files = ['check_stale_lockfile_slave.py', | 626 | files = ["check_stale_lockfile_slave.py", "check_stale_lockfile_master.py"] |
2310 | 658 | 'check_stale_lockfile_master.py'] | 627 | hookenv.log("installing stale lockfile nrpe plugin") |
2307 | 659 | hookenv.log('installing stale lockfile nrpe plugin') | ||
2311 | 660 | for file in files: | 628 | for file in files: |
2313 | 661 | source = os.path.join(hookenv.charm_dir(), 'files', file) | 629 | source = os.path.join(hookenv.charm_dir(), "files", file) |
2314 | 662 | dest = os.path.join(script_dir, file) | 630 | dest = os.path.join(script_dir, file) |
2315 | 663 | if not os.path.exists(os.path.dirname(dest)): | 631 | if not os.path.exists(os.path.dirname(dest)): |
2316 | 664 | os.makedirs(os.path.dirname(dest)) | 632 | os.makedirs(os.path.dirname(dest)) |
2317 | 665 | shutil.copy(source, dest) | 633 | shutil.copy(source, dest) |
2321 | 666 | os.chown(dest, | 634 | os.chown(dest, pwd.getpwnam("ubuntu").pw_uid, grp.getgrnam("ubuntu").gr_gid) |
2319 | 667 | pwd.getpwnam('ubuntu').pw_uid, | ||
2320 | 668 | grp.getgrnam('ubuntu').gr_gid) | ||
2322 | 669 | os.chmod(dest, 0o755) | 635 | os.chmod(dest, 0o755) |
2323 | 670 | 636 | ||
2324 | 671 | for mode in modes: | 637 | for mode in modes: |
2328 | 672 | nrpe_plugin = os.path.join(script_dir, | 638 | nrpe_plugin = os.path.join( |
2329 | 673 | 'check_stale_lockfile_' + # noqa:W504 | 639 | script_dir, "check_stale_lockfile_" + mode + ".py" # noqa:W504 |
2330 | 674 | mode + '.py') | 640 | ) |
2331 | 675 | nagios.add_check( | 641 | nagios.add_check( |
2332 | 676 | [ | 642 | [ |
2333 | 677 | nrpe_plugin, | 643 | nrpe_plugin, |
2336 | 678 | '-f', '/tmp/glance_sync_' + mode + '.lock', | 644 | "-f", |
2337 | 679 | '-w 72000', '-c 14400', | 645 | "/tmp/glance_sync_" + mode + ".lock", |
2338 | 646 | "-w 72000", | ||
2339 | 647 | "-c 14400", | ||
2340 | 680 | ], | 648 | ], |
2344 | 681 | name='glance_sync_' + mode + '_lockfile', | 649 | name="glance_sync_" + mode + "_lockfile", |
2345 | 682 | description='Verify age of image sync lockfile', | 650 | description="Verify age of image sync lockfile", |
2346 | 683 | context=config['nagios_context'], | 651 | context=config["nagios_context"], |
2347 | 684 | unit=hookenv.local_unit(), | 652 | unit=hookenv.local_unit(), |
2348 | 685 | ) | 653 | ) |
2349 | 686 | 654 | ||
2351 | 687 | hookenv.status_set('active', 'Unit is ready') | 655 | hookenv.status_set("active", "Unit is ready") |
2352 | 688 | 656 | ||
2353 | 689 | 657 | ||
2355 | 690 | @when('config.changed.trusted_ssl_ca') | 658 | @when("config.changed.trusted_ssl_ca") |
2356 | 691 | def fix_ssl(): | 659 | def fix_ssl(): |
2357 | 692 | """ | 660 | """ |
2358 | 693 | Write user supplied SSL CA from 'trusted_ssl_ca' to | 661 | Write user supplied SSL CA from 'trusted_ssl_ca' to |
2359 | 694 | 'cert_file', and run `update-ca-certificates`. | 662 | 'cert_file', and run `update-ca-certificates`. |
2360 | 695 | """ | 663 | """ |
2362 | 696 | cert_file = '/usr/local/share/ca-certificates/openstack-service-checks.crt' | 664 | cert_file = "/usr/local/share/ca-certificates/openstack-service-checks.crt" |
2363 | 697 | config = hookenv.config() | 665 | config = hookenv.config() |
2366 | 698 | trusted_ssl_ca = config.get('trusted_ssl_ca').strip() | 666 | trusted_ssl_ca = config.get("trusted_ssl_ca").strip() |
2367 | 699 | hookenv.log('writing ssl ca cert:{}'.format(trusted_ssl_ca)) | 667 | hookenv.log("writing ssl ca cert:{}".format(trusted_ssl_ca)) |
2368 | 700 | cert_content = base64.b64decode(trusted_ssl_ca).decode() | 668 | cert_content = base64.b64decode(trusted_ssl_ca).decode() |
2370 | 701 | with open(cert_file, 'w') as f: | 669 | with open(cert_file, "w") as f: |
2371 | 702 | print(cert_content, file=f) | 670 | print(cert_content, file=f) |
2372 | 703 | try: | 671 | try: |
2374 | 704 | subprocess.call(['/usr/sbin/update-ca-certificates']) | 672 | subprocess.call(["/usr/sbin/update-ca-certificates"]) |
2375 | 705 | except subprocess.CalledProcessError as e: | 673 | except subprocess.CalledProcessError as e: |
2378 | 706 | hookenv.log('ERROR: fix_ssl() failed with {}'.format(e)) | 674 | hookenv.log("ERROR: fix_ssl() failed with {}".format(e)) |
2379 | 707 | hookenv.status_set('error', 'CA cert update failed') | 675 | hookenv.status_set("error", "CA cert update failed") |
2380 | diff --git a/src/tests/functional/tests/test_glance_sync.py b/src/tests/functional/tests/test_glance_sync.py | |||
2381 | index e03511b..0d6e981 100644 | |||
2382 | --- a/src/tests/functional/tests/test_glance_sync.py | |||
2383 | +++ b/src/tests/functional/tests/test_glance_sync.py | |||
2384 | @@ -48,9 +48,10 @@ class BaseGlanceSyncTest(unittest.TestCase): | |||
2385 | 48 | ) | 48 | ) |
2386 | 49 | b64_ssh_key = base64.b64encode(cls.slave_ssh_key.encode()) | 49 | b64_ssh_key = base64.b64encode(cls.slave_ssh_key.encode()) |
2387 | 50 | master_config = {"authorized_keys": b64_ssh_key.decode()} | 50 | master_config = {"authorized_keys": b64_ssh_key.decode()} |
2391 | 51 | slave_config = {"master_creds": master_creds, | 51 | slave_config = { |
2392 | 52 | "sync_source": "ubuntu@{}:/srv/glance_sync/data".format(cls.master_ip), | 52 | "master_creds": master_creds, |
2393 | 53 | } | 53 | "sync_source": "ubuntu@{}:/srv/glance_sync/data".format(cls.master_ip), |
2394 | 54 | } | ||
2395 | 54 | model.set_application_config(cls.master_app, master_config) | 55 | model.set_application_config(cls.master_app, master_config) |
2396 | 55 | model.set_application_config(cls.slave_app, slave_config) | 56 | model.set_application_config(cls.slave_app, slave_config) |
2397 | 56 | 57 | ||
2398 | @@ -72,23 +73,24 @@ class CharmOperationTest(BaseGlanceSyncTest): | |||
2399 | 72 | # Upload a Glance image to use for test, doesn't need to be big or even real | 73 | # Upload a Glance image to use for test, doesn't need to be big or even real |
2400 | 73 | openstack.enable_logging(debug=True) | 74 | openstack.enable_logging(debug=True) |
2401 | 74 | conn_master = openstack.connection.Connection( | 75 | conn_master = openstack.connection.Connection( |
2403 | 75 | region_name='RegionOne', | 76 | region_name="RegionOne", |
2404 | 76 | auth=dict( | 77 | auth=dict( |
2407 | 77 | auth_url='http://{}:35357/v3'.format(self.keystone_master_ip), | 78 | auth_url="http://{}:35357/v3".format(self.keystone_master_ip), |
2408 | 78 | username='admin', | 79 | username="admin", |
2409 | 79 | password=self.master_password, | 80 | password=self.master_password, |
2413 | 80 | project_name='admin', | 81 | project_name="admin", |
2414 | 81 | user_domain_name='admin_domain', | 82 | user_domain_name="admin_domain", |
2415 | 82 | project_domain_name='admin_domain', | 83 | project_domain_name="admin_domain", |
2416 | 83 | ), | 84 | ), |
2419 | 84 | compute_api_version='2', | 85 | compute_api_version="2", |
2420 | 85 | identity_interface='public') | 86 | identity_interface="public", |
2421 | 87 | ) | ||
2422 | 86 | image_attrs = { | 88 | image_attrs = { |
2428 | 87 | 'name': 'zaza_test_image', | 89 | "name": "zaza_test_image", |
2429 | 88 | 'data': 'some_test_data', | 90 | "data": "some_test_data", |
2430 | 89 | 'disk_format': 'raw', | 91 | "disk_format": "raw", |
2431 | 90 | 'container_format': 'bare', | 92 | "container_format": "bare", |
2432 | 91 | 'visibility': 'public', | 93 | "visibility": "public", |
2433 | 92 | } | 94 | } |
2434 | 93 | self.image = conn_master.image.upload_image(**image_attrs) | 95 | self.image = conn_master.image.upload_image(**image_attrs) |
2435 | 94 | # Has image.id, image.created, image.name for future use | 96 | # Has image.id, image.created, image.name for future use |
2436 | @@ -97,21 +99,22 @@ class CharmOperationTest(BaseGlanceSyncTest): | |||
2437 | 97 | def test_02_check_slave(self): | 99 | def test_02_check_slave(self): |
2438 | 98 | """Check that the previously uploaded image lands on the slave. | 100 | """Check that the previously uploaded image lands on the slave. |
2439 | 99 | 101 | ||
2442 | 100 | Run a loop till timeout, that checks Glance in the slave region for the presence of the | 102 | Run a loop until timeout, that checks glance in the slave region |
2443 | 101 | image uploaded to the master. | 103 | for the presence of the image uploaded to the master. |
2444 | 102 | """ | 104 | """ |
2445 | 103 | conn_slave = openstack.connection.Connection( | 105 | conn_slave = openstack.connection.Connection( |
2447 | 104 | region_name='RegionOne', | 106 | region_name="RegionOne", |
2448 | 105 | auth=dict( | 107 | auth=dict( |
2451 | 106 | auth_url='http://{}:35357/v3'.format(self.keystone_slave_ip), | 108 | auth_url="http://{}:35357/v3".format(self.keystone_slave_ip), |
2452 | 107 | username='admin', | 109 | username="admin", |
2453 | 108 | password=self.slave_password, | 110 | password=self.slave_password, |
2457 | 109 | project_name='admin', | 111 | project_name="admin", |
2458 | 110 | user_domain_name='admin_domain', | 112 | user_domain_name="admin_domain", |
2459 | 111 | project_domain_name='admin_domain', | 113 | project_domain_name="admin_domain", |
2460 | 112 | ), | 114 | ), |
2463 | 113 | compute_api_version='2', | 115 | compute_api_version="2", |
2464 | 114 | identity_interface='public') | 116 | identity_interface="public", |
2465 | 117 | ) | ||
2466 | 115 | timeout = time.time() + TEST_TIMEOUT | 118 | timeout = time.time() + TEST_TIMEOUT |
2467 | 116 | while time.time() < timeout: | 119 | while time.time() < timeout: |
2468 | 117 | image_list = conn_slave.get_image("zaza_test_image") | 120 | image_list = conn_slave.get_image("zaza_test_image") |
2469 | diff --git a/src/tests/unit/__init__.py b/src/tests/unit/__init__.py | |||
2470 | index 03acc40..28e9795 100644 | |||
2471 | --- a/src/tests/unit/__init__.py | |||
2472 | +++ b/src/tests/unit/__init__.py | |||
2473 | @@ -1,2 +1,3 @@ | |||
2474 | 1 | import sys | 1 | import sys |
2476 | 2 | sys.path.append('.') | 2 | |
2477 | 3 | sys.path.append(".") | ||
2478 | diff --git a/src/tox.ini b/src/tox.ini | |||
2479 | index cd1b1f7..45ced91 100644 | |||
2480 | --- a/src/tox.ini | |||
2481 | +++ b/src/tox.ini | |||
2482 | @@ -25,7 +25,7 @@ passenv = | |||
2483 | 25 | [testenv:lint] | 25 | [testenv:lint] |
2484 | 26 | commands = | 26 | commands = |
2485 | 27 | flake8 | 27 | flake8 |
2487 | 28 | #TODO black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod)/" . | 28 | black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod)/" . |
2488 | 29 | deps = | 29 | deps = |
2489 | 30 | black | 30 | black |
2490 | 31 | flake8 | 31 | flake8 |
2491 | @@ -45,8 +45,7 @@ exclude = | |||
2492 | 45 | mod, | 45 | mod, |
2493 | 46 | .build | 46 | .build |
2494 | 47 | 47 | ||
2497 | 48 | max-line-length = 120 | 48 | max-line-length = 88 |
2496 | 49 | #TODO max-line-length = 88 | ||
2498 | 50 | max-complexity = 10 | 49 | max-complexity = 10 |
2499 | 51 | 50 | ||
2500 | 52 | [testenv:black] | 51 | [testenv:black] |
LGTM