Merge lp:~corey.bryant/ceilometer/2014.2.2 into lp:~ubuntu-server-dev/ceilometer/juno

Proposed by Corey Bryant on 2015-02-13
Status: Merged
Merged at revision: 126
Proposed branch: lp:~corey.bryant/ceilometer/2014.2.2
Merge into: lp:~ubuntu-server-dev/ceilometer/juno
Diff against target: 433 lines (+7/-407)
3 files modified
debian/changelog (+7/-0)
debian/patches/mongodb-autoreconnect.patch (+0/-406)
debian/patches/series (+0/-1)
To merge this branch: bzr merge lp:~corey.bryant/ceilometer/2014.2.2
Reviewer Review Type Date Requested Status
Ubuntu Server Developers 2015-02-13 Pending
Review via email: mp+249657@code.launchpad.net
To post a comment you must log in.
lp:~corey.bryant/ceilometer/2014.2.2 updated on 2015-02-13
126. By Corey Bryant on 2015-02-13

/d/p/mongodb-autoreconnect.patch: Dropped as it is breaking
ceilometer-dbsync (LP: #1421663).

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2015-02-09 19:57:46 +0000
3+++ debian/changelog 2015-02-13 15:02:15 +0000
4@@ -1,3 +1,10 @@
5+ceilometer (2014.2.2-0ubuntu2) UNRELEASED; urgency=medium
6+
7+ * /d/p/mongodb-autoreconnect.patch: Dropped as it is breaking
8+ ceilometer-dbsync (LP: #1421663).
9+
10+ -- Corey Bryant <corey.bryant@canonical.com> Fri, 13 Feb 2015 08:54:31 -0500
11+
12 ceilometer (2014.2.2-0ubuntu1) utopic; urgency=medium
13
14 [ Corey Bryant ]
15
16=== removed file 'debian/patches/mongodb-autoreconnect.patch'
17--- debian/patches/mongodb-autoreconnect.patch 2015-02-09 13:55:53 +0000
18+++ debian/patches/mongodb-autoreconnect.patch 1970-01-01 00:00:00 +0000
19@@ -1,406 +0,0 @@
20-From cfc2de1d1994b5a1fc9e2f94a6ba4e20303c3ed1 Mon Sep 17 00:00:00 2001
21-From: Igor Degtiarov <idegtiarov@mirantis.com>
22-Date: Thu, 23 Oct 2014 14:05:38 +0300
23-Subject: [PATCH] [MongoDB] Fix bug with reconnection to new master node
24-
25-Fixes bug with raising AutoReconnect exception when MongoDB ReplicaSet
26-loses connection to primary node.
27-
28-Closes-Bug: #1309555
29-
30-Conflicts:
31- ceilometer/event/storage/impl_db2.py
32- ceilometer/event/storage/impl_mongodb.py
33- ceilometer/storage/__init__.py
34- ceilometer/storage/mongo/utils.py
35- ceilometer/tests/storage/test_pymongo_base.py
36-
37-Conflicts are due to refactoring of the storage drivers and this patch
38-has been modified to account for the refactor. The test case within the
39-file test_pymongo_base.py was removed since equivalent test coverage
40-was included in the cherry picked commit in the test_storage_scenarios.py
41-file.
42-
43-Change-Id: Id0e81ba60b28d09adff6a10d04b412f25257d8ce
44-(cherry-picked from commit 21d882c96cbbaeb8b78ff91e06e3615be97bff07)
45----
46- ceilometer/alarm/storage/impl_db2.py | 2 +-
47- ceilometer/alarm/storage/impl_mongodb.py | 2 +-
48- ceilometer/storage/__init__.py | 4 +
49- ceilometer/storage/impl_db2.py | 2 +-
50- ceilometer/storage/impl_mongodb.py | 2 +-
51- ceilometer/storage/mongo/utils.py | 119 +++++++++++++++++----
52- ceilometer/tests/storage/test_pymongo_base.py | 42 --------
53- ceilometer/tests/storage/test_storage_scenarios.py | 88 +++++++++++++++
54- 8 files changed, 196 insertions(+), 65 deletions(-)
55-
56-diff --git a/ceilometer/alarm/storage/impl_db2.py b/ceilometer/alarm/storage/impl_db2.py
57-index 9ca37f2..92db547 100644
58---- a/ceilometer/alarm/storage/impl_db2.py
59-+++ b/ceilometer/alarm/storage/impl_db2.py
60-@@ -73,5 +73,5 @@ class Connection(pymongo_base.Connection):
61- # not been implemented. However calling this method is important for
62- # removal of all the empty dbs created during the test runs since
63- # test run is against mongodb on Jenkins
64-- self.conn.drop_database(self.db)
65-+ self.conn.drop_database(self.db.name)
66- self.conn.close()
67-diff --git a/ceilometer/alarm/storage/impl_mongodb.py b/ceilometer/alarm/storage/impl_mongodb.py
68-index 19fff00..60c0ca4 100644
69---- a/ceilometer/alarm/storage/impl_mongodb.py
70-+++ b/ceilometer/alarm/storage/impl_mongodb.py
71-@@ -63,6 +63,6 @@ class Connection(pymongo_base.Connection):
72- self.upgrade()
73-
74- def clear(self):
75-- self.conn.drop_database(self.db)
76-+ self.conn.drop_database(self.db.name)
77- # Connection will be reopened automatically if needed
78- self.conn.close()
79-diff --git a/ceilometer/storage/__init__.py b/ceilometer/storage/__init__.py
80-index 36d91cc..b338363 100644
81---- a/ceilometer/storage/__init__.py
82-+++ b/ceilometer/storage/__init__.py
83-@@ -53,6 +53,10 @@ STORAGE_OPTS = [
84- default=None,
85- help='The connection string used to connect to the alarm '
86- 'database. (if unset, connection is used)'),
87-+ cfg.StrOpt('mongodb_replica_set',
88-+ default='',
89-+ help="The connection string used to connect to mongo database, "
90-+ "if mongodb replica set was chosen."),
91- ]
92-
93- cfg.CONF.register_opts(STORAGE_OPTS, group='database')
94-diff --git a/ceilometer/storage/impl_db2.py b/ceilometer/storage/impl_db2.py
95-index 4326162..e5c819e 100644
96---- a/ceilometer/storage/impl_db2.py
97-+++ b/ceilometer/storage/impl_db2.py
98-@@ -200,7 +200,7 @@ class Connection(pymongo_base.Connection):
99- # not been implemented. However calling this method is important for
100- # removal of all the empty dbs created during the test runs since
101- # test run is against mongodb on Jenkins
102-- self.conn.drop_database(self.db)
103-+ self.conn.drop_database(self.db.name)
104- self.conn.close()
105-
106- def record_metering_data(self, data):
107-diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py
108-index b9ce7a9..490eb52 100644
109---- a/ceilometer/storage/impl_mongodb.py
110-+++ b/ceilometer/storage/impl_mongodb.py
111-@@ -470,7 +470,7 @@ class Connection(pymongo_base.Connection):
112- )
113-
114- def clear(self):
115-- self.conn.drop_database(self.db)
116-+ self.conn.drop_database(self.db.name)
117- # Connection will be reopened automatically if needed
118- self.conn.close()
119-
120-diff --git a/ceilometer/storage/mongo/utils.py b/ceilometer/storage/mongo/utils.py
121-index 5aa8832..3d851e0 100644
122---- a/ceilometer/storage/mongo/utils.py
123-+++ b/ceilometer/storage/mongo/utils.py
124-@@ -179,26 +179,21 @@ class ConnectionPool(object):
125-
126- @staticmethod
127- def _mongo_connect(url):
128-- max_retries = cfg.CONF.database.max_retries
129-- retry_interval = cfg.CONF.database.retry_interval
130-- attempts = 0
131-- while True:
132-- try:
133-- client = pymongo.MongoClient(url, safe=True)
134-- except pymongo.errors.ConnectionFailure as e:
135-- if 0 <= max_retries <= attempts:
136-- LOG.error(_('Unable to connect to the database after '
137-- '%(retries)d retries. Giving up.') %
138-- {'retries': max_retries})
139-- raise
140-- LOG.warn(_('Unable to connect to the database server: '
141-- '%(errmsg)s. Trying again in %(retry_interval)d '
142-- 'seconds.') %
143-- {'errmsg': e, 'retry_interval': retry_interval})
144-- attempts += 1
145-- time.sleep(retry_interval)
146-+ try:
147-+ if cfg.CONF.database.mongodb_replica_set:
148-+ client = MongoProxy(
149-+ Prefection(
150-+ pymongo.MongoReplicaSetClient(
151-+ url,
152-+ replicaSet=cfg.CONF.database.mongodb_replica_set)))
153- else:
154-- return client
155-+ client = MongoProxy(
156-+ Prefection(pymongo.MongoClient(url, safe=True)))
157-+ return client
158-+ except pymongo.errors.ConnectionFailure as e:
159-+ LOG.warn(_('Unable to connect to the database server: '
160-+ '%(errmsg)s.') % {'errmsg': e})
161-+ raise
162-
163-
164- class QueryTransformer(object):
165-@@ -321,3 +316,89 @@ class QueryTransformer(object):
166- return self._handle_not_op(negated_tree)
167-
168- return self._handle_simple_op(operator_node, nodes)
169-+
170-+
171-+def safe_mongo_call(call):
172-+ def closure(*args, **kwargs):
173-+ max_retries = cfg.CONF.database.max_retries
174-+ retry_interval = cfg.CONF.database.retry_interval
175-+ attempts = 0
176-+ while True:
177-+ try:
178-+ return call(*args, **kwargs)
179-+ except pymongo.errors.AutoReconnect as err:
180-+ if 0 <= max_retries <= attempts:
181-+ LOG.error(_('Unable to reconnect to the primary mongodb '
182-+ 'after %(retries)d retries. Giving up.') %
183-+ {'retries': max_retries})
184-+ raise
185-+ LOG.warn(_('Unable to reconnect to the primary mongodb: '
186-+ '%(errmsg)s. Trying again in %(retry_interval)d '
187-+ 'seconds.') %
188-+ {'errmsg': err, 'retry_interval': retry_interval})
189-+ attempts += 1
190-+ time.sleep(retry_interval)
191-+ return closure
192-+
193-+
194-+class MongoConn(object):
195-+ def __init__(self, method):
196-+ self.method = method
197-+
198-+ @safe_mongo_call
199-+ def __call__(self, *args, **kwargs):
200-+ return self.method(*args, **kwargs)
201-+
202-+MONGO_METHODS = set([typ for typ in dir(pymongo.collection.Collection)
203-+ if not typ.startswith('_')])
204-+MONGO_METHODS.update(set([typ for typ in dir(pymongo.MongoClient)
205-+ if not typ.startswith('_')]))
206-+MONGO_METHODS.update(set([typ for typ in dir(pymongo)
207-+ if not typ.startswith('_')]))
208-+
209-+
210-+class MongoProxy(object):
211-+ def __init__(self, conn):
212-+ self.conn = conn
213-+
214-+ def __getitem__(self, item):
215-+ """Create and return proxy around the method in the connection.
216-+
217-+ :param item: name of the connection
218-+ """
219-+ return MongoProxy(self.conn[item])
220-+
221-+ def __getattr__(self, item):
222-+ """Wrap MongoDB connection.
223-+
224-+ If item is the name of an executable method, for example find or
225-+ insert, wrap this method in the MongoConn.
226-+ Else wrap getting attribute with MongoProxy.
227-+ """
228-+ if item == 'name':
229-+ return getattr(self.conn, item)
230-+ if item in MONGO_METHODS:
231-+ return MongoConn(getattr(self.conn, item))
232-+ return MongoProxy(getattr(self.conn, item))
233-+
234-+ def __call__(self, *args, **kwargs):
235-+ return self.conn(*args, **kwargs)
236-+
237-+
238-+class Prefection(pymongo.collection.Collection):
239-+ def __init__(self, conn):
240-+ self.conn = conn
241-+
242-+ def find(self, *args, **kwargs):
243-+ # We need this modifying method to check a connection for MongoDB
244-+ # in context of MongoProxy approach. Initially 'find' returns Cursor
245-+ # object and doesn't connect to db while Cursor is not used.
246-+ found = self.find(*args, **kwargs)
247-+ try:
248-+ found[0]
249-+ except IndexError:
250-+ pass
251-+ return found
252-+
253-+ def __getattr__(self, item):
254-+ return getattr(self.conn, item)
255-diff --git a/ceilometer/tests/storage/test_pymongo_base.py b/ceilometer/tests/storage/test_pymongo_base.py
256-index c40bc7c..28c40a9 100644
257---- a/ceilometer/tests/storage/test_pymongo_base.py
258-+++ b/ceilometer/tests/storage/test_pymongo_base.py
259-@@ -12,17 +12,13 @@
260- """Tests the mongodb and db2 common functionality
261- """
262-
263--import contextlib
264- import copy
265- import datetime
266-
267- import mock
268--import pymongo
269-
270--from ceilometer.openstack.common.gettextutils import _
271- from ceilometer.publisher import utils
272- from ceilometer import sample
273--from ceilometer.storage.mongo import utils as pymongo_utils
274- from ceilometer.tests import db as tests_db
275- from ceilometer.tests.storage import test_storage_scenarios
276-
277-@@ -168,41 +164,3 @@ class CompatibilityTest(test_storage_scenarios.DBTestBase,
278- def test_counter_unit(self):
279- meters = list(self.conn.get_meters())
280- self.assertEqual(1, len(meters))
281--
282-- def test_mongodb_connect_raises_after_custom_number_of_attempts(self):
283-- retry_interval = 13
284-- max_retries = 37
285-- self.CONF.set_override(
286-- 'retry_interval', retry_interval, group='database')
287-- self.CONF.set_override(
288-- 'max_retries', max_retries, group='database')
289-- # PyMongo is being used to connect even to DB2, but it only
290-- # accepts URLs with the 'mongodb' scheme. This replacement is
291-- # usually done in the DB2 connection implementation, but since
292-- # we don't call that, we have to do it here.
293-- self.CONF.set_override(
294-- 'connection', self.db_manager.url.replace('db2:', 'mongodb:', 1),
295-- group='database')
296--
297-- pool = pymongo_utils.ConnectionPool()
298-- with contextlib.nested(
299-- mock.patch(
300-- 'pymongo.MongoClient',
301-- side_effect=pymongo.errors.ConnectionFailure('foo')),
302-- mock.patch.object(pymongo_utils.LOG, 'error'),
303-- mock.patch.object(pymongo_utils.LOG, 'warn'),
304-- mock.patch.object(pymongo_utils.time, 'sleep')
305-- ) as (MockMongo, MockLOGerror, MockLOGwarn, Mocksleep):
306-- self.assertRaises(pymongo.errors.ConnectionFailure,
307-- pool.connect, self.CONF.database.connection)
308-- Mocksleep.assert_has_calls([mock.call(retry_interval)
309-- for i in range(max_retries)])
310-- MockLOGwarn.assert_any_call(
311-- _('Unable to connect to the database server: %(errmsg)s.'
312-- ' Trying again in %(retry_interval)d seconds.') %
313-- {'errmsg': 'foo',
314-- 'retry_interval': retry_interval})
315-- MockLOGerror.assert_called_with(
316-- _('Unable to connect to the database after '
317-- '%(retries)d retries. Giving up.') %
318-- {'retries': max_retries})
319-diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py
320-index 5318e99..8024378 100644
321---- a/ceilometer/tests/storage/test_storage_scenarios.py
322-+++ b/ceilometer/tests/storage/test_storage_scenarios.py
323-@@ -23,7 +23,9 @@ import datetime
324- import operator
325-
326- import mock
327-+from oslo.config import cfg
328- from oslo.utils import timeutils
329-+import pymongo
330-
331- import ceilometer
332- from ceilometer.alarm.storage import models as alarm_models
333-@@ -3099,3 +3101,89 @@ class BigIntegerTest(tests_db.TestBase,
334- msg = utils.meter_message_from_counter(
335- s, self.CONF.publisher.metering_secret)
336- self.conn.record_metering_data(msg)
337-+
338-+
339-+class MongoAutoReconnectTest(DBTestBase,
340-+ tests_db.MixinTestsWithBackendScenarios):
341-+ cfg.CONF.set_override('retry_interval', 1, group='database')
342-+
343-+ @tests_db.run_with('mongodb')
344-+ def test_mongo_client(self):
345-+ if cfg.CONF.database.mongodb_replica_set:
346-+ self.assertIsInstance(self.conn.conn.conn.conn,
347-+ pymongo.MongoReplicaSetClient)
348-+ else:
349-+ self.assertIsInstance(self.conn.conn.conn.conn,
350-+ pymongo.MongoClient)
351-+
352-+ @staticmethod
353-+ def create_side_effect(method, test_exception):
354-+ def side_effect(*args, **kwargs):
355-+ if test_exception.pop():
356-+ raise pymongo.errors.AutoReconnect
357-+ else:
358-+ return method(*args, **kwargs)
359-+ return side_effect
360-+
361-+ @tests_db.run_with('mongodb')
362-+ def test_mongo_find(self):
363-+ raise_exc = [False, True]
364-+ method = self.conn.db.resource.find
365-+
366-+ with mock.patch('pymongo.collection.Collection.find',
367-+ mock.Mock()) as mock_find:
368-+ mock_find.side_effect = self.create_side_effect(method, raise_exc)
369-+ mock_find.__name__ = 'find'
370-+ resources = list(self.conn.get_resources())
371-+ self.assertEqual(9, len(resources))
372-+
373-+ @tests_db.run_with('mongodb')
374-+ def test_mongo_insert(self):
375-+ raise_exc = [False, True]
376-+ method = self.conn.db.meter.insert
377-+
378-+ with mock.patch('pymongo.collection.Collection.insert',
379-+ mock.Mock(return_value=method)) as mock_insert:
380-+ mock_insert.side_effect = self.create_side_effect(method,
381-+ raise_exc)
382-+ mock_insert.__name__ = 'insert'
383-+ self.create_and_store_sample(
384-+ timestamp=datetime.datetime(2014, 10, 15, 14, 39),
385-+ source='test-proxy')
386-+ meters = list(self.conn.db.meter.find())
387-+ self.assertEqual(12, len(meters))
388-+
389-+ @tests_db.run_with('mongodb')
390-+ def test_mongo_find_and_modify(self):
391-+ raise_exc = [False, True]
392-+ method = self.conn.db.resource.find_and_modify
393-+
394-+ with mock.patch('pymongo.collection.Collection.find_and_modify',
395-+ mock.Mock()) as mock_fam:
396-+ mock_fam.side_effect = self.create_side_effect(method, raise_exc)
397-+ mock_fam.__name__ = 'find_and_modify'
398-+ self.create_and_store_sample(
399-+ timestamp=datetime.datetime(2014, 10, 15, 14, 39),
400-+ source='test-proxy')
401-+ data = self.conn.db.resource.find(
402-+ {'last_sample_timestamp':
403-+ datetime.datetime(2014, 10, 15, 14, 39)})[0]['source']
404-+ self.assertEqual('test-proxy', data)
405-+
406-+ @tests_db.run_with('mongodb')
407-+ def test_mongo_update(self):
408-+ raise_exc = [False, True]
409-+ method = self.conn.db.resource.update
410-+
411-+ with mock.patch('pymongo.collection.Collection.update',
412-+ mock.Mock()) as mock_update:
413-+ mock_update.side_effect = self.create_side_effect(method,
414-+ raise_exc)
415-+ mock_update.__name__ = 'update'
416-+ self.create_and_store_sample(
417-+ timestamp=datetime.datetime(2014, 10, 15, 17, 39),
418-+ source='test-proxy-update')
419-+ data = self.conn.db.resource.find(
420-+ {'last_sample_timestamp':
421-+ datetime.datetime(2014, 10, 15, 17, 39)})[0]['source']
422-+ self.assertEqual('test-proxy-update', data)
423---
424-2.1.0
425-
426
427=== modified file 'debian/patches/series'
428--- debian/patches/series 2015-02-09 13:55:53 +0000
429+++ debian/patches/series 2015-02-13 15:02:15 +0000
430@@ -1,3 +1,2 @@
431 fix-requirements.patch
432 skip-db-tests.patch
433-mongodb-autoreconnect.patch

Subscribers

People subscribed via source and target branches