1
=== modified file 'ci-utils/ci_utils/amqp_utils.py'
2
--- ci-utils/ci_utils/amqp_utils.py	2014-10-23 12:09:49 +0000
3
+++ ci-utils/ci_utils/amqp_utils.py	2015-01-08 14:43:53 +0000
4
@@ -211,6 +211,15 @@
5
211
    return send(progress_trigger, json.dumps(data))
211
    return send(progress_trigger, json.dumps(data))
6
212
212
7
213
213
8
214
def progress_errored(progress_trigger, data=None):
9
215
    if data is None:
10
216
        data = {}
11
217
    data['state'] = 'COMPLETED'
12
218
    data['result'] = 'ERROR'
13
219
    data['exit'] = True
14
220
    return send(progress_trigger, json.dumps(data))
15
221
16
222
17
214
def progress_failed(progress_trigger, data=None):
223
def progress_failed(progress_trigger, data=None):
18
215
    if data is None:
224
    if data is None:
19
216
        data = {}
225
        data = {}
20
217
226
21
=== modified file 'ci-utils/ci_utils/amqp_worker.py'
22
--- ci-utils/ci_utils/amqp_worker.py	2014-10-29 16:43:39 +0000
23
+++ ci-utils/ci_utils/amqp_worker.py	2015-01-08 14:43:53 +0000
24
@@ -213,12 +213,18 @@
25
213
        return t
213
        return t
26
214
214
27
215
    def _handle_exception(self, e, store, trigger, logger, ret):
215
    def _handle_exception(self, e, store, trigger, logger, ret):
28
216
        error = False
29
216
        try:
217
        try:
30
217
            err = 'Unexpected exception occurred'
218
            err = 'Unexpected exception occurred'
31
218
            if isinstance(e, KeyboardInterrupt):
219
            if isinstance(e, KeyboardInterrupt):
32
219
                err = 'Worker terminated'
220
                err = 'Worker terminated'
33
220
            elif isinstance(e, subprocess.CalledProcessError) and logger:
221
            elif isinstance(e, subprocess.CalledProcessError) and logger:
34
221
                err = 'error running {}: {}'.format(e.cmd, e.output)
222
                err = 'error running {}: {}'.format(e.cmd, e.output)
35
223
                error = True
36
224
            # All exceptions except those that get whitelisted here are
37
225
            # considered infrastructure failures.
38
226
            else:
39
227
                error = True
40
222
            ret['error_message'] = err
228
            ret['error_message'] = err
41
223
            ret['traceback'] = traceback.format_exc()
229
            ret['traceback'] = traceback.format_exc()
42
224
            if logger:
230
            if logger:
43
@@ -226,7 +232,10 @@
44
226
            self._store_worker_log(store, logger, ret)
232
            self._store_worker_log(store, logger, ret)
45
227
        finally:
233
        finally:
46
228
            if trigger:
234
            if trigger:
48
229
                amqp_utils.progress_failed(trigger, ret)
235
                if error:
49
236
                    amqp_utils.progress_errored(trigger, ret)
50
237
                else:
51
238
                    amqp_utils.progress_failed(trigger, ret)
52
230
239
53
231
    def _on_message(self, msg):
240
    def _on_message(self, msg):
54
232
        log.info('on_message: %s', msg.body)
241
        log.info('on_message: %s', msg.body)
55
233
242
56
=== modified file 'ci-utils/ci_utils/tests/test_amqp_worker.py'
57
--- ci-utils/ci_utils/tests/test_amqp_worker.py	2014-10-07 10:04:01 +0000
58
+++ ci-utils/ci_utils/tests/test_amqp_worker.py	2015-01-08 14:43:53 +0000
59
@@ -21,7 +21,10 @@
60
21
21
61
22
import mock
22
import mock
62
23
23
64
24
from ci_utils import amqp_worker
24
from ci_utils import (
65
25
    amqp_worker,
66
26
    data_store,
67
27
)
68
25
28
69
26
29
70
27
class _worker(amqp_worker.AMQPWorker):
30
class _worker(amqp_worker.AMQPWorker):
71
@@ -111,6 +114,31 @@
72
111
        self.assertEqual('queue-name.output.log',
114
        self.assertEqual('queue-name.output.log',
73
112
                         retval['artifacts'][0]['name'])
115
                         retval['artifacts'][0]['name'])
74
113
116
75
117
    def _assert_infrastructure_failure(self, err, worker, msg, amqp_utils):
76
118
        #self.assertEqual(1, amqp_utils.progress_update.call_count)
77
119
        msg.channel.basic_ack.assert_called_once_with('foo')
78
120
        self.assertEqual(0, amqp_utils.progress_failed.call_count)
79
121
        self.assertEqual(1, amqp_utils.progress_errored.call_count)
80
122
        self.assertEqual(
81
123
            'queue-name', amqp_utils.progress_errored.call_args[0][0])
82
124
        self.assertEqual(1, worker.store.put_file.call_count)
83
125
84
126
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
85
127
    def testOnMessageDataStoreException(self, amqp_utils):
86
128
        '''Test on message logic for an unexpected failure'''
87
129
        msg = mock.Mock()
88
130
        msg.delivery_tag = 'foo'
89
131
        msg.body = json.dumps({
90
132
            'ticket_id': 1,
91
133
            'progress_trigger': 'queue-name',
92
134
        })
93
135
        err = 'this is the worker error'
94
136
        worker = _worker(None, None, data_store.DataStoreException, err)
95
137
        worker._on_message(msg)
96
138
        self._assert_infrastructure_failure(err, worker, msg, amqp_utils)
97
139
        self.assertIn(
98
140
            err, amqp_utils.progress_errored.call_args[0][1]['traceback'])
99
141
100
114
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
142
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
101
115
    def testOnMessageUnexpected(self, amqp_utils):
143
    def testOnMessageUnexpected(self, amqp_utils):
102
116
        '''Test on message logic for an unexpected failure'''
144
        '''Test on message logic for an unexpected failure'''
103
@@ -123,14 +151,9 @@
104
123
        err = 'this is the worker error'
151
        err = 'this is the worker error'
105
124
        worker = _worker(None, None, RuntimeError, err)
152
        worker = _worker(None, None, RuntimeError, err)
106
125
        worker._on_message(msg)
153
        worker._on_message(msg)
112
126
        self.assertEqual(1, amqp_utils.progress_update.call_count)
154
        self._assert_infrastructure_failure(err, worker, msg, amqp_utils)
108
127
        msg.channel.basic_ack.assert_called_once_with('foo')
109
128
        self.assertEqual(1, amqp_utils.progress_failed.call_count)
110
129
        self.assertEqual(
111
130
            'queue-name', amqp_utils.progress_failed.call_args[0][0])
113
131
        self.assertIn(
155
        self.assertIn(
116
132
            err, amqp_utils.progress_failed.call_args[0][1]['traceback'])
156
            err, amqp_utils.progress_errored.call_args[0][1]['traceback'])
115
133
        self.assertEqual(1, worker.store.put_file.call_count)
117
134
157
118
135
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
158
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
119
136
    def testOnMessageCalledProcessError(self, amqp_utils):
159
    def testOnMessageCalledProcessError(self, amqp_utils):
120
@@ -154,15 +177,11 @@
121
154
177
122
155
        worker = _calledprocess_worker(None, None, None, None)
178
        worker = _calledprocess_worker(None, None, None, None)
123
156
        worker._on_message(msg)
179
        worker._on_message(msg)
130
157
180
        self._assert_infrastructure_failure(err, worker, msg, amqp_utils)
125
158
        self.assertEqual(1, amqp_utils.progress_update.call_count)
126
159
        msg.channel.basic_ack.assert_called_once_with('foo')
127
160
        self.assertEqual(1, amqp_utils.progress_failed.call_count)
128
161
        self.assertEqual(
129
162
            'queue-name', amqp_utils.progress_failed.call_args[0][0])
131
163
        self.assertIn('CalledProcessError',
181
        self.assertIn('CalledProcessError',
134
164
                      amqp_utils.progress_failed.call_args[0][1]['traceback'])
182
                      amqp_utils.progress_errored.call_args[0][1]['traceback'])
135
165
        err = amqp_utils.progress_failed.call_args[0][1]['error_message']
183
136
184
        err = amqp_utils.progress_errored.call_args[0][1]['error_message']
137
166
        self.assertIn(str(worker.cmd), err)
185
        self.assertIn(str(worker.cmd), err)
138
167
        self.assertIn(str(worker.output), err)
186
        self.assertIn(str(worker.output), err)
139
168
        self.assertEqual(1, worker.store.put_file.call_count)
187
        self.assertEqual(1, worker.store.put_file.call_count)
140
@@ -199,9 +218,10 @@
141
199
        worker = _worker(amqp_utils, 1)
218
        worker = _worker(amqp_utils, 1)
142
200
        worker._on_message(msg)
219
        worker._on_message(msg)
143
201
        msg.channel.basic_ack.assert_called_once_with('foo')
220
        msg.channel.basic_ack.assert_called_once_with('foo')
145
202
        self.assertEqual(1, amqp_utils.progress_failed.call_count)
221
        self.assertEqual(0, amqp_utils.progress_failed.call_count)
146
222
        self.assertEqual(1, amqp_utils.progress_errored.call_count)
147
203
        self.assertEqual(
223
        self.assertEqual(
149
204
            'queue-name', amqp_utils.progress_failed.call_args[0][0])
224
            'queue-name', amqp_utils.progress_errored.call_args[0][0])
150
205
225
151
206
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
226
    @mock.patch('ci_utils.amqp_worker.amqp_utils')
152
207
    def testNoQueue(self, amqp_utils):
227
    def testNoQueue(self, amqp_utils):
Status:	Merged
Merged at revision:	918
Proposed branch:	lp:~joetalbott/uci-engine/split_infra_failures
Merge into:	lp:uci-engine
Diff against target:	152 lines (+57/-19) 3 files modified ci-utils/ci_utils/amqp_utils.py (+9/-0) ci-utils/ci_utils/amqp_worker.py (+10/-1) ci-utils/ci_utils/tests/test_amqp_worker.py (+38/-18)
To merge this branch:	bzr merge lp:~joetalbott/uci-engine/split_infra_failures
Related bugs:	Link a bug report
Reviewer	Date Requested	Status
Paul Larson		Approve on 2015-01-08
Francis Ginther		Approve on 2015-01-08
Vincent Ladeuil (community)		Approve on 2015-01-08
Larry E Works (community)	2015-01-07	Approve on 2015-01-07
Review via email: mp+245789@code.launchpad.net