Merge lp:~pwlars/snappy-proposed-selftest-agent/rename-agent-code into lp:snappy-proposed-selftest-agent
- rename-agent-code
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Paul Larson |
Approved revision: | 10 |
Merged at revision: | 7 |
Proposed branch: | lp:~pwlars/snappy-proposed-selftest-agent/rename-agent-code |
Merge into: | lp:snappy-proposed-selftest-agent |
Diff against target: |
733 lines (+308/-307) 15 files modified
README.rst (+5/-5) called-by-tarmac.py (+2/-2) core-selftest-agent.py (+0/-7) core_selftest_agent/__init__.py (+0/-116) core_selftest_agent/constants.py (+0/-49) core_selftest_agent/tests/__init__.py (+0/-16) core_selftest_agent/tests/test_worker.py (+0/-53) core_selftest_agent/worker.py (+0/-52) setup.py (+7/-7) snappy-proposed-selftest-agent.py (+7/-0) snappy_proposed_selftest_agent/__init__.py (+117/-0) snappy_proposed_selftest_agent/constants.py (+49/-0) snappy_proposed_selftest_agent/tests/__init__.py (+16/-0) snappy_proposed_selftest_agent/tests/test_worker.py (+53/-0) snappy_proposed_selftest_agent/worker.py (+52/-0) |
To merge this branch: | bzr merge lp:~pwlars/snappy-proposed-selftest-agent/rename-agent-code |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Para Siva (community) | Approve | ||
Paul Larson | Needs Resubmitting | ||
Review via email: mp+260295@code.launchpad.net |
Commit message
Rename core-selftest-agent to snappy-
Description of the change
Rename core-selftest-agent to snappy-
- 8. By Paul Larson
-
add back snappy-
proposed- selftest- agent.py and snappy_ proposed_ selftest_ agent/
Paul Larson (pwlars) wrote : | # |
argh, I forgot to do bzr before the mv when renaming the module and the cli. I've just added them back, which makes this a bit long, but it's just a rename and s/core-
Para Siva (psivaa) wrote : | # |
Thanks for adding the worker code, Paul. I've got some inline comments about the queue names etc, since we're renaming. Let me know if they dont make sense.
- 9. By Paul Larson
-
change conf file name
- 10. By Paul Larson
-
Add .v1 to exchanges as discussed
Paul Larson (pwlars) wrote : | # |
I was going to handle the exchange .v1 addition we discussed this morning in a separate mp, but no good reason not to do it here. Also change the .conf file name as is being done elsewhere.
Preview Diff
1 | === modified file 'README.rst' | |||
2 | --- README.rst 2015-05-20 22:00:00 +0000 | |||
3 | +++ README.rst 2015-05-27 17:01:56 +0000 | |||
4 | @@ -1,10 +1,10 @@ | |||
7 | 1 | Core Selftest Agent | 1 | Snappy Proposed Selftest Agent |
8 | 2 | ################### | 2 | ############################## |
9 | 3 | 3 | ||
10 | 4 | A micro-service that watches for britney-announcer events of new | 4 | A micro-service that watches for britney-announcer events of new |
11 | 5 | packages to test, checks if they are in the manifest for core images in | 5 | packages to test, checks if they are in the manifest for core images in |
14 | 6 | the channel we want to test, and triggers core-image-builder to start | 6 | the channel we want to test, and triggers snappy-proposed-image-builder |
15 | 7 | building an image with the proposed changes. This service is only | 7 | to start building an image with the proposed changes. This service is only |
16 | 8 | responsible for triggering test runs of the snappy selftests. | 8 | responsible for triggering test runs of the snappy selftests. |
17 | 9 | 9 | ||
18 | 10 | Get the Source | 10 | Get the Source |
19 | @@ -12,7 +12,7 @@ | |||
20 | 12 | 12 | ||
21 | 13 | Branch the code:: | 13 | Branch the code:: |
22 | 14 | 14 | ||
24 | 15 | $ bzr branch lp:core-selftest-agent | 15 | $ bzr branch lp:snappy-proposed-selftest-agent |
25 | 16 | 16 | ||
26 | 17 | Install the Service | 17 | Install the Service |
27 | 18 | =================== | 18 | =================== |
28 | 19 | 19 | ||
29 | === modified file 'called-by-tarmac.py' | |||
30 | --- called-by-tarmac.py 2015-05-22 16:26:11 +0000 | |||
31 | +++ called-by-tarmac.py 2015-05-27 17:01:56 +0000 | |||
32 | @@ -1,6 +1,6 @@ | |||
33 | 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
34 | 2 | # | 2 | # |
36 | 3 | # core-selftest-agent | 3 | # snappy-proposed-selftest-agent |
37 | 4 | # Copyright (C) 2015 Canonical | 4 | # Copyright (C) 2015 Canonical |
38 | 5 | # | 5 | # |
39 | 6 | # This program is free software: you can redistribute it and/or modify | 6 | # This program is free software: you can redistribute it and/or modify |
40 | @@ -33,7 +33,7 @@ | |||
41 | 33 | import sys | 33 | import sys |
42 | 34 | import tempfile | 34 | import tempfile |
43 | 35 | 35 | ||
45 | 36 | SERVICE_NAME = 'core-selftest-agent' | 36 | SERVICE_NAME = 'snappy-proposed-selftest-agent' |
46 | 37 | VENV_DIR = 'venv-{}'.format(SERVICE_NAME) | 37 | VENV_DIR = 'venv-{}'.format(SERVICE_NAME) |
47 | 38 | PIP_DIR = 'pip-cache-'.format(SERVICE_NAME) | 38 | PIP_DIR = 'pip-cache-'.format(SERVICE_NAME) |
48 | 39 | PIP_CACHE_BRANCH = 'lp:~canonical-ci-engineering/{}/' \ | 39 | PIP_CACHE_BRANCH = 'lp:~canonical-ci-engineering/{}/' \ |
49 | 40 | 40 | ||
50 | === removed file 'core-selftest-agent.py' | |||
51 | --- core-selftest-agent.py 2015-05-20 17:20:50 +0000 | |||
52 | +++ core-selftest-agent.py 1970-01-01 00:00:00 +0000 | |||
53 | @@ -1,7 +0,0 @@ | |||
54 | 1 | #!/usr/bin/env python3 | ||
55 | 2 | |||
56 | 3 | from core_selftest_agent import main | ||
57 | 4 | |||
58 | 5 | |||
59 | 6 | if __name__ == '__main__': | ||
60 | 7 | main() | ||
61 | 8 | 0 | ||
62 | === removed directory 'core_selftest_agent' | |||
63 | === removed file 'core_selftest_agent/__init__.py' | |||
64 | --- core_selftest_agent/__init__.py 2015-05-22 19:49:10 +0000 | |||
65 | +++ core_selftest_agent/__init__.py 1970-01-01 00:00:00 +0000 | |||
66 | @@ -1,116 +0,0 @@ | |||
67 | 1 | # core-selftest-agent | ||
68 | 2 | # Copyright (C) 2015 Canonical | ||
69 | 3 | # | ||
70 | 4 | # This program is free software: you can redistribute it and/or modify | ||
71 | 5 | # it under the terms of the GNU General Public License as published by | ||
72 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
73 | 7 | # (at your option) any later version. | ||
74 | 8 | # | ||
75 | 9 | # This program is distributed in the hope that it will be useful, | ||
76 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
77 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
78 | 12 | # GNU General Public License for more details. | ||
79 | 13 | # | ||
80 | 14 | # You should have received a copy of the GNU General Public License | ||
81 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
82 | 16 | # | ||
83 | 17 | |||
84 | 18 | import argparse | ||
85 | 19 | import configparser | ||
86 | 20 | import kombu | ||
87 | 21 | import logging | ||
88 | 22 | import os | ||
89 | 23 | |||
90 | 24 | from uservice_utils.logging import ( | ||
91 | 25 | configure_service_logging, | ||
92 | 26 | ExtraLogger, | ||
93 | 27 | ) | ||
94 | 28 | |||
95 | 29 | from core_selftest_agent import constants | ||
96 | 30 | from core_selftest_agent.worker import CoreSelftestAgentWorker | ||
97 | 31 | |||
98 | 32 | |||
99 | 33 | class CoreImageResultPublisher(object): | ||
100 | 34 | """A callable that can notify a results exchange that the test is running | ||
101 | 35 | |||
102 | 36 | This will be a fanout exchange so multiple consumers can pick up these | ||
103 | 37 | messages and decide whether to take action or ignore them. | ||
104 | 38 | """ | ||
105 | 39 | |||
106 | 40 | def __init__(self, connection): | ||
107 | 41 | self.connection = connection | ||
108 | 42 | exchange = kombu.Exchange(constants.RESULTS_EXCHANGE, | ||
109 | 43 | type='fanout', durable=False) | ||
110 | 44 | self.queue = kombu.Queue(constants.RESULTS_QUEUE, exchange) | ||
111 | 45 | |||
112 | 46 | def __call__(self, payload): | ||
113 | 47 | """Take 'payload' and enqueue it on the rabbit exchange.""" | ||
114 | 48 | queue = self.connection.SimpleQueue(self.queue) | ||
115 | 49 | queue.put(payload) | ||
116 | 50 | queue.close() | ||
117 | 51 | |||
118 | 52 | |||
119 | 53 | class CoreImageBuilder(object): | ||
120 | 54 | """A callable that knows how to queue message for the image builder | ||
121 | 55 | |||
122 | 56 | For now we're using a simple queue, but this can easily be extended to use | ||
123 | 57 | a full-blown topic exchange in the future. | ||
124 | 58 | """ | ||
125 | 59 | |||
126 | 60 | def __init__(self, connection): | ||
127 | 61 | self.connection = connection | ||
128 | 62 | |||
129 | 63 | def __call__(self, payload): | ||
130 | 64 | """Take 'payload' and enqueue it on the rabbit queue.""" | ||
131 | 65 | queue = self.connection.SimpleQueue(constants.IMAGE_BUILD_QUEUE) | ||
132 | 66 | queue.put(payload) | ||
133 | 67 | queue.close() | ||
134 | 68 | |||
135 | 69 | |||
136 | 70 | def read_config(): | ||
137 | 71 | parser = argparse.ArgumentParser(description='Core Selftest Agent.') | ||
138 | 72 | parser.add_argument( | ||
139 | 73 | '-c', | ||
140 | 74 | '--conf', | ||
141 | 75 | default='core-service.conf', | ||
142 | 76 | help='Configuration file path' | ||
143 | 77 | ) | ||
144 | 78 | args = parser.parse_args() | ||
145 | 79 | |||
146 | 80 | # Load configuration options. | ||
147 | 81 | config = configparser.ConfigParser() | ||
148 | 82 | config.read(args.conf) | ||
149 | 83 | return config | ||
150 | 84 | |||
151 | 85 | |||
152 | 86 | def get_logger(name, extra={}): | ||
153 | 87 | if logging.getLoggerClass() != ExtraLogger: | ||
154 | 88 | logging.setLoggerClass(ExtraLogger) | ||
155 | 89 | logger = logging.getLogger(name) | ||
156 | 90 | e = constants.LOGGING_EXTRA.copy() | ||
157 | 91 | e.update(extra) | ||
158 | 92 | logger.set_extra_args(e) | ||
159 | 93 | return logger | ||
160 | 94 | |||
161 | 95 | |||
162 | 96 | def main(): | ||
163 | 97 | config = read_config() | ||
164 | 98 | log_path = os.path.abspath( | ||
165 | 99 | os.path.join(__file__, '../../logs/core-selftest-agent.log')) | ||
166 | 100 | configure_service_logging( | ||
167 | 101 | log_path, | ||
168 | 102 | config['logstash'] if 'logstash' in config else None | ||
169 | 103 | ) | ||
170 | 104 | |||
171 | 105 | ampq_uris = config.get('amqp', 'uris').split() | ||
172 | 106 | try: | ||
173 | 107 | with kombu.Connection(ampq_uris) as connection: | ||
174 | 108 | monitor = CoreSelftestAgentWorker( | ||
175 | 109 | connection, | ||
176 | 110 | constants, | ||
177 | 111 | CoreImageBuilder(connection), | ||
178 | 112 | CoreImageResultPublisher(connection), | ||
179 | 113 | ) | ||
180 | 114 | monitor.run() | ||
181 | 115 | except KeyboardInterrupt: | ||
182 | 116 | print("Bye!") | ||
183 | 117 | 0 | ||
184 | === removed file 'core_selftest_agent/constants.py' | |||
185 | --- core_selftest_agent/constants.py 2015-05-26 14:57:41 +0000 | |||
186 | +++ core_selftest_agent/constants.py 1970-01-01 00:00:00 +0000 | |||
187 | @@ -1,49 +0,0 @@ | |||
188 | 1 | # core-selftest-agent | ||
189 | 2 | # Copyright (C) 2015 Canonical | ||
190 | 3 | # | ||
191 | 4 | # This program is free software: you can redistribute it and/or modify | ||
192 | 5 | # it under the terms of the GNU General Public License as published by | ||
193 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
194 | 7 | # (at your option) any later version. | ||
195 | 8 | # | ||
196 | 9 | # This program is distributed in the hope that it will be useful, | ||
197 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
198 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
199 | 12 | # GNU General Public License for more details. | ||
200 | 13 | # | ||
201 | 14 | # You should have received a copy of the GNU General Public License | ||
202 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
203 | 16 | # | ||
204 | 17 | |||
205 | 18 | """Constants for this service.""" | ||
206 | 19 | import socket | ||
207 | 20 | |||
208 | 21 | # Queue names are constant, rather than being defined in the config. | ||
209 | 22 | |||
210 | 23 | # We listen to a fanout exchange for candidate packages to test | ||
211 | 24 | INPUT_EXCHANGE = "candidates.exchange" | ||
212 | 25 | |||
213 | 26 | # The queue needs to be unique to selftest agents | ||
214 | 27 | INPUT_QUEUE = "core.selftest.candidates" | ||
215 | 28 | |||
216 | 29 | # The queue we send events on to request image build and test | ||
217 | 30 | IMAGE_BUILD_QUEUE = "snappy-proposed-migration.package.v1" | ||
218 | 31 | |||
219 | 32 | # We need to notify the results exchange that the test is in progress | ||
220 | 33 | RESULTS_EXCHANGE = "results.exchange" | ||
221 | 34 | RESULTS_QUEUE = "core.selftest.results" | ||
222 | 35 | |||
223 | 36 | # The queue we put fatally error'd payloads into: | ||
224 | 37 | DEAD_LETTER_QUEUE = "core.deadletters.v1" | ||
225 | 38 | |||
226 | 39 | SOLUTION_NAME = "core-image-testing" | ||
227 | 40 | |||
228 | 41 | SERVICE_NAME = "core-selftest-agent" | ||
229 | 42 | |||
230 | 43 | HOSTNAME = socket.gethostname() | ||
231 | 44 | |||
232 | 45 | LOGGING_EXTRA = { | ||
233 | 46 | 'solution': SOLUTION_NAME, | ||
234 | 47 | 'service': SERVICE_NAME, | ||
235 | 48 | 'hostname': HOSTNAME, | ||
236 | 49 | } | ||
237 | 50 | 0 | ||
238 | === removed directory 'core_selftest_agent/tests' | |||
239 | === removed file 'core_selftest_agent/tests/__init__.py' | |||
240 | --- core_selftest_agent/tests/__init__.py 2015-05-22 19:49:10 +0000 | |||
241 | +++ core_selftest_agent/tests/__init__.py 1970-01-01 00:00:00 +0000 | |||
242 | @@ -1,16 +0,0 @@ | |||
243 | 1 | # core-selftest-agent | ||
244 | 2 | # Copyright (C) 2015 Canonical | ||
245 | 3 | # | ||
246 | 4 | # This program is free software: you can redistribute it and/or modify | ||
247 | 5 | # it under the terms of the GNU General Public License as published by | ||
248 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
249 | 7 | # (at your option) any later version. | ||
250 | 8 | # | ||
251 | 9 | # This program is distributed in the hope that it will be useful, | ||
252 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
253 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
254 | 12 | # GNU General Public License for more details. | ||
255 | 13 | # | ||
256 | 14 | # You should have received a copy of the GNU General Public License | ||
257 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
258 | 16 | # | ||
259 | 17 | 0 | ||
260 | === removed file 'core_selftest_agent/tests/test_worker.py' | |||
261 | --- core_selftest_agent/tests/test_worker.py 2015-05-26 14:57:41 +0000 | |||
262 | +++ core_selftest_agent/tests/test_worker.py 1970-01-01 00:00:00 +0000 | |||
263 | @@ -1,53 +0,0 @@ | |||
264 | 1 | # core-selftest-agent | ||
265 | 2 | # Copyright (C) 2015 Canonical | ||
266 | 3 | # | ||
267 | 4 | # This program is free software: you can redistribute it and/or modify | ||
268 | 5 | # it under the terms of the GNU General Public License as published by | ||
269 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
270 | 7 | # (at your option) any later version. | ||
271 | 8 | # | ||
272 | 9 | # This program is distributed in the hope that it will be useful, | ||
273 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
274 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
275 | 12 | # GNU General Public License for more details. | ||
276 | 13 | # | ||
277 | 14 | # You should have received a copy of the GNU General Public License | ||
278 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
279 | 16 | # | ||
280 | 17 | |||
281 | 18 | import kombu | ||
282 | 19 | from testtools import TestCase | ||
283 | 20 | |||
284 | 21 | from core_selftest_agent import constants | ||
285 | 22 | from core_selftest_agent.worker import CoreSelftestAgentWorker | ||
286 | 23 | |||
287 | 24 | |||
288 | 25 | class WorkerTests(TestCase): | ||
289 | 26 | def test_can_read_message(self): | ||
290 | 27 | conn = kombu.Connection('memory:///') | ||
291 | 28 | queue_message(conn, {'test': 'value'}) | ||
292 | 29 | |||
293 | 30 | build_worker = LoggingConsumer() | ||
294 | 31 | result_worker = LoggingConsumer() | ||
295 | 32 | q = CoreSelftestAgentWorker(conn, constants, | ||
296 | 33 | build_worker, result_worker) | ||
297 | 34 | next(q.consume(limit=1, timeout=5.0)) | ||
298 | 35 | self.assertEqual(build_worker.messages_seen, [dict(test='value')]) | ||
299 | 36 | self.assertEqual(result_worker.messages_seen, [dict(test='value')]) | ||
300 | 37 | |||
301 | 38 | |||
302 | 39 | class LoggingConsumer(object): | ||
303 | 40 | """A consumer callback object that acks and logs all received payloads.""" | ||
304 | 41 | def __init__(self): | ||
305 | 42 | self.messages_seen = [] | ||
306 | 43 | |||
307 | 44 | def __call__(self, message): | ||
308 | 45 | self.messages_seen.append(message) | ||
309 | 46 | |||
310 | 47 | |||
311 | 48 | def queue_message(conn, message): | ||
312 | 49 | exchange = kombu.Exchange(constants.INPUT_EXCHANGE, | ||
313 | 50 | type='fanout', durable=False) | ||
314 | 51 | queue = kombu.Queue(constants.INPUT_QUEUE, exchange) | ||
315 | 52 | with conn.SimpleQueue(queue) as q: | ||
316 | 53 | q.put(message) | ||
317 | 54 | 0 | ||
318 | === removed file 'core_selftest_agent/worker.py' | |||
319 | --- core_selftest_agent/worker.py 2015-05-22 19:49:10 +0000 | |||
320 | +++ core_selftest_agent/worker.py 1970-01-01 00:00:00 +0000 | |||
321 | @@ -1,52 +0,0 @@ | |||
322 | 1 | # core-selftest-agent | ||
323 | 2 | # Copyright (C) 2015 Canonical | ||
324 | 3 | # | ||
325 | 4 | # This program is free software: you can redistribute it and/or modify | ||
326 | 5 | # it under the terms of the GNU General Public License as published by | ||
327 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
328 | 7 | # (at your option) any later version. | ||
329 | 8 | # | ||
330 | 9 | # This program is distributed in the hope that it will be useful, | ||
331 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
332 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
333 | 12 | # GNU General Public License for more details. | ||
334 | 13 | # | ||
335 | 14 | # You should have received a copy of the GNU General Public License | ||
336 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
337 | 16 | # | ||
338 | 17 | |||
339 | 18 | import kombu | ||
340 | 19 | |||
341 | 20 | from kombu.mixins import ConsumerMixin | ||
342 | 21 | |||
343 | 22 | |||
344 | 23 | class CoreSelftestAgentWorker(ConsumerMixin): | ||
345 | 24 | |||
346 | 25 | def __init__(self, connection, config, build_worker, result_worker): | ||
347 | 26 | self.config = config | ||
348 | 27 | self.connection = connection | ||
349 | 28 | self.build_worker = build_worker | ||
350 | 29 | self.result_worker = result_worker | ||
351 | 30 | |||
352 | 31 | exchange = kombu.Exchange(self.config.INPUT_EXCHANGE, | ||
353 | 32 | type='fanout', durable=False) | ||
354 | 33 | self.queue = kombu.Queue(self.config.INPUT_QUEUE, exchange) | ||
355 | 34 | |||
356 | 35 | def get_consumers(self, Consumer, channel): | ||
357 | 36 | """Return consumers instances for all configured queues.""" | ||
358 | 37 | queues = [self.queue] | ||
359 | 38 | return [kombu.Consumer( | ||
360 | 39 | channel, queues=queues, callbacks=[self.process])] | ||
361 | 40 | |||
362 | 41 | def process(self, body, message): | ||
363 | 42 | """ Process incoming messages about candidate packages | ||
364 | 43 | |||
365 | 44 | TODO: Check if we care about this package, if so, send a message | ||
366 | 45 | to the image builder to create the image and start the tests, and | ||
367 | 46 | a message to the results exchange to mark it as in progress. | ||
368 | 47 | |||
369 | 48 | For now, just blindly post messages for these | ||
370 | 49 | """ | ||
371 | 50 | |||
372 | 51 | self.build_worker(body) | ||
373 | 52 | self.result_worker(body) | ||
374 | 53 | 0 | ||
375 | === modified file 'setup.py' | |||
376 | --- setup.py 2015-05-22 19:49:10 +0000 | |||
377 | +++ setup.py 2015-05-27 17:01:56 +0000 | |||
378 | @@ -1,6 +1,6 @@ | |||
379 | 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
380 | 2 | 2 | ||
382 | 3 | # core-selftest-agent | 3 | # snappy-proposed-selftest-agent |
383 | 4 | # Copyright (C) 2015 Canonical | 4 | # Copyright (C) 2015 Canonical |
384 | 5 | # | 5 | # |
385 | 6 | # This program is free software: you can redistribute it and/or modify | 6 | # This program is free software: you can redistribute it and/or modify |
386 | @@ -27,15 +27,15 @@ | |||
387 | 27 | 27 | ||
388 | 28 | 28 | ||
389 | 29 | setup( | 29 | setup( |
391 | 30 | name='core-selftest-agent', | 30 | name='snappy-proposed-selftest-agent', |
392 | 31 | version=VERSION, | 31 | version=VERSION, |
395 | 32 | description=('A microservice for triggering snappy selftest on changes to ' | 32 | description=('A microservice for triggering snappy selftest on changes ' |
396 | 33 | 'core packages.'), | 33 | 'to core packages.'), |
397 | 34 | author='Canonical CI Engineering Team', | 34 | author='Canonical CI Engineering Team', |
398 | 35 | author_email='canonical-ci-engineering@lists.launchpad.net', | 35 | author_email='canonical-ci-engineering@lists.launchpad.net', |
400 | 36 | url='https://launchpad.net/core-selftest-agent', | 36 | url='https://launchpad.net/snappy-proposed-selftest-agent', |
401 | 37 | license='GPLv3', | 37 | license='GPLv3', |
402 | 38 | packages=find_packages(), | 38 | packages=find_packages(), |
405 | 39 | scripts=['core-selftest-agent.py'], | 39 | scripts=['snappy-proposed-selftest-agent.py'], |
406 | 40 | test_suite='core_selftest_agent.tests', | 40 | test_suite='snappy_proposed_selftest_agent.tests', |
407 | 41 | ) | 41 | ) |
408 | 42 | 42 | ||
409 | === added file 'snappy-proposed-selftest-agent.py' | |||
410 | --- snappy-proposed-selftest-agent.py 1970-01-01 00:00:00 +0000 | |||
411 | +++ snappy-proposed-selftest-agent.py 2015-05-27 17:01:56 +0000 | |||
412 | @@ -0,0 +1,7 @@ | |||
413 | 1 | #!/usr/bin/env python3 | ||
414 | 2 | |||
415 | 3 | from snappy_proposed_selftest_agent import main | ||
416 | 4 | |||
417 | 5 | |||
418 | 6 | if __name__ == '__main__': | ||
419 | 7 | main() | ||
420 | 0 | 8 | ||
421 | === added directory 'snappy_proposed_selftest_agent' | |||
422 | === added file 'snappy_proposed_selftest_agent/__init__.py' | |||
423 | --- snappy_proposed_selftest_agent/__init__.py 1970-01-01 00:00:00 +0000 | |||
424 | +++ snappy_proposed_selftest_agent/__init__.py 2015-05-27 17:01:56 +0000 | |||
425 | @@ -0,0 +1,117 @@ | |||
426 | 1 | # snappy-proposed-selftest-agent | ||
427 | 2 | # Copyright (C) 2015 Canonical | ||
428 | 3 | # | ||
429 | 4 | # This program is free software: you can redistribute it and/or modify | ||
430 | 5 | # it under the terms of the GNU General Public License as published by | ||
431 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
432 | 7 | # (at your option) any later version. | ||
433 | 8 | # | ||
434 | 9 | # This program is distributed in the hope that it will be useful, | ||
435 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
436 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
437 | 12 | # GNU General Public License for more details. | ||
438 | 13 | # | ||
439 | 14 | # You should have received a copy of the GNU General Public License | ||
440 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
441 | 16 | # | ||
442 | 17 | |||
443 | 18 | import argparse | ||
444 | 19 | import configparser | ||
445 | 20 | import kombu | ||
446 | 21 | import logging | ||
447 | 22 | import os | ||
448 | 23 | |||
449 | 24 | from uservice_utils.logging import ( | ||
450 | 25 | configure_service_logging, | ||
451 | 26 | ExtraLogger, | ||
452 | 27 | ) | ||
453 | 28 | |||
454 | 29 | from snappy_proposed_selftest_agent import constants | ||
455 | 30 | from snappy_proposed_selftest_agent.worker import CoreSelftestAgentWorker | ||
456 | 31 | |||
457 | 32 | |||
458 | 33 | class CoreImageResultPublisher(object): | ||
459 | 34 | """A callable that can notify a results exchange that the test is running | ||
460 | 35 | |||
461 | 36 | This will be a fanout exchange so multiple consumers can pick up these | ||
462 | 37 | messages and decide whether to take action or ignore them. | ||
463 | 38 | """ | ||
464 | 39 | |||
465 | 40 | def __init__(self, connection): | ||
466 | 41 | self.connection = connection | ||
467 | 42 | exchange = kombu.Exchange(constants.RESULTS_EXCHANGE, | ||
468 | 43 | type='fanout', durable=False) | ||
469 | 44 | self.queue = kombu.Queue(constants.RESULTS_QUEUE, exchange) | ||
470 | 45 | |||
471 | 46 | def __call__(self, payload): | ||
472 | 47 | """Take 'payload' and enqueue it on the rabbit exchange.""" | ||
473 | 48 | queue = self.connection.SimpleQueue(self.queue) | ||
474 | 49 | queue.put(payload) | ||
475 | 50 | queue.close() | ||
476 | 51 | |||
477 | 52 | |||
478 | 53 | class CoreImageBuilder(object): | ||
479 | 54 | """A callable that knows how to queue message for the image builder | ||
480 | 55 | |||
481 | 56 | For now we're using a simple queue, but this can easily be extended to use | ||
482 | 57 | a full-blown topic exchange in the future. | ||
483 | 58 | """ | ||
484 | 59 | |||
485 | 60 | def __init__(self, connection): | ||
486 | 61 | self.connection = connection | ||
487 | 62 | |||
488 | 63 | def __call__(self, payload): | ||
489 | 64 | """Take 'payload' and enqueue it on the rabbit queue.""" | ||
490 | 65 | queue = self.connection.SimpleQueue(constants.IMAGE_BUILD_QUEUE) | ||
491 | 66 | queue.put(payload) | ||
492 | 67 | queue.close() | ||
493 | 68 | |||
494 | 69 | |||
495 | 70 | def read_config(): | ||
496 | 71 | parser = argparse.ArgumentParser(description='Core Selftest Agent.') | ||
497 | 72 | parser.add_argument( | ||
498 | 73 | '-c', | ||
499 | 74 | '--conf', | ||
500 | 75 | default='snappy-proposed-service.conf', | ||
501 | 76 | help='Configuration file path' | ||
502 | 77 | ) | ||
503 | 78 | args = parser.parse_args() | ||
504 | 79 | |||
505 | 80 | # Load configuration options. | ||
506 | 81 | config = configparser.ConfigParser() | ||
507 | 82 | config.read(args.conf) | ||
508 | 83 | return config | ||
509 | 84 | |||
510 | 85 | |||
511 | 86 | def get_logger(name, extra={}): | ||
512 | 87 | if logging.getLoggerClass() != ExtraLogger: | ||
513 | 88 | logging.setLoggerClass(ExtraLogger) | ||
514 | 89 | logger = logging.getLogger(name) | ||
515 | 90 | e = constants.LOGGING_EXTRA.copy() | ||
516 | 91 | e.update(extra) | ||
517 | 92 | logger.set_extra_args(e) | ||
518 | 93 | return logger | ||
519 | 94 | |||
520 | 95 | |||
521 | 96 | def main(): | ||
522 | 97 | config = read_config() | ||
523 | 98 | log_path = os.path.abspath( | ||
524 | 99 | os.path.join(__file__, | ||
525 | 100 | '../../logs/snappy-proposed-selftest-agent.log')) | ||
526 | 101 | configure_service_logging( | ||
527 | 102 | log_path, | ||
528 | 103 | config['logstash'] if 'logstash' in config else None | ||
529 | 104 | ) | ||
530 | 105 | |||
531 | 106 | ampq_uris = config.get('amqp', 'uris').split() | ||
532 | 107 | try: | ||
533 | 108 | with kombu.Connection(ampq_uris) as connection: | ||
534 | 109 | monitor = CoreSelftestAgentWorker( | ||
535 | 110 | connection, | ||
536 | 111 | constants, | ||
537 | 112 | CoreImageBuilder(connection), | ||
538 | 113 | CoreImageResultPublisher(connection), | ||
539 | 114 | ) | ||
540 | 115 | monitor.run() | ||
541 | 116 | except KeyboardInterrupt: | ||
542 | 117 | print("Bye!") | ||
543 | 0 | 118 | ||
544 | === added file 'snappy_proposed_selftest_agent/constants.py' | |||
545 | --- snappy_proposed_selftest_agent/constants.py 1970-01-01 00:00:00 +0000 | |||
546 | +++ snappy_proposed_selftest_agent/constants.py 2015-05-27 17:01:56 +0000 | |||
547 | @@ -0,0 +1,49 @@ | |||
548 | 1 | # snappy-proposed-selftest-agent | ||
549 | 2 | # Copyright (C) 2015 Canonical | ||
550 | 3 | # | ||
551 | 4 | # This program is free software: you can redistribute it and/or modify | ||
552 | 5 | # it under the terms of the GNU General Public License as published by | ||
553 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
554 | 7 | # (at your option) any later version. | ||
555 | 8 | # | ||
556 | 9 | # This program is distributed in the hope that it will be useful, | ||
557 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
558 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
559 | 12 | # GNU General Public License for more details. | ||
560 | 13 | # | ||
561 | 14 | # You should have received a copy of the GNU General Public License | ||
562 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
563 | 16 | # | ||
564 | 17 | |||
565 | 18 | """Constants for this service.""" | ||
566 | 19 | import socket | ||
567 | 20 | |||
568 | 21 | # Queue names are constant, rather than being defined in the config. | ||
569 | 22 | |||
570 | 23 | # We listen to a fanout exchange for candidate packages to test | ||
571 | 24 | INPUT_EXCHANGE = "candidates.exchange.v1" | ||
572 | 25 | |||
573 | 26 | # The queue needs to be unique to selftest agents | ||
574 | 27 | INPUT_QUEUE = "core.selftest.candidates" | ||
575 | 28 | |||
576 | 29 | # The queue we send events on to request image build and test | ||
577 | 30 | IMAGE_BUILD_QUEUE = "snappy-proposed-migration.package.v1" | ||
578 | 31 | |||
579 | 32 | # We need to notify the results exchange that the test is in progress | ||
580 | 33 | RESULTS_EXCHANGE = "results.exchange.v1" | ||
581 | 34 | RESULTS_QUEUE = "core.selftest.results" | ||
582 | 35 | |||
583 | 36 | # The queue we put fatally error'd payloads into: | ||
584 | 37 | DEAD_LETTER_QUEUE = "core.deadletters.v1" | ||
585 | 38 | |||
586 | 39 | SOLUTION_NAME = "snappy-proposed-image-testing" | ||
587 | 40 | |||
588 | 41 | SERVICE_NAME = "snappy-proposed-selftest-agent" | ||
589 | 42 | |||
590 | 43 | HOSTNAME = socket.gethostname() | ||
591 | 44 | |||
592 | 45 | LOGGING_EXTRA = { | ||
593 | 46 | 'solution': SOLUTION_NAME, | ||
594 | 47 | 'service': SERVICE_NAME, | ||
595 | 48 | 'hostname': HOSTNAME, | ||
596 | 49 | } | ||
597 | 0 | 50 | ||
598 | === added directory 'snappy_proposed_selftest_agent/tests' | |||
599 | === added file 'snappy_proposed_selftest_agent/tests/__init__.py' | |||
600 | --- snappy_proposed_selftest_agent/tests/__init__.py 1970-01-01 00:00:00 +0000 | |||
601 | +++ snappy_proposed_selftest_agent/tests/__init__.py 2015-05-27 17:01:56 +0000 | |||
602 | @@ -0,0 +1,16 @@ | |||
603 | 1 | # snappy-proposed-selftest-agent | ||
604 | 2 | # Copyright (C) 2015 Canonical | ||
605 | 3 | # | ||
606 | 4 | # This program is free software: you can redistribute it and/or modify | ||
607 | 5 | # it under the terms of the GNU General Public License as published by | ||
608 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
609 | 7 | # (at your option) any later version. | ||
610 | 8 | # | ||
611 | 9 | # This program is distributed in the hope that it will be useful, | ||
612 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
613 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
614 | 12 | # GNU General Public License for more details. | ||
615 | 13 | # | ||
616 | 14 | # You should have received a copy of the GNU General Public License | ||
617 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
618 | 16 | # | ||
619 | 0 | 17 | ||
620 | === added file 'snappy_proposed_selftest_agent/tests/test_worker.py' | |||
621 | --- snappy_proposed_selftest_agent/tests/test_worker.py 1970-01-01 00:00:00 +0000 | |||
622 | +++ snappy_proposed_selftest_agent/tests/test_worker.py 2015-05-27 17:01:56 +0000 | |||
623 | @@ -0,0 +1,53 @@ | |||
624 | 1 | # snappy-proposed-selftest-agent | ||
625 | 2 | # Copyright (C) 2015 Canonical | ||
626 | 3 | # | ||
627 | 4 | # This program is free software: you can redistribute it and/or modify | ||
628 | 5 | # it under the terms of the GNU General Public License as published by | ||
629 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
630 | 7 | # (at your option) any later version. | ||
631 | 8 | # | ||
632 | 9 | # This program is distributed in the hope that it will be useful, | ||
633 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
634 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
635 | 12 | # GNU General Public License for more details. | ||
636 | 13 | # | ||
637 | 14 | # You should have received a copy of the GNU General Public License | ||
638 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
639 | 16 | # | ||
640 | 17 | |||
641 | 18 | import kombu | ||
642 | 19 | from testtools import TestCase | ||
643 | 20 | |||
644 | 21 | from snappy_proposed_selftest_agent import constants | ||
645 | 22 | from snappy_proposed_selftest_agent.worker import CoreSelftestAgentWorker | ||
646 | 23 | |||
647 | 24 | |||
648 | 25 | class WorkerTests(TestCase): | ||
649 | 26 | def test_can_read_message(self): | ||
650 | 27 | conn = kombu.Connection('memory:///') | ||
651 | 28 | queue_message(conn, {'test': 'value'}) | ||
652 | 29 | |||
653 | 30 | build_worker = LoggingConsumer() | ||
654 | 31 | result_worker = LoggingConsumer() | ||
655 | 32 | q = CoreSelftestAgentWorker(conn, constants, | ||
656 | 33 | build_worker, result_worker) | ||
657 | 34 | next(q.consume(limit=1, timeout=5.0)) | ||
658 | 35 | self.assertEqual(build_worker.messages_seen, [dict(test='value')]) | ||
659 | 36 | self.assertEqual(result_worker.messages_seen, [dict(test='value')]) | ||
660 | 37 | |||
661 | 38 | |||
662 | 39 | class LoggingConsumer(object): | ||
663 | 40 | """A consumer callback object that acks and logs all received payloads.""" | ||
664 | 41 | def __init__(self): | ||
665 | 42 | self.messages_seen = [] | ||
666 | 43 | |||
667 | 44 | def __call__(self, message): | ||
668 | 45 | self.messages_seen.append(message) | ||
669 | 46 | |||
670 | 47 | |||
671 | 48 | def queue_message(conn, message): | ||
672 | 49 | exchange = kombu.Exchange(constants.INPUT_EXCHANGE, | ||
673 | 50 | type='fanout', durable=False) | ||
674 | 51 | queue = kombu.Queue(constants.INPUT_QUEUE, exchange) | ||
675 | 52 | with conn.SimpleQueue(queue) as q: | ||
676 | 53 | q.put(message) | ||
677 | 0 | 54 | ||
678 | === added file 'snappy_proposed_selftest_agent/worker.py' | |||
679 | --- snappy_proposed_selftest_agent/worker.py 1970-01-01 00:00:00 +0000 | |||
680 | +++ snappy_proposed_selftest_agent/worker.py 2015-05-27 17:01:56 +0000 | |||
681 | @@ -0,0 +1,52 @@ | |||
682 | 1 | # snappy-proposed-selftest-agent | ||
683 | 2 | # Copyright (C) 2015 Canonical | ||
684 | 3 | # | ||
685 | 4 | # This program is free software: you can redistribute it and/or modify | ||
686 | 5 | # it under the terms of the GNU General Public License as published by | ||
687 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
688 | 7 | # (at your option) any later version. | ||
689 | 8 | # | ||
690 | 9 | # This program is distributed in the hope that it will be useful, | ||
691 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
692 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
693 | 12 | # GNU General Public License for more details. | ||
694 | 13 | # | ||
695 | 14 | # You should have received a copy of the GNU General Public License | ||
696 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
697 | 16 | # | ||
698 | 17 | |||
699 | 18 | import kombu | ||
700 | 19 | |||
701 | 20 | from kombu.mixins import ConsumerMixin | ||
702 | 21 | |||
703 | 22 | |||
704 | 23 | class CoreSelftestAgentWorker(ConsumerMixin): | ||
705 | 24 | |||
706 | 25 | def __init__(self, connection, config, build_worker, result_worker): | ||
707 | 26 | self.config = config | ||
708 | 27 | self.connection = connection | ||
709 | 28 | self.build_worker = build_worker | ||
710 | 29 | self.result_worker = result_worker | ||
711 | 30 | |||
712 | 31 | exchange = kombu.Exchange(self.config.INPUT_EXCHANGE, | ||
713 | 32 | type='fanout', durable=False) | ||
714 | 33 | self.queue = kombu.Queue(self.config.INPUT_QUEUE, exchange) | ||
715 | 34 | |||
716 | 35 | def get_consumers(self, Consumer, channel): | ||
717 | 36 | """Return consumers instances for all configured queues.""" | ||
718 | 37 | queues = [self.queue] | ||
719 | 38 | return [kombu.Consumer( | ||
720 | 39 | channel, queues=queues, callbacks=[self.process])] | ||
721 | 40 | |||
722 | 41 | def process(self, body, message): | ||
723 | 42 | """ Process incoming messages about candidate packages | ||
724 | 43 | |||
725 | 44 | TODO: Check if we care about this package, if so, send a message | ||
726 | 45 | to the image builder to create the image and start the tests, and | ||
727 | 46 | a message to the results exchange to mark it as in progress. | ||
728 | 47 | |||
729 | 48 | For now, just blindly post messages for these | ||
730 | 49 | """ | ||
731 | 50 | |||
732 | 51 | self.build_worker(body) | ||
733 | 52 | self.result_worker(body) |
+1, Thanks for the comment about the worker code as incoming MPs