Merge lp:~canonical-platform-qa/qakit/test-execution-gathering into lp:qakit
- test-execution-gathering
- Merge into trunk
Proposed by
Allan LeSage
Status: | Merged |
---|---|
Approved by: | Christopher Lee |
Approved revision: | 31 |
Merged at revision: | 22 |
Proposed branch: | lp:~canonical-platform-qa/qakit/test-execution-gathering |
Merge into: | lp:qakit |
Diff against target: |
811 lines (+720/-3) 7 files modified
qakit/metrics/practitest/instances.py (+102/-0) qakit/metrics/practitest/testsets.py (+144/-0) qakit/metrics/test_execution.py (+131/-0) qakit/metrics/tests/test_practitest_instances.py (+113/-0) qakit/metrics/tests/test_practitest_testsets.py (+80/-0) qakit/metrics/tests/test_test_execution.py (+99/-0) qakit/practitest/practitest.py (+51/-3) |
To merge this branch: | bzr merge lp:~canonical-platform-qa/qakit/test-execution-gathering |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Christopher Lee (community) | Approve | ||
Review via email: mp+275778@code.launchpad.net |
Commit message
Gather test execution data from PractiTest.
Description of the change
Gather test execution data from PractiTest and deposit in a mongodb database.
To post a comment you must log in.
- 25. By Allan LeSage
-
De-shebangify.
- 26. By Allan LeSage
-
Adjust count insertion.
- 27. By Allan LeSage
-
Adjust a warning log message.
- 28. By Allan LeSage
-
Adjust a docstring to call out list.
- 29. By Allan LeSage
-
Move no last testset logic up a level, some whitespace and naming changes thrown in.
- 30. By Allan LeSage
-
Resolve flake8 config conflict, remove NOQA.
- 31. By Allan LeSage
-
Just raise.
Revision history for this message
Allan LeSage (allanlesage) wrote : | # |
Addressed in numbered revisions, thanks Chris.
Revision history for this message
Christopher Lee (veebers) wrote : | # |
LGTM, thanks for the changes.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === added directory 'qakit/metrics/practitest' | |||
2 | === added file 'qakit/metrics/practitest/instances.py' | |||
3 | --- qakit/metrics/practitest/instances.py 1970-01-01 00:00:00 +0000 | |||
4 | +++ qakit/metrics/practitest/instances.py 2015-10-29 22:05:12 +0000 | |||
5 | @@ -0,0 +1,102 @@ | |||
6 | 1 | # UESQA Metrics | ||
7 | 2 | # Copyright (C) 2015 Canonical | ||
8 | 3 | # | ||
9 | 4 | # This program is free software: you can redistribute it and/or modify | ||
10 | 5 | # it under the terms of the GNU General Public License as published by | ||
11 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
12 | 7 | # (at your option) any later version. | ||
13 | 8 | # | ||
14 | 9 | # This program is distributed in the hope that it will be useful, | ||
15 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | 12 | # GNU General Public License for more details. | ||
18 | 13 | # | ||
19 | 14 | # You should have received a copy of the GNU General Public License | ||
20 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
21 | 16 | |||
22 | 17 | |||
23 | 18 | import logging | ||
24 | 19 | |||
25 | 20 | import qakit.practitest.practitest as practitest | ||
26 | 21 | |||
27 | 22 | logger = logging.getLogger(__name__) | ||
28 | 23 | |||
29 | 24 | |||
30 | 25 | def append_testset_id(instance): | ||
31 | 26 | """Append TestSet ID to the given PractiTest instance dict. | ||
32 | 27 | |||
33 | 28 | This makes querying much easier later, else it has to be parsed | ||
34 | 29 | out of a string. | ||
35 | 30 | |||
36 | 31 | :param instance: a PractiTest instance | ||
37 | 32 | |||
38 | 33 | """ | ||
39 | 34 | instance['testset_id'] = int(instance['id'].split(':')[0]) | ||
40 | 35 | return instance | ||
41 | 36 | |||
42 | 37 | |||
43 | 38 | def append_data_to_instance_dict(instance): | ||
44 | 39 | """Append TestSet ID and last run datetime to the given instance. | ||
45 | 40 | |||
46 | 41 | :param instance: a PractiTest instance to which to append data | ||
47 | 42 | |||
48 | 43 | """ | ||
49 | 44 | instance = practitest.append_last_run_datetime(instance) | ||
50 | 45 | instance = append_testset_id(instance) | ||
51 | 46 | return instance | ||
52 | 47 | |||
53 | 48 | |||
54 | 49 | def update_testset_instances(db, session, testset_id): | ||
55 | 50 | """Retrieve instances from PractiTest for the given TestSet ID. | ||
56 | 51 | |||
57 | 52 | NOTE that old runs get crushed on the PractiTest side: e.g. if a | ||
58 | 53 | test fails and it's re-run, the failure is lost to the API. | ||
59 | 54 | |||
60 | 55 | :param db: MongoClient db in which to store PractiTest results | ||
61 | 56 | :param practitest_session: PractitestSession | ||
62 | 57 | :param testset_id: the PractiTest TestSet for which to retrieve instances. | ||
63 | 58 | |||
64 | 59 | """ | ||
65 | 60 | logger.info("Retrieving instances for TestSet {}.".format(testset_id)) | ||
66 | 61 | try: | ||
67 | 62 | instances = session.get_instances(testset_id) | ||
68 | 63 | except ValueError as e: | ||
69 | 64 | logger.warn('TestSet not found for id: {}: '.format( | ||
70 | 65 | testset_id), str(e)) | ||
71 | 66 | return 0 | ||
72 | 67 | count = 0 | ||
73 | 68 | for instance in instances: | ||
74 | 69 | instance = append_data_to_instance_dict(instance) | ||
75 | 70 | result = db.instances.update({'system_id': instance['system_id']}, | ||
76 | 71 | instance, | ||
77 | 72 | upsert=True) | ||
78 | 73 | count += result['n'] | ||
79 | 74 | logger.info("Retrieved {} instances for TestSet {}.".format(count, | ||
80 | 75 | testset_id)) | ||
81 | 76 | return count | ||
82 | 77 | |||
83 | 78 | |||
84 | 79 | def update_testsets_instances(db, practitest_session, testset_ids=None): | ||
85 | 80 | """Update instances for the given testsets, finding new instances. | ||
86 | 81 | |||
87 | 82 | An instance represents a test execution in PractiTest. Note that if a | ||
88 | 83 | test is executed more than once, only the last execution is reported via | ||
89 | 84 | the API: e.g. if a test fails and then is run again and passes, the | ||
90 | 85 | failure is lost to us. | ||
91 | 86 | |||
92 | 87 | :param db: a MongoClient db in which to store PractiTest instances | ||
93 | 88 | :param practitest_session: a PractitestSession | ||
94 | 89 | :param testset_ids: list of PractiTest TestSet ids for which to get | ||
95 | 90 | instances: numbers appear in header of TestSet webpage. If no ids are | ||
96 | 91 | given, retrieve instances for all *known* TestSets (i.e. all already | ||
97 | 92 | in the given database). | ||
98 | 93 | |||
99 | 94 | """ | ||
100 | 95 | logger.info("Updating TestSets with new instance runs.") | ||
101 | 96 | if testset_ids is None: | ||
102 | 97 | testset_ids = [testset['id'] for testset in db.testsets.find()] | ||
103 | 98 | count = 0 | ||
104 | 99 | for testset_id in testset_ids: | ||
105 | 100 | count += update_testset_instances( | ||
106 | 101 | db, practitest_session, testset_id) | ||
107 | 102 | logger.info("Updated {} instances.".format(count)) | ||
108 | 0 | 103 | ||
109 | === added file 'qakit/metrics/practitest/testsets.py' | |||
110 | --- qakit/metrics/practitest/testsets.py 1970-01-01 00:00:00 +0000 | |||
111 | +++ qakit/metrics/practitest/testsets.py 2015-10-29 22:05:12 +0000 | |||
112 | @@ -0,0 +1,144 @@ | |||
113 | 1 | # UESQA Metrics | ||
114 | 2 | # Copyright (C) 2015 Canonical | ||
115 | 3 | # | ||
116 | 4 | # This program is free software: you can redistribute it and/or modify | ||
117 | 5 | # it under the terms of the GNU General Public License as published by | ||
118 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
119 | 7 | # (at your option) any later version. | ||
120 | 8 | # | ||
121 | 9 | # This program is distributed in the hope that it will be useful, | ||
122 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
123 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
124 | 12 | # GNU General Public License for more details. | ||
125 | 13 | # | ||
126 | 14 | # You should have received a copy of the GNU General Public License | ||
127 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
128 | 16 | |||
129 | 17 | import logging | ||
130 | 18 | import pymongo | ||
131 | 19 | from qakit.practitest.practitest import append_last_run_datetime | ||
132 | 20 | |||
133 | 21 | logger = logging.getLogger(__name__) | ||
134 | 22 | |||
135 | 23 | |||
136 | 24 | def retrieve_testset(db, practitest_session, testset_id): | ||
137 | 25 | """Retrieve a testset of the given ID, storing it in our database. | ||
138 | 26 | |||
139 | 27 | :param db: a MongoClient collection in which testsets are stored | ||
140 | 28 | :param practitest_session: a PractiTestSession | ||
141 | 29 | :param testset_id: the ID of the PractiTest testset to retrieve | ||
142 | 30 | :raises ValueError: if testset_id is not found in PractiTest | ||
143 | 31 | |||
144 | 32 | """ | ||
145 | 33 | try: | ||
146 | 34 | testset = practitest_session.get_testset(testset_id) | ||
147 | 35 | except ValueError: | ||
148 | 36 | logger.warn("Testset {} not found!".format(testset_id)) | ||
149 | 37 | raise | ||
150 | 38 | testset = append_last_run_datetime(testset) | ||
151 | 39 | result = db.testsets.update( | ||
152 | 40 | {'system_id': testset['system_id']}, | ||
153 | 41 | testset, | ||
154 | 42 | upsert=True) | ||
155 | 43 | if result['updatedExisting']: | ||
156 | 44 | logger.info("Updated testset {}.".format(testset_id)) | ||
157 | 45 | else: | ||
158 | 46 | logger.info("Inserted testset {}.".format(testset_id)) | ||
159 | 47 | return result | ||
160 | 48 | |||
161 | 49 | |||
162 | 50 | def _get_last_known_testset_id(db): | ||
163 | 51 | """Get the last known testset ID from a db. | ||
164 | 52 | |||
165 | 53 | :param db: a MongoClient collection in which testsets are stored | ||
166 | 54 | |||
167 | 55 | """ | ||
168 | 56 | testsets = db.testsets.find() | ||
169 | 57 | try: | ||
170 | 58 | return testsets.sort('id', pymongo.DESCENDING).limit(1)[0]['id'] | ||
171 | 59 | except IndexError: | ||
172 | 60 | raise ValueError("No testsets found in db.") | ||
173 | 61 | |||
174 | 62 | |||
175 | 63 | def update_testsets(db, practitest_session, testset_ids=None): | ||
176 | 64 | """Update PractiTest testsets with new data. | ||
177 | 65 | |||
178 | 66 | Note that we update testsets themselves only--instances (i.e. test | ||
179 | 67 | executions) should be updated separately. | ||
180 | 68 | |||
181 | 69 | :param db: a MongoClient collection in which testsets are stored | ||
182 | 70 | :param practitest_session: a PractiTestSession | ||
183 | 71 | :param testset_ids: a list of testsets to retrieve; if not specified, | ||
184 | 72 | all known testsets in the given db are updated | ||
185 | 73 | |||
186 | 74 | """ | ||
187 | 75 | logger.info("Updating existing testsets.") | ||
188 | 76 | if testset_ids is None: | ||
189 | 77 | testset_ids = [testset['id'] for testset in db.testsets.find()] | ||
190 | 78 | count = 0 | ||
191 | 79 | for testset_id in testset_ids: | ||
192 | 80 | try: | ||
193 | 81 | result = retrieve_testset(db, practitest_session, testset_id) | ||
194 | 82 | count += result['n'] | ||
195 | 83 | except ValueError: | ||
196 | 84 | logger.warn( | ||
197 | 85 | "Testset {} not found, assume deleted.".format(testset_id)) | ||
198 | 86 | logger.info("Updated {} testsets.".format(count)) | ||
199 | 87 | |||
200 | 88 | |||
201 | 89 | def _scan_for_testsets(db, practitest_session, testset_id_range): | ||
202 | 90 | """Scan a range of testset IDs, retrieving each if it exists in PractiTest. | ||
203 | 91 | |||
204 | 92 | Use this to scout ahead while discovering new testsets. | ||
205 | 93 | |||
206 | 94 | :param db: a MongoClient collection in which testsets are stored | ||
207 | 95 | :param practitest_session: a PractiTestSession | ||
208 | 96 | :param testset_id_range: a range of testset IDs | ||
209 | 97 | |||
210 | 98 | """ | ||
211 | 99 | count = 0 | ||
212 | 100 | for testset_id in testset_id_range: | ||
213 | 101 | try: | ||
214 | 102 | result = retrieve_testset(db, practitest_session, testset_id) | ||
215 | 103 | count += result['n'] | ||
216 | 104 | except ValueError: | ||
217 | 105 | logger.debug("Testset {} not found.".format(testset_id)) | ||
218 | 106 | return count | ||
219 | 107 | |||
220 | 108 | |||
221 | 109 | def discover_new_testsets(db, practitest_session, increment=50): | ||
222 | 110 | """Discover and retrieve new testsets in PractiTest. | ||
223 | 111 | |||
224 | 112 | PractiTest API won't give a list of IDs of testsets belonging to a | ||
225 | 113 | project. IDs are assigned incrementally and occasionally deleted; | ||
226 | 114 | scan a given number of IDs ahead and attempt to find a testset at | ||
227 | 115 | each ID, continue until a full scan comes up empty. | ||
228 | 116 | |||
229 | 117 | :param db: a MongoClient collection in which testsets are stored | ||
230 | 118 | :param practitest_session: a PractiTestSession | ||
231 | 119 | :param increment: the increment by which to scan ahead, defaults to 50 | ||
232 | 120 | |||
233 | 121 | """ | ||
234 | 122 | logger.info("Discovering new testsets in PractiTest.") | ||
235 | 123 | try: | ||
236 | 124 | logger.info("Last known testset is {}.".format(last_testset_id)) | ||
237 | 125 | last_testset_id = _get_last_known_testset_id(db) | ||
238 | 126 | except ValueError as e: | ||
239 | 127 | logger.info("No testsets in db.") | ||
240 | 128 | last_testset_id = 0 | ||
241 | 129 | total_count = 0 | ||
242 | 130 | testsets_found = True | ||
243 | 131 | next_testset_id = last_testset_id + 1 | ||
244 | 132 | while testsets_found: | ||
245 | 133 | testset_id_range = range(next_testset_id, next_testset_id+increment) | ||
246 | 134 | scan_count = _scan_for_testsets( | ||
247 | 135 | db, | ||
248 | 136 | practitest_session, | ||
249 | 137 | testset_id_range) | ||
250 | 138 | if scan_count is 0: | ||
251 | 139 | testsets_found = False | ||
252 | 140 | else: | ||
253 | 141 | next_testset_id = testset_id_range[-1] + 1 | ||
254 | 142 | total_count += scan_count | ||
255 | 143 | logger.info("Retrieved {} testsets.".format(total_count)) | ||
256 | 144 | return total_count | ||
257 | 0 | 145 | ||
258 | === added file 'qakit/metrics/test_execution.py' | |||
259 | --- qakit/metrics/test_execution.py 1970-01-01 00:00:00 +0000 | |||
260 | +++ qakit/metrics/test_execution.py 2015-10-29 22:05:12 +0000 | |||
261 | @@ -0,0 +1,131 @@ | |||
262 | 1 | #!/usr/bin/python3 | ||
263 | 2 | # UESQA Metrics | ||
264 | 3 | # Copyright (C) 2015 Canonical | ||
265 | 4 | # | ||
266 | 5 | # This program is free software: you can redistribute it and/or modify | ||
267 | 6 | # it under the terms of the GNU General Public License as published by | ||
268 | 7 | # the Free Software Foundation, either version 3 of the License, or | ||
269 | 8 | # (at your option) any later version. | ||
270 | 9 | # | ||
271 | 10 | # This program is distributed in the hope that it will be useful, | ||
272 | 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
273 | 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
274 | 13 | # GNU General Public License for more details. | ||
275 | 14 | # | ||
276 | 15 | # You should have received a copy of the GNU General Public License | ||
277 | 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
278 | 17 | |||
279 | 18 | |||
280 | 19 | import argparse | ||
281 | 20 | import configparser | ||
282 | 21 | import logging | ||
283 | 22 | import sys | ||
284 | 23 | |||
285 | 24 | import pymongo | ||
286 | 25 | |||
287 | 26 | import qakit.config as qakit_config | ||
288 | 27 | import qakit.metrics.practitest.instances as instances | ||
289 | 28 | import qakit.metrics.practitest.testsets as testsets | ||
290 | 29 | import qakit.metrics.util as util | ||
291 | 30 | import qakit.practitest.practitest as practitest | ||
292 | 31 | |||
293 | 32 | logger = logging.getLogger(__name__) | ||
294 | 33 | |||
295 | 34 | |||
296 | 35 | def update_testsets(db, practitest_session, testset_ids=None): | ||
297 | 36 | """Discover new and update existing PractiTest testsets. | ||
298 | 37 | |||
299 | 38 | :param db: a MongoClient collection in which PractiTest data is stored | ||
300 | 39 | :param practitest_session: a PractiTestSession | ||
301 | 40 | :param testsets: a list of PractiTest testset ids to update; if not | ||
302 | 41 | specified, new testsets are discovered and all existing testsets are | ||
303 | 42 | updated | ||
304 | 43 | |||
305 | 44 | """ | ||
306 | 45 | if not testset_ids: | ||
307 | 46 | testset_ids = [testset['id'] for testset in db.testsets.find()] | ||
308 | 47 | # (newly-discovered testsets will be updated upon discovery) | ||
309 | 48 | testsets.discover_new_testsets(db, practitest_session) | ||
310 | 49 | testsets.update_testsets(db, practitest_session, testset_ids=testset_ids) | ||
311 | 50 | |||
312 | 51 | |||
313 | 52 | def update_instances(db, practitest_session, testset_ids=None): | ||
314 | 53 | """Discover new and update existing instances for the given testset IDs. | ||
315 | 54 | |||
316 | 55 | :param db: a MongoClient collection in which PractiTest data is stored | ||
317 | 56 | :param practitest_session: a PractiTestSession | ||
318 | 57 | :param testsets: a list of PractiTest testset ids to update; if not | ||
319 | 58 | specified, all known testsets are updated | ||
320 | 59 | |||
321 | 60 | """ | ||
322 | 61 | if not testset_ids: | ||
323 | 62 | testset_ids = [testset['id'] for testset in db.testsets.find()] | ||
324 | 63 | instances.update_testsets_instances( | ||
325 | 64 | db, | ||
326 | 65 | practitest_session, | ||
327 | 66 | testset_ids=testset_ids) | ||
328 | 67 | |||
329 | 68 | |||
330 | 69 | def update_test_execution_data(db, practitest_session, testset_ids=None): | ||
331 | 70 | """Update PractiTest test execution data: testsets and instances. | ||
332 | 71 | |||
333 | 72 | :param db: a MongoClient collection in which PractiTest data is stored | ||
334 | 73 | :param practitest_session: a PractiTestSession | ||
335 | 74 | :param testsets: a list of PractiTest testset ids to update; if not | ||
336 | 75 | specified, new testsets are discovered and all existing testsets are | ||
337 | 76 | updated | ||
338 | 77 | |||
339 | 78 | """ | ||
340 | 79 | update_testsets(db, practitest_session, testset_ids=testset_ids) | ||
341 | 80 | update_instances(db, practitest_session, testset_ids=testset_ids) | ||
342 | 81 | |||
343 | 82 | |||
344 | 83 | def _read_config(config_filepath): | ||
345 | 84 | """Parse the config at the given filepath, returning a config dict. | ||
346 | 85 | |||
347 | 86 | :param config_filepath: the filepath to a configuration file | ||
348 | 87 | |||
349 | 88 | """ | ||
350 | 89 | config_file = configparser.ConfigParser() | ||
351 | 90 | config_file.read(config_filepath) | ||
352 | 91 | config = {} | ||
353 | 92 | for var_name in ('PRACTITEST_PROJECT_ID', | ||
354 | 93 | 'PRACTITEST_API_KEY', | ||
355 | 94 | 'PRACTITEST_API_SECRET_KEY'): | ||
356 | 95 | new_var_name = var_name.lower().replace('practitest_', '') | ||
357 | 96 | config[new_var_name] = config_file['DEFAULT'][var_name] | ||
358 | 97 | return config | ||
359 | 98 | |||
360 | 99 | |||
361 | 100 | def _parse_arguments(): | ||
362 | 101 | parser = argparse.ArgumentParser( | ||
363 | 102 | "Retrieve PractiTest results, storing them in a MongoDB database.") | ||
364 | 103 | parser.add_argument( | ||
365 | 104 | nargs='*', | ||
366 | 105 | dest='testsets', | ||
367 | 106 | help='TestSet from which to retrieve instances', | ||
368 | 107 | type=int) | ||
369 | 108 | return parser.parse_args() | ||
370 | 109 | |||
371 | 110 | |||
372 | 111 | def main(): | ||
373 | 112 | util.setup_logging() | ||
374 | 113 | config_dict = _read_config(qakit_config.get_config_file_location()) | ||
375 | 114 | args = _parse_arguments() | ||
376 | 115 | conn = pymongo.MongoClient() | ||
377 | 116 | db = conn.metrics | ||
378 | 117 | practitest_session = practitest.PractitestSession( | ||
379 | 118 | config_dict['project_id'], | ||
380 | 119 | config_dict['api_key'], | ||
381 | 120 | config_dict['api_secret_key']) | ||
382 | 121 | update_test_execution_data( | ||
383 | 122 | db, | ||
384 | 123 | practitest_session, | ||
385 | 124 | testset_ids=args.testsets) | ||
386 | 125 | |||
387 | 126 | |||
388 | 127 | if __name__ == '__main__': | ||
389 | 128 | try: | ||
390 | 129 | sys.exit(main()) | ||
391 | 130 | except Exception as e: | ||
392 | 131 | logger.error(e, exc_info=True) | ||
393 | 0 | 132 | ||
394 | === added file 'qakit/metrics/tests/test_practitest_instances.py' | |||
395 | --- qakit/metrics/tests/test_practitest_instances.py 1970-01-01 00:00:00 +0000 | |||
396 | +++ qakit/metrics/tests/test_practitest_instances.py 2015-10-29 22:05:12 +0000 | |||
397 | @@ -0,0 +1,113 @@ | |||
398 | 1 | # UESQA Metrics | ||
399 | 2 | # Copyright (C) 2015 Canonical | ||
400 | 3 | # | ||
401 | 4 | # This program is free software: you can redistribute it and/or modify | ||
402 | 5 | # it under the terms of the GNU General Public License as published by | ||
403 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
404 | 7 | # (at your option) any later version. | ||
405 | 8 | # | ||
406 | 9 | # This program is distributed in the hope that it will be useful, | ||
407 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
408 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
409 | 12 | # GNU General Public License for more details. | ||
410 | 13 | # | ||
411 | 14 | # You should have received a copy of the GNU General Public License | ||
412 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
413 | 16 | |||
414 | 17 | import unittest | ||
415 | 18 | from unittest import mock | ||
416 | 19 | |||
417 | 20 | from bson import ObjectId | ||
418 | 21 | |||
419 | 22 | import qakit.metrics.practitest.instances as instances | ||
420 | 23 | |||
421 | 24 | |||
422 | 25 | INSTANCE = { | ||
423 | 26 | "_id": ObjectId("5590253dca21169c04a38b51"), | ||
424 | 27 | "tester": { | ||
425 | 28 | "value": "iahmad", | ||
426 | 29 | "name": "Tester" | ||
427 | 30 | }, | ||
428 | 31 | "last_run_week_number": 20, | ||
429 | 32 | "name": "Complete Edges Intro", | ||
430 | 33 | "last_run": { | ||
431 | 34 | "value": "21-May-2015 20:59", | ||
432 | 35 | "name": "Last Run" | ||
433 | 36 | }, | ||
434 | 37 | "___f_10578": { | ||
435 | 38 | "value": None, | ||
436 | 39 | "name": "Build #" | ||
437 | 40 | }, | ||
438 | 41 | "id": "10:1", | ||
439 | 42 | "system_id": 1393092, | ||
440 | 43 | "run_status": { | ||
441 | 44 | "value": "FAILED", | ||
442 | 45 | "name": "Run Status" | ||
443 | 46 | } | ||
444 | 47 | } | ||
445 | 48 | |||
446 | 49 | |||
447 | 50 | TESTSETS = [ | ||
448 | 51 | {'id': 761}, | ||
449 | 52 | {'id': 762}, | ||
450 | 53 | {'id': 763}, | ||
451 | 54 | {'id': 764}, | ||
452 | 55 | {'id': 765}, | ||
453 | 56 | {'id': 766}, | ||
454 | 57 | {'id': 767} | ||
455 | 58 | ] | ||
456 | 59 | |||
457 | 60 | |||
458 | 61 | class AppendTestSetIdTestCase(unittest.TestCase): | ||
459 | 62 | |||
460 | 63 | def test_testset_id_appended_correctly(self): | ||
461 | 64 | instance = instances.append_testset_id(INSTANCE) | ||
462 | 65 | self.assertEqual(10, instance['testset_id']) | ||
463 | 66 | |||
464 | 67 | |||
465 | 68 | class AppendDataToInstanceDictTestCase(unittest.TestCase): | ||
466 | 69 | |||
467 | 70 | @mock.patch('qakit.practitest.practitest.append_last_run_datetime') | ||
468 | 71 | @mock.patch('qakit.metrics.practitest.instances.append_testset_id') | ||
469 | 72 | def test_appends_called(self, | ||
470 | 73 | mock_append_testset_id, | ||
471 | 74 | mock_append_last_run_datetime): | ||
472 | 75 | # I agree, this should never fail :) | ||
473 | 76 | mock_append_last_run_datetime.return_value = 'fake-instance' | ||
474 | 77 | instances.append_data_to_instance_dict(INSTANCE) | ||
475 | 78 | mock_append_last_run_datetime.assert_called_with(INSTANCE) | ||
476 | 79 | mock_append_testset_id.assert_called_with('fake-instance') | ||
477 | 80 | |||
478 | 81 | |||
479 | 82 | @mock.patch('qakit.metrics.practitest.instances.update_testset_instances') | ||
480 | 83 | class UpdateTestSetInstancesTestCase(unittest.TestCase): | ||
481 | 84 | |||
482 | 85 | def test_no_testset_ids_specified(self, mock_update_testset_instances): | ||
483 | 86 | db = mock.Mock( | ||
484 | 87 | testsets=mock.Mock( | ||
485 | 88 | find=mock.Mock(return_value=TESTSETS))) | ||
486 | 89 | instances.update_testsets_instances( | ||
487 | 90 | db, | ||
488 | 91 | 'fake-practitest-session', | ||
489 | 92 | None) | ||
490 | 93 | self.assertEqual(7, mock_update_testset_instances.call_count) | ||
491 | 94 | |||
492 | 95 | def test_one_testset_specified(self, mock_update_testset_instances): | ||
493 | 96 | instances.update_testsets_instances( | ||
494 | 97 | 'fake-db', | ||
495 | 98 | 'fake-practitest-session', | ||
496 | 99 | [761]) | ||
497 | 100 | self.assertEqual(1, mock_update_testset_instances.call_count) | ||
498 | 101 | mock_update_testset_instances.assert_called_with( | ||
499 | 102 | 'fake-db', 'fake-practitest-session', 761) | ||
500 | 103 | |||
501 | 104 | def test_several_testsets_specified(self, mock_update_testset_instances): | ||
502 | 105 | testset_ids = [761, 762, 763] | ||
503 | 106 | instances.update_testsets_instances( | ||
504 | 107 | 'fake-db', | ||
505 | 108 | 'fake-practitest-session', | ||
506 | 109 | testset_ids) | ||
507 | 110 | self.assertEqual(3, mock_update_testset_instances.call_count) | ||
508 | 111 | for testset_id in testset_ids: | ||
509 | 112 | mock_update_testset_instances.assert_any_call( | ||
510 | 113 | 'fake-db', 'fake-practitest-session', testset_id) | ||
511 | 0 | 114 | ||
512 | === added file 'qakit/metrics/tests/test_practitest_testsets.py' | |||
513 | --- qakit/metrics/tests/test_practitest_testsets.py 1970-01-01 00:00:00 +0000 | |||
514 | +++ qakit/metrics/tests/test_practitest_testsets.py 2015-10-29 22:05:12 +0000 | |||
515 | @@ -0,0 +1,80 @@ | |||
516 | 1 | # UESQA Metrics | ||
517 | 2 | # Copyright (C) 2015 Canonical | ||
518 | 3 | # | ||
519 | 4 | # This program is free software: you can redistribute it and/or modify | ||
520 | 5 | # it under the terms of the GNU General Public License as published by | ||
521 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
522 | 7 | # (at your option) any later version. | ||
523 | 8 | # | ||
524 | 9 | # This program is distributed in the hope that it will be useful, | ||
525 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
526 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
527 | 12 | # GNU General Public License for more details. | ||
528 | 13 | # | ||
529 | 14 | # You should have received a copy of the GNU General Public License | ||
530 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
531 | 16 | |||
532 | 17 | import unittest | ||
533 | 18 | from unittest import mock | ||
534 | 19 | |||
535 | 20 | import qakit.metrics.practitest.testsets as testsets | ||
536 | 21 | |||
537 | 22 | TESTSETS = [ | ||
538 | 23 | {'id': 761}, | ||
539 | 24 | {'id': 762}, | ||
540 | 25 | {'id': 763}, | ||
541 | 26 | {'id': 764}, | ||
542 | 27 | {'id': 765}, | ||
543 | 28 | {'id': 766}, | ||
544 | 29 | {'id': 767} | ||
545 | 30 | ] | ||
546 | 31 | |||
547 | 32 | |||
548 | 33 | class GetLastKnownTestsetIdTestCase(unittest.TestCase): | ||
549 | 34 | |||
550 | 35 | def testset_exists(self): | ||
551 | 36 | db = mock.Mock( | ||
552 | 37 | testsets=mock.Mock( | ||
553 | 38 | find=mock.Mock( | ||
554 | 39 | return_value=mock.Mock( | ||
555 | 40 | sort=mock.Mock( | ||
556 | 41 | return_value=mock.Mock( | ||
557 | 42 | limit=mock.Mock( | ||
558 | 43 | return_value=TESTSETS[:1]))))))) | ||
559 | 44 | self.assertEqual( | ||
560 | 45 | 761, | ||
561 | 46 | testsets._get_last_known_testset_id(db)) | ||
562 | 47 | |||
563 | 48 | def test_no_testsets_raises_valueerror(self): | ||
564 | 49 | testsets_ = mock.MagicMock() | ||
565 | 50 | get_item = testsets_.sort.return_value.limit.return_value.__getitem__ | ||
566 | 51 | get_item.side_effect = IndexError | ||
567 | 52 | db = mock.Mock( | ||
568 | 53 | testsets=mock.Mock( | ||
569 | 54 | find=mock.Mock( | ||
570 | 55 | return_value=testsets_))) | ||
571 | 56 | with self.assertRaises(ValueError): | ||
572 | 57 | testsets._get_last_known_testset_id(db) | ||
573 | 58 | |||
574 | 59 | |||
575 | 60 | @mock.patch('qakit.metrics.practitest.testsets.retrieve_testset') | ||
576 | 61 | class UpdateTestsetsTestCase(unittest.TestCase): | ||
577 | 62 | |||
578 | 63 | def test_no_testsets_specified(self, mock_retrieve_testset): | ||
579 | 64 | db = mock.Mock( | ||
580 | 65 | testsets=mock.Mock( | ||
581 | 66 | find=mock.Mock( | ||
582 | 67 | return_value=TESTSETS))) | ||
583 | 68 | mock_retrieve_testset.return_value = {'n': 1} | ||
584 | 69 | testsets.update_testsets(db, 'fake-practitest-session') | ||
585 | 70 | for testset_id in [testset['id'] for testset in TESTSETS]: | ||
586 | 71 | mock_retrieve_testset.assert_any_call( | ||
587 | 72 | db, 'fake-practitest-session', testset_id) | ||
588 | 73 | |||
589 | 74 | def test_testsets_specified(self, mock_retrieve_testset): | ||
590 | 75 | mock_retrieve_testset.return_value = {'n': 1} | ||
591 | 76 | testsets_ = [testset['id'] for testset in TESTSETS] | ||
592 | 77 | testsets.update_testsets('fake-db', 'fake-practitest-session', testsets_) | ||
593 | 78 | for testset_id in [testset['id'] for testset in TESTSETS]: | ||
594 | 79 | mock_retrieve_testset.assert_any_call( | ||
595 | 80 | 'fake-db', 'fake-practitest-session', testset_id) | ||
596 | 0 | 81 | ||
597 | === added file 'qakit/metrics/tests/test_test_execution.py' | |||
598 | --- qakit/metrics/tests/test_test_execution.py 1970-01-01 00:00:00 +0000 | |||
599 | +++ qakit/metrics/tests/test_test_execution.py 2015-10-29 22:05:12 +0000 | |||
600 | @@ -0,0 +1,99 @@ | |||
601 | 1 | # UESQA Metrics | ||
602 | 2 | # Copyright (C) 2015 Canonical | ||
603 | 3 | # | ||
604 | 4 | # This program is free software: you can redistribute it and/or modify | ||
605 | 5 | # it under the terms of the GNU General Public License as published by | ||
606 | 6 | # the Free Software Foundation, either version 3 of the License, or | ||
607 | 7 | # (at your option) any later version. | ||
608 | 8 | # | ||
609 | 9 | # This program is distributed in the hope that it will be useful, | ||
610 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
611 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
612 | 12 | # GNU General Public License for more details. | ||
613 | 13 | # | ||
614 | 14 | # You should have received a copy of the GNU General Public License | ||
615 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
616 | 16 | |||
617 | 17 | import unittest | ||
618 | 18 | from unittest import mock | ||
619 | 19 | |||
620 | 20 | import qakit.metrics.test_execution as test_execution | ||
621 | 21 | |||
622 | 22 | |||
623 | 23 | TESTSETS = [ | ||
624 | 24 | {'id': 761}, | ||
625 | 25 | {'id': 762}, | ||
626 | 26 | {'id': 763}, | ||
627 | 27 | {'id': 764}, | ||
628 | 28 | {'id': 765}, | ||
629 | 29 | {'id': 766}, | ||
630 | 30 | {'id': 767} | ||
631 | 31 | ] | ||
632 | 32 | |||
633 | 33 | |||
634 | 34 | @mock.patch('qakit.metrics.test_execution.update_testsets') | ||
635 | 35 | @mock.patch('qakit.metrics.test_execution.update_instances') | ||
636 | 36 | class UpdateTestExecutionDataTestCase(unittest.TestCase): | ||
637 | 37 | |||
638 | 38 | def test_update_both_testsets_and_instances( | ||
639 | 39 | self, | ||
640 | 40 | mock_update_instances, | ||
641 | 41 | mock_update_testsets): | ||
642 | 42 | test_execution.update_test_execution_data( | ||
643 | 43 | 'fake-db', 'fake-practitest-session', 'fake-testset-ids') | ||
644 | 44 | for f in mock_update_instances, mock_update_testsets: | ||
645 | 45 | f.assert_called_with( | ||
646 | 46 | 'fake-db', | ||
647 | 47 | 'fake-practitest-session', | ||
648 | 48 | testset_ids='fake-testset-ids') | ||
649 | 49 | |||
650 | 50 | |||
651 | 51 | @mock.patch('qakit.metrics.practitest.testsets.update_testsets') | ||
652 | 52 | class UpdateTestSetsTestCase(unittest.TestCase): | ||
653 | 53 | |||
654 | 54 | def test_testset_ids_specified(self, mock_update_testsets): | ||
655 | 55 | test_execution.update_testsets( | ||
656 | 56 | 'fake-db', 'fake-practitest-session', 'fake-testset-ids') | ||
657 | 57 | mock_update_testsets.assert_called_with( | ||
658 | 58 | 'fake-db', | ||
659 | 59 | 'fake-practitest-session', | ||
660 | 60 | testset_ids='fake-testset-ids') | ||
661 | 61 | |||
662 | 62 | @mock.patch('qakit.metrics.practitest.testsets.discover_new_testsets') | ||
663 | 63 | def test_no_testset_ids_specified( | ||
664 | 64 | self, | ||
665 | 65 | mock_discover_new_testsets, | ||
666 | 66 | mock_update_testsets): | ||
667 | 67 | db = mock.Mock( | ||
668 | 68 | testsets=mock.Mock( | ||
669 | 69 | find=mock.Mock(return_value=TESTSETS))) | ||
670 | 70 | test_execution.update_testsets( | ||
671 | 71 | db, 'fake-practitest-session', testset_ids=None) | ||
672 | 72 | mock_discover_new_testsets.assert_called_with( | ||
673 | 73 | db, 'fake-practitest-session') | ||
674 | 74 | testset_ids = [testset['id'] for testset in TESTSETS] | ||
675 | 75 | mock_update_testsets.assert_called_with( | ||
676 | 76 | db, 'fake-practitest-session', testset_ids=testset_ids) | ||
677 | 77 | |||
678 | 78 | |||
679 | 79 | @mock.patch('qakit.metrics.practitest.instances.update_testsets_instances') | ||
680 | 80 | class UpdateInstancesTestCase(unittest.TestCase): | ||
681 | 81 | |||
682 | 82 | def test_testset_ids_specified(self, mock_update_testsets_instances): | ||
683 | 83 | test_execution.update_instances( | ||
684 | 84 | 'fake-db', 'fake-practitest-session', 'fake-testset-ids') | ||
685 | 85 | mock_update_testsets_instances.assert_called_with( | ||
686 | 86 | 'fake-db', | ||
687 | 87 | 'fake-practitest-session', | ||
688 | 88 | testset_ids='fake-testset-ids') | ||
689 | 89 | |||
690 | 90 | def test_no_testset_ids_specified(self, mock_update_testsets_instances): | ||
691 | 91 | db = mock.Mock( | ||
692 | 92 | testsets=mock.Mock( | ||
693 | 93 | find=mock.Mock(return_value=TESTSETS))) | ||
694 | 94 | test_execution.update_instances(db, 'fake-practitest-session', None) | ||
695 | 95 | testset_ids = [testset['id'] for testset in TESTSETS] | ||
696 | 96 | mock_update_testsets_instances.assert_called_with( | ||
697 | 97 | db, | ||
698 | 98 | 'fake-practitest-session', | ||
699 | 99 | testset_ids=testset_ids) | ||
700 | 0 | 100 | ||
701 | === modified file 'qakit/practitest/practitest.py' | |||
702 | --- qakit/practitest/practitest.py 2015-09-03 13:32:10 +0000 | |||
703 | +++ qakit/practitest/practitest.py 2015-10-29 22:05:12 +0000 | |||
704 | @@ -15,6 +15,7 @@ | |||
705 | 15 | # You should have received a copy of the GNU General Public License | 15 | # You should have received a copy of the GNU General Public License |
706 | 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
707 | 17 | 17 | ||
708 | 18 | import datetime | ||
709 | 18 | import json | 19 | import json |
710 | 19 | import logging | 20 | import logging |
711 | 20 | import requests | 21 | import requests |
712 | @@ -28,6 +29,35 @@ | |||
713 | 28 | PROD_PRACTITEST = 'https://prod.practitest.com/api' | 29 | PROD_PRACTITEST = 'https://prod.practitest.com/api' |
714 | 29 | 30 | ||
715 | 30 | 31 | ||
716 | 32 | def practitest_time_to_datetime(practitest_time): | ||
717 | 33 | """Convert from PractiTest time to Python datetime. | ||
718 | 34 | |||
719 | 35 | PractiTest times look like '13-Apr-2015 22:02'. | ||
720 | 36 | |||
721 | 37 | """ | ||
722 | 38 | return datetime.datetime.strptime( | ||
723 | 39 | practitest_time, '%d-%b-%Y %H:%M') | ||
724 | 40 | |||
725 | 41 | |||
726 | 42 | def append_last_run_datetime(practitest_dict): | ||
727 | 43 | """Append a Python datetime for 'last_run' to a given PractiTest dict. | ||
728 | 44 | |||
729 | 45 | PractiTest gives us its datetimes as strings; translating to a Python | ||
730 | 46 | datetime makes querying, sorting, etc. much easier. | ||
731 | 47 | |||
732 | 48 | :param practitest_dict: a PractiTest JSON dict such as an instance or a | ||
733 | 49 | testset | ||
734 | 50 | |||
735 | 51 | """ | ||
736 | 52 | try: | ||
737 | 53 | practitest_dict['last_run_datetime'] = practitest_time_to_datetime( | ||
738 | 54 | practitest_dict['last_run']['value']) | ||
739 | 55 | except ValueError: | ||
740 | 56 | # no last_run, no harm done | ||
741 | 57 | pass | ||
742 | 58 | return practitest_dict | ||
743 | 59 | |||
744 | 60 | |||
745 | 31 | class PractitestSession: | 61 | class PractitestSession: |
746 | 32 | 62 | ||
747 | 33 | def __init__(self, project_id, api_key, api_secret_key, user_email=None): | 63 | def __init__(self, project_id, api_key, api_secret_key, user_email=None): |
748 | @@ -54,12 +84,14 @@ | |||
749 | 54 | def _get(self, url, params={}): | 84 | def _get(self, url, params={}): |
750 | 55 | """Get the given url, retrying on failure.""" | 85 | """Get the given url, retrying on failure.""" |
751 | 56 | params['project_id'] = self.project_id | 86 | params['project_id'] = self.project_id |
753 | 57 | return requests.get( | 87 | response = requests.get( |
754 | 58 | url=url, | 88 | url=url, |
755 | 59 | headers=auth.compose_headers( | 89 | headers=auth.compose_headers( |
756 | 60 | self.api_key, | 90 | self.api_key, |
757 | 61 | self.api_secret_key), | 91 | self.api_secret_key), |
758 | 62 | data=json.dumps(params)) | 92 | data=json.dumps(params)) |
759 | 93 | response.raise_for_status() | ||
760 | 94 | return response | ||
761 | 63 | 95 | ||
762 | 64 | 96 | ||
763 | 65 | def _get_all(self, url, params={}): | 97 | def _get_all(self, url, params={}): |
764 | @@ -69,6 +101,8 @@ | |||
765 | 69 | return int(pagination['total_entities']) - total_read > 0 | 101 | return int(pagination['total_entities']) - total_read > 0 |
766 | 70 | 102 | ||
767 | 71 | json_data = [] | 103 | json_data = [] |
768 | 104 | response = None | ||
769 | 105 | params['page'] = 1 | ||
770 | 72 | while True: | 106 | while True: |
771 | 73 | response = self._get(url, params) | 107 | response = self._get(url, params) |
772 | 74 | if not response.ok: | 108 | if not response.ok: |
773 | @@ -79,6 +113,7 @@ | |||
774 | 79 | params['page'] = str(pagination['page'] + 1) | 113 | params['page'] = str(pagination['page'] + 1) |
775 | 80 | else: | 114 | else: |
776 | 81 | break | 115 | break |
777 | 116 | response.raise_for_status() | ||
778 | 82 | return json_data | 117 | return json_data |
779 | 83 | 118 | ||
780 | 84 | 119 | ||
781 | @@ -159,7 +194,14 @@ | |||
782 | 159 | 194 | ||
783 | 160 | """ | 195 | """ |
784 | 161 | url = PROD_PRACTITEST + '/sets/{}.json'.format(id) | 196 | url = PROD_PRACTITEST + '/sets/{}.json'.format(id) |
786 | 162 | return self._get(url).json() | 197 | try: |
787 | 198 | return self._get(url).json() | ||
788 | 199 | except requests.exceptions.HTTPError as e: | ||
789 | 200 | if "TestSet was not found" in e.response.text: | ||
790 | 201 | raise ValueError("TestSet {} not found!".format(id)) | ||
791 | 202 | else: | ||
792 | 203 | raise | ||
793 | 204 | |||
794 | 163 | 205 | ||
795 | 164 | def create_testset(self, name, priority, device, | 206 | def create_testset(self, name, priority, device, |
796 | 165 | level, release, build, buildinfo): | 207 | level, release, build, buildinfo): |
797 | @@ -187,7 +229,13 @@ | |||
798 | 187 | 229 | ||
799 | 188 | """ | 230 | """ |
800 | 189 | url = PROD_PRACTITEST + '/sets/{}/instances.json'.format(id) | 231 | url = PROD_PRACTITEST + '/sets/{}/instances.json'.format(id) |
802 | 190 | return self._get_all(url) | 232 | try: |
803 | 233 | return self._get_all(url) | ||
804 | 234 | except requests.exceptions.HTTPError as e: | ||
805 | 235 | if "TestSet was not found" in e.response.text: | ||
806 | 236 | raise ValueError("TestSet {} not found!".format(id)) | ||
807 | 237 | else: | ||
808 | 238 | raise | ||
809 | 191 | 239 | ||
810 | 192 | def create_instances(self, set_id, test_ids): | 240 | def create_instances(self, set_id, test_ids): |
811 | 193 | """ | 241 | """ |
Looking good, a couple of inline comments.