Merge lp:~abentley/charmworld/remove-store-ingest-job into lp:~juju-jitsu/charmworld/trunk
- remove-store-ingest-job
- Merge into trunk
Proposed by
Aaron Bentley
Status: | Merged |
---|---|
Approved by: | Curtis Hovey |
Approved revision: | 293 |
Merged at revision: | 283 |
Proposed branch: | lp:~abentley/charmworld/remove-store-ingest-job |
Merge into: | lp:~juju-jitsu/charmworld/trunk |
Diff against target: |
664 lines (+204/-226) 4 files modified
charmworld/jobs/ingest.py (+142/-171) charmworld/jobs/tests/test_bzr.py (+24/-26) charmworld/jobs/tests/test_proof.py (+13/-10) charmworld/jobs/tests/test_store.py (+25/-19) |
To merge this branch: | bzr merge lp:~abentley/charmworld/remove-store-ingest-job |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Curtis Hovey (community) | code | Approve | |
Review via email: mp+171162@code.launchpad.net |
Commit message
Remove remaining sub-Jobs from ingest.
Description of the change
This branch completes the work for bug #1190015: ingest is really a set of functions that are run consecutively, but is structured as a list of jobs.
It removes StoreIngestJob, ChangelogIngestJob and ProofIngestJob. The remaining code has some room for improvement, espeicially some very short functions, but this can be addressed piecemeal.
IngestJob and DBIngestJob are retained because they are parents of the remaining job, UpdateCharmJob.
As a driveby, the 'retry' parameter was removed from do_bzr_update, since it was never used.
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'charmworld/jobs/ingest.py' |
2 | --- charmworld/jobs/ingest.py 2013-06-24 15:45:29 +0000 |
3 | +++ charmworld/jobs/ingest.py 2013-06-24 19:49:29 +0000 |
4 | @@ -88,14 +88,14 @@ |
5 | self.db = db |
6 | |
7 | |
8 | -def do_bzr_update(charm_data, db, fs, log, root_dir=None, retry=True): |
9 | +def do_bzr_update(charm_data, db, fs, log, root_dir=None): |
10 | """Fetch a branch from bzr, and augment charm data.""" |
11 | if not root_dir: |
12 | root_dir = CHARM_DIR |
13 | root_dir = root_dir |
14 | if not os.path.exists(root_dir): |
15 | os.makedirs(root_dir) |
16 | - update_charm_files(root_dir, fs, charm_data, log, retry) |
17 | + update_charm_files(root_dir, fs, charm_data, log) |
18 | |
19 | |
20 | def update_charm_branch(root_dir, fs, charm_data, branch_dir, log, |
21 | @@ -153,7 +153,7 @@ |
22 | return charm_data |
23 | |
24 | |
25 | -def update_charm_files(root_dir, fs, charm_data, log, retry): |
26 | +def update_charm_files(root_dir, fs, charm_data, log, retry=True): |
27 | if charm_data['branch_deleted']: |
28 | return |
29 | branch_dir = os.path.abspath( |
30 | @@ -225,13 +225,10 @@ |
31 | fs = getfs(db) |
32 | try: |
33 | do_bzr_update(charm_data, db, fs, log) |
34 | - if not run_job(StoreIngestJob(), charm_data): |
35 | - return False |
36 | - if not run_job(ProofIngestJob(), charm_data): |
37 | - return False |
38 | + update_from_store(charm_data, CharmStore(), log) |
39 | + update_proof_data(charm_data, log) |
40 | update_jenkins_data(db, charm_data, log) |
41 | - if not run_job(ChangelogIngestJob(), charm_data): |
42 | - return False |
43 | + update_from_revisions(charm_data) |
44 | update_date_created(charm_data, log) |
45 | scan_charm(charm_data, db, fs, log) |
46 | except Exception as e: |
47 | @@ -258,78 +255,69 @@ |
48 | CharmSource(self.db, index_client).save(charm_data) |
49 | |
50 | |
51 | -class ChangelogIngestJob(IngestJob): |
52 | - |
53 | - name = 'changelog' |
54 | - |
55 | - def __init__(self, limit=10, since=None): |
56 | - self.limit = limit |
57 | - self.since = since |
58 | - |
59 | - def setup(self): |
60 | - super(ChangelogIngestJob, self).setup() |
61 | - if self.since is None: |
62 | - days_of_revisions = settings.get('days_of_revisions') |
63 | - if days_of_revisions is not None: |
64 | - cutoff = datetime.utcnow() - timedelta(int(days_of_revisions)) |
65 | - self.since = calendar.timegm(cutoff.timetuple()) |
66 | - |
67 | - @staticmethod |
68 | - def _rev_info(r, branch): |
69 | - d = { |
70 | - 'authors': r.get_apparent_authors(), |
71 | - "revno": branch.revision_id_to_revno(r.revision_id), |
72 | - "committer": r.committer, |
73 | - "created": r.timestamp, |
74 | - "message": r.message |
75 | - } |
76 | - return d |
77 | - |
78 | - def run(self, charm_data): |
79 | - if charm_data['branch_deleted']: |
80 | - return |
81 | - branch_dir = charm_data["branch_dir"] |
82 | - charm_data.update(self.get_changes(branch_dir)) |
83 | - |
84 | - def get_changes(self, branch_dir): |
85 | - charm_data = {} |
86 | - branch = Branch.open(branch_dir) |
87 | - branch.lock_read() |
88 | - try: |
89 | - revisions = self.get_revisions(branch) |
90 | - charm_data["changes"] = changes = [] |
91 | - for r in revisions: |
92 | - changes.append(self._rev_info(r, branch)) |
93 | - if len(revisions) == 0: |
94 | - last_change = None |
95 | - first_change = None |
96 | - else: |
97 | - last_change = changes[0] |
98 | - first = branch.repository.get_revision(branch.get_rev_id(1)) |
99 | - first_change = self._rev_info(first, branch) |
100 | - charm_data.update({ |
101 | - 'last_change': last_change, |
102 | - 'first_change': first_change, |
103 | - }) |
104 | - return charm_data |
105 | - finally: |
106 | - branch.unlock() |
107 | - |
108 | - def get_revisions(self, branch): |
109 | - # We only want the last 10 changes, in descending order. |
110 | - graph = branch.repository.get_graph() |
111 | - cur_rev_id = branch.last_revision() |
112 | - ancestry_iter = graph.iter_lefthand_ancestry(cur_rev_id) |
113 | - revs = [] |
114 | - for num, revision_id in enumerate(ancestry_iter): |
115 | - if revision_id == NULL_REVISION: |
116 | +def _rev_info(r, branch): |
117 | + d = { |
118 | + 'authors': r.get_apparent_authors(), |
119 | + "revno": branch.revision_id_to_revno(r.revision_id), |
120 | + "committer": r.committer, |
121 | + "created": r.timestamp, |
122 | + "message": r.message |
123 | + } |
124 | + return d |
125 | + |
126 | + |
127 | +def update_from_revisions(charm_data, limit=10, since=None): |
128 | + if charm_data['branch_deleted']: |
129 | + return |
130 | + if since is None: |
131 | + days_of_revisions = settings.get('days_of_revisions') |
132 | + if days_of_revisions is not None: |
133 | + cutoff = datetime.utcnow() - timedelta(int(days_of_revisions)) |
134 | + since = calendar.timegm(cutoff.timetuple()) |
135 | + branch_dir = charm_data["branch_dir"] |
136 | + charm_data.update(get_changes(branch_dir, limit, since)) |
137 | + |
138 | + |
139 | +def get_changes(branch_dir, limit, since): |
140 | + charm_data = {} |
141 | + branch = Branch.open(branch_dir) |
142 | + branch.lock_read() |
143 | + try: |
144 | + revisions = get_revisions(branch, limit, since) |
145 | + charm_data["changes"] = changes = [] |
146 | + for r in revisions: |
147 | + changes.append(_rev_info(r, branch)) |
148 | + if len(revisions) == 0: |
149 | + last_change = None |
150 | + first_change = None |
151 | + else: |
152 | + last_change = changes[0] |
153 | + first = branch.repository.get_revision(branch.get_rev_id(1)) |
154 | + first_change = _rev_info(first, branch) |
155 | + charm_data.update({ |
156 | + 'last_change': last_change, |
157 | + 'first_change': first_change, |
158 | + }) |
159 | + return charm_data |
160 | + finally: |
161 | + branch.unlock() |
162 | + |
163 | + |
164 | +def get_revisions(branch, limit, since): |
165 | + # We only want the last 10 changes, in descending order. |
166 | + graph = branch.repository.get_graph() |
167 | + cur_rev_id = branch.last_revision() |
168 | + ancestry_iter = graph.iter_lefthand_ancestry(cur_rev_id) |
169 | + revs = [] |
170 | + for num, revision_id in enumerate(ancestry_iter): |
171 | + if revision_id == NULL_REVISION: |
172 | + break |
173 | + revision = branch.repository.get_revision(revision_id) |
174 | + if num >= limit: |
175 | + if since is None or revision.timestamp < since: |
176 | break |
177 | - revision = branch.repository.get_revision(revision_id) |
178 | - if num >= self.limit: |
179 | - if self.since is None or revision.timestamp < self.since: |
180 | - break |
181 | - revs.append(revision) |
182 | - return revs |
183 | + revs.append(revision) |
184 | + return revs |
185 | |
186 | |
187 | def update_jenkins_data(db, charm, log): |
188 | @@ -414,66 +402,62 @@ |
189 | return (result_id, result['result']) |
190 | |
191 | |
192 | -class ProofIngestJob(IngestJob): |
193 | - |
194 | - name = 'proof' |
195 | - |
196 | - @contextlib.contextmanager |
197 | - def _get_proof_lib(self, new_path): |
198 | - if new_path not in sys.path: |
199 | - sys.path.append(new_path) |
200 | - try: |
201 | - import lib.proof as prooflib |
202 | - yield prooflib |
203 | - except ImportError: |
204 | - yield None |
205 | - finally: |
206 | - if new_path in sys.path: |
207 | - sys.path.remove(new_path) |
208 | - |
209 | - def setup(self, proofer=None, proof_path=None): |
210 | - if not proofer: |
211 | - proofer = self.get_proofer(proof_path) |
212 | - self.proofer = proofer |
213 | - |
214 | - def get_proofer(self, proof_path=None): |
215 | - proofer = None |
216 | - if not proof_path: |
217 | - proof_path = CHARM_PROOF_PATH |
218 | - |
219 | - # Use config.CHARM_PROOF_PATH for testing. Monkeypatching is scary. |
220 | - if not os.path.isdir(proof_path): |
221 | - err_msg = ("proof error before processing began: could not find " |
222 | - "charm proof path.") |
223 | - self.log.exception(err_msg) |
224 | - self.log.exception("CHARM_PROOF_PATH: %s", proof_path) |
225 | - with self._get_proof_lib(proof_path) as prooflib: |
226 | - if not prooflib: |
227 | - err_msg = ("proof error before processing began: could not " |
228 | - "import charm proof lib.") |
229 | - self.log.exception(err_msg) |
230 | - self.log.exception( |
231 | - "CHARM_PROOF_PATH: %s", proof_path) |
232 | - else: |
233 | - proofer = prooflib.run |
234 | - return proofer |
235 | - |
236 | - def run(self, charm): |
237 | - if not self.proofer: |
238 | - self.log.exception("proof aborted.") |
239 | - raise Exception("No proofer") |
240 | - if charm['branch_deleted']: |
241 | - return |
242 | - proof = {} |
243 | - lint, exit_code = self.proofer(charm['branch_dir']) |
244 | - for line in lint: |
245 | - if not ':' in line: |
246 | - continue |
247 | - level, msg = line.split(':', 1) |
248 | - if level == "W" and 'name' in msg: |
249 | - continue |
250 | - proof.setdefault(level.lower(), []).append(msg) |
251 | - charm['proof'] = proof |
252 | +@contextlib.contextmanager |
253 | +def get_proof_lib(new_path): |
254 | + if new_path not in sys.path: |
255 | + sys.path.append(new_path) |
256 | + try: |
257 | + import lib.proof as prooflib |
258 | + yield prooflib |
259 | + except ImportError: |
260 | + yield None |
261 | + finally: |
262 | + if new_path in sys.path: |
263 | + sys.path.remove(new_path) |
264 | + |
265 | + |
266 | +def get_proofer(log, proof_path=None): |
267 | + proofer = None |
268 | + if not proof_path: |
269 | + proof_path = CHARM_PROOF_PATH |
270 | + |
271 | + # Use config.CHARM_PROOF_PATH for testing. Monkeypatching is scary. |
272 | + if not os.path.isdir(proof_path): |
273 | + err_msg = ("proof error before processing began: could not find " |
274 | + "charm proof path.") |
275 | + log.exception(err_msg) |
276 | + log.exception("CHARM_PROOF_PATH: %s", proof_path) |
277 | + with get_proof_lib(proof_path) as prooflib: |
278 | + if not prooflib: |
279 | + err_msg = ("proof error before processing began: could not " |
280 | + "import charm proof lib.") |
281 | + log.exception(err_msg) |
282 | + log.exception( |
283 | + "CHARM_PROOF_PATH: %s", proof_path) |
284 | + else: |
285 | + proofer = prooflib.run |
286 | + return proofer |
287 | + |
288 | + |
289 | +def update_proof_data(charm, log, _proofer=None): |
290 | + proofer = _proofer |
291 | + if proofer is None: |
292 | + proofer = get_proofer(log) |
293 | + if not proofer: |
294 | + log.exception("proof aborted.") |
295 | + raise Exception("No proofer") |
296 | + if charm['branch_deleted']: |
297 | + return |
298 | + proof = {} |
299 | + lint, exit_code = proofer(charm['branch_dir']) |
300 | + for line in lint: |
301 | + if not ':' in line: |
302 | + continue |
303 | + level, msg = line.split(':', 1) |
304 | + if level == "W" and 'name' in msg: |
305 | + continue |
306 | + proof.setdefault(level.lower(), []).append(msg) |
307 | + charm['proof'] = proof |
308 | |
309 | |
310 | def scan_charm(charm_data, db, fs, log): |
311 | @@ -639,37 +623,24 @@ |
312 | charm['date_created'] = timestamp(date_created.replace(microsecond=0)) |
313 | |
314 | |
315 | -class StoreIngestJob(IngestJob): |
316 | - |
317 | - name = 'store' |
318 | - |
319 | - def __init__(self, store=None): |
320 | - super(StoreIngestJob, self).__init__() |
321 | - if store is None: |
322 | - store = CharmStore() |
323 | - self.store = store |
324 | - |
325 | - def run(self, charm): |
326 | - old_address = None |
327 | - for address in addresses(charm): |
328 | - if old_address is not None: |
329 | - self.log.info("rechecking %s with %s", old_address, address) |
330 | - old_address = address |
331 | - data = self._store_get(address) |
332 | - if 'errors' not in data and 'warnings' not in data: |
333 | - break |
334 | - |
335 | - if 'errors' in data or 'warnings' in data: |
336 | - self.log.warning("store error on %s %s" % (address, data)) |
337 | - |
338 | - data["store_checked"] = datetime.now().ctime() |
339 | - |
340 | - charm['store_data'] = data |
341 | - charm['store_url'] = make_store_url(data['revision'], address) |
342 | - update_download_count(self.store, charm) |
343 | - |
344 | - def _store_get(self, address): |
345 | - return CharmStore().get_charm_info(address) |
346 | +def update_from_store(charm, store, log): |
347 | + old_address = None |
348 | + for address in addresses(charm): |
349 | + if old_address is not None: |
350 | + log.info("rechecking %s with %s", old_address, address) |
351 | + old_address = address |
352 | + data = store.get_charm_info(address) |
353 | + if 'errors' not in data and 'warnings' not in data: |
354 | + break |
355 | + |
356 | + if 'errors' in data or 'warnings' in data: |
357 | + log.warning("store error on %s %s" % (address, data)) |
358 | + |
359 | + data["store_checked"] = datetime.now().ctime() |
360 | + |
361 | + charm['store_data'] = data |
362 | + charm['store_url'] = make_store_url(data['revision'], address) |
363 | + update_download_count(store, charm) |
364 | |
365 | |
366 | # XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for |
367 | |
368 | === modified file 'charmworld/jobs/tests/test_bzr.py' |
369 | --- charmworld/jobs/tests/test_bzr.py 2013-06-24 15:45:29 +0000 |
370 | +++ charmworld/jobs/tests/test_bzr.py 2013-06-24 19:49:29 +0000 |
371 | @@ -6,15 +6,20 @@ |
372 | from logging import getLogger |
373 | import os |
374 | import shutil |
375 | +import time |
376 | |
377 | from bzrlib.bzrdir import BzrDir |
378 | from mock import patch |
379 | |
380 | |
381 | +from charmworld.jobs import ingest |
382 | from charmworld.jobs.ingest import ( |
383 | add_files, |
384 | do_bzr_update, |
385 | - ChangelogIngestJob, |
386 | + get_changes, |
387 | + get_revisions, |
388 | + _rev_info, |
389 | + update_from_revisions, |
390 | ) |
391 | from charmworld.models import getfs |
392 | from charmworld.testing import ( |
393 | @@ -169,24 +174,23 @@ |
394 | |
395 | def test_get_revisions(self): |
396 | tree = self.make_locked_tree() |
397 | - job = ChangelogIngestJob() |
398 | - revisions = job.get_revisions(tree.branch) |
399 | + revisions = get_revisions(tree.branch, 10, 0) |
400 | self.assertEqual([], revisions) |
401 | for x in range(9): |
402 | self.num_commit(tree, x) |
403 | - revisions = job.get_revisions(tree.branch) |
404 | + revisions = get_revisions(tree.branch, 10, 0) |
405 | self.assertEqual(["revision %d" % num for num in range(8, -1, -1)], |
406 | [revision.message for revision in revisions]) |
407 | self.num_commit(tree, 9) |
408 | - revisions = job.get_revisions(tree.branch) |
409 | + revisions = get_revisions(tree.branch, 10, time.time()) |
410 | self.assertEqual(["revision %d" % num for num in range(9, -1, -1)], |
411 | [revision.message for revision in revisions]) |
412 | self.num_commit(tree, 10) |
413 | - revisions = job.get_revisions(tree.branch) |
414 | + revisions = get_revisions(tree.branch, 10, time.time()) |
415 | self.assertEqual(["revision %d" % num for num in range(10, 0, -1)], |
416 | [revision.message for revision in revisions]) |
417 | self.num_commit(tree, 11) |
418 | - revisions = job.get_revisions(tree.branch) |
419 | + revisions = get_revisions(tree.branch, 10, time.time()) |
420 | self.assertEqual(["revision %d" % num for num in range(11, 1, -1)], |
421 | [revision.message for revision in revisions]) |
422 | |
423 | @@ -194,9 +198,9 @@ |
424 | tree = self.make_locked_tree() |
425 | for x in range(15): |
426 | self.num_commit(tree, x) |
427 | - revisions = ChangelogIngestJob(limit=5).get_revisions(tree.branch) |
428 | + revisions = get_revisions(tree.branch, 5, time.time()) |
429 | self.assertEqual(5, len(revisions)) |
430 | - revisions = ChangelogIngestJob(limit=20).get_revisions(tree.branch) |
431 | + revisions = get_revisions(tree.branch, 20, time.time()) |
432 | self.assertEqual(15, len(revisions)) |
433 | |
434 | def test_get_revisions_honours_since(self): |
435 | @@ -204,11 +208,9 @@ |
436 | tree.commit('', timestamp=0, committer='') |
437 | tree.commit('', timestamp=1, committer='') |
438 | tree.commit('', timestamp=2, committer='') |
439 | - job = ChangelogIngestJob(since=0, limit=0) |
440 | - revisions = job.get_revisions(tree.branch) |
441 | + revisions = get_revisions(tree.branch, 0, 0) |
442 | self.assertEqual(3, len(revisions)) |
443 | - job = ChangelogIngestJob(since=1, limit=0) |
444 | - revisions = job.get_revisions(tree.branch) |
445 | + revisions = get_revisions(tree.branch, 0, 1) |
446 | self.assertEqual(2, len(revisions)) |
447 | |
448 | def test_limit_overrides_since(self): |
449 | @@ -216,8 +218,7 @@ |
450 | tree.commit('', timestamp=0, committer='') |
451 | tree.commit('', timestamp=1, committer='') |
452 | tree.commit('', timestamp=2, committer='') |
453 | - job = ChangelogIngestJob(since=1, limit=3) |
454 | - revisions = job.get_revisions(tree.branch) |
455 | + revisions = get_revisions(tree.branch, since=1, limit=3) |
456 | self.assertEqual(3, len(revisions)) |
457 | |
458 | def test_rev_info(self): |
459 | @@ -225,7 +226,7 @@ |
460 | revision_id = tree.commit('message1', committer=JRH) |
461 | rev = tree.branch.repository.get_revision(revision_id) |
462 | rev.timestamp = 34 |
463 | - rev_info = ChangelogIngestJob._rev_info(rev, tree.branch) |
464 | + rev_info = _rev_info(rev, tree.branch) |
465 | self.assertEqual({ |
466 | 'authors': [JRH], |
467 | 'committer': JRH, |
468 | @@ -236,8 +237,7 @@ |
469 | |
470 | @staticmethod |
471 | def get_rev_info(branch, revision_id): |
472 | - rev = branch.repository.get_revision(revision_id) |
473 | - return ChangelogIngestJob._rev_info(rev, branch) |
474 | + return _rev_info(branch.repository.get_revision(revision_id), branch) |
475 | |
476 | def test_rev_info_supplied_authors(self): |
477 | tree = self.make_locked_tree() |
478 | @@ -250,15 +250,14 @@ |
479 | def test_get_changes(self): |
480 | tree = self.make_locked_tree() |
481 | first_id = self.num_commit(tree, 1) |
482 | - job = ChangelogIngestJob() |
483 | - charm_data = job.get_changes('tree') |
484 | + charm_data = get_changes('tree', 10, time.time()) |
485 | self.assertItemsEqual(['changes', 'first_change', 'last_change'], |
486 | charm_data.keys()) |
487 | self.assertEqual(charm_data['first_change'], charm_data['last_change']) |
488 | self.assertEqual([charm_data['first_change']], charm_data['changes']) |
489 | for num in range(1, 11): |
490 | self.num_commit(tree, 1) |
491 | - charm_data = job.get_changes('tree') |
492 | + charm_data = get_changes('tree', 10, time.time()) |
493 | self.assertNotEqual(charm_data['first_change'], |
494 | charm_data['last_change']) |
495 | self.assertEqual(charm_data['last_change'], charm_data['changes'][0]) |
496 | @@ -269,16 +268,15 @@ |
497 | |
498 | def test_get_changes_no_revisions(self): |
499 | self.make_locked_tree() |
500 | - charm_data = ChangelogIngestJob().get_changes('tree') |
501 | + charm_data = get_changes('tree', 10, time.time()) |
502 | self.assertEqual([], charm_data['changes']) |
503 | self.assertIs(None, charm_data['first_change']) |
504 | self.assertIs(None, charm_data['last_change']) |
505 | |
506 | def test_branch_deleted(self): |
507 | - # ChangelogIngestJob does not do anything if the Launchpad branch |
508 | + # update_from_revisions does not do anything if the Launchpad branch |
509 | # of a charm is deleted. |
510 | - job = ChangelogIngestJob() |
511 | charm = factory.get_charm_json(branch_deleted=True) |
512 | - with patch.object(job, 'get_changes') as mock: |
513 | - job.run(charm) |
514 | + with patch.object(ingest, 'get_changes') as mock: |
515 | + update_from_revisions(charm) |
516 | self.assertFalse(mock.called) |
517 | |
518 | === modified file 'charmworld/jobs/tests/test_proof.py' |
519 | --- charmworld/jobs/tests/test_proof.py 2013-06-07 12:36:30 +0000 |
520 | +++ charmworld/jobs/tests/test_proof.py 2013-06-24 19:49:29 +0000 |
521 | @@ -1,9 +1,14 @@ |
522 | # Copyright 2012, 2013 Canonical Ltd. This software is licensed under the |
523 | # GNU Affero General Public License version 3 (see the file LICENSE). |
524 | |
525 | -from mock import patch |
526 | - |
527 | -from charmworld.jobs.ingest import ProofIngestJob |
528 | +from logging import getLogger |
529 | + |
530 | +from mock import MagicMock |
531 | + |
532 | +from charmworld.jobs.ingest import ( |
533 | + get_proofer, |
534 | + update_proof_data, |
535 | +) |
536 | from charmworld.testing import factory |
537 | from charmworld.testing import JobTestBase |
538 | |
539 | @@ -14,9 +19,8 @@ |
540 | # If the charm-proof library cannot be found, an error is logged |
541 | # and the proof loop aborted. |
542 | nonexistant_path = '/does/not/exist/' |
543 | - job = ProofIngestJob() |
544 | handler = self.get_handler('charm.proof') |
545 | - job.setup(proof_path=nonexistant_path) |
546 | + get_proofer(getLogger('charm.proof'), nonexistant_path) |
547 | log_messages = [record.getMessage() for record in handler.buffer] |
548 | err = ("proof error before processing began: " |
549 | "could not find charm proof path.") |
550 | @@ -26,9 +30,8 @@ |
551 | def test_no_proof_for_deleted_branch(self): |
552 | # If the branch of a charm is deleted from Launchpad, the proof |
553 | # job does nothing. |
554 | - job = ProofIngestJob() |
555 | - job.setup() |
556 | + self.proofer = get_proofer(getLogger('charm.proof')) |
557 | charm = factory.get_charm_json(branch_deleted=True) |
558 | - with patch.object(job, 'proofer') as mock: |
559 | - job.run(charm) |
560 | - self.assertFalse(mock.called) |
561 | + proofer = MagicMock() |
562 | + update_proof_data(charm, getLogger('charm.proof'), proofer) |
563 | + self.assertFalse(proofer.called) |
564 | |
565 | === modified file 'charmworld/jobs/tests/test_store.py' |
566 | --- charmworld/jobs/tests/test_store.py 2013-05-27 12:45:51 +0000 |
567 | +++ charmworld/jobs/tests/test_store.py 2013-06-24 19:49:29 +0000 |
568 | @@ -1,17 +1,25 @@ |
569 | # Copyright 2012, 2013 Canonical Ltd. This software is licensed under the |
570 | # GNU Affero General Public License version 3 (see the file LICENSE). |
571 | |
572 | +from logging import getLogger |
573 | from mock import patch |
574 | from requests import Response |
575 | |
576 | -from charmworld.jobs.ingest import StoreIngestJob |
577 | -from charmworld.jobs.ingest import addresses |
578 | +from charmworld.charmstore import CharmStore |
579 | +from charmworld.jobs.ingest import ( |
580 | + addresses, |
581 | + update_from_store, |
582 | +) |
583 | from charmworld.testing import factory |
584 | from charmworld.testing import JobTestBase |
585 | |
586 | |
587 | class TestStoreJob(JobTestBase): |
588 | |
589 | + def setUp(self): |
590 | + super(TestStoreJob, self).setUp() |
591 | + self.log = getLogger('charm.store') |
592 | + |
593 | def _mock_data(self, warning=False, error=False): |
594 | data = {'revision': 1} |
595 | if warning: |
596 | @@ -20,7 +28,7 @@ |
597 | data['errors'] = 'Error forced by mock.' |
598 | return data |
599 | |
600 | - def falsify_session(self, job, content='[[0]]'): |
601 | + def falsify_session(self, store, content='[[0]]'): |
602 | |
603 | class FakeSession: |
604 | |
605 | @@ -30,17 +38,16 @@ |
606 | response._content = content |
607 | return response |
608 | |
609 | - job.store.session = FakeSession |
610 | + store.session = FakeSession |
611 | |
612 | def test_error_handling_error(self): |
613 | handler = self.get_handler('charm.store') |
614 | - job = StoreIngestJob() |
615 | - job.setup() |
616 | - self.falsify_session(job) |
617 | + store = CharmStore() |
618 | + self.falsify_session(store) |
619 | ignore, charm = factory.makeCharm(self.db, promulgated=True) |
620 | - with patch.object(job, '_store_get', |
621 | + with patch.object(store, 'get_charm_info', |
622 | lambda x: self._mock_data(error=True)): |
623 | - job.run(charm) |
624 | + update_from_store(charm, store, self.log) |
625 | |
626 | first_address = "cs:%s/%s" % (charm["series"], charm["name"]) |
627 | second_address = "cs:~%s/%s/%s" % ( |
628 | @@ -54,13 +61,12 @@ |
629 | |
630 | def test_error_handling_warning(self): |
631 | handler = self.get_handler('charm.store') |
632 | - job = StoreIngestJob() |
633 | - job.setup() |
634 | - self.falsify_session(job) |
635 | + store = CharmStore() |
636 | + self.falsify_session(store) |
637 | ignore, charm = factory.makeCharm(self.db, promulgated=True) |
638 | - with patch.object(job, '_store_get', |
639 | + with patch.object(store, 'get_charm_info', |
640 | lambda x: self._mock_data(warning=True)): |
641 | - job.run(charm) |
642 | + update_from_store(charm, store, self.log) |
643 | |
644 | first_address = "cs:%s/%s" % (charm["series"], charm["name"]) |
645 | second_address = "cs:~%s/%s/%s" % ( |
646 | @@ -73,13 +79,13 @@ |
647 | self.assertIn(err_msg, log_messages) |
648 | |
649 | def test_run(self): |
650 | - job = StoreIngestJob() |
651 | - job.setup() |
652 | - self.falsify_session(job, '[["2010-12-24", 4], ["2010-12-25", 1]]') |
653 | + store = CharmStore() |
654 | + self.falsify_session(store, '[["2010-12-24", 4], ["2010-12-25", 1]]') |
655 | ignore, charm = factory.makeCharm(self.db, promulgated=True) |
656 | del charm['store_url'] |
657 | - with patch.object(job, '_store_get', lambda x: self._mock_data()): |
658 | - job.run(charm) |
659 | + with patch.object(store, 'get_charm_info', |
660 | + lambda x: self._mock_data()): |
661 | + update_from_store(charm, store, self.log) |
662 | data = charm['store_data'] |
663 | self.assertIn('store_checked', data.keys()) |
664 | self.assertEqual(1, data['revision']) |
Thank you.