Merge lp:~cjwatson/launchpad/codeimport-worker-refactor into lp:launchpad
- codeimport-worker-refactor
- Merge into devel
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 18231 |
Proposed branch: | lp:~cjwatson/launchpad/codeimport-worker-refactor |
Merge into: | lp:launchpad |
Prerequisite: | lp:~cjwatson/launchpad/codeimport-source-details-refactor |
Diff against target: |
547 lines (+114/-96) 7 files modified
lib/lp/code/xmlrpc/codeimportscheduler.py (+4/-4) lib/lp/code/xmlrpc/tests/test_codeimportscheduler.py (+5/-5) lib/lp/codehosting/codeimport/tests/test_worker.py (+16/-16) lib/lp/codehosting/codeimport/tests/test_workermonitor.py (+6/-6) lib/lp/codehosting/codeimport/worker.py (+74/-56) lib/lp/codehosting/codeimport/workermonitor.py (+5/-5) lib/lp/testing/factory.py (+4/-4) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad/codeimport-worker-refactor |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+308122@code.launchpad.net |
Commit message
Refactor various bits of the code import worker to be less Bazaar-specific.
Description of the change
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/code/xmlrpc/codeimportscheduler.py' | |||
2 | --- lib/lp/code/xmlrpc/codeimportscheduler.py 2016-10-11 13:46:57 +0000 | |||
3 | +++ lib/lp/code/xmlrpc/codeimportscheduler.py 2016-10-11 13:46:57 +0000 | |||
4 | @@ -69,10 +69,10 @@ | |||
5 | 69 | job = self._getJob(job_id) | 69 | job = self._getJob(job_id) |
6 | 70 | arguments = CodeImportSourceDetails.fromCodeImportJob( | 70 | arguments = CodeImportSourceDetails.fromCodeImportJob( |
7 | 71 | job).asArguments() | 71 | job).asArguments() |
12 | 72 | branch = job.code_import.branch | 72 | target = job.code_import.target |
13 | 73 | branch_url = canonical_url(branch) | 73 | target_url = canonical_url(target) |
14 | 74 | log_file_name = '%s.log' % branch.unique_name[1:].replace('/', '-') | 74 | log_file_name = '%s.log' % target.unique_name[1:].replace('/', '-') |
15 | 75 | return (arguments, branch_url, log_file_name) | 75 | return (arguments, target_url, log_file_name) |
16 | 76 | 76 | ||
17 | 77 | @return_fault | 77 | @return_fault |
18 | 78 | def _updateHeartbeat(self, job_id, log_tail): | 78 | def _updateHeartbeat(self, job_id, log_tail): |
19 | 79 | 79 | ||
20 | === modified file 'lib/lp/code/xmlrpc/tests/test_codeimportscheduler.py' | |||
21 | --- lib/lp/code/xmlrpc/tests/test_codeimportscheduler.py 2016-10-11 13:46:57 +0000 | |||
22 | +++ lib/lp/code/xmlrpc/tests/test_codeimportscheduler.py 2016-10-11 13:46:57 +0000 | |||
23 | @@ -57,20 +57,20 @@ | |||
24 | 57 | self.assertEqual(code_import_job.id, job_id) | 57 | self.assertEqual(code_import_job.id, job_id) |
25 | 58 | 58 | ||
26 | 59 | def test_getImportDataForJobID(self): | 59 | def test_getImportDataForJobID(self): |
28 | 60 | # getImportDataForJobID returns the worker arguments, branch url and | 60 | # getImportDataForJobID returns the worker arguments, target url and |
29 | 61 | # log file name for an import corresponding to a particular job. | 61 | # log file name for an import corresponding to a particular job. |
30 | 62 | code_import_job = self.makeCodeImportJob(running=True) | 62 | code_import_job = self.makeCodeImportJob(running=True) |
31 | 63 | code_import = removeSecurityProxy(code_import_job).code_import | 63 | code_import = removeSecurityProxy(code_import_job).code_import |
33 | 64 | code_import_arguments, branch_url, log_file_name = \ | 64 | code_import_arguments, target_url, log_file_name = \ |
34 | 65 | self.api.getImportDataForJobID(code_import_job.id) | 65 | self.api.getImportDataForJobID(code_import_job.id) |
35 | 66 | import_as_arguments = CodeImportSourceDetails.fromCodeImportJob( | 66 | import_as_arguments = CodeImportSourceDetails.fromCodeImportJob( |
36 | 67 | code_import_job).asArguments() | 67 | code_import_job).asArguments() |
37 | 68 | expected_log_file_name = '%s.log' % ( | 68 | expected_log_file_name = '%s.log' % ( |
39 | 69 | code_import.branch.unique_name[1:].replace('/', '-')) | 69 | code_import.target.unique_name[1:].replace('/', '-')) |
40 | 70 | self.assertEqual( | 70 | self.assertEqual( |
42 | 71 | (import_as_arguments, canonical_url(code_import.branch), | 71 | (import_as_arguments, canonical_url(code_import.target), |
43 | 72 | expected_log_file_name), | 72 | expected_log_file_name), |
45 | 73 | (code_import_arguments, branch_url, log_file_name)) | 73 | (code_import_arguments, target_url, log_file_name)) |
46 | 74 | 74 | ||
47 | 75 | def test_getImportDataForJobID_not_found(self): | 75 | def test_getImportDataForJobID_not_found(self): |
48 | 76 | # getImportDataForJobID returns a NoSuchCodeImportJob fault when there | 76 | # getImportDataForJobID returns a NoSuchCodeImportJob fault when there |
49 | 77 | 77 | ||
50 | === modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py' | |||
51 | --- lib/lp/codehosting/codeimport/tests/test_worker.py 2016-10-11 13:46:57 +0000 | |||
52 | +++ lib/lp/codehosting/codeimport/tests/test_worker.py 2016-10-11 13:46:57 +0000 | |||
53 | @@ -78,7 +78,7 @@ | |||
54 | 78 | get_default_bazaar_branch_store, | 78 | get_default_bazaar_branch_store, |
55 | 79 | GitImportWorker, | 79 | GitImportWorker, |
56 | 80 | ImportDataStore, | 80 | ImportDataStore, |
58 | 81 | ImportWorker, | 81 | ToBzrImportWorker, |
59 | 82 | ) | 82 | ) |
60 | 83 | from lp.codehosting.safe_open import ( | 83 | from lp.codehosting.safe_open import ( |
61 | 84 | AcceptAnythingPolicy, | 84 | AcceptAnythingPolicy, |
62 | @@ -428,7 +428,7 @@ | |||
63 | 428 | source_details = self.factory.makeCodeImportSourceDetails() | 428 | source_details = self.factory.makeCodeImportSourceDetails() |
64 | 429 | # That the remote name is like this is part of the interface of | 429 | # That the remote name is like this is part of the interface of |
65 | 430 | # ImportDataStore. | 430 | # ImportDataStore. |
67 | 431 | remote_name = '%08x.tar.gz' % (source_details.branch_id,) | 431 | remote_name = '%08x.tar.gz' % (source_details.target_id,) |
68 | 432 | local_name = '%s.tar.gz' % (self.factory.getUniqueString(),) | 432 | local_name = '%s.tar.gz' % (self.factory.getUniqueString(),) |
69 | 433 | transport = self.get_transport() | 433 | transport = self.get_transport() |
70 | 434 | transport.put_bytes(remote_name, '') | 434 | transport.put_bytes(remote_name, '') |
71 | @@ -442,7 +442,7 @@ | |||
72 | 442 | source_details = self.factory.makeCodeImportSourceDetails() | 442 | source_details = self.factory.makeCodeImportSourceDetails() |
73 | 443 | # That the remote name is like this is part of the interface of | 443 | # That the remote name is like this is part of the interface of |
74 | 444 | # ImportDataStore. | 444 | # ImportDataStore. |
76 | 445 | remote_name = '%08x.tar.gz' % (source_details.branch_id,) | 445 | remote_name = '%08x.tar.gz' % (source_details.target_id,) |
77 | 446 | content = self.factory.getUniqueString() | 446 | content = self.factory.getUniqueString() |
78 | 447 | transport = self.get_transport() | 447 | transport = self.get_transport() |
79 | 448 | transport.put_bytes(remote_name, content) | 448 | transport.put_bytes(remote_name, content) |
80 | @@ -457,7 +457,7 @@ | |||
81 | 457 | source_details = self.factory.makeCodeImportSourceDetails() | 457 | source_details = self.factory.makeCodeImportSourceDetails() |
82 | 458 | # That the remote name is like this is part of the interface of | 458 | # That the remote name is like this is part of the interface of |
83 | 459 | # ImportDataStore. | 459 | # ImportDataStore. |
85 | 460 | remote_name = '%08x.tar.gz' % (source_details.branch_id,) | 460 | remote_name = '%08x.tar.gz' % (source_details.target_id,) |
86 | 461 | content = self.factory.getUniqueString() | 461 | content = self.factory.getUniqueString() |
87 | 462 | transport = self.get_transport() | 462 | transport = self.get_transport() |
88 | 463 | transport.put_bytes(remote_name, content) | 463 | transport.put_bytes(remote_name, content) |
89 | @@ -481,7 +481,7 @@ | |||
90 | 481 | store.put(local_name) | 481 | store.put(local_name) |
91 | 482 | # That the remote name is like this is part of the interface of | 482 | # That the remote name is like this is part of the interface of |
92 | 483 | # ImportDataStore. | 483 | # ImportDataStore. |
94 | 484 | remote_name = '%08x.tar.gz' % (source_details.branch_id,) | 484 | remote_name = '%08x.tar.gz' % (source_details.target_id,) |
95 | 485 | self.assertEquals(content, transport.get_bytes(remote_name)) | 485 | self.assertEquals(content, transport.get_bytes(remote_name)) |
96 | 486 | 486 | ||
97 | 487 | def test_put_ensures_base(self): | 487 | def test_put_ensures_base(self): |
98 | @@ -509,7 +509,7 @@ | |||
99 | 509 | store.put(local_name, self.get_transport(local_prefix)) | 509 | store.put(local_name, self.get_transport(local_prefix)) |
100 | 510 | # That the remote name is like this is part of the interface of | 510 | # That the remote name is like this is part of the interface of |
101 | 511 | # ImportDataStore. | 511 | # ImportDataStore. |
103 | 512 | remote_name = '%08x.tar.gz' % (source_details.branch_id,) | 512 | remote_name = '%08x.tar.gz' % (source_details.target_id,) |
104 | 513 | self.assertEquals(content, transport.get_bytes(remote_name)) | 513 | self.assertEquals(content, transport.get_bytes(remote_name)) |
105 | 514 | 514 | ||
106 | 515 | 515 | ||
107 | @@ -585,8 +585,8 @@ | |||
108 | 585 | transport = store.import_data_store._transport | 585 | transport = store.import_data_store._transport |
109 | 586 | source_details = store.import_data_store.source_details | 586 | source_details = store.import_data_store.source_details |
110 | 587 | self.assertTrue( | 587 | self.assertTrue( |
113 | 588 | transport.has('%08x.tar.gz' % source_details.branch_id), | 588 | transport.has('%08x.tar.gz' % source_details.target_id), |
114 | 589 | "Couldn't find '%08x.tar.gz'" % source_details.branch_id) | 589 | "Couldn't find '%08x.tar.gz'" % source_details.target_id) |
115 | 590 | 590 | ||
116 | 591 | def test_fetchFromArchiveFailure(self): | 591 | def test_fetchFromArchiveFailure(self): |
117 | 592 | # If a tree has not been archived yet, but we try to retrieve it from | 592 | # If a tree has not been archived yet, but we try to retrieve it from |
118 | @@ -631,7 +631,7 @@ | |||
119 | 631 | 631 | ||
120 | 632 | def makeImportWorker(self): | 632 | def makeImportWorker(self): |
121 | 633 | """Make an ImportWorker.""" | 633 | """Make an ImportWorker.""" |
123 | 634 | return ImportWorker( | 634 | return ToBzrImportWorker( |
124 | 635 | self.source_details, self.get_transport('import_data'), | 635 | self.source_details, self.get_transport('import_data'), |
125 | 636 | self.makeBazaarBranchStore(), logging.getLogger("silent"), | 636 | self.makeBazaarBranchStore(), logging.getLogger("silent"), |
126 | 637 | AcceptAnythingPolicy()) | 637 | AcceptAnythingPolicy()) |
127 | @@ -728,7 +728,7 @@ | |||
128 | 728 | import_worker.import_data_store.put('git.db') | 728 | import_worker.import_data_store.put('git.db') |
129 | 729 | # Make sure there's a Bazaar branch in the branch store. | 729 | # Make sure there's a Bazaar branch in the branch store. |
130 | 730 | branch = self.make_branch('branch') | 730 | branch = self.make_branch('branch') |
132 | 731 | ImportWorker.pushBazaarBranch(import_worker, branch) | 731 | ToBzrImportWorker.pushBazaarBranch(import_worker, branch) |
133 | 732 | # Finally, fetching the tree gets the git.db file too. | 732 | # Finally, fetching the tree gets the git.db file too. |
134 | 733 | branch = import_worker.getBazaarBranch() | 733 | branch = import_worker.getBazaarBranch() |
135 | 734 | self.assertEqual( | 734 | self.assertEqual( |
136 | @@ -747,7 +747,7 @@ | |||
137 | 747 | import_worker.import_data_store.put('git-cache.tar.gz') | 747 | import_worker.import_data_store.put('git-cache.tar.gz') |
138 | 748 | # Make sure there's a Bazaar branch in the branch store. | 748 | # Make sure there's a Bazaar branch in the branch store. |
139 | 749 | branch = self.make_branch('branch') | 749 | branch = self.make_branch('branch') |
141 | 750 | ImportWorker.pushBazaarBranch(import_worker, branch) | 750 | ToBzrImportWorker.pushBazaarBranch(import_worker, branch) |
142 | 751 | # Finally, fetching the tree gets the git.db file too. | 751 | # Finally, fetching the tree gets the git.db file too. |
143 | 752 | new_branch = import_worker.getBazaarBranch() | 752 | new_branch = import_worker.getBazaarBranch() |
144 | 753 | self.assertEqual( | 753 | self.assertEqual( |
145 | @@ -767,13 +767,13 @@ | |||
146 | 767 | :source_details: A `CodeImportSourceDetails` describing the import. | 767 | :source_details: A `CodeImportSourceDetails` describing the import. |
147 | 768 | """ | 768 | """ |
148 | 769 | tree_transport = get_transport(config.codeimport.foreign_tree_store) | 769 | tree_transport = get_transport(config.codeimport.foreign_tree_store) |
150 | 770 | prefix = '%08x' % source_details.branch_id | 770 | prefix = '%08x' % source_details.target_id |
151 | 771 | if tree_transport.has('.'): | 771 | if tree_transport.has('.'): |
152 | 772 | for filename in tree_transport.list_dir('.'): | 772 | for filename in tree_transport.list_dir('.'): |
153 | 773 | if filename.startswith(prefix): | 773 | if filename.startswith(prefix): |
154 | 774 | tree_transport.delete(filename) | 774 | tree_transport.delete(filename) |
155 | 775 | branchstore = get_default_bazaar_branch_store() | 775 | branchstore = get_default_bazaar_branch_store() |
157 | 776 | branch_name = '%08x' % source_details.branch_id | 776 | branch_name = '%08x' % source_details.target_id |
158 | 777 | if branchstore.transport.has(branch_name): | 777 | if branchstore.transport.has(branch_name): |
159 | 778 | branchstore.transport.delete_tree(branch_name) | 778 | branchstore.transport.delete_tree(branch_name) |
160 | 779 | 779 | ||
161 | @@ -822,7 +822,7 @@ | |||
162 | 822 | """Get the Bazaar branch 'worker' stored into its BazaarBranchStore. | 822 | """Get the Bazaar branch 'worker' stored into its BazaarBranchStore. |
163 | 823 | """ | 823 | """ |
164 | 824 | branch_url = worker.bazaar_branch_store._getMirrorURL( | 824 | branch_url = worker.bazaar_branch_store._getMirrorURL( |
166 | 825 | worker.source_details.branch_id) | 825 | worker.source_details.target_id) |
167 | 826 | return Branch.open(branch_url) | 826 | return Branch.open(branch_url) |
168 | 827 | 827 | ||
169 | 828 | def test_import(self): | 828 | def test_import(self): |
170 | @@ -885,7 +885,7 @@ | |||
171 | 885 | self.addCleanup(lambda: shutil.rmtree(tree_path)) | 885 | self.addCleanup(lambda: shutil.rmtree(tree_path)) |
172 | 886 | 886 | ||
173 | 887 | branch_url = get_default_bazaar_branch_store()._getMirrorURL( | 887 | branch_url = get_default_bazaar_branch_store()._getMirrorURL( |
175 | 888 | source_details.branch_id) | 888 | source_details.target_id) |
176 | 889 | branch = Branch.open(branch_url) | 889 | branch = Branch.open(branch_url) |
177 | 890 | 890 | ||
178 | 891 | self.assertEqual(self.foreign_commit_count, branch.revno()) | 891 | self.assertEqual(self.foreign_commit_count, branch.revno()) |
179 | @@ -1345,7 +1345,7 @@ | |||
180 | 1345 | self.assertEqual( | 1345 | self.assertEqual( |
181 | 1346 | CodeImportWorkerExitCode.SUCCESS, worker.run()) | 1346 | CodeImportWorkerExitCode.SUCCESS, worker.run()) |
182 | 1347 | branch_url = self.bazaar_store._getMirrorURL( | 1347 | branch_url = self.bazaar_store._getMirrorURL( |
184 | 1348 | worker.source_details.branch_id) | 1348 | worker.source_details.target_id) |
185 | 1349 | branch = Branch.open(branch_url) | 1349 | branch = Branch.open(branch_url) |
186 | 1350 | self.assertEquals(self.revid, branch.last_revision()) | 1350 | self.assertEquals(self.revid, branch.last_revision()) |
187 | 1351 | 1351 | ||
188 | 1352 | 1352 | ||
189 | === modified file 'lib/lp/codehosting/codeimport/tests/test_workermonitor.py' | |||
190 | --- lib/lp/codehosting/codeimport/tests/test_workermonitor.py 2016-10-11 13:46:57 +0000 | |||
191 | +++ lib/lp/codehosting/codeimport/tests/test_workermonitor.py 2016-10-11 13:46:57 +0000 | |||
192 | @@ -241,21 +241,21 @@ | |||
193 | 241 | return worker_monitor.getWorkerArguments().addCallback( | 241 | return worker_monitor.getWorkerArguments().addCallback( |
194 | 242 | self.assertEqual, args) | 242 | self.assertEqual, args) |
195 | 243 | 243 | ||
198 | 244 | def test_getWorkerArguments_sets_branch_url_and_logfilename(self): | 244 | def test_getWorkerArguments_sets_target_url_and_logfilename(self): |
199 | 245 | # getWorkerArguments sets the _branch_url (for use in oops reports) | 245 | # getWorkerArguments sets the _target_url (for use in oops reports) |
200 | 246 | # and _log_file_name (for upload to the librarian) attributes on the | 246 | # and _log_file_name (for upload to the librarian) attributes on the |
201 | 247 | # WorkerMonitor from the data returned by getImportDataForJobID. | 247 | # WorkerMonitor from the data returned by getImportDataForJobID. |
203 | 248 | branch_url = self.factory.getUniqueString() | 248 | target_url = self.factory.getUniqueString() |
204 | 249 | log_file_name = self.factory.getUniqueString() | 249 | log_file_name = self.factory.getUniqueString() |
205 | 250 | worker_monitor = self.makeWorkerMonitorWithJob( | 250 | worker_monitor = self.makeWorkerMonitorWithJob( |
207 | 251 | 1, (['a'], branch_url, log_file_name)) | 251 | 1, (['a'], target_url, log_file_name)) |
208 | 252 | 252 | ||
209 | 253 | def check_branch_log(ignored): | 253 | def check_branch_log(ignored): |
210 | 254 | # Looking at the _ attributes here is in slightly poor taste, but | 254 | # Looking at the _ attributes here is in slightly poor taste, but |
211 | 255 | # much much easier than them by logging and parsing an oops, etc. | 255 | # much much easier than them by logging and parsing an oops, etc. |
212 | 256 | self.assertEqual( | 256 | self.assertEqual( |
215 | 257 | (branch_url, log_file_name), | 257 | (target_url, log_file_name), |
216 | 258 | (worker_monitor._branch_url, worker_monitor._log_file_name)) | 258 | (worker_monitor._target_url, worker_monitor._log_file_name)) |
217 | 259 | 259 | ||
218 | 260 | return worker_monitor.getWorkerArguments().addCallback( | 260 | return worker_monitor.getWorkerArguments().addCallback( |
219 | 261 | check_branch_log) | 261 | check_branch_log) |
220 | 262 | 262 | ||
221 | === modified file 'lib/lp/codehosting/codeimport/worker.py' | |||
222 | --- lib/lp/codehosting/codeimport/worker.py 2016-10-11 13:46:57 +0000 | |||
223 | +++ lib/lp/codehosting/codeimport/worker.py 2016-10-11 13:46:57 +0000 | |||
224 | @@ -15,6 +15,7 @@ | |||
225 | 15 | 'ForeignTreeStore', | 15 | 'ForeignTreeStore', |
226 | 16 | 'GitImportWorker', | 16 | 'GitImportWorker', |
227 | 17 | 'ImportWorker', | 17 | 'ImportWorker', |
228 | 18 | 'ToBzrImportWorker', | ||
229 | 18 | 'get_default_bazaar_branch_store', | 19 | 'get_default_bazaar_branch_store', |
230 | 19 | ] | 20 | ] |
231 | 20 | 21 | ||
232 | @@ -268,18 +269,20 @@ | |||
233 | 268 | of the information suitable for passing around on executables' command | 269 | of the information suitable for passing around on executables' command |
234 | 269 | lines. | 270 | lines. |
235 | 270 | 271 | ||
238 | 271 | :ivar branch_id: The id of the branch associated to this code import, used | 272 | :ivar target_id: The id of the Bazaar branch associated with this code |
239 | 272 | for locating the existing import and the foreign tree. | 273 | import, used for locating the existing import and the foreign tree. |
240 | 273 | :ivar rcstype: 'cvs', 'git', 'bzr-svn', 'bzr' as appropriate. | 274 | :ivar rcstype: 'cvs', 'git', 'bzr-svn', 'bzr' as appropriate. |
241 | 274 | :ivar url: The branch URL if rcstype in ['bzr-svn', 'git', 'bzr'], None | 275 | :ivar url: The branch URL if rcstype in ['bzr-svn', 'git', 'bzr'], None |
242 | 275 | otherwise. | 276 | otherwise. |
243 | 276 | :ivar cvs_root: The $CVSROOT if rcstype == 'cvs', None otherwise. | 277 | :ivar cvs_root: The $CVSROOT if rcstype == 'cvs', None otherwise. |
244 | 277 | :ivar cvs_module: The CVS module if rcstype == 'cvs', None otherwise. | 278 | :ivar cvs_module: The CVS module if rcstype == 'cvs', None otherwise. |
245 | 279 | :ivar stacked_on_url: The URL of the branch that the associated branch | ||
246 | 280 | is stacked on, if any. | ||
247 | 278 | """ | 281 | """ |
248 | 279 | 282 | ||
250 | 280 | def __init__(self, branch_id, rcstype, url=None, cvs_root=None, | 283 | def __init__(self, target_id, rcstype, url=None, cvs_root=None, |
251 | 281 | cvs_module=None, stacked_on_url=None): | 284 | cvs_module=None, stacked_on_url=None): |
253 | 282 | self.branch_id = branch_id | 285 | self.target_id = target_id |
254 | 283 | self.rcstype = rcstype | 286 | self.rcstype = rcstype |
255 | 284 | self.url = url | 287 | self.url = url |
256 | 285 | self.cvs_root = cvs_root | 288 | self.cvs_root = cvs_root |
257 | @@ -289,7 +292,7 @@ | |||
258 | 289 | @classmethod | 292 | @classmethod |
259 | 290 | def fromArguments(cls, arguments): | 293 | def fromArguments(cls, arguments): |
260 | 291 | """Convert command line-style arguments to an instance.""" | 294 | """Convert command line-style arguments to an instance.""" |
262 | 292 | branch_id = int(arguments.pop(0)) | 295 | target_id = int(arguments.pop(0)) |
263 | 293 | rcstype = arguments.pop(0) | 296 | rcstype = arguments.pop(0) |
264 | 294 | if rcstype in ['bzr-svn', 'git', 'bzr']: | 297 | if rcstype in ['bzr-svn', 'git', 'bzr']: |
265 | 295 | url = arguments.pop(0) | 298 | url = arguments.pop(0) |
266 | @@ -305,34 +308,34 @@ | |||
267 | 305 | else: | 308 | else: |
268 | 306 | raise AssertionError("Unknown rcstype %r." % rcstype) | 309 | raise AssertionError("Unknown rcstype %r." % rcstype) |
269 | 307 | return cls( | 310 | return cls( |
271 | 308 | branch_id, rcstype, url, cvs_root, cvs_module, stacked_on_url) | 311 | target_id, rcstype, url, cvs_root, cvs_module, stacked_on_url) |
272 | 309 | 312 | ||
273 | 310 | @classmethod | 313 | @classmethod |
274 | 311 | def fromCodeImportJob(cls, job): | 314 | def fromCodeImportJob(cls, job): |
275 | 312 | """Convert a `CodeImportJob` to an instance.""" | 315 | """Convert a `CodeImportJob` to an instance.""" |
276 | 313 | code_import = job.code_import | 316 | code_import = job.code_import |
280 | 314 | branch = code_import.branch | 317 | target = code_import.target |
281 | 315 | if branch.stacked_on is not None and not branch.stacked_on.private: | 318 | if target.stacked_on is not None and not target.stacked_on.private: |
282 | 316 | stacked_path = branch_id_alias(branch.stacked_on) | 319 | stacked_path = branch_id_alias(target.stacked_on) |
283 | 317 | stacked_on_url = compose_public_url('http', stacked_path) | 320 | stacked_on_url = compose_public_url('http', stacked_path) |
284 | 318 | else: | 321 | else: |
285 | 319 | stacked_on_url = None | 322 | stacked_on_url = None |
286 | 320 | if code_import.rcs_type == RevisionControlSystems.BZR_SVN: | 323 | if code_import.rcs_type == RevisionControlSystems.BZR_SVN: |
287 | 321 | return cls( | 324 | return cls( |
289 | 322 | branch.id, 'bzr-svn', str(code_import.url), | 325 | target.id, 'bzr-svn', str(code_import.url), |
290 | 323 | stacked_on_url=stacked_on_url) | 326 | stacked_on_url=stacked_on_url) |
291 | 324 | elif code_import.rcs_type == RevisionControlSystems.CVS: | 327 | elif code_import.rcs_type == RevisionControlSystems.CVS: |
292 | 325 | return cls( | 328 | return cls( |
294 | 326 | branch.id, 'cvs', | 329 | target.id, 'cvs', |
295 | 327 | cvs_root=str(code_import.cvs_root), | 330 | cvs_root=str(code_import.cvs_root), |
296 | 328 | cvs_module=str(code_import.cvs_module)) | 331 | cvs_module=str(code_import.cvs_module)) |
297 | 329 | elif code_import.rcs_type == RevisionControlSystems.GIT: | 332 | elif code_import.rcs_type == RevisionControlSystems.GIT: |
298 | 330 | return cls( | 333 | return cls( |
300 | 331 | branch.id, 'git', str(code_import.url), | 334 | target.id, 'git', str(code_import.url), |
301 | 332 | stacked_on_url=stacked_on_url) | 335 | stacked_on_url=stacked_on_url) |
302 | 333 | elif code_import.rcs_type == RevisionControlSystems.BZR: | 336 | elif code_import.rcs_type == RevisionControlSystems.BZR: |
303 | 334 | return cls( | 337 | return cls( |
305 | 335 | branch.id, 'bzr', str(code_import.url), | 338 | target.id, 'bzr', str(code_import.url), |
306 | 336 | stacked_on_url=stacked_on_url) | 339 | stacked_on_url=stacked_on_url) |
307 | 337 | else: | 340 | else: |
308 | 338 | raise AssertionError("Unknown rcstype %r." % code_import.rcs_type) | 341 | raise AssertionError("Unknown rcstype %r." % code_import.rcs_type) |
309 | @@ -340,7 +343,7 @@ | |||
310 | 340 | def asArguments(self): | 343 | def asArguments(self): |
311 | 341 | """Return a list of arguments suitable for passing to a child process. | 344 | """Return a list of arguments suitable for passing to a child process. |
312 | 342 | """ | 345 | """ |
314 | 343 | result = [str(self.branch_id), self.rcstype] | 346 | result = [str(self.target_id), self.rcstype] |
315 | 344 | if self.rcstype in ['bzr-svn', 'git', 'bzr']: | 347 | if self.rcstype in ['bzr-svn', 'git', 'bzr']: |
316 | 345 | result.append(self.url) | 348 | result.append(self.url) |
317 | 346 | if self.stacked_on_url is not None: | 349 | if self.stacked_on_url is not None: |
318 | @@ -374,7 +377,7 @@ | |||
319 | 374 | """ | 377 | """ |
320 | 375 | self.source_details = source_details | 378 | self.source_details = source_details |
321 | 376 | self._transport = transport | 379 | self._transport = transport |
323 | 377 | self._branch_id = source_details.branch_id | 380 | self._target_id = source_details.target_id |
324 | 378 | 381 | ||
325 | 379 | def _getRemoteName(self, local_name): | 382 | def _getRemoteName(self, local_name): |
326 | 380 | """Convert `local_name` to the name used to store a file. | 383 | """Convert `local_name` to the name used to store a file. |
327 | @@ -394,7 +397,7 @@ | |||
328 | 394 | if dot_index < 0: | 397 | if dot_index < 0: |
329 | 395 | raise AssertionError("local_name must have an extension.") | 398 | raise AssertionError("local_name must have an extension.") |
330 | 396 | ext = local_name[dot_index:] | 399 | ext = local_name[dot_index:] |
332 | 397 | return '%08x%s' % (self._branch_id, ext) | 400 | return '%08x%s' % (self._target_id, ext) |
333 | 398 | 401 | ||
334 | 399 | def fetch(self, filename, dest_transport=None): | 402 | def fetch(self, filename, dest_transport=None): |
335 | 400 | """Retrieve `filename` from the store. | 403 | """Retrieve `filename` from the store. |
336 | @@ -505,57 +508,23 @@ | |||
337 | 505 | class ImportWorker: | 508 | class ImportWorker: |
338 | 506 | """Oversees the actual work of a code import.""" | 509 | """Oversees the actual work of a code import.""" |
339 | 507 | 510 | ||
350 | 508 | # Where the Bazaar working tree will be stored. | 511 | def __init__(self, source_details, logger, opener_policy): |
341 | 509 | BZR_BRANCH_PATH = 'bzr_branch' | ||
342 | 510 | |||
343 | 511 | # Should `getBazaarBranch` create a working tree? | ||
344 | 512 | needs_bzr_tree = True | ||
345 | 513 | |||
346 | 514 | required_format = BzrDirFormat.get_default_format() | ||
347 | 515 | |||
348 | 516 | def __init__(self, source_details, import_data_transport, | ||
349 | 517 | bazaar_branch_store, logger, opener_policy): | ||
351 | 518 | """Construct an `ImportWorker`. | 512 | """Construct an `ImportWorker`. |
352 | 519 | 513 | ||
353 | 520 | :param source_details: A `CodeImportSourceDetails` object. | 514 | :param source_details: A `CodeImportSourceDetails` object. |
354 | 521 | :param bazaar_branch_store: A `BazaarBranchStore`. The import worker | ||
355 | 522 | uses this to fetch and store the Bazaar branches that are created | ||
356 | 523 | and updated during the import process. | ||
357 | 524 | :param logger: A `Logger` to pass to cscvs. | 515 | :param logger: A `Logger` to pass to cscvs. |
358 | 525 | :param opener_policy: Policy object that decides what branches can | 516 | :param opener_policy: Policy object that decides what branches can |
359 | 526 | be imported | 517 | be imported |
360 | 527 | """ | 518 | """ |
361 | 528 | self.source_details = source_details | 519 | self.source_details = source_details |
362 | 529 | self.bazaar_branch_store = bazaar_branch_store | ||
363 | 530 | self.import_data_store = ImportDataStore( | ||
364 | 531 | import_data_transport, self.source_details) | ||
365 | 532 | self._logger = logger | 520 | self._logger = logger |
366 | 533 | self._opener_policy = opener_policy | 521 | self._opener_policy = opener_policy |
367 | 534 | 522 | ||
368 | 535 | def getBazaarBranch(self): | ||
369 | 536 | """Return the Bazaar `Branch` that we are importing into.""" | ||
370 | 537 | if os.path.isdir(self.BZR_BRANCH_PATH): | ||
371 | 538 | shutil.rmtree(self.BZR_BRANCH_PATH) | ||
372 | 539 | return self.bazaar_branch_store.pull( | ||
373 | 540 | self.source_details.branch_id, self.BZR_BRANCH_PATH, | ||
374 | 541 | self.required_format, self.needs_bzr_tree, | ||
375 | 542 | stacked_on_url=self.source_details.stacked_on_url) | ||
376 | 543 | |||
377 | 544 | def pushBazaarBranch(self, bazaar_branch): | ||
378 | 545 | """Push the updated Bazaar branch to the server. | ||
379 | 546 | |||
380 | 547 | :return: True if revisions were transferred. | ||
381 | 548 | """ | ||
382 | 549 | return self.bazaar_branch_store.push( | ||
383 | 550 | self.source_details.branch_id, bazaar_branch, | ||
384 | 551 | self.required_format, | ||
385 | 552 | stacked_on_url=self.source_details.stacked_on_url) | ||
386 | 553 | |||
387 | 554 | def getWorkingDirectory(self): | 523 | def getWorkingDirectory(self): |
388 | 555 | """The directory we should change to and store all scratch files in. | 524 | """The directory we should change to and store all scratch files in. |
389 | 556 | """ | 525 | """ |
390 | 557 | base = config.codeimportworker.working_directory_root | 526 | base = config.codeimportworker.working_directory_root |
392 | 558 | dirname = 'worker-for-branch-%s' % self.source_details.branch_id | 527 | dirname = 'worker-for-branch-%s' % self.source_details.target_id |
393 | 559 | return os.path.join(base, dirname) | 528 | return os.path.join(base, dirname) |
394 | 560 | 529 | ||
395 | 561 | def run(self): | 530 | def run(self): |
396 | @@ -594,7 +563,56 @@ | |||
397 | 594 | raise NotImplementedError() | 563 | raise NotImplementedError() |
398 | 595 | 564 | ||
399 | 596 | 565 | ||
401 | 597 | class CSCVSImportWorker(ImportWorker): | 566 | class ToBzrImportWorker(ImportWorker): |
402 | 567 | """Oversees the actual work of a code import to Bazaar.""" | ||
403 | 568 | |||
404 | 569 | # Where the Bazaar working tree will be stored. | ||
405 | 570 | BZR_BRANCH_PATH = 'bzr_branch' | ||
406 | 571 | |||
407 | 572 | # Should `getBazaarBranch` create a working tree? | ||
408 | 573 | needs_bzr_tree = True | ||
409 | 574 | |||
410 | 575 | required_format = BzrDirFormat.get_default_format() | ||
411 | 576 | |||
412 | 577 | def __init__(self, source_details, import_data_transport, | ||
413 | 578 | bazaar_branch_store, logger, opener_policy): | ||
414 | 579 | """Construct a `ToBzrImportWorker`. | ||
415 | 580 | |||
416 | 581 | :param source_details: A `CodeImportSourceDetails` object. | ||
417 | 582 | :param bazaar_branch_store: A `BazaarBranchStore`. The import worker | ||
418 | 583 | uses this to fetch and store the Bazaar branches that are created | ||
419 | 584 | and updated during the import process. | ||
420 | 585 | :param logger: A `Logger` to pass to cscvs. | ||
421 | 586 | :param opener_policy: Policy object that decides what branches can | ||
422 | 587 | be imported | ||
423 | 588 | """ | ||
424 | 589 | super(ToBzrImportWorker, self).__init__( | ||
425 | 590 | source_details, logger, opener_policy) | ||
426 | 591 | self.bazaar_branch_store = bazaar_branch_store | ||
427 | 592 | self.import_data_store = ImportDataStore( | ||
428 | 593 | import_data_transport, self.source_details) | ||
429 | 594 | |||
430 | 595 | def getBazaarBranch(self): | ||
431 | 596 | """Return the Bazaar `Branch` that we are importing into.""" | ||
432 | 597 | if os.path.isdir(self.BZR_BRANCH_PATH): | ||
433 | 598 | shutil.rmtree(self.BZR_BRANCH_PATH) | ||
434 | 599 | return self.bazaar_branch_store.pull( | ||
435 | 600 | self.source_details.target_id, self.BZR_BRANCH_PATH, | ||
436 | 601 | self.required_format, self.needs_bzr_tree, | ||
437 | 602 | stacked_on_url=self.source_details.stacked_on_url) | ||
438 | 603 | |||
439 | 604 | def pushBazaarBranch(self, bazaar_branch): | ||
440 | 605 | """Push the updated Bazaar branch to the server. | ||
441 | 606 | |||
442 | 607 | :return: True if revisions were transferred. | ||
443 | 608 | """ | ||
444 | 609 | return self.bazaar_branch_store.push( | ||
445 | 610 | self.source_details.target_id, bazaar_branch, | ||
446 | 611 | self.required_format, | ||
447 | 612 | stacked_on_url=self.source_details.stacked_on_url) | ||
448 | 613 | |||
449 | 614 | |||
450 | 615 | class CSCVSImportWorker(ToBzrImportWorker): | ||
451 | 598 | """An ImportWorker for imports that use CSCVS. | 616 | """An ImportWorker for imports that use CSCVS. |
452 | 599 | 617 | ||
453 | 600 | As well as invoking cscvs to do the import, this class also needs to | 618 | As well as invoking cscvs to do the import, this class also needs to |
454 | @@ -674,7 +692,7 @@ | |||
455 | 674 | return CodeImportWorkerExitCode.SUCCESS_NOCHANGE | 692 | return CodeImportWorkerExitCode.SUCCESS_NOCHANGE |
456 | 675 | 693 | ||
457 | 676 | 694 | ||
459 | 677 | class PullingImportWorker(ImportWorker): | 695 | class PullingImportWorker(ToBzrImportWorker): |
460 | 678 | """An import worker for imports that can be done by a bzr plugin. | 696 | """An import worker for imports that can be done by a bzr plugin. |
461 | 679 | 697 | ||
462 | 680 | Subclasses need to implement `probers`. | 698 | Subclasses need to implement `probers`. |
463 | @@ -806,7 +824,7 @@ | |||
464 | 806 | return config.codeimport.git_revisions_import_limit | 824 | return config.codeimport.git_revisions_import_limit |
465 | 807 | 825 | ||
466 | 808 | def getBazaarBranch(self): | 826 | def getBazaarBranch(self): |
468 | 809 | """See `ImportWorker.getBazaarBranch`. | 827 | """See `ToBzrImportWorker.getBazaarBranch`. |
469 | 810 | 828 | ||
470 | 811 | In addition to the superclass' behaviour, we retrieve bzr-git's | 829 | In addition to the superclass' behaviour, we retrieve bzr-git's |
471 | 812 | caches, both legacy and modern, from the import data store and put | 830 | caches, both legacy and modern, from the import data store and put |
472 | @@ -828,7 +846,7 @@ | |||
473 | 828 | return branch | 846 | return branch |
474 | 829 | 847 | ||
475 | 830 | def pushBazaarBranch(self, bazaar_branch): | 848 | def pushBazaarBranch(self, bazaar_branch): |
477 | 831 | """See `ImportWorker.pushBazaarBranch`. | 849 | """See `ToBzrImportWorker.pushBazaarBranch`. |
478 | 832 | 850 | ||
479 | 833 | In addition to the superclass' behaviour, we store bzr-git's cache | 851 | In addition to the superclass' behaviour, we store bzr-git's cache |
480 | 834 | directory at .bzr/repository/git in the import data store. | 852 | directory at .bzr/repository/git in the import data store. |
481 | 835 | 853 | ||
482 | === modified file 'lib/lp/codehosting/codeimport/workermonitor.py' | |||
483 | --- lib/lp/codehosting/codeimport/workermonitor.py 2013-01-07 02:40:55 +0000 | |||
484 | +++ lib/lp/codehosting/codeimport/workermonitor.py 2016-10-11 13:46:57 +0000 | |||
485 | @@ -134,7 +134,7 @@ | |||
486 | 134 | self.codeimport_endpoint = codeimport_endpoint | 134 | self.codeimport_endpoint = codeimport_endpoint |
487 | 135 | self._call_finish_job = True | 135 | self._call_finish_job = True |
488 | 136 | self._log_file = tempfile.TemporaryFile() | 136 | self._log_file = tempfile.TemporaryFile() |
490 | 137 | self._branch_url = None | 137 | self._target_url = None |
491 | 138 | self._log_file_name = 'no-name-set.txt' | 138 | self._log_file_name = 'no-name-set.txt' |
492 | 139 | self._access_policy = access_policy | 139 | self._access_policy = access_policy |
493 | 140 | 140 | ||
494 | @@ -143,7 +143,7 @@ | |||
495 | 143 | context = { | 143 | context = { |
496 | 144 | 'twisted_failure': failure, | 144 | 'twisted_failure': failure, |
497 | 145 | 'http_request': errorlog.ScriptRequest( | 145 | 'http_request': errorlog.ScriptRequest( |
499 | 146 | [('code_import_job_id', self._job_id)], self._branch_url), | 146 | [('code_import_job_id', self._job_id)], self._target_url), |
500 | 147 | } | 147 | } |
501 | 148 | report = config.create(context) | 148 | report = config.create(context) |
502 | 149 | 149 | ||
503 | @@ -169,15 +169,15 @@ | |||
504 | 169 | def getWorkerArguments(self): | 169 | def getWorkerArguments(self): |
505 | 170 | """Get arguments for the worker for the import we are working on. | 170 | """Get arguments for the worker for the import we are working on. |
506 | 171 | 171 | ||
508 | 172 | This also sets the _branch_url and _log_file_name attributes for use | 172 | This also sets the _target_url and _log_file_name attributes for use |
509 | 173 | in the _logOopsFromFailure and finishJob methods respectively. | 173 | in the _logOopsFromFailure and finishJob methods respectively. |
510 | 174 | """ | 174 | """ |
511 | 175 | deferred = self.codeimport_endpoint.callRemote( | 175 | deferred = self.codeimport_endpoint.callRemote( |
512 | 176 | 'getImportDataForJobID', self._job_id) | 176 | 'getImportDataForJobID', self._job_id) |
513 | 177 | 177 | ||
514 | 178 | def _processResult(result): | 178 | def _processResult(result): |
517 | 179 | code_import_arguments, branch_url, log_file_name = result | 179 | code_import_arguments, target_url, log_file_name = result |
518 | 180 | self._branch_url = branch_url | 180 | self._target_url = target_url |
519 | 181 | self._log_file_name = log_file_name | 181 | self._log_file_name = log_file_name |
520 | 182 | self._logger.info( | 182 | self._logger.info( |
521 | 183 | 'Found source details: %s', code_import_arguments) | 183 | 'Found source details: %s', code_import_arguments) |
522 | 184 | 184 | ||
523 | === modified file 'lib/lp/testing/factory.py' | |||
524 | --- lib/lp/testing/factory.py 2016-10-03 17:00:56 +0000 | |||
525 | +++ lib/lp/testing/factory.py 2016-10-11 13:46:57 +0000 | |||
526 | @@ -492,11 +492,11 @@ | |||
527 | 492 | epoch = datetime(2009, 1, 1, tzinfo=pytz.UTC) | 492 | epoch = datetime(2009, 1, 1, tzinfo=pytz.UTC) |
528 | 493 | return epoch + timedelta(minutes=self.getUniqueInteger()) | 493 | return epoch + timedelta(minutes=self.getUniqueInteger()) |
529 | 494 | 494 | ||
531 | 495 | def makeCodeImportSourceDetails(self, branch_id=None, rcstype=None, | 495 | def makeCodeImportSourceDetails(self, target_id=None, rcstype=None, |
532 | 496 | url=None, cvs_root=None, cvs_module=None, | 496 | url=None, cvs_root=None, cvs_module=None, |
533 | 497 | stacked_on_url=None): | 497 | stacked_on_url=None): |
536 | 498 | if branch_id is None: | 498 | if target_id is None: |
537 | 499 | branch_id = self.getUniqueInteger() | 499 | target_id = self.getUniqueInteger() |
538 | 500 | if rcstype is None: | 500 | if rcstype is None: |
539 | 501 | rcstype = 'bzr-svn' | 501 | rcstype = 'bzr-svn' |
540 | 502 | if rcstype in ['bzr-svn', 'bzr']: | 502 | if rcstype in ['bzr-svn', 'bzr']: |
541 | @@ -516,7 +516,7 @@ | |||
542 | 516 | else: | 516 | else: |
543 | 517 | raise AssertionError("Unknown rcstype %r." % rcstype) | 517 | raise AssertionError("Unknown rcstype %r." % rcstype) |
544 | 518 | return CodeImportSourceDetails( | 518 | return CodeImportSourceDetails( |
546 | 519 | branch_id, rcstype, url, cvs_root, cvs_module, | 519 | target_id, rcstype, url, cvs_root, cvs_module, |
547 | 520 | stacked_on_url=stacked_on_url) | 520 | stacked_on_url=stacked_on_url) |
548 | 521 | 521 | ||
549 | 522 | 522 |