Merge lp:~dorian-kemps/unifield-server/US-6448 into lp:unifield-server
- US-6448
- Merge into trunk
Proposed by
jftempo
Status: | Needs review |
---|---|
Proposed branch: | lp:~dorian-kemps/unifield-server/US-6448 |
Merge into: | lp:unifield-server |
Diff against target: |
280 lines (+58/-56) 2 files modified
bin/addons/msf_tools/automated_import.py (+0/-1) bin/addons/msf_tools/automated_import_job.py (+58/-55) |
To merge this branch: | bzr merge lp:~dorian-kemps/unifield-server/US-6448 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
UniField Reviewer Team | Pending | ||
Review via email: mp+388477@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bin/addons/msf_tools/automated_import.py' | |||
2 | --- bin/addons/msf_tools/automated_import.py 2019-08-02 14:52:35 +0000 | |||
3 | +++ bin/addons/msf_tools/automated_import.py 2020-07-31 12:01:18 +0000 | |||
4 | @@ -357,7 +357,6 @@ | |||
5 | 357 | 357 | ||
6 | 358 | return res | 358 | return res |
7 | 359 | 359 | ||
8 | 360 | |||
9 | 361 | def _generate_ir_cron(self, import_brw): | 360 | def _generate_ir_cron(self, import_brw): |
10 | 362 | """ | 361 | """ |
11 | 363 | Returns the values for the ir.cron to create according to automated.import values | 362 | Returns the values for the ir.cron to create according to automated.import values |
12 | 364 | 363 | ||
13 | === modified file 'bin/addons/msf_tools/automated_import_job.py' | |||
14 | --- bin/addons/msf_tools/automated_import_job.py 2019-11-13 16:23:09 +0000 | |||
15 | +++ bin/addons/msf_tools/automated_import_job.py 2020-07-31 12:01:18 +0000 | |||
16 | @@ -30,6 +30,8 @@ | |||
17 | 30 | import logging | 30 | import logging |
18 | 31 | import posixpath | 31 | import posixpath |
19 | 32 | import traceback | 32 | import traceback |
20 | 33 | import threading | ||
21 | 34 | import pooler | ||
22 | 33 | 35 | ||
23 | 34 | from osv import osv | 36 | from osv import osv |
24 | 35 | from osv import fields | 37 | from osv import fields |
25 | @@ -311,8 +313,22 @@ | |||
26 | 311 | def manual_process_import(self, cr, uid, ids, context=None): | 313 | def manual_process_import(self, cr, uid, ids, context=None): |
27 | 312 | if isinstance(ids, (int, long)): | 314 | if isinstance(ids, (int, long)): |
28 | 313 | ids = [ids] | 315 | ids = [ids] |
29 | 316 | data_obj = self.pool.get('ir.model.data') | ||
30 | 317 | |||
31 | 314 | wiz = self.read(cr, uid, ids[0], ['import_id'], context) | 318 | wiz = self.read(cr, uid, ids[0], ['import_id'], context) |
33 | 315 | return self.process_import(cr, uid, wiz['import_id'][0], started_job_id=ids[0], context=None) | 319 | |
34 | 320 | # Background import | ||
35 | 321 | thread = threading.Thread(target=self.process_import, args=(cr, uid, wiz['import_id'][0], ids[0], None)) | ||
36 | 322 | thread.start() | ||
37 | 323 | |||
38 | 324 | return { | ||
39 | 325 | 'type': 'ir.actions.act_window', | ||
40 | 326 | 'res_model': self._name, | ||
41 | 327 | 'res_id': ids[0], | ||
42 | 328 | 'view_type': 'form', | ||
43 | 329 | 'view_mode': 'form,tree', | ||
44 | 330 | 'context': context, | ||
45 | 331 | } | ||
46 | 316 | 332 | ||
47 | 317 | def process_import(self, cr, uid, import_id, started_job_id=False, context=None): | 333 | def process_import(self, cr, uid, import_id, started_job_id=False, context=None): |
48 | 318 | """ | 334 | """ |
49 | @@ -326,7 +342,6 @@ | |||
50 | 326 | :return: True | 342 | :return: True |
51 | 327 | """ | 343 | """ |
52 | 328 | import_obj = self.pool.get('automated.import') | 344 | import_obj = self.pool.get('automated.import') |
53 | 329 | data_obj = self.pool.get('ir.model.data') | ||
54 | 330 | 345 | ||
55 | 331 | if context is None: | 346 | if context is None: |
56 | 332 | context = {} | 347 | context = {} |
57 | @@ -334,7 +349,9 @@ | |||
58 | 334 | if isinstance(import_id, (int, long)): | 349 | if isinstance(import_id, (int, long)): |
59 | 335 | import_id = [import_id] | 350 | import_id = [import_id] |
60 | 336 | 351 | ||
62 | 337 | import_data = import_obj.browse(cr, uid, import_id[0], context=context) | 352 | new_cr = pooler.get_db(cr.dbname).cursor() |
63 | 353 | |||
64 | 354 | import_data = import_obj.browse(new_cr, uid, import_id[0], context=context) | ||
65 | 338 | no_file = False | 355 | no_file = False |
66 | 339 | already_done = [] | 356 | already_done = [] |
67 | 340 | job_id = False | 357 | job_id = False |
68 | @@ -344,12 +361,14 @@ | |||
69 | 344 | nb_processed = 0 | 361 | nb_processed = 0 |
70 | 345 | if started_job_id: | 362 | if started_job_id: |
71 | 346 | job_id = started_job_id | 363 | job_id = started_job_id |
72 | 364 | self.write(new_cr, uid, job_id, {'state': 'in_progress'}, context=context) | ||
73 | 347 | prev_job_id = False | 365 | prev_job_id = False |
74 | 348 | else: | 366 | else: |
75 | 349 | prev_job_id = job_id | 367 | prev_job_id = job_id |
79 | 350 | job_id = self.create(cr, uid, {'import_id': import_data.id, 'state': 'in_progress'}, context=context) | 368 | job_id = self.create(new_cr, uid, {'import_id': import_data.id, 'state': 'in_progress'}, context=context) |
80 | 351 | cr.commit() # keep trace of the job in case of error | 369 | new_cr.commit() # keep trace of the job in case of error |
81 | 352 | job = self.browse(cr, uid, job_id, context=context) | 370 | |
82 | 371 | job = self.browse(new_cr, uid, job_id, context=context) | ||
83 | 353 | started_job_id = False | 372 | started_job_id = False |
84 | 354 | md5 = False | 373 | md5 = False |
85 | 355 | error = None | 374 | error = None |
86 | @@ -361,17 +380,18 @@ | |||
87 | 361 | context.update({'no_raise_if_ok': True, 'auto_import_ok': True}) | 380 | context.update({'no_raise_if_ok': True, 'auto_import_ok': True}) |
88 | 362 | try: | 381 | try: |
89 | 363 | if import_data.ftp_ok and import_data.ftp_protocol == 'ftp': | 382 | if import_data.ftp_ok and import_data.ftp_protocol == 'ftp': |
91 | 364 | ftp_connec = self.pool.get('automated.import').ftp_test_connection(cr, uid, import_data.id, context=context) | 383 | ftp_connec = self.pool.get('automated.import').ftp_test_connection(new_cr, uid, import_data.id, context=context) |
92 | 365 | elif import_data.ftp_ok and import_data.ftp_protocol == 'sftp': | 384 | elif import_data.ftp_ok and import_data.ftp_protocol == 'sftp': |
94 | 366 | sftp = self.pool.get('automated.import').sftp_test_connection(cr, uid, import_data.id, context=context) | 385 | sftp = self.pool.get('automated.import').sftp_test_connection(new_cr, uid, import_data.id, context=context) |
95 | 367 | except Exception, e: | 386 | except Exception, e: |
96 | 368 | if job.id: | 387 | if job.id: |
97 | 369 | if isinstance(e, osv.except_osv): | 388 | if isinstance(e, osv.except_osv): |
98 | 370 | msg = e.value | 389 | msg = e.value |
99 | 371 | else: | 390 | else: |
100 | 372 | msg = e | 391 | msg = e |
103 | 373 | self.write(cr, uid, job_id, {'state': 'error', 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), 'start_time': start_time, 'comment': tools.ustr(msg)}, context=context) | 392 | self.write(new_cr, uid, job_id, {'state': 'error', 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), 'start_time': start_time, 'comment': tools.ustr(msg)}, context=context) |
104 | 374 | cr.commit() | 393 | new_cr.commit() |
105 | 394 | new_cr.close(True) | ||
106 | 375 | raise | 395 | raise |
107 | 376 | 396 | ||
108 | 377 | try: | 397 | try: |
109 | @@ -381,6 +401,7 @@ | |||
110 | 381 | except osv.except_osv as e: | 401 | except osv.except_osv as e: |
111 | 382 | error = tools.ustr(e) | 402 | error = tools.ustr(e) |
112 | 383 | no_file = True | 403 | no_file = True |
113 | 404 | new_cr.close(True) | ||
114 | 384 | # In case of manual processing, raise the error | 405 | # In case of manual processing, raise the error |
115 | 385 | if job.file_to_import: | 406 | if job.file_to_import: |
116 | 386 | raise e | 407 | raise e |
117 | @@ -411,18 +432,18 @@ | |||
118 | 411 | error = _('No file to import in %s !') % import_data.src_path | 432 | error = _('No file to import in %s !') % import_data.src_path |
119 | 412 | else: | 433 | else: |
120 | 413 | # files already processed in previous loop: delete the in_progress job | 434 | # files already processed in previous loop: delete the in_progress job |
122 | 414 | self.unlink(cr, 1, [job_id], context=context) | 435 | self.unlink(new_cr, 1, [job_id], context=context) |
123 | 415 | job_id = prev_job_id | 436 | job_id = prev_job_id |
124 | 416 | break | 437 | break |
125 | 417 | 438 | ||
127 | 418 | elif md5 and self.search_exist(cr, uid, [('import_id', '=', import_data.id), ('file_sum', '=', md5)], context=context): | 439 | elif md5 and self.search_exist(new_cr, uid, [('import_id', '=', import_data.id), ('file_sum', '=', md5)], context=context): |
128 | 419 | error = _('A file with same checksum has been already imported !') | 440 | error = _('A file with same checksum has been already imported !') |
129 | 420 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=False) | 441 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=False) |
131 | 421 | self.infolog(cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) | 442 | self.infolog(new_cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) |
132 | 422 | 443 | ||
133 | 423 | if error: | 444 | if error: |
136 | 424 | self.infolog(cr, uid, '%s :: %s' % (import_data.name , error)) | 445 | self.infolog(new_cr, uid, '%s :: %s' % (import_data.name , error)) |
137 | 425 | self.write(cr, uid, [job.id], { | 446 | self.write(new_cr, uid, [job.id], { |
138 | 426 | 'filename': filename, | 447 | 'filename': filename, |
139 | 427 | 'file_to_import': data64, | 448 | 'file_to_import': data64, |
140 | 428 | 'start_time': start_time, | 449 | 'start_time': start_time, |
141 | @@ -434,7 +455,7 @@ | |||
142 | 434 | 'state': 'done' if no_file else 'error', | 455 | 'state': 'done' if no_file else 'error', |
143 | 435 | }, context=context) | 456 | }, context=context) |
144 | 436 | continue | 457 | continue |
146 | 437 | else: # file to import given | 458 | else: # file to import given |
147 | 438 | no_file = True | 459 | no_file = True |
148 | 439 | if job.import_id.ftp_source_ok: | 460 | if job.import_id.ftp_source_ok: |
149 | 440 | raise osv.except_osv(_('Error'), _('You cannot manually select a file to import if given source path is set on FTP server')) | 461 | raise osv.except_osv(_('Error'), _('You cannot manually select a file to import if given source path is set on FTP server')) |
150 | @@ -444,18 +465,8 @@ | |||
151 | 444 | md5 = hashlib.md5(job.file_to_import).hexdigest() | 465 | md5 = hashlib.md5(job.file_to_import).hexdigest() |
152 | 445 | 466 | ||
153 | 446 | if job.file_sum != md5: | 467 | if job.file_sum != md5: |
166 | 447 | if self.search_exist(cr, uid, [('file_sum', '=', md5), ('id', '!=', job.id)], context=context): | 468 | if self.search_exist(new_cr, uid, [('file_sum', '=', md5), ('id', '!=', job.id)], context=context): |
167 | 448 | self.write(cr, uid, [job.id], {'file_sum': md5}, context=context) | 469 | self.write(new_cr, uid, [job.id], {'file_sum': md5}, context=context) |
156 | 449 | return { | ||
157 | 450 | 'type': 'ir.actions.act_window', | ||
158 | 451 | 'res_model': self._name, | ||
159 | 452 | 'res_id': job_id, | ||
160 | 453 | 'view_type': 'form', | ||
161 | 454 | 'view_mode': 'form,tree', | ||
162 | 455 | 'target': 'new', | ||
163 | 456 | 'view_id': [data_obj.get_object_reference(cr, uid, 'msf_tools', 'automated_import_job_file_view')[1]], | ||
164 | 457 | 'context': context, | ||
165 | 458 | } | ||
168 | 459 | 470 | ||
169 | 460 | oldest_file = os.path.join(job.import_id.src_path, job.filename) | 471 | oldest_file = os.path.join(job.import_id.src_path, job.filename) |
170 | 461 | filename = job.filename | 472 | filename = job.filename |
171 | @@ -467,7 +478,7 @@ | |||
172 | 467 | try: | 478 | try: |
173 | 468 | if import_data.ftp_source_ok and import_data.ftp_protocol == 'ftp': | 479 | if import_data.ftp_source_ok and import_data.ftp_protocol == 'ftp': |
174 | 469 | prefix = '%s_' % filename.split('.')[0] | 480 | prefix = '%s_' % filename.split('.')[0] |
176 | 470 | suffix = '.xls' if self.pool.get('stock.picking').get_import_filetype(cr, uid, filename) == 'excel' else '.xml' | 481 | suffix = '.xls' if self.pool.get('stock.picking').get_import_filetype(new_cr, uid, filename) == 'excel' else '.xml' |
177 | 471 | temp_file = tempfile.NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) | 482 | temp_file = tempfile.NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) |
178 | 472 | ftp_connec.retrbinary('RETR %s' % oldest_file, temp_file.write) | 483 | ftp_connec.retrbinary('RETR %s' % oldest_file, temp_file.write) |
179 | 473 | temp_file.close() | 484 | temp_file.close() |
180 | @@ -480,12 +491,12 @@ | |||
181 | 480 | processed, rejected, headers = getattr( | 491 | processed, rejected, headers = getattr( |
182 | 481 | self.pool.get(import_data.function_id.model_id.model), | 492 | self.pool.get(import_data.function_id.model_id.model), |
183 | 482 | import_data.function_id.method_to_call | 493 | import_data.function_id.method_to_call |
185 | 483 | )(cr, uid, oldest_file, context=context) | 494 | )(new_cr, uid, oldest_file, context=context) |
186 | 484 | if processed: | 495 | if processed: |
188 | 485 | nb_processed += self.generate_file_report(cr, uid, job, processed, headers, ftp_connec=ftp_connec, sftp=sftp) | 496 | nb_processed += self.generate_file_report(new_cr, uid, job, processed, headers, ftp_connec=ftp_connec, sftp=sftp) |
189 | 486 | 497 | ||
190 | 487 | if rejected: | 498 | if rejected: |
192 | 488 | nb_rejected += self.generate_file_report(cr, uid, job, rejected, headers, rejected=True, ftp_connec=ftp_connec, sftp=sftp) | 499 | nb_rejected += self.generate_file_report(new_cr, uid, job, rejected, headers, rejected=True, ftp_connec=ftp_connec, sftp=sftp) |
193 | 489 | state = 'error' | 500 | state = 'error' |
194 | 490 | for resjected_line in rejected: | 501 | for resjected_line in rejected: |
195 | 491 | line_message = '' | 502 | line_message = '' |
196 | @@ -498,13 +509,13 @@ | |||
197 | 498 | nb_rejected += context.get('rejected_confirmation') | 509 | nb_rejected += context.get('rejected_confirmation') |
198 | 499 | state = 'error' | 510 | state = 'error' |
199 | 500 | 511 | ||
201 | 501 | self.infolog(cr, uid, _('%s :: Import job done with %s records processed and %s rejected') % (import_data.name, len(processed), nb_rejected)) | 512 | self.infolog(new_cr, uid, _('%s :: Import job done with %s records processed and %s rejected') % (import_data.name, len(processed), nb_rejected)) |
202 | 502 | 513 | ||
203 | 503 | if import_data.function_id.model_id.model == 'purchase.order': | 514 | if import_data.function_id.model_id.model == 'purchase.order': |
204 | 504 | po_id = context.get('po_id', False) or self.pool.get('purchase.order').get_po_id_from_file(cr, uid, oldest_file, context=context) or False | 515 | po_id = context.get('po_id', False) or self.pool.get('purchase.order').get_po_id_from_file(cr, uid, oldest_file, context=context) or False |
205 | 505 | if po_id and (nb_processed or nb_rejected): | 516 | if po_id and (nb_processed or nb_rejected): |
208 | 506 | po_name = self.pool.get('purchase.order').read(cr, uid, po_id, ['name'], context=context)['name'] | 517 | po_name = self.pool.get('purchase.order').read(new_cr, uid, po_id, ['name'], context=context)['name'] |
209 | 507 | nb_total_pol = self.pool.get('purchase.order.line').search(cr, uid, [('order_id', '=', po_id)], count=True, context=context) | 518 | nb_total_pol = self.pool.get('purchase.order.line').search(new_cr, uid, [('order_id', '=', po_id)], count=True, context=context) |
210 | 508 | msg = _('%s: ') % po_name | 519 | msg = _('%s: ') % po_name |
211 | 509 | if nb_processed: | 520 | if nb_processed: |
212 | 510 | msg += _('%s out of %s lines have been updated') % (nb_processed, nb_total_pol) | 521 | msg += _('%s out of %s lines have been updated') % (nb_processed, nb_total_pol) |
213 | @@ -513,14 +524,14 @@ | |||
214 | 513 | if nb_rejected: | 524 | if nb_rejected: |
215 | 514 | msg += _('%s out of %s lines have been rejected') % (nb_rejected, nb_total_pol) | 525 | msg += _('%s out of %s lines have been rejected') % (nb_rejected, nb_total_pol) |
216 | 515 | if nb_processed or nb_rejected: | 526 | if nb_processed or nb_rejected: |
218 | 516 | self.pool.get('purchase.order').log(cr, uid, po_id, msg) | 527 | self.pool.get('purchase.order').log(new_cr, uid, po_id, msg) |
219 | 517 | 528 | ||
220 | 518 | if context.get('job_comment'): | 529 | if context.get('job_comment'): |
221 | 519 | for msg_dict in context['job_comment']: | 530 | for msg_dict in context['job_comment']: |
223 | 520 | self.pool.get(msg_dict['res_model']).log(cr, uid, msg_dict['res_id'], msg_dict['msg']) | 531 | self.pool.get(msg_dict['res_model']).log(new_cr, uid, msg_dict['res_id'], msg_dict['msg']) |
224 | 521 | error_message.append(msg_dict['msg']) | 532 | error_message.append(msg_dict['msg']) |
225 | 522 | 533 | ||
227 | 523 | self.write(cr, uid, [job.id], { | 534 | self.write(new_cr, uid, [job.id], { |
228 | 524 | 'filename': filename, | 535 | 'filename': filename, |
229 | 525 | 'start_time': start_time, | 536 | 'start_time': start_time, |
230 | 526 | 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), | 537 | 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), |
231 | @@ -533,13 +544,13 @@ | |||
232 | 533 | }, context=context) | 544 | }, context=context) |
233 | 534 | is_success = True if not rejected else False | 545 | is_success = True if not rejected else False |
234 | 535 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=is_success) | 546 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=is_success) |
237 | 536 | self.infolog(cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) | 547 | self.infolog(new_cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) |
238 | 537 | cr.commit() | 548 | new_cr.commit() |
239 | 538 | except Exception as e: | 549 | except Exception as e: |
241 | 539 | cr.rollback() | 550 | new_cr.rollback() |
242 | 540 | trace_b = tools.ustr(traceback.format_exc()) | 551 | trace_b = tools.ustr(traceback.format_exc()) |
245 | 541 | self.infolog(cr, uid, '%s :: %s' % (import_data.name, trace_b)) | 552 | self.infolog(new_cr, uid, '%s :: %s' % (import_data.name, trace_b)) |
246 | 542 | self.write(cr, uid, [job.id], { | 553 | self.write(new_cr, uid, [job.id], { |
247 | 543 | 'filename': False, | 554 | 'filename': False, |
248 | 544 | 'start_time': start_time, | 555 | 'start_time': start_time, |
249 | 545 | 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), | 556 | 'end_time': time.strftime('%Y-%m-%d %H:%M:%S'), |
250 | @@ -551,26 +562,18 @@ | |||
251 | 551 | 'state': 'error', | 562 | 'state': 'error', |
252 | 552 | }, context=context) | 563 | }, context=context) |
253 | 553 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=False) | 564 | move_to_process_path(import_data, ftp_connec, sftp, filename, success=False) |
255 | 554 | self.infolog(cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) | 565 | self.infolog(new_cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename)) |
256 | 555 | finally: | 566 | finally: |
257 | 556 | if orig_file_name: | 567 | if orig_file_name: |
258 | 557 | self.end_processing_filename(orig_file_name) | 568 | self.end_processing_filename(orig_file_name) |
259 | 558 | 569 | ||
260 | 570 | new_cr.commit() | ||
261 | 571 | new_cr.close(True) | ||
262 | 572 | |||
263 | 559 | if 'row' in context: | 573 | if 'row' in context: |
264 | 560 | # causing LmF when running job manually | 574 | # causing LmF when running job manually |
265 | 561 | context.pop('row') | 575 | context.pop('row') |
266 | 562 | 576 | ||
267 | 563 | return { | ||
268 | 564 | 'type': 'ir.actions.act_window', | ||
269 | 565 | 'res_model': self._name, | ||
270 | 566 | 'res_id': job_id, | ||
271 | 567 | 'view_type': 'form', | ||
272 | 568 | 'view_mode': 'form,tree', | ||
273 | 569 | 'target': 'current', | ||
274 | 570 | 'context': context, | ||
275 | 571 | } | ||
276 | 572 | |||
277 | 573 | |||
278 | 574 | def generate_file_report(self, cr, uid, job_brw, data_lines, headers, rejected=False, ftp_connec=None, sftp=None): | 577 | def generate_file_report(self, cr, uid, job_brw, data_lines, headers, rejected=False, ftp_connec=None, sftp=None): |
279 | 575 | """ | 578 | """ |
280 | 576 | Create a csv file that contains the processed lines and put this csv file | 579 | Create a csv file that contains the processed lines and put this csv file |