Merge lp:~jfb-tempo-consulting/unifield-server/US-11601 into lp:unifield-server

[jftempo]

[lp:~jfb-tempo-consulting/unifield-server/US-11601]

6511. By jftempo on 2023-08-04

US-11715 [FIX] Dav move file: manage time out

=== modified file 'bin/addons/account_hq_entries/wizard/esc_lines_import.py'
--- bin/addons/account_hq_entries/wizard/esc_lines_import.py	2023-06-23 11:55:48 +0000
+++ bin/addons/account_hq_entries/wizard/esc_lines_import.py	2023-08-04 14:37:16 +0000
@@ -9,6 +9,7 @@
 from tools.translate import _
 import time
 import base64
+import os
 import re
 from psycopg2 import IntegrityError
 
@@ -47,6 +48,7 @@
 
     _defaults = {
         'state': 'draft',
+        'start_date': lambda *x: fields.datetime.now()
     }
 
     def __init__(self, pool, cr):
@@ -126,7 +128,7 @@
         if not self.browse(cr, uid, ids[0], context=context).file:
             raise osv.except_osv(_('Warning'), _('No file to import'))
 
-        threading.Thread(target=self.load_bg, args=(cr.dbname, uid, ids[0], context)).start()
+        threading.Thread(target=self.load_bg, args=(cr.dbname, uid, ids[0], False, context)).start()
         self.write(cr, uid, ids[0], {'state': 'inprogress', 'progress': 0}, context=context)
 
         view_id = self.pool.get('ir.model.data').get_object_reference(cr, uid, 'account_hq_entries', 'esc_line_import_progress_wizard')[1]
@@ -142,8 +144,21 @@
             'context': context,
         }
 
-
-    def load_bg(self, dbname, uid, wiz_id, context=None):
+    def auto_import(self, cr, uid, file_to_import, context=None):
+
+        import_id = self.create(cr, uid, {
+            'file': base64.encodestring(open(file_to_import, 'r').read()),
+            'filename': os.path.split(file_to_import)[1],
+            'state': 'inprogress',
+        })
+        cr.commit()
+        self.load_bg(cr.dbname, uid, import_id, auto_import=True, context=context)
+        curr_state = self.browse(cr, uid, import_id, fields_to_fetch=['state'], context=context).state
+        self.write(cr, uid, import_id, {'state': 'ack'}, context=context)
+        return import_id, curr_state
+
+
+    def load_bg(self, dbname, uid, wiz_id, auto_import=False, context=None):
         def manage_error(line_index, msg, row):
             errors.append(_('Line %s, %s') % (line_index, _(msg)))
             line_data = []
@@ -162,6 +177,7 @@
 
         created_ids = {}
         consignee_instances = {}
+
         try:
             cr = pooler.get_db(dbname).cursor()
             cr2 = pooler.get_db(dbname).cursor()

=== modified file 'bin/addons/msf_instance/msf_instance.py'
--- bin/addons/msf_instance/msf_instance.py	2023-07-12 14:08:51 +0000
+++ bin/addons/msf_instance/msf_instance.py	2023-08-04 14:37:16 +0000
@@ -823,7 +823,7 @@
                         temp_create = True
 
                     if not upload_ok:
-                        upload_ok, error = dav.upload(temp_fileobj, temp_drive_file, buffer_size=buffer_size, log=True, progress_obj=progress_obj)
+                        upload_ok, error = dav.upload(temp_fileobj, temp_drive_file, buffer_size=buffer_size, log=True, progress_obj=progress_obj, continuation=True)
 
                     # please don't change the following to else:
                     if upload_ok:

=== modified file 'bin/addons/msf_tools/automated_import.py'
--- bin/addons/msf_tools/automated_import.py	2022-08-19 12:59:43 +0000
+++ bin/addons/msf_tools/automated_import.py	2023-08-04 14:37:16 +0000
@@ -372,10 +372,13 @@
             return self.local_connection.get_file_content(path)
         return self.connection.get_file_content(path)
 
-    def move_to_process_path(self, filename, success):
+    def move_to_process_path(self, filename, success, local_src=False):
         """
         Move the file `file` from `src_path` to `dest_path`
+        local_src : if empty use the source_path + filename
+                    else local_src + filename
         :return: return True
+
         """
 
 
@@ -388,25 +391,32 @@
 
         logging.getLogger('automated.import').info(_('Moving %s to %s') % (filename, dest_path))
 
-        if self.source_is_remote and dest_is_remote:
-            # from remote to remote (rename)
-            src_file_name = posixpath.join(self.source, filename)
-            dest_file_name = posixpath.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), filename))
-            self.connection.rename(src_file_name, dest_file_name)
-        elif not self.source_is_remote and dest_is_remote:
-            # from local to remote
-            local_file = os.path.join(self.source, filename)
-            remote_file = posixpath.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), filename))
-            self.connection.push(local_file, remote_file)
-            os.remove(local_file)
-        elif self.source_is_remote and not dest_is_remote:
+        if dest_is_remote:
+            if self.source_is_remote and not local_src:
+                # from remote to remote (rename)
+                src_file_name = posixpath.join(self.source, filename)
+                dest_file_name = posixpath.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), filename))
+                self.connection.rename(src_file_name, dest_file_name)
+            else:
+                # from local to remote
+                if local_src:
+                    local_file = os.path.join(local_src, filename)
+                else:
+                    local_file = os.path.join(self.source, filename)
+                remote_file = posixpath.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), filename))
+                self.connection.push(local_file, remote_file)
+                os.remove(local_file)
+        elif self.source_is_remote and not local_src:
             # from remote to local
             src_file_name = posixpath.join(self.source, filename)
             destfile = os.path.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), self.connection.remove_special_chars(filename)))
             self.connection.get(src_file_name, destfile, delete=True)
         else:
             # from local to local
-            src_file_name = os.path.join(self.source, filename)
+            if local_src:
+                src_file_name = os.path.join(local_src, filename)
+            else:
+                src_file_name = os.path.join(self.source, filename)
             destfile = os.path.join(dest_path, '%s_%s' % (time.strftime('%Y%m%d_%H%M%S'), self.connection.remove_special_chars(filename)))
             shutil.move(src_file_name, destfile)
 

=== modified file 'bin/addons/msf_tools/automated_import_data.xml'
--- bin/addons/msf_tools/automated_import_data.xml	2022-08-16 16:02:19 +0000
+++ bin/addons/msf_tools/automated_import_data.xml	2023-08-04 14:37:16 +0000
@@ -155,5 +155,11 @@
             <field name="method_to_call">import_data_from_csv</field>
         </record>
 
+        <record id="auto_import_fnct_iil" model="automated.import.function">
+            <field name="name">Import IIL</field>
+            <field name="model_id" model="ir.model" search="[('model', '=', 'esc.line.import')]" />
+            <field name="method_to_call">auto_import</field>
+        </record>
+
     </data>
 </openerp>

=== modified file 'bin/addons/msf_tools/automated_import_job.py'
--- bin/addons/msf_tools/automated_import_job.py	2023-04-21 09:37:00 +0000
+++ bin/addons/msf_tools/automated_import_job.py	2023-08-04 14:37:16 +0000
@@ -38,6 +38,7 @@
 from mission_stock.mission_stock import UnicodeWriter
 
 from threading import RLock
+from service.web_services import report_spool
 
 
 class automated_import_job(osv.osv):
@@ -347,35 +348,93 @@
                         remote.connection.get(oldest_file, tmp_dest_file)
                         oldest_file = tmp_dest_file
 
-                    processed, rejected, headers = getattr(
+                    import_results = getattr(
                         self.pool.get(import_data.function_id.model_id.model),
                         import_data.function_id.method_to_call
                     )(cr, uid, oldest_file, context=context)
-                    if processed:
-                        nb_processed += self.generate_file_report(cr, uid, job, processed, headers, remote=remote)
-
-                    if rejected:
-                        nb_rejected = nb_processed  # US-7624 If one row is not correct, all processed rows are rejected
-                        self.generate_file_report(cr, uid, job, rejected, headers, remote=remote, rejected=True)
-                        state = 'error'
-                        for resjected_line in rejected:
-                            line_message = ''
-                            if resjected_line[0]:
-                                line_message = _('Line %s: ') % resjected_line[0]
-                            line_message += resjected_line[2]
-                            error_message.append(line_message)
-
-                        if import_data.function_id.method_to_call == 'auto_import_destination':
-                            error_message.append(_("no data will be imported until all the error messages are corrected"))
-                            tools.cache.clean_caches_for_db(cr.dbname)
-                            tools.read_cache.clean_caches_for_db(cr.dbname)
-
-
-                    if context.get('rejected_confirmation'):
-                        nb_rejected += context.get('rejected_confirmation')
-                        state = 'error'
-
-                    self.infolog(cr, uid, _('%s :: Import job done with %s records processed and %s rejected') % (import_data.name, len(processed), nb_rejected))
+
+                    if import_data.function_id.model_id.model == 'esc.line.import':
+                        import_wiz_id, import_state = import_results
+                        import_wiz_obj = self.pool.get('esc.line.import').browse(cr, uid, import_wiz_id, context=context)
+                        nb_processed = import_wiz_obj.created
+                        nb_rejected = import_wiz_obj.nberrors
+                        is_success = bool(nb_processed) # if at least one IIL created
+                        import_error = ''
+                        if nb_rejected:
+                            import_error = import_wiz_obj.error
+
+                        # Generate Report
+                        r_filename = '%s_iil_%s_%s.txt' % (
+                            time.strftime('%Y%m%d_%H%M%S'),
+                            import_wiz_id,
+                            nb_rejected and 'with_rejected_lines' or 'ok'
+                        )
+
+                        report_file_name = remote.get_report_file_name(r_filename)
+                        fp_file = open(report_file_name, 'wb')
+                        fp_file.write("File: %s\r\nStart: %s\r\nEnd: %s\r\nState: %s\r\nFile lines processed: %s\r\nIIL Created: %s\r\nLines rejected: %s\r\n\r\n%s" % (
+                            import_wiz_obj.filename,
+                            import_wiz_obj.start_date,
+                            import_wiz_obj.end_date,
+                            import_state,
+                            import_wiz_obj.total,
+                            import_wiz_obj.created,
+                            import_wiz_obj.nberrors,
+                            import_error
+                        ))
+                        fp_file.close()
+                        remote.push_report(report_file_name, r_filename)
+
+                        if nb_rejected:
+                            # export datas :
+                            report_name = "esc_line_import_rejected"
+                            rp_spool = report_spool()
+                            res_export = rp_spool.exp_report(cr.dbname, uid, report_name, [import_wiz_id], {}, context)
+                            file_res = {'state': False}
+                            while not file_res.get('state'):
+                                file_res = rp_spool.exp_report_get(cr.dbname, uid, res_export)
+                                time.sleep(0.5)
+
+                            if file_res.get('result'):
+                                tmp_dir = tempfile.mkdtemp()
+                                rejected_fn = 'rejected_lines_%s' % filename
+                                rejected_full_filename = os.path.join(tmp_dir, rejected_fn)
+                                fp_rejected = open(rejected_full_filename, 'wb')
+                                fp_rejected.write(base64.decodestring(file_res.get('result')))
+                                fp_rejected.close()
+                                remote.move_to_process_path(rejected_fn, success=False, local_src=tmp_dir)
+
+                    else:
+                        processed, rejected, headers = import_results
+
+                        is_success = True
+                        if processed:
+                            nb_processed += self.generate_file_report(cr, uid, job, processed, headers, remote=remote)
+
+                        if rejected:
+                            is_success = False
+                            nb_rejected = self.generate_file_report(cr, uid, job, rejected, headers, remote=remote, rejected=True)
+                            if import_data.function_id.model_id.model == 'hr.employee':
+                                nb_rejected = nb_processed # US-7624 If one row is not correct, all processed rows are rejected
+                            state = 'error'
+                            for resjected_line in rejected:
+                                line_message = ''
+                                if resjected_line[0]:
+                                    line_message = _('Line %s: ') % resjected_line[0]
+                                line_message += resjected_line[2]
+                                error_message.append(line_message)
+
+                            if import_data.function_id.method_to_call == 'auto_import_destination':
+                                error_message.append(_("no data will be imported until all the error messages are corrected"))
+                                tools.cache.clean_caches_for_db(cr.dbname)
+                                tools.read_cache.clean_caches_for_db(cr.dbname)
+
+
+                        if context.get('rejected_confirmation'):
+                            nb_rejected += context.get('rejected_confirmation')
+                            state = 'error'
+
+                    self.infolog(cr, uid, _('%s :: Import job done with %s records processed and %s rejected') % (import_data.name, nb_processed, nb_rejected))
 
                     if import_data.function_id.model_id.model == 'purchase.order':
                         po_id = context.get('po_id', False) or self.pool.get('purchase.order').get_po_id_from_file(cr, uid, oldest_file, context=context) or False
@@ -407,7 +466,7 @@
                         'file_to_import': data64,
                         'state': state,
                     }, context=context)
-                    is_success = True if not rejected else False
+
                     remote.move_to_process_path(filename, success=is_success)
                     self.infolog(cr, uid, _('%s :: Import file (%s) moved to destination path') % (import_data.name, filename))
                     cr.commit()

=== modified file 'bin/tools/webdav.py'
--- bin/tools/webdav.py	2022-08-18 09:24:04 +0000
+++ bin/tools/webdav.py	2023-08-04 14:37:16 +0000
@@ -13,6 +13,7 @@
 import os
 import posixpath
 import time
+import urlparse
 
 class ConnectionFailed(Exception):
     pass
@@ -33,28 +34,50 @@
         if not port:
             port = 443 if protocol == 'https' else 80
         self.path = path or ''
-        if not self.path.endswith('/'):
-            self.path = '%s/' % self.path
-
-        # oneDrive: need to split /site/ and path
-        # in our config site is /personal/unifield_xxx_yyy/
-        # path is /Documents/Unifield/
-        self.baseurl = '{0}://{1}:{2}/{3}/'.format(protocol, host, port, '/'.join(self.path.split('/')[0:3]) )
-
-        if len(self.path.split('/')) < 5:
-            self.path = '%sDocuments/' % self.path
+
+        self.url = '{0}://{1}:{2}'.format(protocol, host, port)
 
         self.login()
 
     def login(self):
-        ctx_auth = AuthenticationContext(self.baseurl)
+        ctx_auth = AuthenticationContext(self.url)
 
         if ctx_auth.acquire_token_for_user(self.username, cgi.escape(self.password)):
             self.request = ClientRequest(ctx_auth)
-            self.request.context = ClientContext(self.baseurl, ctx_auth)
-
+            self.request.context = ClientContext(self.url, ctx_auth)
             if not ctx_auth.provider.FedAuth or not ctx_auth.provider.rtFa:
                 raise ConnectionFailed(ctx_auth.get_last_error())
+
+
+            # get the server_site url
+            if not self.path.startswith('/'):
+                self.path = '/%s' % self.path
+            options = RequestOptions(self.url)
+            options.method = HttpMethod.Get
+            options.set_header("X-HTTP-Method", "GET")
+            options.set_header('accept', 'application/json;odata=verbose')
+            self.request.context.authenticate_request(options)
+            self.request.context.ensure_form_digest(options)
+
+            result = requests.post(url="%s/%s/_api/contextinfo" % (self.url, self.path), headers=options.headers, auth=options.auth)
+            if result.status_code not in (200, 201):
+                raise requests.exceptions.RequestException("Path %s not found" % self.path)
+            js = result.json()
+            baseurl = js.get('d', {}).get('GetContextWebInformation', {}).get('WebFullUrl')
+            if not baseurl.endswith('/'):
+                baseurl = '%s/' % baseurl
+
+            if not baseurl:
+                raise requests.exceptions.RequestException("Full Url not found %s" % self.path)
+            parsed_base = urlparse.urlparse(baseurl).path
+            self.baseurl = '%s%s' % (self.url, parsed_base)
+            if not self.path.startswith('/'):
+                self.path = '/%s' % self.path
+            if not self.path.endswith('/') and len(self.path) > 1:
+                self.path = '%s/' % (self.path, )
+
+            # set auth ctx with the full correct url or move fails
+            self.request.context = ClientContext(self.baseurl, ctx_auth)
         else:
             raise requests.exceptions.RequestException(ctx_auth.get_last_error())
 
@@ -113,17 +136,22 @@
             raise Exception(self.parse_error(result))
         return True
 
-    def move(self, remote_path, dest):
+    def move(self, remote_path, dest, retry=True):
         webUri = '%s%s' % (self.path, remote_path)
         destUri = '%s%s' % (self.path, dest)
         # falgs=1 to overwrite existing file
         request_url = "%s_api/web/getfilebyserverrelativeurl('%s')/moveto(newurl='%s',flags=1)" % (self.baseurl, webUri, destUri)
         result = self.format_request(request_url, 'POST')
         if result.status_code not in (200, 201):
+            error = self.parse_error(result)
+            if retry and ('timed out' in error or '2130575252' in error):
+                logging.getLogger('cloud.backup').info('OneDrive move: session time out')
+                self.login()
+                return self.move(remote_path, dest, retry=False)
             raise Exception(self.parse_error(result))
         return True
 
-    def upload(self, fileobj, remote_path, buffer_size=None, log=False, progress_obj=False):
+    def upload(self, fileobj, remote_path, buffer_size=None, log=False, progress_obj=False, continuation=False):
         if not self.session_uuid:
             self.session_uuid = uuid.uuid1()
 
@@ -137,6 +165,9 @@
             except:
                 size = None
 
+        if not continuation:
+            self.session_offset = -1
+
         if self.session_offset != -1:
             fileobj.seek(self.session_offset)
 
@@ -190,6 +221,7 @@
                         progress_obj.write({'name': percent})
 
                 logger.info('OneDrive: %d bytes sent on %s bytes %s' % (self.session_offset, size or 'unknown', percent_txt))
+        self.session_offset = -1
         return (True, '')
 
     def list(self, remote_path):
@@ -203,13 +235,14 @@
         self.request.context.authenticate_request(options)
         self.request.context.ensure_form_digest(options)
         result = requests.get(url=request_url, headers=options.headers, auth=options.auth)
+        if result.status_code not in (200, 201):
+            raise requests.exceptions.RequestException(self.parse_error(result))
 
         result = result.json()
         files=[]
         for i in range(len(result['d']['results'])):
             item = result['d']['results'][i]
             files.append(item)
-
         return files