Merge lp:~florent.x/openobject-server/6.1-fallback-search-sort into lp:openobject-server/6.1

Proposed by Florent
Status: Needs review
Proposed branch: lp:~florent.x/openobject-server/6.1-fallback-search-sort
Merge into: lp:openobject-server/6.1
Diff against target: 348 lines (+237/-11)
2 files modified
openerp/osv/expression.py (+110/-4)
openerp/osv/orm.py (+127/-7)
To merge this branch: bzr merge lp:~florent.x/openobject-server/6.1-fallback-search-sort
Reviewer Review Type Date Requested Status
OpenERP Core Team Pending
Review via email: mp+124963@code.launchpad.net

Description of the change

I have adapted the code for 6.1.
It is adapted from the work done by XRG and published in the C2C branch for 6.0

You need to add these lines in "openerp-server.conf" to trigger this behavior:

[orm]
fallback_search = True

(Or add an attribute `_fallback_search = True` on some Models)

It works smoothly without visible drawbacks.

To post a comment you must log in.
4276. By Florent

[FIX] oversight in previous patch.

4277. By Florent

[FIX] don't pass a keyword argument user= to the _column.get method because the argument is sometimes 'uid'.

Unmerged revisions

4277. By Florent

[FIX] don't pass a keyword argument user= to the _column.get method because the argument is sometimes 'uid'.

4276. By Florent

[FIX] oversight in previous patch.

4275. By Florent

[IMP] fallback search support, adapted from xrg.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'openerp/osv/expression.py'
2--- openerp/osv/expression.py 2012-01-24 12:42:52 +0000
3+++ openerp/osv/expression.py 2012-09-22 09:11:20 +0000
4@@ -123,7 +123,7 @@
5
6 import logging
7
8-from openerp.tools import flatten, reverse_enumerate
9+from openerp.tools import flatten, reverse_enumerate, config
10 import fields
11 import openerp.modules
12 from openerp.osv.orm import MAGIC_COLUMNS
13@@ -340,6 +340,21 @@
14 (select_field, from_table, select_field))
15 return [r[0] for r in cr.fetchall()]
16
17+def _m2o_cmp(a, b):
18+ if a is False:
19+ if b:
20+ return -1
21+ return 0
22+ if not b:
23+ return 1
24+ elif isinstance(b, (int, long)):
25+ return cmp(a[0], b)
26+ elif isinstance(b, basestring):
27+ return cmp(a[1], b)
28+ else:
29+ # Arbitrary: unknown b is greater than all record values
30+ return -1
31+
32 class expression(object):
33 """
34 parse a domain expression
35@@ -347,6 +362,43 @@
36 leafs are still in a ('foo', '=', 'bar') format
37 For more info: http://christophe-simonis-at-tiny.blogspot.com/2008/08/new-new-domain-notation.html
38 """
39+ FALLBACK_OPS = {
40+ '=': lambda a, b: a == b,
41+ '!=': lambda a, b: a != b,
42+ '<>': lambda a, b: a != b,
43+ '<=': lambda a, b: a <= b,
44+ '<': lambda a, b: a < b,
45+ '>': lambda a, b: a > b,
46+ '>=': lambda a, b: a >= b,
47+ '=?': lambda a, b: b is None or b is False or a == b,
48+ #'=like': lambda a, b: , need regexp?
49+ #'=ilike': lambda a, b: ,
50+ 'like': lambda a, b: b in a,
51+ 'not like': lambda a, b: b not in a,
52+ 'ilike': lambda a, b: not b or (a and (b.lower() in a.lower())),
53+ 'not ilike': lambda a, b: b and (not a or b.lower() not in a.lower()),
54+ 'in': lambda a, b: a in b,
55+ 'not in': lambda a, b: a not in b,
56+ }
57+
58+ FALLBACK_OPS_M2O = {
59+ '=': lambda a, b: _m2o_cmp(a, b) == 0,
60+ '!=': _m2o_cmp,
61+ '<>': _m2o_cmp,
62+ '<=': lambda a, b: _m2o_cmp(a, b) <= 0,
63+ '<': lambda a, b: _m2o_cmp(a, b) < 0,
64+ '>': lambda a, b: _m2o_cmp(a, b) > 0,
65+ '>=': lambda a, b: _m2o_cmp(a, b) >= 0,
66+ '=?': lambda a, b: b is None or b is False or _m2o_cmp(a, b) == 0,
67+ #'=like': lambda a, b: , need regexp?
68+ #'=ilike': lambda a, b: ,
69+ 'like': lambda a, b: not b or (a and b in a[1]),
70+ 'not like': lambda a, b: b and (not a or b not in a[1]),
71+ 'ilike': lambda a, b: not b or (a and b.lower() in a[1].lower()),
72+ 'not ilike': lambda a, b: b and (not a or b.lower() not in a[1].lower()),
73+ 'in': lambda a, b: a and a[1] in b,
74+ 'not in': lambda a, b: not a or a[1] not in b,
75+ }
76
77 def __init__(self, cr, uid, exp, table, context):
78 self.has_unaccent = openerp.modules.registry.RegistryManager.get(cr.dbname).has_unaccent
79@@ -471,11 +523,65 @@
80 if field._properties and not field.store:
81 # this is a function field that is not stored
82 if not field._fnct_search:
83- # the function field doesn't provide a search function and doesn't store
84- # values in the database, so we must ignore it : we generate a dummy leaf
85- self.__exp[i] = TRUE_LEAF
86+ if hasattr(table, '_fallback_search'):
87+ do_fallback = table._fallback_search
88+ else:
89+ do_fallback = config.get_misc('orm', 'fallback_search', None)
90+ if do_fallback is None:
91+ # the function field doesn't provide a search function and doesn't store
92+ # values in the database, so we must ignore it : we generate a dummy leaf
93+ self.__exp[i] = TRUE_LEAF
94+ elif do_fallback:
95+ # Do the slow fallback.
96+ # Try to see if the expression so far is a straight (ANDed)
97+ # combination. In that case, we can restrict the query
98+ if field._type == 'many2one':
99+ op_fn = self.FALLBACK_OPS_M2O.get(operator)
100+ else:
101+ op_fn = self.FALLBACK_OPS.get(operator)
102+ if not op_fn:
103+ raise ValueError('Cannot fallback with operator "%s"!' % operator)
104+ e_so_far = self.__exp[:i]
105+ for e in e_so_far:
106+ if not is_leaf(e, internal=True):
107+ e_so_far = []
108+ break
109+ ids_so_far = table.search(cr, uid, e_so_far, context=context)
110+ if not ids_so_far:
111+ self.__exp[i] = ('id', '=', 0)
112+ else:
113+ ids2 = []
114+ if field._multi:
115+ fget_name = [field_path[0]]
116+ else:
117+ fget_name = field_path[0]
118+ rvals = field.get(cr, table, ids_so_far, fget_name, uid, context=context)
119+ for res_id, rval in rvals.items():
120+ if field._multi:
121+ rval = rval.get(field_path[0])
122+ if rval is None or rval is False:
123+ pass
124+ elif field._type == 'integer':
125+ # workaround the str() of fields.function.get() :(
126+ rval = int(rval)
127+ elif field._type == 'float':
128+ assert isinstance(rval, float), "%s: %r" % (type(rval), rval)
129+ if field.digits:
130+ rval = round(rval, field.digits[1] or field.digits_compute(cr)[1])
131+
132+ # TODO: relational fields don't work here, must implement
133+ # special operators between their (id, name) and right
134+
135+ if op_fn(rval, right):
136+ ids2.append(res_id)
137+ self.__exp[i] = ('id', 'in', ids2)
138+ else:
139+ raise NotImplementedError("Cannot compute %s.%s field for filtering" %
140+ (table._name, left))
141 else:
142 subexp = field.search(cr, uid, table, left, [self.__exp[i]], context=context)
143+ # Reminder: the field.search() API returns an expression, not a dataset,
144+ # which means that [] => True clause
145 if not subexp:
146 self.__exp[i] = TRUE_LEAF
147 else:
148
149=== modified file 'openerp/osv/orm.py'
150--- openerp/osv/orm.py 2012-08-26 20:26:55 +0000
151+++ openerp/osv/orm.py 2012-09-22 09:11:20 +0000
152@@ -251,6 +251,46 @@
153 self.value = value
154 self.args = (name, value)
155
156+class PythonOrderBy(list):
157+ """Replacement class for _generate_order_by() requesting python sorting."""
158+
159+ def get_sort_key(self):
160+ """Return a function to use in sort(key=...)."""
161+ if len(self) != 1:
162+ raise NotImplementedError("Cannot sort by: %s" % ','.join(self))
163+
164+ _getters = []
165+ for field in [item.split(' ', 1)[0] for item in self]:
166+ if field.endswith(':'):
167+ field = field[:-1]
168+ # the visual string of m2o
169+ _getters.append(lambda item: item[field] and item[field][1] or None)
170+ else:
171+ _getters.append(lambda item: item[field])
172+ def sort_key(k):
173+ return tuple([g(k) for g in _getters])
174+ return sort_key
175+
176+ def get_fields(self):
177+ """Return the field components of this order-by list."""
178+ def _clean(item):
179+ field = item.split(' ', 1)[0]
180+ if field.endswith(':'):
181+ field = field[:-1]
182+ return field
183+ return [_clean(item) for item in self]
184+
185+ def get_reverse(self):
186+ """Return True if the order is reversed (default is False)."""
187+ reverse = None
188+ for item in self:
189+ rev = item.partition(' ')[2].lower() == 'desc'
190+ if reverse is None:
191+ reverse = rev
192+ elif reverse != rev:
193+ raise NotImplementedError("Cannot use multiple order directions on python sorting!")
194+ return reverse
195+
196 class BrowseRecordError(Exception):
197 pass
198
199@@ -653,6 +693,14 @@
200 To create a class that should not be instantiated, the _register class attribute
201 may be set to False.
202 """
203+ # @attribute _fallback_search enables *slow*, fallback search for those
204+ # function fields that do not provide a _fnct_search() .
205+ # Use with care, as this may perform a read() of the full dataset
206+ # of that model, in order to compute the search condition.
207+ # Takes 3 values:
208+ # None the default 'ignore' behavior,
209+ # True use the slow method
210+ # False stop and raise an exception on those fields
211 __metaclass__ = MetaModel
212 _register = False # Set to false if the model shouldn't be automatically discovered.
213 _name = None
214@@ -3350,12 +3398,21 @@
215 for parent in self._inherits:
216 res.update(self.pool.get(parent).fields_get(cr, user, allfields, context))
217
218+ all_selectable = False
219+ if getattr(self, '_fallback_search', False) == True:
220+ all_selectable = True
221+ elif config.get_misc('orm', 'fallback_search', None) == True:
222+ all_selectable = True
223+
224 for f, field in self._columns.iteritems():
225 if allfields and f not in allfields:
226 continue
227
228 res[f] = fields.field_to_dict(self, cr, user, field, context=context)
229
230+ if all_selectable:
231+ res[f]['selectable'] = True
232+
233 if not write_access:
234 res[f]['readonly'] = True
235 res[f]['states'] = {}
236@@ -4529,6 +4586,7 @@
237 order_by_clause = self._order
238 if order_spec:
239 order_by_elements = []
240+ python_order = False
241 self._check_qorder(order_spec)
242 for order_part in order_spec.split(','):
243 order_split = order_part.strip().split(' ')
244@@ -4541,25 +4599,70 @@
245 order_column = self._columns[order_field]
246 if order_column._classic_read:
247 inner_clause = '"%s"."%s"' % (self._table, order_field)
248- elif order_column._type == 'many2one':
249+ elif order_column._type == 'many2one' and (
250+ order_column._classic_write or
251+ getattr(order_column, 'store', False)):
252 inner_clause = self._generate_m2o_order_by(order_field, query)
253+ elif order_by_elements:
254+ # We already have a sortable field: ignore this field
255+ continue
256 else:
257- continue # ignore non-readable or "non-joinable" fields
258+ if hasattr(self, '_fallback_search'):
259+ do_fallback = self._fallback_search
260+ else:
261+ do_fallback = config.get_misc('orm', 'fallback_search', None)
262+ if do_fallback is None:
263+ continue # ignore non-readable or "non-joinable" fields
264+ elif do_fallback is True:
265+ inner_clause = order_field
266+ if order_column._type in ('many2one',):
267+ inner_clause += ':'
268+ python_order = True
269+ else:
270+ raise except_orm(
271+ _('Error!'),
272+ _('Object model %s does not support order by function field "%s"!') %
273+ (self._name, order_field))
274 elif order_field in self._inherit_fields:
275 parent_obj = self.pool.get(self._inherit_fields[order_field][3])
276 order_column = parent_obj._columns[order_field]
277 if order_column._classic_read:
278 inner_clause = self._inherits_join_calc(order_field, query)
279- elif order_column._type == 'many2one':
280+ elif order_column._type == 'many2one' and (
281+ order_column._classic_write or
282+ getattr(order_column, 'store', False)):
283 inner_clause = self._generate_m2o_order_by(order_field, query)
284+ elif order_by_elements:
285+ # We already have a valid domain: ignore this field
286+ continue
287 else:
288- continue # ignore non-readable or "non-joinable" fields
289+ if hasattr(self, '_fallback_search'):
290+ do_fallback = self._fallback_search
291+ elif hasattr(parent_obj, '_fallback_search'):
292+ do_fallback = parent_obj._fallback_search
293+ else:
294+ do_fallback = config.get_misc('orm', 'fallback_search', None)
295+ if do_fallback is None:
296+ continue # ignore non-readable or "non-joinable" fields
297+ elif do_fallback is True:
298+ inner_clause = order_field
299+ if order_column._type in ('many2one',):
300+ inner_clause += ':'
301+ python_order = True
302+ else:
303+ raise except_orm(
304+ _('Error!'),
305+ _('Object model %s does not support order by function field "%s"!') %
306+ (self._name, order_field))
307 if inner_clause:
308 if isinstance(inner_clause, list):
309 for clause in inner_clause:
310 order_by_elements.append("%s %s" % (clause, order_direction))
311 else:
312 order_by_elements.append("%s %s" % (inner_clause, order_direction))
313+ if python_order and order_by_elements:
314+ # the fallback search cannot order on multiple fields
315+ return PythonOrderBy(order_by_elements)
316 if order_by_elements:
317 order_by_clause = ",".join(order_by_elements)
318
319@@ -4596,9 +4699,26 @@
320 cr.execute('SELECT count("%s".id) FROM ' % self._table + from_clause + where_str + limit_str + offset_str, where_clause_params)
321 res = cr.fetchall()
322 return res[0][0]
323- cr.execute('SELECT "%s".id FROM ' % self._table + from_clause + where_str + order_by + limit_str + offset_str, where_clause_params)
324- res = cr.fetchall()
325- return [x[0] for x in res]
326+ elif isinstance(order_by, PythonOrderBy):
327+ # Fall back to pythonic sorting (+ offset, limit)
328+ cr.execute('SELECT "%s".id FROM ' % self._table +
329+ from_clause + where_str, # no offset or limit
330+ where_clause_params)
331+ res = cr.fetchall()
332+ data_res = self.read(cr, user, [res_id for (res_id,) in res],
333+ fields=order_by.get_fields(), context=context)
334+ data_res.sort(key=order_by.get_sort_key(), reverse=order_by.get_reverse())
335+ if offset:
336+ data_res = data_res[offset:]
337+ if limit:
338+ data_res = data_res[:limit]
339+ return [r['id'] for r in data_res]
340+ else:
341+ cr.execute('SELECT "%s".id FROM ' % self._table +
342+ from_clause + where_str + order_by + limit_str + offset_str,
343+ where_clause_params)
344+ res = cr.fetchall()
345+ return [res_id for (res_id,) in res]
346
347 # returns the different values ever entered for one field
348 # this is used, for example, in the client when the user hits enter on