Merge lp:~thisfred/u1db/combine-mapping-3 into lp:u1db

[Eric Casteleijn]

Added the combine() mapping.

Added the combine() mapping.

[lp:~thisfred/u1db/combine-mapping-3]

368. By Eric Casteleijn on 2012-07-25

removed debug prints

369. By Eric Casteleijn on 2012-07-26

removed debugging

[Samuele Pedroni (pedronis)]

review: Approve

=== modified file 'src/u1db_query.c'
--- src/u1db_query.c	2012-07-25 20:48:18 +0000
+++ src/u1db_query.c	2012-07-26 12:03:19 +0000
@@ -213,6 +213,7 @@
 static int op_split_words(
     parse_tree *tree, json_object *obj, string_list *result);
 static int op_bool(parse_tree *tree, json_object *obj, string_list *result);
+static int op_combine(parse_tree *tree, json_object *obj, string_list *result);
 
 static const int JUST_EXPRESSION[1] = {EXPRESSION};
 static const int EXPRESSION_INTEGER[2] = {EXPRESSION, INTEGER};
@@ -228,7 +229,8 @@
     op_lower, "lower", json_type_string, 1, JUST_EXPRESSION,
     op_number, "number", json_type_int, 2, EXPRESSION_INTEGER,
     op_split_words, "split_words", json_type_string, 1, JUST_EXPRESSION,
-    op_bool, "bool", json_type_boolean, 1, JUST_EXPRESSION};
+    op_bool, "bool", json_type_boolean, 1, JUST_EXPRESSION,
+    op_combine, "combine", json_type_string, -1, JUST_EXPRESSION};
 
 static int
 extract_field_values(json_object *obj, const string_list *field_path,
@@ -443,6 +445,21 @@
 }
 
 static int
+op_combine(parse_tree *tree, json_object *obj, string_list *result)
+{
+    parse_tree *node = NULL;
+    int status = U1DB_OK;
+
+    node = tree->first_child;
+    for (node = tree->first_child; node != NULL; node = node->next_sibling) {
+        status = get_values(node, obj, result);
+        if (status != U1DB_OK)
+            return status;
+    }
+    return status;
+}
+
+static int
 op_split_words(parse_tree *tree, json_object *obj, string_list *result)
 {
     string_list_item *item = NULL;
@@ -1265,13 +1282,13 @@
     while (expression[*start] == ' ')
         (*start)++;
     size = *idx - *start;
-    if (!size)
-        return U1DB_OK;
     term = expression + *start;
     (*idx)++;
     while (expression[*idx] == ' ')
         (*idx)++;
     *start = *idx;
+    if (!size)
+        return U1DB_OK;
     if (size) {
         status = append_child(result);
         if (status != U1DB_OK) {

=== modified file 'u1db/query_parser.py'
--- u1db/query_parser.py	2012-07-25 19:10:50 +0000
+++ u1db/query_parser.py	2012-07-26 12:03:19 +0000
@@ -207,6 +207,26 @@
         return list(result)
 
 
+class Combine(Transformation):
+    """Combine multiple expressions into a single index."""
+
+    name = "combine"
+    # variable number of args
+    arity = -1
+
+    def __init__(self, *inner):
+        super(Combine, self).__init__(inner)
+
+    def get(self, raw_doc):
+        inner_values = []
+        for inner in self.inner:
+            inner_values.extend(inner.get(raw_doc))
+        return self.transform(inner_values)
+
+    def transform(self, values):
+        return values
+
+
 class IsNull(Transformation):
     """Indicate whether the input is None.
 
@@ -348,3 +368,4 @@
 Parser.register_transormation(Number)
 Parser.register_transormation(Bool)
 Parser.register_transormation(IsNull)
+Parser.register_transormation(Combine)

=== modified file 'u1db/tests/c_backend_wrapper.pyx'
--- u1db/tests/c_backend_wrapper.pyx	2012-07-23 18:00:03 +0000
+++ u1db/tests/c_backend_wrapper.pyx	2012-07-26 12:03:19 +0000
@@ -138,6 +138,8 @@
     int U1DB_INDEX_DOES_NOT_EXIST
     int U1DB_INVALID_GENERATION
     int U1DB_INVALID_TRANSACTION_ID
+    int U1DB_INVALID_TRANSFORMATION_FUNCTION
+    int U1DB_UNKNOWN_OPERATION
     int U1DB_INTERNAL_ERROR
     int U1DB_TARGET_UNAVAILABLE
 
@@ -612,6 +614,10 @@
         raise errors.DocumentTooBig
     if status == U1DB_USER_QUOTA_EXCEEDED:
         raise errors.UserQuotaExceeded
+    if status == U1DB_INVALID_TRANSFORMATION_FUNCTION:
+        raise errors.IndexDefinitionParseError
+    if status == U1DB_UNKNOWN_OPERATION:
+        raise errors.IndexDefinitionParseError
     raise RuntimeError('%s (status: %s)' % (context, status))
 
 

=== modified file 'u1db/tests/test_backends.py'
--- u1db/tests/test_backends.py	2012-07-24 16:22:38 +0000
+++ u1db/tests/test_backends.py	2012-07-26 12:03:19 +0000
@@ -1456,6 +1456,13 @@
         self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord')
         self.assertEqual([], self.db.get_from_index('test-idx', '*'))
 
+    def test_nested_unknown_operation(self):
+        self.db.create_doc_from_json(nested_doc)
+        # sub exists, but sub.foo does not:
+        self.assertRaises(
+            errors.IndexDefinitionParseError, self.db.create_index, 'test-idx',
+            'unknown_operation(field1)')
+
     def test_index_list1(self):
         self.db.create_index("index", "name")
         content = '{"name": ["foo", "bar"]}'
@@ -1617,6 +1624,31 @@
         rows = self.db.get_from_index("index", "*")
         self.assertEqual([], rows)
 
+    def test_get_from_index_with_combine(self):
+        self.db.create_index("index", "combine(foo, bar)")
+        content = '{"foo": "value1", "bar": "value2"}'
+        doc = self.db.create_doc_from_json(content)
+        rows = self.db.get_from_index("index", "value1")
+        self.assertEqual([doc], rows)
+        rows = self.db.get_from_index("index", "value2")
+        self.assertEqual([doc], rows)
+
+    def test_get_complex_combine(self):
+        self.db.create_index(
+            "index", "combine(number(foo, 5), lower(bar), split_words(baz))")
+        content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}'
+        doc = self.db.create_doc_from_json(content)
+        content = '{"foo": "not a number", "bar": "something"}'
+        doc2 = self.db.create_doc_from_json(content)
+        rows = self.db.get_from_index("index", "00012")
+        self.assertEqual([doc], rows)
+        rows = self.db.get_from_index("index", "allcaps")
+        self.assertEqual([doc], rows)
+        rows = self.db.get_from_index("index", "nox")
+        self.assertEqual([doc], rows)
+        rows = self.db.get_from_index("index", "something")
+        self.assertEqual([doc2], rows)
+
     def test_get_index_keys_from_index(self):
         self.db.create_index('test-idx', 'key')
         content1 = '{"key": "value1"}'

=== modified file 'u1db/tests/test_query_parser.py'
--- u1db/tests/test_query_parser.py	2012-07-25 16:20:27 +0000
+++ u1db/tests/test_query_parser.py	2012-07-26 12:03:19 +0000
@@ -59,6 +59,12 @@
             self.parser.parse('lower(split_words(field1))'),
             query_parser.Lower)
 
+    def test_nested_branching_mapping(self):
+        self.assertIsInstance(
+            self.parser.parse(
+                'combine(lower(field1), split_words(field2), '
+                'number(field3, 5))'), query_parser.Combine)
+
     def test_single_mapping_multiple_fields(self):
         self.assertIsInstance(
             self.parser.parse('number(field1, 5)'), query_parser.Number)