Merge lp:~thisfred/u1db/ergo-sum into lp:u1db

[Eric Casteleijn]

This adds a sum operator that acts like combine, except instead of indexing a bunch of string expressions, it indexes the sum of a bunch of integer expressions.

This adds a sum operator that acts like combine, except instead of indexing a bunch of string expressions, it indexes the sum of a bunch of integer expressions.

[lp:~thisfred/u1db/ergo-sum]

367. By Eric Casteleijn on 2012-07-29

updated api doc

Unmerged revisions

367. By Eric Casteleijn on 2012-07-29

updated api doc

366. By Eric Casteleijn on 2012-07-29

added sum operator

=== modified file 'html-docs/high-level-api.rst'
--- html-docs/high-level-api.rst	2012-07-26 13:34:58 +0000
+++ html-docs/high-level-api.rst	2012-07-29 17:35:21 +0000
@@ -231,6 +231,9 @@
    field is absent, otherwise false
  * ``combine(index_expression1, index_expression2, ...)`` - Combine the values
    of an arbitrary number of sub expressions into a single index.
+ * ``sum(index_expression1, index_expression2, ...)`` - Sum the values
+   of an arbitrary number of (integer) sub expressions into a single index
+   value.
 
 So, the index expression ``splitwords(lower(field.name))`` applied to
 a document with ID "doc3" and content::

=== modified file 'src/u1db_query.c'
--- src/u1db_query.c	2012-07-29 17:35:21 +0000
+++ src/u1db_query.c	2012-07-29 17:35:21 +0000
@@ -233,6 +233,7 @@
     parse_tree *tree, json_object *obj, string_list *result);
 static int op_bool(parse_tree *tree, json_object *obj, string_list *result);
 static int op_combine(parse_tree *tree, json_object *obj, string_list *result);
+static int op_sum(parse_tree *tree, json_object *obj, string_list *result);
 
 static const int JUST_EXPRESSION[1] = {EXPRESSION};
 static const int EXPRESSION_INTEGER[2] = {EXPRESSION, INTEGER};
@@ -249,7 +250,8 @@
     op_number, "number", json_type_int, 2, EXPRESSION_INTEGER,
     op_split_words, "split_words", json_type_string, 1, JUST_EXPRESSION,
     op_bool, "bool", json_type_boolean, 1, JUST_EXPRESSION,
-    op_combine, "combine", json_type_string, -1, JUST_EXPRESSION};
+    op_combine, "combine", json_type_string, -1, JUST_EXPRESSION,
+    op_sum, "sum", json_type_int, -1, JUST_EXPRESSION};
 
 static int
 extract_value(json_object *val, int value_type, string_list *values)
@@ -368,14 +370,15 @@
 }
 
 static int
-get_values(parse_tree *tree, json_object *obj, string_list *values)
+get_values(parse_tree *tree, json_object *obj, int json_type,
+           string_list *values)
 {
     int status = U1DB_OK;
     if (tree->op) {
         status = ((op_function)tree->op)(tree, obj, values);
     } else {
         status = extract_field_values(
-            obj, tree->field_path->head, json_type_string, values);
+            obj, tree->field_path->head, json_type, values);
     }
     return status;
 }
@@ -392,7 +395,7 @@
     status = init_list(&values);
     if (status != U1DB_OK)
         return status;
-    status = get_values(tree->first_child, obj, values);
+    status = get_values(tree->first_child, obj, json_type_string, values);
     if (status != U1DB_OK)
         goto finish;
     for (item = values->head; item != NULL; item = item->next)
@@ -434,8 +437,7 @@
     status = init_list(&values);
     if (status != U1DB_OK)
         return status;
-    status = extract_field_values(
-        obj, node->field_path->head, json_type_int, values);
+    status = get_values(tree->first_child, obj, json_type_int, values);
     if (status != U1DB_OK)
         goto finish;
     node = node->next_sibling;
@@ -493,9 +495,8 @@
     parse_tree *node = NULL;
     int status = U1DB_OK;
 
-    node = tree->first_child;
     for (node = tree->first_child; node != NULL; node = node->next_sibling) {
-        status = get_values(node, obj, result);
+        status = get_values(node, obj, json_type_string, result);
         if (status != U1DB_OK)
             return status;
     }
@@ -503,6 +504,35 @@
 }
 
 static int
+op_sum(parse_tree *tree, json_object *obj, string_list *result)
+{
+    parse_tree *node = NULL;
+    int status = U1DB_OK;
+    int sum = 0;
+    string_list *intermediate_values = NULL;
+    string_list_item *item = NULL;
+    char string_value[MAX_INT_STR_LEN];
+
+    status = init_list(&intermediate_values);
+    if (status != U1DB_OK)
+        goto finish;
+    for (node = tree->first_child; node != NULL; node = node->next_sibling) {
+        status = get_values(node, obj, json_type_int, intermediate_values);
+        if (status != U1DB_OK)
+            goto finish;
+    }
+    for (item = intermediate_values->head; item != NULL; item = item->next) {
+        sum += atoi(item->data);
+    }
+    snprintf(string_value, MAX_INT_STR_LEN, "%d", sum);
+    status = append(result, string_value);
+finish:
+    if (intermediate_values != NULL)
+        destroy_list(intermediate_values);
+    return status;
+}
+
+static int
 op_split_words(parse_tree *tree, json_object *obj, string_list *result)
 {
     string_list_item *item = NULL;
@@ -514,7 +544,7 @@
     status = init_list(&values);
     if (status != U1DB_OK)
         return status;
-    status = get_values(tree->first_child, obj, values);
+    status = get_values(tree->first_child, obj, json_type_string, values);
     if (status != U1DB_OK)
         goto finish;
     for (item = values->head; item != NULL; item = item->next)
@@ -556,11 +586,6 @@
     status = init_list(&values);
     if (status != U1DB_OK)
         return status;
-    status = get_values(tree->first_child, obj, values);
-    if (status != U1DB_OK)
-        goto finish;
-    //just return all the strings which have been filtered and converted from
-    //booleans by extract_field_values.
 
     status = extract_field_values(
         obj, tree->first_child->field_path->head, json_type_boolean, values);
@@ -1566,7 +1591,7 @@
     status = init_list(&values);
     if (status != U1DB_OK)
         goto finish;
-    status = get_values(tree, ctx->obj, values);
+    status = get_values(tree, ctx->obj, json_type_string, values);
     if (status != U1DB_OK)
         goto finish;
     for (item = values->head; item != NULL; item = item->next)

=== modified file 'u1db/query_parser.py'
--- u1db/query_parser.py	2012-07-29 17:35:21 +0000
+++ u1db/query_parser.py	2012-07-29 17:35:21 +0000
@@ -239,6 +239,26 @@
         return values
 
 
+class Sum(Transformation):
+    """Put the sum of multiple expressions into a single index."""
+
+    name = "sum"
+    # variable number of args
+    arity = -1
+
+    def __init__(self, *inner):
+        super(Sum, self).__init__(inner)
+
+    def get(self, raw_doc):
+        inner_values = []
+        for inner in self.inner:
+            inner_values.extend(inner.get(raw_doc))
+        return self.transform(inner_values)
+
+    def transform(self, values):
+        return [sum([v for v in values if isinstance(v, int)])]
+
+
 class IsNull(Transformation):
     """Indicate whether the input is None.
 
@@ -372,3 +392,4 @@
 Parser.register_transormation(Bool)
 Parser.register_transormation(IsNull)
 Parser.register_transormation(Combine)
+Parser.register_transormation(Sum)

=== modified file 'u1db/tests/test_backends.py'
--- u1db/tests/test_backends.py	2012-07-29 17:35:21 +0000
+++ u1db/tests/test_backends.py	2012-07-29 17:35:21 +0000
@@ -1660,6 +1660,21 @@
         rows = self.db.get_from_index("index", "value2")
         self.assertEqual([doc], rows)
 
+    def test_get_from_index_with_sum(self):
+        self.db.create_index("index", "number(sum(foo, bar), 5)")
+        content = '{"foo": 12, "bar": [23, 7]}'
+        doc = self.db.create_doc_from_json(content)
+        content = '{"foo": 12, "bar": [23, 7]}'
+        rows = self.db.get_from_index("index", "00042")
+        self.assertEqual([doc], rows)
+
+    def test_get_from_index_with_sum_ignores_non_numeric(self):
+        self.db.create_index("index", "number(sum(foo, bar), 5)")
+        content = '{"foo": "12", "bar": [23, 7]}'
+        doc = self.db.create_doc_from_json(content)
+        rows = self.db.get_from_index("index", "00030")
+        self.assertEqual([doc], rows)
+
     def test_get_complex_combine(self):
         self.db.create_index(
             "index", "combine(number(foo, 5), lower(bar), split_words(baz))")

=== modified file 'u1db/tests/test_query_parser.py'
--- u1db/tests/test_query_parser.py	2012-07-29 17:35:21 +0000
+++ u1db/tests/test_query_parser.py	2012-07-29 17:35:21 +0000
@@ -65,6 +65,12 @@
                 'combine(lower(field1), split_words(field2), '
                 'number(field3, 5))'), query_parser.Combine)
 
+    def test_nested_branching_mapping2(self):
+        self.assertIsInstance(
+            self.parser.parse(
+                'number(sum(field1, sub.field2, field3), 6)'),
+                query_parser.Number)
+
     def test_single_mapping_multiple_fields(self):
         self.assertIsInstance(
             self.parser.parse('number(field1, 5)'), query_parser.Number)