Remove Django-style WHERE; consider Python expressions

.travis.yml

@@ -35,6 +35,7 @@ install:
  # - cd $TRAVIS_BUILD_DIR
  # $TRAVIS_BUILD_DIR is set to the location of the cloned repository:
  # for example: /home/travis/build/gramps-project/gramps
+ - git clone -b master https://github.com/srossross/meta
  - python setup.py build
 
 before_script:
@@ -45,7 +46,7 @@ before_script:
 script:
  # --exclude=TestUser because of older version of mock
  #                    without configure_mock
- - GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test  --exclude=user_test gramps
+ - PYTHONPATH=meta GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test  --exclude=user_test gramps
 
 after_success:
  - codecov

gramps/gen/db/base.py

@@ -1287,7 +1287,7 @@ class DbReadBase(object):
                     if compare(item, op, value):
                         return True
                 return False
-            if op == "=":
+            if op in ["=", "=="]:
                 matched = v == value
             elif op == ">":
                 matched = v > value
@@ -1430,15 +1430,15 @@ class DbReadBase(object):
         name = self.get_table_func(table,"class_func").get_field_alias(name)
         return name.replace(".", "__")
 
-    Person = property(lambda self:QuerySet(self, "Person"))
+    Person = property(lambda self: QuerySet(self, "Person"))
-    Family = property(lambda self:QuerySet(self, "Family"))
+    Family = property(lambda self: QuerySet(self, "Family"))
-    Note = property(lambda self:QuerySet(self, "Note"))
+    Note = property(lambda self: QuerySet(self, "Note"))
-    Citation = property(lambda self:QuerySet(self, "Citation"))
+    Citation = property(lambda self: QuerySet(self, "Citation"))
-    Source = property(lambda self:QuerySet(self, "Source"))
+    Source = property(lambda self: QuerySet(self, "Source"))
-    Repository = property(lambda self:QuerySet(self, "Repository"))
+    Repository = property(lambda self: QuerySet(self, "Repository"))
-    Place = property(lambda self:QuerySet(self, "Place"))
+    Place = property(lambda self: QuerySet(self, "Place"))
-    Event = property(lambda self:QuerySet(self, "Event"))
+    Event = property(lambda self: QuerySet(self, "Event"))
-    Tag = property(lambda self:QuerySet(self, "Tag"))
+    Tag = property(lambda self: QuerySet(self, "Tag"))
 
 class DbWriteBase(DbReadBase):
     """
@@ -2089,41 +2089,6 @@ class DbWriteBase(DbReadBase):
         """
         return getattr(self, table_name)
 
-class Operator(object):
-    """
-    Base for QuerySet operators.
-    """
-    op = "OP"
-    def __init__(self, *expressions, **kwargs):
-        if self.op in ["AND", "OR"]:
-            exprs = [expression.list for expression
-                     in expressions]
-            for key in kwargs:
-                exprs.append(
-                    _select_field_operator_value(key, "=", kwargs[key]))
-        else: # "NOT"
-            if expressions:
-                exprs = expressions.list
-            else:
-                key, value = list(kwargs.items())[0]
-                exprs = _select_field_operator_value(key, "=", value)
-        self.list = [self.op, exprs]
-
-class AND(Operator):
-    op = "AND"
-
-class OR(Operator):
-    """
-    OR operator for QuerySet logical WHERE expressions.
-    """
-    op = "OR"
-
-class NOT(Operator):
-    """
-    NOT operator for QuerySet logical WHERE expressions.
-    """
-    op = "NOT"
-
 class QuerySet(object):
     """
     A container for selection criteria before being actually
@@ -2164,20 +2129,15 @@ class QuerySet(object):
         self.needs_to_run = True
         return self
 
-    def _add_where_clause(self, *args, **kwargs):
+    def _add_where_clause(self, *args):
         """
         Add a condition to the where clause.
         """
         # First, handle AND, OR, NOT args:
         and_expr = []
-        for arg in args:
+        for expr in args:
-            expr = arg.list
             and_expr.append(expr)
         # Next, handle kwargs:
-        for keyword in kwargs:
-            and_expr.append(
-                _select_field_operator_value(
-                    keyword, "=", kwargs[keyword]))
         if and_expr:
             if self.where_by:
                 self.where_by = ["AND", [self.where_by] + and_expr]
@@ -2260,20 +2220,32 @@ class QuerySet(object):
         self.database = proxy_class(self.database, *args, **kwargs)
         return self
 
-    def filter(self, *args, **kwargs):
+    def where(self, where_clause):
+        """
+        Apply a where_clause (closure) to the selection process.
+        """
+        from gramps.gen.db.where import eval_where
+        # if there is already a generator, then error:
+        if self.generator:
+            raise Exception("Queries in invalid order")
+        where_by = eval_where(where_clause)
+        self._add_where_clause(where_by)
+        return self
+
+    def filter(self, *args):
         """
         Apply a filter to the database.
         """
         from gramps.gen.proxy import FilterProxyDb
         from gramps.gen.filters import GenericFilter
+        from gramps.gen.db.where import eval_where
         for i in range(len(args)):
             arg = args[i]
             if isinstance(arg, GenericFilter):
                 self.database = FilterProxyDb(self.database, arg, *args[i+1:])
-                if arg.where_by:
+                if hasattr(arg, "where"):
-                    self._add_where_clause(arg.where_by)
+                    where_by = eval_where(arg.where)
-            elif isinstance(arg, Operator):
+                    self._add_where_clause(where_by)
-                self._add_where_clause(arg)
             elif callable(arg):
                 if self.generator and self.needs_to_run:
                     ## error
@@ -2285,8 +2257,6 @@ class QuerySet(object):
                 self.generator = filter(arg, self.generator)
             else:
                 pass # ignore, may have been arg from previous Filter
-        if kwargs:
-            self._add_where_clause(**kwargs)
         return self
 
     def map(self, f):
@@ -2329,33 +2299,3 @@ class QuerySet(object):
                     item.add_tag(tag.handle)
                     commit_func(item, trans)
 
-def _to_dot_format(field):
-    """
-    Convert a field keyword arg into a proper
-    dotted field name.
-    """
-    return field.replace("__", ".")
-
-def _select_field_operator_value(field, op, value):
-    """
-    Convert a field keyword arg into proper
-    field, op, and value.
-    """
-    alias = {
-        "LT": "<",
-        "GT": ">",
-        "LTE": "<=",
-        "GTE": ">=",
-        "IS_NOT": "IS NOT",
-        "IS_NULL": "IS NULL",
-        "IS_NOT_NULL": "IS NOT NULL",
-        "NE": "<>",
-    }
-    for operator in ["LIKE", "IN"] + list(alias.keys()):
-        operator = "__" + operator
-        if field.endswith(operator):
-            op = field[-len(operator) + 2:]
-            field = field[:-len(operator)]
-            op = alias.get(op, op)
-    field = _to_dot_format(field)
-    return (field, op, value)

gramps/gen/db/test/test_where.py

@@ -0,0 +1,102 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2016       Gramps Development Team
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+#
+
+from gramps.gen.db.where import eval_where
+import unittest
+
+##########
+# Tests:
+
+def make_closure(surname):
+    """
+    Test closure.
+    """
+    from gramps.gen.lib import Person
+    return (lambda person: 
+            (person.primary_name.surname_list[0].surname == surname and
+             person.gender == Person.MALE))
+
+class Thing(object):
+    def __init__(self):
+        self.list = ["I0", "I1", "I2"]
+
+    def where(self):
+        return lambda person: person.gramps_id == self.list[1]
+
+class ClosureTest(unittest.TestCase):
+    def check(self, test):
+        result = eval_where(test[0])
+        self.assertTrue(result == test[1], "%s is not %s" % (result, test[1]))
+
+    def test_01(self):
+        self.check(
+            (lambda family: (family.private and 
+                             family.mother_handle.gramps_id != "I0001"), 
+             ['AND', [['private', '==', True], 
+                      ['mother_handle.gramps_id', '!=', 'I0001']]]))
+
+    def test_02(self):
+        self.check(
+            (lambda person: LIKE(person.gramps_id, "I0001"),
+             ['gramps_id', 'LIKE', 'I0001']))
+
+    def test_03(self):
+        self.check(
+            (lambda note: note.gramps_id == "N0001",
+             ['gramps_id', '==', 'N0001']))
+
+    def test_04(self):
+        self.check(
+            (lambda person: person.event_ref_list.ref.gramps_id == "E0001",
+             ['event_ref_list.ref.gramps_id', '==', 'E0001']))
+
+    def test_05(self):
+        self.check(
+            (lambda person: LIKE(person.gramps_id, "I0001") or person.private,
+             ["OR", [['gramps_id', 'LIKE', 'I0001'],
+                     ["private", "==", True]]]))
+
+    def test_06(self):
+        self.check(
+            (lambda person: person.event_ref_list <= 0,
+             ["event_ref_list", "<=", 0]))
+
+    def test_07(self):
+        self.check(
+            (lambda person: person.primary_name.surname_list[0].surname == "Smith",
+             ["primary_name.surname_list.0.surname", "==", "Smith"]))
+
+    def test_08(self):
+        self.check(
+            (make_closure("Smith"),
+             ["AND", [["primary_name.surname_list.0.surname", "==", "Smith"],
+                      ["gender", "==", 1]]]))
+
+    def test_09(self):
+        self.check(
+            [Thing().where(), ["gramps_id", "==", "I1"]])
+
+    def test_10(self):
+        self.check(
+            (lambda person: LIKE(person.gramps_id, "I000%"),
+             ["gramps_id", "LIKE", "I000%"]))
+
+if __name__ == "__main__":
+    unittest.main()

gramps/gen/db/where.py

@@ -0,0 +1,151 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2016       Gramps Development Team
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+#
+
+from meta.asttools import Visitor
+from meta.decompiler import _ast, decompile_func
+
+import copy
+
+class ParseFilter(Visitor):
+    def visitName(self, node):
+        return node.id
+    
+    def visitNum(self, node):
+        return node.n
+        
+    def visitlong(self, node):
+        return node
+
+    def process_expression(self, expr):
+        if isinstance(expr, str):
+            # boolean
+            return [self.process_field(expr), "==", True]
+        elif len(expr) == 3:
+            # (field, op, value)
+            return [self.process_field(expr[0]), 
+                    expr[1], 
+                    self.process_value(expr[2])]
+        else:
+            # list of exprs
+            return [self.process_expression(exp) for
+                    exp in expr]
+
+    def process_value(self, value):
+        try:
+            return eval(value, self.env)
+        except:
+            return value
+
+    def process_field(self, field):
+        field = field.replace("[", ".").replace("]", "")
+        if field.startswith(self.parameter + "."):
+            return field[len(self.parameter) + 1:]
+        else:
+            return field
+
+    def visitCall(self, node):
+        """
+        Handle LIKE()
+        """
+        return [self.process_field(self.visit(node.args[0])), 
+                self.visit(node.func),
+                self.process_value(self.visit(node.args[1]))]
+
+    def visitStr(self, node):
+        return node.s
+
+    def visitlist(self, list):
+        return [self.visit(node) for node in list]
+
+    def visitCompare(self, node):
+        return [self.process_field(self.visit(node.left)), 
+                " ".join(self.visit(node.ops)), 
+                self.process_value(self.visit(node.comparators[0]))]
+
+    def visitAttribute(self, node):
+        return "%s.%s" % (self.visit(node.value), node.attr)
+
+    def get_boolean_op(self, node):
+        if isinstance(node, _ast.And):
+            return "AND"
+        elif isinstance(node, _ast.Or):
+            return "OR"
+        else:
+            raise Exception("invalid boolean")
+
+    def visitNotEq(self, node):
+        return "!="
+
+    def visitLtE(self, node):
+        return "<="
+
+    def visitGtE(self, node):
+        return ">="
+
+    def visitEq(self, node):
+        return "=="
+
+    def visitBoolOp(self, node):
+        """
+        BoolOp: boolean operator
+        """
+        op = self.get_boolean_op(node.op)
+        values = list(node.values)
+        return [op, self.process_expression(
+            [self.visit(value) for value in values])]
+
+    def visitLambda(self, node):
+        self.parameter = self.visit(node.args)[0]
+        return self.visit(node.body)
+
+    def visitarguments(self, node):
+        return [self.visit(arg) for arg in node.args]
+
+    def visitarg(self, node):
+        return node.arg
+
+    def visitSubscript(self, node):
+        return "%s[%s]" % (self.visit(node.value), 
+                          self.visit(node.slice))
+
+    def visitIndex(self, node):
+        return self.visit(node.value)
+
+def make_env(closure):
+    """
+    Create an environment from the closure.
+    """
+    env = copy.copy(closure.__globals__)
+    if closure.__closure__:
+        for i in range(len(closure.__closure__)):
+            env[closure.__code__.co_freevars[i]] = closure.__closure__[i].cell_contents
+    return env
+
+def eval_where(closure):
+    """
+    Given a closure, parse and evaluate it.
+    Return a WHERE expression.
+    """
+    parser = ParseFilter()
+    parser.env = make_env(closure)
+    ast_top = decompile_func(closure)
+    result = parser.visit(ast_top)
+    return result
+

gramps/plugins/database/test/db_test.py

@@ -123,12 +123,12 @@ class BSDDBTest(unittest.TestCase):
         self.assertTrue(len(result) == 60, len(result))
 
     def test_queryset_2(self):
-        result = list(self.db.Person.filter(gramps_id__LIKE="I000%").select())
+        result = list(self.db.Person.where(lambda person: LIKE(person.gramps_id, "I000%")).select())
         self.assertTrue(len(result) == 10, len(result))
 
     def test_queryset_3(self):
         result = list(self.db.Family
-                      .filter(mother_handle__gramps_id__LIKE="I003%")
+                      .where(lambda family: LIKE(family.mother_handle.gramps_id, "I003%"))
                       .select())
         self.assertTrue(len(result) == 6, result)
 
@@ -138,7 +138,7 @@ class BSDDBTest(unittest.TestCase):
 
     def test_queryset_4b(self):
         result = list(self.db.Family
-                      .filter(mother_handle__event_ref_list__ref__gramps_id='E0156')
+                      .where(lambda family: family.mother_handle.event_ref_list.ref.gramps_id == 'E0156')
                       .select())
         self.assertTrue(len(result) == 1, len(result))
 
@@ -154,9 +154,8 @@ class BSDDBTest(unittest.TestCase):
                         [r["mother_handle.event_ref_list.0"] for r in result])
 
     def test_queryset_7(self):
-        from gramps.gen.db import NOT
         result = list(self.db.Family
-                      .filter(NOT(mother_handle__event_ref_list__0=None))
+                      .where(lambda family: family.mother_handle.event_ref_list[0] != None)
                       .select())
         self.assertTrue(len(result) == 21, len(result))
 
@@ -188,22 +187,28 @@ class BSDDBTest(unittest.TestCase):
         self.assertTrue(result == 60, result)
 
     def test_tag_1(self):
-        self.db.Person.filter(gramps_id="I0001").tag("Test")
+        self.db.Person.where(lambda person: person.gramps_id == "I0001").tag("Test")
-        result = self.db.Person.filter(tag_list__name="Test").count()
+        result = self.db.Person.where(lambda person: person.tag_list.name == "Test").count()
         self.assertTrue(result == 1, result)
 
-    # def test_filter_1(self):
+    def test_filter_1(self):
-    #     from gramps.gen.filters.rules.person import (IsDescendantOf,
+        from gramps.gen.filters.rules.person import (IsDescendantOf,
-    #                                                  IsAncestorOf)
+                                                     IsAncestorOf)
-    #     from gramps.gen.filters import GenericFilter
+        from gramps.gen.filters import GenericFilter
-    #     filter = GenericFilter()
+        filter = GenericFilter()
-    #     filter.set_logical_op("or")
+        filter.set_logical_op("or")
-    #     filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id,
+        filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id,
-    #                                     True]))
+                                        True]))
-    #     filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id,
+        filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id,
-    #                                   True]))
+                                      True]))
-    #     result = self.db.Person.filter(filter).count()
+        result = self.db.Person.filter(filter).count()
-    #     self.assertTrue(result == 15, result)
+        self.assertTrue(result == 15, result)
+        filter.where = lambda person: person.private == True
+        result = self.db.Person.filter(filter).count()
+        self.assertTrue(result == 1, result)
+        filter.where = lambda person: person.private != True
+        result = self.db.Person.filter(filter).count()
+        self.assertTrue(result == 14, result)
 
     def test_filter_2(self):
         result = self.db.Person.filter(lambda p: p.private).count()