Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Django-style WHERE; consider Python expressions

Browse Source
This commit is contained in:
Doug Blank
2016-04-17 16:54:55 -04:00
parent 3f7b441a54
commit f093c8bd79
5 changed files with 308 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@@ -35,6 +35,7 @@ install:
# - cd $TRAVIS_BUILD_DIR
# $TRAVIS_BUILD_DIR is set to the location of the cloned repository:
# for example: /home/travis/build/gramps-project/gramps
- git clone -b master https://github.com/srossross/meta
- python setup.py build
before_script:
@@ -45,7 +46,7 @@ before_script:
script:
# --exclude=TestUser because of older version of mock
# without configure_mock
- GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps
- PYTHONPATH=meta GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps
after_success:
- codecov

118
gramps/gen/db/base.py
View File

@@ -1287,7 +1287,7 @@ class DbReadBase(object):
if compare(item, op, value):
return True
return False
if op == "=":
if op in ["=", "=="]:
matched = v == value
elif op == ">":
matched = v > value
@@ -1430,15 +1430,15 @@ class DbReadBase(object):
name = self.get_table_func(table,"class_func").get_field_alias(name)
return name.replace(".", "__")
Person = property(lambda self:QuerySet(self, "Person"))
Family = property(lambda self:QuerySet(self, "Family"))
Note = property(lambda self:QuerySet(self, "Note"))
Citation = property(lambda self:QuerySet(self, "Citation"))
Source = property(lambda self:QuerySet(self, "Source"))
Repository = property(lambda self:QuerySet(self, "Repository"))
Place = property(lambda self:QuerySet(self, "Place"))
Event = property(lambda self:QuerySet(self, "Event"))
Tag = property(lambda self:QuerySet(self, "Tag"))
Person = property(lambda self: QuerySet(self, "Person"))
Family = property(lambda self: QuerySet(self, "Family"))
Note = property(lambda self: QuerySet(self, "Note"))
Citation = property(lambda self: QuerySet(self, "Citation"))
Source = property(lambda self: QuerySet(self, "Source"))
Repository = property(lambda self: QuerySet(self, "Repository"))
Place = property(lambda self: QuerySet(self, "Place"))
Event = property(lambda self: QuerySet(self, "Event"))
Tag = property(lambda self: QuerySet(self, "Tag"))
class DbWriteBase(DbReadBase):
"""
@@ -2089,41 +2089,6 @@ class DbWriteBase(DbReadBase):
"""
return getattr(self, table_name)
class Operator(object):
"""
Base for QuerySet operators.
"""
op = "OP"
def __init__(self, *expressions, **kwargs):
if self.op in ["AND", "OR"]:
exprs = [expression.list for expression
in expressions]
for key in kwargs:
exprs.append(
_select_field_operator_value(key, "=", kwargs[key]))
else: # "NOT"
if expressions:
exprs = expressions.list
else:
key, value = list(kwargs.items())[0]
exprs = _select_field_operator_value(key, "=", value)
self.list = [self.op, exprs]
class AND(Operator):
op = "AND"
class OR(Operator):
"""
OR operator for QuerySet logical WHERE expressions.
"""
op = "OR"
class NOT(Operator):
"""
NOT operator for QuerySet logical WHERE expressions.
"""
op = "NOT"
class QuerySet(object):
"""
A container for selection criteria before being actually
@@ -2164,20 +2129,15 @@ class QuerySet(object):
self.needs_to_run = True
return self
def _add_where_clause(self, *args, **kwargs):
def _add_where_clause(self, *args):
"""
Add a condition to the where clause.
"""
# First, handle AND, OR, NOT args:
and_expr = []
for arg in args:
expr = arg.list
for expr in args:
and_expr.append(expr)
# Next, handle kwargs:
for keyword in kwargs:
and_expr.append(
_select_field_operator_value(
keyword, "=", kwargs[keyword]))
if and_expr:
if self.where_by:
self.where_by = ["AND", [self.where_by] + and_expr]
@@ -2260,20 +2220,32 @@ class QuerySet(object):
self.database = proxy_class(self.database, *args, **kwargs)
return self
def filter(self, *args, **kwargs):
def where(self, where_clause):
"""
Apply a where_clause (closure) to the selection process.
"""
from gramps.gen.db.where import eval_where
# if there is already a generator, then error:
if self.generator:
raise Exception("Queries in invalid order")
where_by = eval_where(where_clause)
self._add_where_clause(where_by)
return self
def filter(self, *args):
"""
Apply a filter to the database.
"""
from gramps.gen.proxy import FilterProxyDb
from gramps.gen.filters import GenericFilter
from gramps.gen.db.where import eval_where
for i in range(len(args)):
arg = args[i]
if isinstance(arg, GenericFilter):
self.database = FilterProxyDb(self.database, arg, *args[i+1:])
if arg.where_by:
self._add_where_clause(arg.where_by)
elif isinstance(arg, Operator):
self._add_where_clause(arg)
if hasattr(arg, "where"):
where_by = eval_where(arg.where)
self._add_where_clause(where_by)
elif callable(arg):
if self.generator and self.needs_to_run:
## error
@@ -2285,8 +2257,6 @@ class QuerySet(object):
self.generator = filter(arg, self.generator)
else:
pass # ignore, may have been arg from previous Filter
if kwargs:
self._add_where_clause(**kwargs)
return self
def map(self, f):
@@ -2329,33 +2299,3 @@ class QuerySet(object):
item.add_tag(tag.handle)
commit_func(item, trans)
def _to_dot_format(field):
"""
Convert a field keyword arg into a proper
dotted field name.
"""
return field.replace("__", ".")
def _select_field_operator_value(field, op, value):
"""
Convert a field keyword arg into proper
field, op, and value.
"""
alias = {
"LT": "<",
"GT": ">",
"LTE": "<=",
"GTE": ">=",
"IS_NOT": "IS NOT",
"IS_NULL": "IS NULL",
"IS_NOT_NULL": "IS NOT NULL",
"NE": "<>",
}
for operator in ["LIKE", "IN"] + list(alias.keys()):
operator = "__" + operator
if field.endswith(operator):
op = field[-len(operator) + 2:]
field = field[:-len(operator)]
op = alias.get(op, op)
field = _to_dot_format(field)
return (field, op, value)

102
gramps/gen/db/test/test_where.py Normal file
View File

@@ -0,0 +1,102 @@
#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2016 Gramps Development Team
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
from gramps.gen.db.where import eval_where
import unittest
##########
# Tests:
def make_closure(surname):
"""
Test closure.
"""
from gramps.gen.lib import Person
return (lambda person:
(person.primary_name.surname_list[0].surname == surname and
person.gender == Person.MALE))
class Thing(object):
def __init__(self):
self.list = ["I0", "I1", "I2"]
def where(self):
return lambda person: person.gramps_id == self.list[1]
class ClosureTest(unittest.TestCase):
def check(self, test):
result = eval_where(test[0])
self.assertTrue(result == test[1], "%s is not %s" % (result, test[1]))
def test_01(self):
self.check(
(lambda family: (family.private and
family.mother_handle.gramps_id != "I0001"),
['AND', [['private', '==', True],
['mother_handle.gramps_id', '!=', 'I0001']]]))
def test_02(self):
self.check(
(lambda person: LIKE(person.gramps_id, "I0001"),
['gramps_id', 'LIKE', 'I0001']))
def test_03(self):
self.check(
(lambda note: note.gramps_id == "N0001",
['gramps_id', '==', 'N0001']))
def test_04(self):
self.check(
(lambda person: person.event_ref_list.ref.gramps_id == "E0001",
['event_ref_list.ref.gramps_id', '==', 'E0001']))
def test_05(self):
self.check(
(lambda person: LIKE(person.gramps_id, "I0001") or person.private,
["OR", [['gramps_id', 'LIKE', 'I0001'],
["private", "==", True]]]))
def test_06(self):
self.check(
(lambda person: person.event_ref_list <= 0,
["event_ref_list", "<=", 0]))
def test_07(self):
self.check(
(lambda person: person.primary_name.surname_list[0].surname == "Smith",
["primary_name.surname_list.0.surname", "==", "Smith"]))
def test_08(self):
self.check(
(make_closure("Smith"),
["AND", [["primary_name.surname_list.0.surname", "==", "Smith"],
["gender", "==", 1]]]))
def test_09(self):
self.check(
[Thing().where(), ["gramps_id", "==", "I1"]])
def test_10(self):
self.check(
(lambda person: LIKE(person.gramps_id, "I000%"),
["gramps_id", "LIKE", "I000%"]))
if __name__ == "__main__":
unittest.main()

151
gramps/gen/db/where.py Normal file
View File

@@ -0,0 +1,151 @@
#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2016 Gramps Development Team
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
from meta.asttools import Visitor
from meta.decompiler import _ast, decompile_func
import copy
class ParseFilter(Visitor):
def visitName(self, node):
return node.id
def visitNum(self, node):
return node.n
def visitlong(self, node):
return node
def process_expression(self, expr):
if isinstance(expr, str):
# boolean
return [self.process_field(expr), "==", True]
elif len(expr) == 3:
# (field, op, value)
return [self.process_field(expr[0]),
expr[1],
self.process_value(expr[2])]
else:
# list of exprs
return [self.process_expression(exp) for
exp in expr]
def process_value(self, value):
try:
return eval(value, self.env)
except:
return value
def process_field(self, field):
field = field.replace("[", ".").replace("]", "")
if field.startswith(self.parameter + "."):
return field[len(self.parameter) + 1:]
else:
return field
def visitCall(self, node):
"""
Handle LIKE()
"""
return [self.process_field(self.visit(node.args[0])),
self.visit(node.func),
self.process_value(self.visit(node.args[1]))]
def visitStr(self, node):
return node.s
def visitlist(self, list):
return [self.visit(node) for node in list]
def visitCompare(self, node):
return [self.process_field(self.visit(node.left)),
" ".join(self.visit(node.ops)),
self.process_value(self.visit(node.comparators[0]))]
def visitAttribute(self, node):
return "%s.%s" % (self.visit(node.value), node.attr)
def get_boolean_op(self, node):
if isinstance(node, _ast.And):
return "AND"
elif isinstance(node, _ast.Or):
return "OR"
else:
raise Exception("invalid boolean")
def visitNotEq(self, node):
return "!="
def visitLtE(self, node):
return "<="
def visitGtE(self, node):
return ">="
def visitEq(self, node):
return "=="
def visitBoolOp(self, node):
"""
BoolOp: boolean operator
"""
op = self.get_boolean_op(node.op)
values = list(node.values)
return [op, self.process_expression(
[self.visit(value) for value in values])]
def visitLambda(self, node):
self.parameter = self.visit(node.args)[0]
return self.visit(node.body)
def visitarguments(self, node):
return [self.visit(arg) for arg in node.args]
def visitarg(self, node):
return node.arg
def visitSubscript(self, node):
return "%s[%s]" % (self.visit(node.value),
self.visit(node.slice))
def visitIndex(self, node):
return self.visit(node.value)
def make_env(closure):
"""
Create an environment from the closure.
"""
env = copy.copy(closure.__globals__)
if closure.__closure__:
for i in range(len(closure.__closure__)):
env[closure.__code__.co_freevars[i]] = closure.__closure__[i].cell_contents
return env
def eval_where(closure):
"""
Given a closure, parse and evaluate it.
Return a WHERE expression.
"""
parser = ParseFilter()
parser.env = make_env(closure)
ast_top = decompile_func(closure)
result = parser.visit(ast_top)
return result

43
gramps/plugins/database/test/db_test.py
View File

@@ -123,12 +123,12 @@ class BSDDBTest(unittest.TestCase):
self.assertTrue(len(result) == 60, len(result))
def test_queryset_2(self):
result = list(self.db.Person.filter(gramps_id__LIKE="I000%").select())
result = list(self.db.Person.where(lambda person: LIKE(person.gramps_id, "I000%")).select())
self.assertTrue(len(result) == 10, len(result))
def test_queryset_3(self):
result = list(self.db.Family
.filter(mother_handle__gramps_id__LIKE="I003%")
.where(lambda family: LIKE(family.mother_handle.gramps_id, "I003%"))
.select())
self.assertTrue(len(result) == 6, result)
@@ -138,7 +138,7 @@ class BSDDBTest(unittest.TestCase):
def test_queryset_4b(self):
result = list(self.db.Family
.filter(mother_handle__event_ref_list__ref__gramps_id='E0156')
.where(lambda family: family.mother_handle.event_ref_list.ref.gramps_id == 'E0156')
.select())
self.assertTrue(len(result) == 1, len(result))
@@ -154,9 +154,8 @@ class BSDDBTest(unittest.TestCase):
[r["mother_handle.event_ref_list.0"] for r in result])
def test_queryset_7(self):
from gramps.gen.db import NOT
result = list(self.db.Family
.filter(NOT(mother_handle__event_ref_list__0=None))
.where(lambda family: family.mother_handle.event_ref_list[0] != None)
.select())
self.assertTrue(len(result) == 21, len(result))
@@ -188,22 +187,28 @@ class BSDDBTest(unittest.TestCase):
self.assertTrue(result == 60, result)
def test_tag_1(self):
self.db.Person.filter(gramps_id="I0001").tag("Test")
result = self.db.Person.filter(tag_list__name="Test").count()
self.db.Person.where(lambda person: person.gramps_id == "I0001").tag("Test")
result = self.db.Person.where(lambda person: person.tag_list.name == "Test").count()
self.assertTrue(result == 1, result)
# def test_filter_1(self):
# from gramps.gen.filters.rules.person import (IsDescendantOf,
# IsAncestorOf)
# from gramps.gen.filters import GenericFilter
# filter = GenericFilter()
# filter.set_logical_op("or")
# filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id,
# True]))
# filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id,
# True]))
# result = self.db.Person.filter(filter).count()
# self.assertTrue(result == 15, result)
def test_filter_1(self):
from gramps.gen.filters.rules.person import (IsDescendantOf,
IsAncestorOf)
from gramps.gen.filters import GenericFilter
filter = GenericFilter()
filter.set_logical_op("or")
filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id,
True]))
filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id,
True]))
result = self.db.Person.filter(filter).count()
self.assertTrue(result == 15, result)
filter.where = lambda person: person.private == True
result = self.db.Person.filter(filter).count()
self.assertTrue(result == 1, result)
filter.where = lambda person: person.private != True
result = self.db.Person.filter(filter).count()
self.assertTrue(result == 14, result)
def test_filter_2(self):
result = self.db.Person.filter(lambda p: p.private).count()