Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

9742: Rewrite cursors to avoid using table maps

Browse Source
This commit is contained in:
Nick Hall 2016-10-23 19:39:00 +01:00
parent 2947e84501
commit f036206961
7 changed files with 329 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
gramps/gen/db/dbconst.py
View File

@ -56,6 +56,7 @@ DBCACHE = 0x4000000 # Size of the shared memory buffer pool
DBLOCKS = 100000 # Maximum number of locks supported
DBOBJECTS = 100000 # Maximum number of simultaneously locked objects
DBUNDO = 1000 # Maximum size of undo buffer
ARRAYSIZE = 1000 # The arraysize for a SQL cursor
PERSON_KEY = 0
FAMILY_KEY = 1

33
gramps/gen/db/generic.py
View File

@ -2,6 +2,7 @@
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2015-2016 Gramps Development Team
# Copyright (C) 2016 Nick Hall
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -305,14 +306,14 @@ class MetaCursor:
pass
class Cursor:
def __init__(self, map):
self.map = map
def __init__(self, iterator):
self.iterator = iterator
self._iter = self.__iter__()
def __enter__(self):
return self
def __iter__(self):
for item in self.map.keys():
yield (item, self.map[item])
for handle, data in self.iterator():
yield (handle, data)
def __next__(self):
try:
return self._iter.__next__()
@ -321,8 +322,8 @@ class Cursor:
def __exit__(self, *args, **kwargs):
pass
def iter(self):
for item in self.map.keys():
yield (item, self.map[item])
for handle, data in self.iterator():
yield (handle, data)
def first(self):
self._iter = self.__iter__()
try:
@ -1295,37 +1296,37 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback):
return Note.create(data)
def get_place_cursor(self):
return Cursor(self.place_map)
return Cursor(self._iter_raw_place_data)
def get_place_tree_cursor(self, *args, **kwargs):
return TreeCursor(self, self.place_map)
def get_person_cursor(self):
return Cursor(self.person_map)
return Cursor(self._iter_raw_person_data)
def get_family_cursor(self):
return Cursor(self.family_map)
return Cursor(self._iter_raw_family_data)
def get_event_cursor(self):
return Cursor(self.event_map)
return Cursor(self._iter_raw_event_data)
def get_note_cursor(self):
return Cursor(self.note_map)
return Cursor(self._iter_raw_note_data)
def get_tag_cursor(self):
return Cursor(self.tag_map)
return Cursor(self._iter_raw_tag_data)
def get_repository_cursor(self):
return Cursor(self.repository_map)
return Cursor(self._iter_raw_repository_data)
def get_media_cursor(self):
return Cursor(self.media_map)
return Cursor(self._iter_raw_media_data)
def get_citation_cursor(self):
return Cursor(self.citation_map)
return Cursor(self._iter_raw_citation_data)
def get_source_cursor(self):
return Cursor(self.source_map)
return Cursor(self._iter_raw_source_data)
def add_person(self, person, trans, set_gid=True):
if not person.handle:

75
gramps/plugins/db/dbapi/dbapi.py
View File

@ -2,6 +2,7 @@
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
# Copyright (C) 2016 Nick Hall
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -1370,6 +1371,80 @@ class DBAPI(DbGeneric):
for row in rows:
yield row[0]
def _iter_raw_data(self, obj_key):
"""
Return an iterator over raw data in the database.
"""
table = KEY_TO_NAME_MAP[obj_key]
sql = "SELECT handle, blob_data FROM %s" % table
with self.dbapi.cursor() as cursor:
cursor.execute(sql)
rows = cursor.fetchmany()
while rows:
for row in rows:
yield (row[0].encode('utf8'), pickle.loads(row[1]))
rows = cursor.fetchmany()
def _iter_raw_person_data(self):
"""
Return an iterator over raw Person data.
"""
return self._iter_raw_data(PERSON_KEY)
def _iter_raw_family_data(self):
"""
Return an iterator over raw Family data.
"""
return self._iter_raw_data(FAMILY_KEY)
def _iter_raw_event_data(self):
"""
Return an iterator over raw Event data.
"""
return self._iter_raw_data(EVENT_KEY)
def _iter_raw_place_data(self):
"""
Return an iterator over raw Place data.
"""
return self._iter_raw_data(PLACE_KEY)
def _iter_raw_repository_data(self):
"""
Return an iterator over raw Repository data.
"""
return self._iter_raw_data(REPOSITORY_KEY)
def _iter_raw_source_data(self):
"""
Return an iterator over raw Source data.
"""
return self._iter_raw_data(SOURCE_KEY)
def _iter_raw_citation_data(self):
"""
Return an iterator over raw Citation data.
"""
return self._iter_raw_data(CITATION_KEY)
def _iter_raw_media_data(self):
"""
Return an iterator over raw Media data.
"""
return self._iter_raw_data(MEDIA_KEY)
def _iter_raw_note_data(self):
"""
Return an iterator over raw Note data.
"""
return self._iter_raw_data(NOTE_KEY)
def _iter_raw_tag_data(self):
"""
Return an iterator over raw Tag data.
"""
return self._iter_raw_data(TAG_KEY)
def reindex_reference_map(self, callback):
"""
Reindex all primary records in the database.

69
gramps/plugins/db/dbapi/mysql.py
View File

@ -2,6 +2,7 @@
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
# Copyright (C) 2016 Nick Hall
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -18,9 +19,21 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#-------------------------------------------------------------------------
#
# Standard python modules
#
#-------------------------------------------------------------------------
import MySQLdb
import re
#-------------------------------------------------------------------------
#
# Gramps modules
#
#-------------------------------------------------------------------------
from gramps.gen.db.dbconst import ARRAYSIZE
MySQLdb.paramstyle = 'qmark' ## Doesn't work
class MySQL:
@ -40,9 +53,9 @@ class MySQL:
return summary
def __init__(self, *args, **kwargs):
self.connection = MySQLdb.connect(*args, **kwargs)
self.connection.autocommit(True)
self.cursor = self.connection.cursor()
self.__connection = MySQLdb.connect(*args, **kwargs)
self.__connection.autocommit(True)
self.__cursor = self.__connection.cursor()
def _hack_query(self, query):
## Workaround: no qmark support:
@ -70,27 +83,61 @@ class MySQL:
def execute(self, query, args=[]):
query = self._hack_query(query)
self.cursor.execute(query, args)
self.__cursor.execute(query, args)
def fetchone(self):
return self.cursor.fetchone()
return self.__cursor.fetchone()
def fetchall(self):
return self.cursor.fetchall()
return self.__cursor.fetchall()
def commit(self):
self.cursor.execute("COMMIT;");
self.__cursor.execute("COMMIT;");
def begin(self):
self.cursor.execute("BEGIN;");
self.__cursor.execute("BEGIN;");
def rollback(self):
self.connection.rollback()
self.__connection.rollback()
def table_exists(self, table):
self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables "
self.__cursor.execute("SELECT COUNT(*) "
"FROM information_schema.tables "
"WHERE table_name='%s';" % table)
return self.fetchone()[0] != 0
def close(self):
self.connection.close()
self.__connection.close()
def cursor(self):
return Cursor(self.__connection)
class Cursor:
def __init__(self, connection):
self.__connection = connection
def __enter__(self):
self.__cursor = self.__connection.cursor()
self.__cursor.arraysize = ARRAYSIZE
return self
def __exit__(self, *args, **kwargs):
self.__cursor.close()
def execute(self, *args, **kwargs):
"""
Executes an SQL statement.
:param args: arguments to be passed to the sqlite3 execute statement
:type args: list
:param kwargs: arguments to be passed to the sqlite3 execute statement
:type kwargs: list
"""
self.__cursor.execute(*args, **kwargs)
def fetchmany(self):
"""
Fetches the next set of rows of a query result, returning a list. An
empty list is returned when no more rows are available.
"""
return self.__cursor.fetchmany()

75
gramps/plugins/db/dbapi/postgresql.py
View File

@ -2,6 +2,7 @@
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
# Copyright (C) 2016 Nick Hall
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -18,9 +19,21 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#-------------------------------------------------------------------------
#
# Standard python modules
#
#-------------------------------------------------------------------------
import psycopg2
import re
#-------------------------------------------------------------------------
#
# Gramps modules
#
#-------------------------------------------------------------------------
from gramps.gen.db.dbconst import ARRAYSIZE
psycopg2.paramstyle = 'format'
class Postgresql:
@ -40,9 +53,9 @@ class Postgresql:
return summary
def __init__(self, *args, **kwargs):
self.connection = psycopg2.connect(*args, **kwargs)
self.connection.autocommit = True
self.cursor = self.connection.cursor()
self.__connection = psycopg2.connect(*args, **kwargs)
self.__connection.autocommit = True
self.__cursor = self.__connection.cursor()
def _hack_query(self, query):
query = query.replace("?", "%s")
@ -71,33 +84,71 @@ class Postgresql:
else:
args = None
try:
self.cursor.execute(sql, args, **kwargs)
self.__cursor.execute(sql, args, **kwargs)
except:
self.cursor.execute("rollback")
self.__cursor.execute("rollback")
raise
def fetchone(self):
try:
return self.cursor.fetchone()
return self.__cursor.fetchone()
except:
return None
def fetchall(self):
return self.cursor.fetchall()
return self.__cursor.fetchall()
def begin(self):
self.cursor.execute("BEGIN;")
self.__cursor.execute("BEGIN;")
def commit(self):
self.cursor.execute("COMMIT;")
self.__cursor.execute("COMMIT;")
def rollback(self):
self.connection.rollback()
self.__connection.rollback()
def table_exists(self, table):
self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables "
self.__cursor.execute("SELECT COUNT(*) "
"FROM information_schema.tables "
"WHERE table_name=?;", [table])
return self.fetchone()[0] != 0
def close(self):
self.connection.close()
self.__connection.close()
def cursor(self):
return Cursor(self.__connection)
class Cursor:
def __init__(self, connection):
self.__connection = connection
def __enter__(self):
self.__cursor = self.__connection.cursor()
self.__cursor.arraysize = ARRAYSIZE
return self
def __exit__(self, *args, **kwargs):
self.__cursor.close()
def execute(self, *args, **kwargs):
"""
Executes an SQL statement.
:param args: arguments to be passed to the sqlite3 execute statement
:type args: list
:param kwargs: arguments to be passed to the sqlite3 execute statement
:type kwargs: list
"""
self.__cursor.execute(*args, **kwargs)
def fetchmany(self):
"""
Fetches the next set of rows of a query result, returning a list. An
empty list is returned when no more rows are available.
"""
try:
return self.__cursor.fetchmany()
except:
return None

73
gramps/plugins/db/dbapi/sqlite.py
View File

@ -2,6 +2,7 @@
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
# Copyright (C) 2016 Nick Hall
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -31,6 +32,13 @@ import sqlite3
import logging
import re
#-------------------------------------------------------------------------
#
# Gramps modules
#
#-------------------------------------------------------------------------
from gramps.gen.db.dbconst import ARRAYSIZE
sqlite3.paramstyle = 'qmark'
#-------------------------------------------------------------------------
@ -72,10 +80,9 @@ class Sqlite:
:type kwargs: list
"""
self.log = logging.getLogger(".sqlite")
self.connection = sqlite3.connect(*args, **kwargs)
self.cursor = self.connection.cursor()
self.queries = {}
self.connection.create_function("regexp", 2, regexp)
self.__connection = sqlite3.connect(*args, **kwargs)
self.__cursor = self.__connection.cursor()
self.__connection.create_function("regexp", 2, regexp)
def execute(self, *args, **kwargs):
"""
@ -87,21 +94,21 @@ class Sqlite:
:type kwargs: list
"""
self.log.debug(args)
self.cursor.execute(*args, **kwargs)
self.__cursor.execute(*args, **kwargs)
def fetchone(self):
"""
Fetches the next row of a query result set, returning a single sequence,
or None when no more data is available.
"""
return self.cursor.fetchone()
return self.__cursor.fetchone()
def fetchall(self):
"""
Fetches the next set of rows of a query result, returning a list. An
empty list is returned when no more rows are available.
"""
return self.cursor.fetchall()
return self.__cursor.fetchall()
def begin(self):
"""
@ -116,14 +123,14 @@ class Sqlite:
Commit the current transaction.
"""
self.log.debug("COMMIT;")
self.connection.commit()
self.__connection.commit()
def rollback(self):
"""
Roll back any changes to the database since the last call to commit().
"""
self.log.debug("ROLLBACK;")
self.connection.rollback()
self.__connection.rollback()
def table_exists(self, table):
"""
@ -134,7 +141,8 @@ class Sqlite:
:returns: True if the table exists, false otherwise.
:rtype: bool
"""
self.execute("SELECT COUNT(*) FROM sqlite_master "
self.execute("SELECT COUNT(*) "
"FROM sqlite_master "
"WHERE type='table' AND name='%s';" % table)
return self.fetchone()[0] != 0
@ -143,7 +151,50 @@ class Sqlite:
Close the current database.
"""
self.log.debug("closing database...")
self.connection.close()
self.__connection.close()
def cursor(self):
"""
Return a new cursor.
"""
return Cursor(self.__connection)
#-------------------------------------------------------------------------
#
# Cursor class
#
#-------------------------------------------------------------------------
class Cursor:
def __init__(self, connection):
self.__connection = connection
def __enter__(self):
self.__cursor = self.__connection.cursor()
self.__cursor.arraysize = ARRAYSIZE
return self
def __exit__(self, *args, **kwargs):
self.__cursor.close()
def execute(self, *args, **kwargs):
"""
Executes an SQL statement.
:param args: arguments to be passed to the sqlite3 execute statement
:type args: list
:param kwargs: arguments to be passed to the sqlite3 execute statement
:type kwargs: list
"""
self.__cursor.execute(*args, **kwargs)
def fetchmany(self):
"""
Fetches the next set of rows of a query result, returning a list. An
empty list is returned when no more rows are available.
"""
return self.__cursor.fetchmany()
def regexp(expr, value):
"""

51
gramps/plugins/db/dbapi/test/db_test.py
View File

@ -342,6 +342,57 @@ class DbTest(unittest.TestCase):
for gramps_id in self.gids['Note']:
self.assertTrue(self.db.has_note_gramps_id(gramps_id))
################################################################
#
# Test get_*_cursor methods
#
################################################################
def __get_cursor_test(self, cursor_func, raw_func):
with cursor_func() as cursor:
for handle, data1 in cursor:
data2 = raw_func(handle)
self.assertEqual(data1, data2)
def test_get_person_cursor(self):
self.__get_cursor_test(self.db.get_person_cursor,
self.db.get_raw_person_data)
def test_get_family_cursor(self):
self.__get_cursor_test(self.db.get_family_cursor,
self.db.get_raw_family_data)
def test_get_event_cursor(self):
self.__get_cursor_test(self.db.get_event_cursor,
self.db.get_raw_event_data)
def test_get_place_cursor(self):
self.__get_cursor_test(self.db.get_place_cursor,
self.db.get_raw_place_data)
def test_get_repository_cursor(self):
self.__get_cursor_test(self.db.get_repository_cursor,
self.db.get_raw_repository_data)
def test_get_source_cursor(self):
self.__get_cursor_test(self.db.get_source_cursor,
self.db.get_raw_source_data)
def test_get_citation_cursor(self):
self.__get_cursor_test(self.db.get_citation_cursor,
self.db.get_raw_citation_data)
def test_get_media_cursor(self):
self.__get_cursor_test(self.db.get_media_cursor,
self.db.get_raw_media_data)
def test_get_note_cursor(self):
self.__get_cursor_test(self.db.get_note_cursor,
self.db.get_raw_note_data)
def test_get_tag_cursor(self):
self.__get_cursor_test(self.db.get_tag_cursor,
self.db.get_raw_tag_data)
if __name__ == "__main__":
unittest.main()