Create module for database utilities

This commit is contained in:
Nick Hall 2016-11-22 22:35:43 +00:00
parent 5358e8dd93
commit 382987f366
26 changed files with 222 additions and 188 deletions

View File

@ -46,7 +46,7 @@ import re
from gramps.gen.recentfiles import recent_files
from gramps.gen.utils.file import rm_tempdir, get_empty_tempdir
from .clidbman import CLIDbManager, NAME_FILE, find_locker_name
from gramps.gen.db.utils import make_database
from gramps.gen.plug import BasePluginManager
from gramps.gen.plug.report import CATEGORY_BOOK, CATEGORY_CODE, BookList
from .plug import cl_report, cl_book
@ -501,7 +501,7 @@ class ArgHandler:
else:
self.imp_db_path = get_empty_tempdir("import_dbdir")
dbid = config.get('database.backend')
newdb = self.dbstate.make_database(dbid)
newdb = make_database(dbid)
newdb.write_version(self.imp_db_path)
try:

View File

@ -49,6 +49,7 @@ from gramps.gen.plug import BasePluginManager
from gramps.gen.config import config
from gramps.gen.constfunc import win
from gramps.gen.db.dbconst import DBLOGNAME
from gramps.gen.db.utils import make_database
from gramps.gen.const import GRAMPS_LOCALE as glocale
_ = glocale.translation.gettext
@ -160,7 +161,7 @@ class CLIDbManager:
dbid = file.read().strip()
if not self.is_locked(dirpath):
try:
database = self.dbstate.make_database(dbid)
database = make_database(dbid)
database.load(dirpath, None, update=False)
retval = database.get_summary()
database.close(update=False)
@ -307,7 +308,7 @@ class CLIDbManager:
# write the version number into metadata
if dbid is None:
dbid = "bsddb"
newdb = self.dbstate.make_database(dbid)
newdb = make_database(dbid)
newdb.write_version(new_path)
(tval, last) = time_val(new_path)
@ -376,7 +377,7 @@ class CLIDbManager:
## Use bsddb, for now, because we assumed that above.
dbid = "bsddb" ## config.get('database.backend')
dbase = self.dbstate.make_database(dbid)
dbase = make_database(dbid)
dbase.load(new_path, user.callback)
import_function = plugin.get_import_function()

View File

@ -48,6 +48,7 @@ from gramps.gen.display.name import displayer as name_displayer
from gramps.gen.config import config
from gramps.gen.const import PLUGINS_DIR, USER_PLUGINS
from gramps.gen.db.dbconst import DBBACKEND
from gramps.gen.db.utils import make_database
from gramps.gen.errors import DbError
from gramps.gen.dbstate import DbState
from gramps.gen.db.exceptions import (DbUpgradeRequiredError,
@ -163,7 +164,7 @@ class CLIDbLoader:
else:
dbid = "bsddb"
db = self.dbstate.make_database(dbid)
db = make_database(dbid)
self.dbstate.change_database(db)
self.dbstate.db.disable_signals()

View File

@ -69,38 +69,4 @@ from .dbconst import *
from .txn import *
from .exceptions import *
from .undoredo import *
def find_surname_name(key, data):
"""
Creating a surname from raw name, to use for sort and index
returns a byte string
"""
return __index_surname(data[5])
def __index_surname(surn_list):
"""
All non pa/matronymic surnames are used in indexing.
pa/matronymic not as they change for every generation!
returns a byte string
"""
from gramps.gen.lib import NameOriginType
if surn_list:
surn = " ".join([x[0] for x in surn_list if not (x[3][0] in [
NameOriginType.PATRONYMIC, NameOriginType.MATRONYMIC]) ])
else:
surn = ""
return surn
def open_database(database, force_unlock=False):
"""
Shortcut for external uses of databases.
"""
from ..dbstate import DbState
return DbState().open_database(database, force_unlock)
def make_database(dbid):
"""
Shortcut for external uses of databases.
"""
from ..dbstate import DbState
return DbState().make_database(dbid)
from .utils import *

175
gramps/gen/db/utils.py Normal file
View File

@ -0,0 +1,175 @@
#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2000-2006 Donald N. Allingham
# Copyright (C) 2011 Tim G L Lyons
# Copyright (C) 2012 Doug Blank <doug.blank@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
"""
Database utilites
"""
#------------------------------------------------------------------------
#
# Python modules
#
#------------------------------------------------------------------------
import os
import logging
#------------------------------------------------------------------------
#
# Gramps modules
#
#------------------------------------------------------------------------
from ..plug import BasePluginManager
from ..const import PLUGINS_DIR, USER_PLUGINS
from ..config import config
from .dbconst import DBLOGNAME
#-------------------------------------------------------------------------
#
# set up logging
#
#-------------------------------------------------------------------------
_LOG = logging.getLogger(DBLOGNAME)
def make_database(plugin_id):
"""
Make a database, given a plugin id.
"""
pmgr = BasePluginManager.get_instance()
pdata = pmgr.get_plugin(plugin_id)
if not pdata:
# This might happen if using gramps from outside, and
# we haven't loaded plugins yet
pmgr.reg_plugins(PLUGINS_DIR, None, None)
pmgr.reg_plugins(USER_PLUGINS, None, None, load_on_reg=True)
pdata = pmgr.get_plugin(plugin_id)
if pdata:
mod = pmgr.load_plugin(pdata)
if mod:
database = getattr(mod, pdata.databaseclass)
db = database()
import inspect
caller_frame = inspect.stack()[1]
_LOG.debug("Database class instance created Class:%s instance:%s. "
"Called from File %s, line %s, in %s"
% ((db.__class__.__name__, hex(id(db)))
+ (os.path.split(caller_frame[1])[1],)
+ tuple(caller_frame[i] for i in range(2, 4))))
return db
else:
raise Exception("can't load database backend: '%s'" % plugin_id)
else:
raise Exception("no such database backend: '%s'" % plugin_id)
def open_database(dbname, force_unlock=False, callback=None):
"""
Open a database by name and return the database.
"""
data = lookup_family_tree(dbname)
database = None
if data:
dbpath, locked, locked_by, backend = data
if (not locked) or (locked and force_unlock):
database = make_database(backend)
database.load(dbpath, callback=callback)
return database
def lookup_family_tree(dbname):
"""
Find a Family Tree given its name, and return properties.
"""
dbdir = os.path.expanduser(config.get('database.path'))
for dpath in os.listdir(dbdir):
dirpath = os.path.join(dbdir, dpath)
path_name = os.path.join(dirpath, "name.txt")
if os.path.isfile(path_name):
with open(path_name, 'r', encoding='utf8') as file:
name = file.readline().strip()
if dbname == name:
locked = False
locked_by = None
backend = None
fname = os.path.join(dirpath, "database.txt")
if os.path.isfile(fname):
with open(fname, 'r', encoding='utf8') as ifile:
backend = ifile.read().strip()
else:
backend = "bsddb"
try:
fname = os.path.join(dirpath, "lock")
with open(fname, 'r', encoding='utf8') as ifile:
locked_by = ifile.read().strip()
locked = True
except (OSError, IOError):
pass
return (dirpath, locked, locked_by, backend)
return None
def import_as_dict(filename, user, skp_imp_adds=True):
"""
Import the filename into a InMemoryDB and return it.
"""
db = make_database("inmemorydb")
db.load(None)
db.set_feature("skip-import-additions", skp_imp_adds)
(name, ext) = os.path.splitext(os.path.basename(filename))
format = ext[1:].lower()
pmgr = BasePluginManager.get_instance()
import_list = pmgr.get_reg_importers()
for pdata in import_list:
if format == pdata.extension:
mod = pmgr.load_plugin(pdata)
if not mod:
for item in pmgr.get_fail_list():
name, error_tuple, pdata = item
# (filename, (exception-type, exception, traceback), pdata)
etype, exception, traceback = error_tuple
#print("ERROR:", name, exception)
return False
import_function = getattr(mod, pdata.import_function)
results = import_function(db, filename, user)
if results is None:
return None
return db
return None
def find_surname_name(key, data):
"""
Creating a surname from raw name, to use for sort and index
returns a byte string
"""
return __index_surname(data[5])
def __index_surname(surn_list):
"""
All non pa/matronymic surnames are used in indexing.
pa/matronymic not as they change for every generation!
returns a byte string
"""
from gramps.gen.lib import NameOriginType
if surn_list:
surn = " ".join([x[0] for x in surn_list if not (x[3][0] in [
NameOriginType.PATRONYMIC, NameOriginType.MATRONYMIC])])
else:
surn = ""
return surn

View File

@ -43,6 +43,7 @@ from .proxy.proxybase import ProxyDbBase
from .utils.callback import Callback
from .config import config
from gramps.gen.db.dbconst import DBLOGNAME
from gramps.gen.db.utils import make_database
#-------------------------------------------------------------------------
#
@ -68,7 +69,7 @@ class DbState(Callback):
place holder until a real DB is assigned.
"""
Callback.__init__(self)
self.db = self.make_database("dummydb")
self.db = make_database("dummydb")
self.open = False # Deprecated - use DbState.is_open()
self.stack = []
@ -134,7 +135,7 @@ class DbState(Callback):
self.emit('no-database', ())
if self.is_open():
self.db.close()
self.db = self.make_database("dummydb")
self.db = make_database("dummydb")
self.open = False
self.emit('database-changed', (self.db, ))
@ -178,82 +179,3 @@ class DbState(Callback):
"""
self.db = self.stack.pop()
self.emit('database-changed', (self.db, ))
def make_database(self, plugin_id):
"""
Make a database, given a plugin id.
"""
from .plug import BasePluginManager
from .const import PLUGINS_DIR, USER_PLUGINS
pmgr = BasePluginManager.get_instance()
pdata = pmgr.get_plugin(plugin_id)
if not pdata:
# This might happen if using gramps from outside, and
# we haven't loaded plugins yet
pmgr.reg_plugins(PLUGINS_DIR, self, None)
pmgr.reg_plugins(USER_PLUGINS, self, None, load_on_reg=True)
pdata = pmgr.get_plugin(plugin_id)
if pdata:
mod = pmgr.load_plugin(pdata)
if mod:
database = getattr(mod, pdata.databaseclass)
db = database()
import inspect
caller_frame = inspect.stack()[1]
_LOG.debug("Database class instance created Class:%s instance:%s. "
"Called from File %s, line %s, in %s"
% ((db.__class__.__name__, hex(id(db)))
+ (os.path.split(caller_frame[1])[1],)
+ tuple(caller_frame[i] for i in range(2, 4))))
return db
else:
raise Exception("can't load database backend: '%s'" % plugin_id)
else:
raise Exception("no such database backend: '%s'" % plugin_id)
def open_database(self, dbname, force_unlock=False, callback=None):
"""
Open a database by name and return the database.
"""
data = self.lookup_family_tree(dbname)
database = None
if data:
dbpath, locked, locked_by, backend = data
if (not locked) or (locked and force_unlock):
database = self.make_database(backend)
database.load(dbpath, callback=callback)
return database
def lookup_family_tree(self, dbname):
"""
Find a Family Tree given its name, and return properties.
"""
dbdir = os.path.expanduser(config.get('database.path'))
for dpath in os.listdir(dbdir):
dirpath = os.path.join(dbdir, dpath)
path_name = os.path.join(dirpath, "name.txt")
if os.path.isfile(path_name):
with open(path_name, 'r', encoding='utf8') as file:
name = file.readline().strip()
if dbname == name:
locked = False
locked_by = None
backend = None
fname = os.path.join(dirpath, "database.txt")
if os.path.isfile(fname):
with open(fname, 'r', encoding='utf8') as ifile:
backend = ifile.read().strip()
else:
backend = "bsddb"
try:
fname = os.path.join(dirpath, "lock")
with open(fname, 'r', encoding='utf8') as ifile:
locked_by = ifile.read().strip()
locked = True
except (OSError, IOError):
pass
return (dirpath, locked, locked_by, backend)
return None

View File

@ -24,7 +24,7 @@ Unittest that tests event-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests family-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests media-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests note-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests person-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilter
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests place-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -24,7 +24,7 @@ Unittest that tests repository-specific filter rules
import unittest
import os
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.filters import GenericFilterFactory
from gramps.gen.const import DATA_DIR

View File

@ -22,7 +22,8 @@
import unittest
from gramps.gen.db import make_database, DbTxn
from gramps.gen.db import DbTxn
from gramps.gen.db.utils import make_database
from ..import (Person, Surname, Name, NameType, Family, FamilyRelType,
Event, EventType, Source, Place, PlaceName, Citation, Date,

View File

@ -25,7 +25,7 @@ import os
from .. import (Person, Family, Event, Source, Place, Citation,
Repository, Media, Note, Tag)
from gramps.gen.merge.diff import import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.cli.user import User
from gramps.gen.const import DATA_DIR

View File

@ -22,13 +22,8 @@
This package implements an object difference engine.
"""
import os
from gramps.cli.user import User
from ..dbstate import DbState
from gramps.cli.grampscli import CLIManager
from ..plug import BasePluginManager
from gramps.gen.db import make_database
from ..db.utils import import_as_dict
from ..const import GRAMPS_LOCALE as glocale
_ = glocale.translation.gettext
@ -82,39 +77,6 @@ def parse(string):
retval.append(current)
return retval
def import_as_dict(filename, user=None, skp_imp_adds=True):
"""
Import the filename into a InMemoryDB and return it.
"""
if user is None:
user = User()
db = make_database("inmemorydb")
db.load(None)
db.set_feature("skip-import-additions", skp_imp_adds)
dbstate = DbState()
climanager = CLIManager(dbstate, setloader=False, user=user)
climanager.do_reg_plugins(dbstate, None)
pmgr = BasePluginManager.get_instance()
(name, ext) = os.path.splitext(os.path.basename(filename))
format = ext[1:].lower()
import_list = pmgr.get_reg_importers()
for pdata in import_list:
if format == pdata.extension:
mod = pmgr.load_plugin(pdata)
if not mod:
for item in pmgr.get_fail_list():
name, error_tuple, pdata = item
# (filename, (exception-type, exception, traceback), pdata)
etype, exception, traceback = error_tuple
#print("ERROR:", name, exception)
return False
import_function = getattr(mod, pdata.import_function)
results = import_function(db, filename, user)
if results is None:
return None
return db
return None
def diff_dates(json1, json2):
"""
Compare two json date objects. Returns True if different.

View File

@ -40,7 +40,7 @@ import unittest
#-------------------------------------------------------------------------
from gramps.gen.const import TEMP_DIR, USER_HOME, USER_PLUGINS, VERSION
from gramps.gen.utils.file import media_path, get_empty_tempdir
from gramps.gen.dbstate import DbState
from gramps.gen.db.utils import make_database
#-------------------------------------------------------------------------
#
@ -58,12 +58,10 @@ class FileTest(unittest.TestCase):
"""
# Create database
dbstate = DbState()
db = dbstate.make_database("bsddb")
db = make_database("bsddb")
path = get_empty_tempdir("utils_file_test")
db.write_version(path)
db.load(path)
dbstate.change_database(db)
# Test without db.mediapath set
self.assertEqual(media_path(db), os.path.normcase(os.path.normpath(

View File

@ -53,6 +53,7 @@ from gi.repository import GObject
#-------------------------------------------------------------------------
from gramps.gen.const import GRAMPS_LOCALE as glocale
from gramps.gen.db.dbconst import DBBACKEND
from gramps.gen.db.utils import make_database
_ = glocale.translation.gettext
from gramps.cli.grampscli import CLIDbLoader
from gramps.gen.config import config
@ -317,7 +318,7 @@ class DbLoader(CLIDbLoader):
else:
dbid = "bsddb"
db = self.dbstate.make_database(dbid)
db = make_database(dbid)
db.disable_signals()
self.dbstate.no_database()

View File

@ -62,6 +62,7 @@ from .ddtargets import DdTargets
from gramps.gen.recentfiles import rename_filename, remove_filename
from .glade import Glade
from gramps.gen.db.exceptions import DbException
from gramps.gen.db.utils import make_database, open_database
from gramps.gen.config import config
from gramps.gui.listmodel import ListModel
from gramps.gen.constfunc import win
@ -652,7 +653,7 @@ class DbManager(CLIDbManager):
self.__start_cursor(_("Extracting archive..."))
dbase = self.dbstate.make_database("bsddb")
dbase = make_database("bsddb")
dbase.load(new_path)
self.__start_cursor(_("Importing archive..."))
@ -768,7 +769,7 @@ class DbManager(CLIDbManager):
Actually convert the db from BSDDB to DB-API.
"""
try:
db = self.dbstate.open_database(name)
db = open_database(name)
except:
ErrorDialog(_("Opening the '%s' database") % name,
_("An attempt to convert the database failed. "
@ -798,7 +799,7 @@ class DbManager(CLIDbManager):
new_text = "%s %s" % (name, _("(Converted #%d)") % count)
new_path, newname = self._create_new_db(new_text, edit_entry=False)
## Create a new database of correct type:
dbase = self.dbstate.make_database("dbapi")
dbase = make_database("dbapi")
dbase.write_version(new_path)
dbase.load(new_path)
## import from XML
@ -914,10 +915,10 @@ class DbManager(CLIDbManager):
fname = os.path.join(dirname, filename)
os.unlink(fname)
newdb = self.dbstate.make_database("bsddb")
newdb = make_database("bsddb")
newdb.write_version(dirname)
dbase = self.dbstate.make_database("bsddb")
dbase = make_database("bsddb")
dbase.set_save_path(dirname)
dbase.load(dirname, None)

View File

@ -36,6 +36,7 @@ from gramps.gen.lib import (Person, Family, Event, Source, Place, Citation,
Repository, Media, Note, Tag)
from gramps.cli.user import User
from gramps.gen.dbstate import DbState
from gramps.gen.db.utils import make_database
from gramps.gui.editors.editreference import EditReference
class MockWindow:
@ -55,7 +56,7 @@ class TestEditReference(unittest.TestCase):
@unittest.skipUnless(MOCKING, "Requires unittest.mock to run")
def test_editreference(self):
dbstate = DbState()
db = dbstate.make_database("bsddb")
db = make_database("bsddb")
path = "/tmp/edit_ref_test"
try:
os.mkdir(path)

View File

@ -23,6 +23,7 @@ import unittest
from .. import DbBsddb, DbTxn
from gramps.cli.clidbman import CLIDbManager
from gramps.gen.dbstate import DbState
from gramps.gen.db.utils import make_database
from gramps.gen.lib import (Source, RepoRef, Citation, Repository, Person,
Family, Event, Place, Media)
@ -37,7 +38,7 @@ class GrampsDbBaseTest(unittest.TestCase):
self.dbstate = DbState()
self.dbman = CLIDbManager(self.dbstate)
dirpath, name = self.dbman.create_new_db_cli("Test: bsddb", dbid="bsddb")
self._db = self.dbstate.make_database("bsddb")
self._db = make_database("bsddb")
self._db.load(dirpath, None)
def tearDown(self):

View File

@ -30,7 +30,8 @@ import unittest
# Gramps modules
#
#-------------------------------------------------------------------------
from gramps.gen.db import make_database, DbTxn
from gramps.gen.db import DbTxn
from gramps.gen.db.utils import make_database
from gramps.gen.lib import (Person, Family, Event, Place, Repository, Source,
Citation, Media, Note, Tag, Researcher, Surname)

View File

@ -29,7 +29,8 @@ from time import localtime, strptime
from unittest.mock import patch
#import logging
from gramps.gen.merge.diff import diff_dbs, import_as_dict
from gramps.gen.db.utils import import_as_dict
from gramps.gen.merge.diff import diff_dbs
from gramps.gen.simple import SimpleAccess
from gramps.gen.utils.id import set_det_id
from gramps.cli.user import User

View File

@ -24,7 +24,7 @@ import os
import shutil
from gramps.test.test_util import Gramps
from gramps.gen.db import open_database
from gramps.gen.db.utils import open_database
ddir = os.path.dirname(__file__)
example = os.path.join(ddir, "..", "..", "..",

View File

@ -60,6 +60,7 @@ gramps/gen/db/cursor.py
gramps/gen/db/dbconst.py
gramps/gen/db/test/db_test.py
gramps/gen/db/txn.py
gramps/gen/db/utils.py
#
# gen.display package
#

View File

@ -37,6 +37,7 @@ import sys
from gramps.gen.dbstate import DbState
from gramps.cli.clidbman import CLIDbManager
from gramps.gen.db.base import DbTxn
from gramps.gen.db.utils import make_database
from gramps.gen.errors import DbError, HandleError
import gramps.gen.const as const
import gramps.gen.lib as RelLib
@ -99,7 +100,7 @@ try:
dbstate = DbState()
dbman = CLIDbManager(dbstate)
dirpath, name = dbman.create_new_db_cli(filename1, dbid="bsddb")
db = dbstate.make_database("bsddb")
db = make_database("bsddb")
db.load(dirpath, None)
print("Add person 1")
add_person( db,"Anton", "Albers",True,False)
@ -120,7 +121,7 @@ try:
dbstate = DbState()
dbman = CLIDbManager(dbstate)
dirpath, name = dbman.create_new_db_cli(filename1, dbid="bsddb")
db = dbstate.make_database("bsddb")
db = make_database("bsddb")
db.load(dirpath, None)
print("Add person 4")
add_person( db,"Felix", "Fowler",True,False)
@ -135,7 +136,7 @@ try:
dbstate = DbState()
dbman = CLIDbManager(dbstate)
dirpath, name = dbman.create_new_db_cli(filename2, dbid="bsddb")
db = dbstate.make_database("bsddb")
db = make_database("bsddb")
db.load(dirpath, None)
print("Add source")