From 926dbca98fddb74e44724a8a79004270d3e7e407 Mon Sep 17 00:00:00 2001 From: Benny Malengier Date: Tue, 15 Jan 2013 08:42:50 +0000 Subject: [PATCH] 6329: Crazy citations after XML import of bacuo from Gramps 3.4.3 svn: r21123 --- gramps/gen/constfunc.py | 7 ++++++ gramps/gen/db/undoredo.py | 17 +++++++++++-- gramps/gen/db/write.py | 24 +++++++++++++------ .../gen/filters/rules/_matchesfilterbase.py | 8 +++++++ gramps/gen/utils/db.py | 1 - gramps/gui/views/treemodels/flatbasemodel.py | 13 ++++++---- gramps/gui/views/treemodels/treebasemodel.py | 16 ++++++++++--- 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/gramps/gen/constfunc.py b/gramps/gen/constfunc.py index 7144ed264..f4425ba15 100644 --- a/gramps/gen/constfunc.py +++ b/gramps/gen/constfunc.py @@ -64,6 +64,13 @@ else: STRTYPE = str UNITYPE = str cuni = conv_to_unicode_direct + +# handle in database is bytes, while internally Gramps wants unicode for py3 +if sys.version_info[0] < 3: + handle2internal = lambda x: x +else: + handle2internal = lambda x: conv_to_unicode(x, 'utf-8') + #------------------------------------------------------------------------- # # Platform determination functions diff --git a/gramps/gen/db/undoredo.py b/gramps/gen/db/undoredo.py index ec5ecd6db..ac8c0ead9 100644 --- a/gramps/gen/db/undoredo.py +++ b/gramps/gen/db/undoredo.py @@ -53,6 +53,7 @@ from ..ggettext import gettext as _ # Gramps modules # #------------------------------------------------------------------------- +from ..constfunc import conv_to_unicode from .dbconst import * from . import BSDDBTxn from ..errors import DbError @@ -67,6 +68,18 @@ DBERRS = (db.DBRunRecoveryError, db.DBAccessError, _SIGBASE = ('person', 'family', 'source', 'event', 'media', 'place', 'repository', 'reference', 'note', 'tag', 'citation') + +#------------------------------------------------------------------------- +# +# Helper functions +# +#------------------------------------------------------------------------- +# handle in database is bytes, while internally Gramps wants unicode for py3 +if sys.version_info[0] < 3: + handle2internal = lambda x: x +else: + handle2internal = lambda x: conv_to_unicode(x, 'utf-8') + #------------------------------------------------------------------------- # # DbUndo class @@ -315,7 +328,7 @@ class DbUndo(object): """ try: if data is None: - emit(signal_root + '-delete', ([handle],)) + emit(signal_root + '-delete', ([handle2internal(handle)],)) db_map.delete(handle, txn=self.txn) else: ex_data = db_map.get(handle, txn=self.txn) @@ -324,7 +337,7 @@ class DbUndo(object): else: signal = signal_root + '-add' db_map.put(handle, data, txn=self.txn) - emit(signal, ([handle],)) + emit(signal, ([handle2internal(handle)],)) except DBERRS as msg: self.db._log_error() diff --git a/gramps/gen/db/write.py b/gramps/gen/db/write.py index 0b0eb8882..6e6f3e659 100644 --- a/gramps/gen/db/write.py +++ b/gramps/gen/db/write.py @@ -80,7 +80,7 @@ from ..utils.callback import Callback from ..utils.cast import (conv_unicode_tosrtkey, conv_dbstr_to_unicode) from ..updatecallback import UpdateCallback from ..errors import DbError -from ..constfunc import win, conv_to_unicode, cuni, UNITYPE +from ..constfunc import win, conv_to_unicode, cuni, UNITYPE, handle2internal _LOG = logging.getLogger(DBLOGNAME) LOG = logging.getLogger(".citation") @@ -163,7 +163,7 @@ KEY_TO_NAME_MAP = {PERSON_KEY: 'person', # Helper functions # #------------------------------------------------------------------------- - + def find_idmap(key, data): """ return id for association of secondary index. returns a byte string @@ -857,6 +857,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): def delete_primary_from_reference_map(self, handle, transaction, txn=None): """ Remove all references to the primary object from the reference_map. + handle should be utf-8 """ primary_cur = self.get_reference_map_primary_cursor() @@ -875,8 +876,12 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): # so we need the second tuple give us a reference that we can # combine with the primary_handle to get the main key. - - main_key = (handle, pickle.loads(data)[1][1]) + if sys.version_info[0] < 3: + #handle should be in python 2 str + main_key = (handle, pickle.loads(data)[1][1]) + else: + #python 3 work internally with unicode + main_key = (handle.decode('utf-8'), pickle.loads(data)[1][1]) # The trick is not to remove while inside the cursor, # but collect them all and remove after the cursor is closed @@ -950,7 +955,12 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): the passed transaction. """ if isinstance(key, tuple): - #create a string key + #create a byte string key, first validity check in python 3! + for val in key: + if sys.version_info[0] >= 3 and isinstance(val, bytes): + raise DbError(_('An attempt is made to safe a reference key ' + 'which is partly bytecode, this is not allowed.\n' + 'Key is %s') % str(key)) key = str(key) if isinstance(key, UNITYPE): key = key.encode('utf-8') @@ -1895,10 +1905,10 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): """ if (obj_type, trans_type) in transaction: if trans_type == TXNDEL: - handles = [handle for handle, data in + handles = [handle2internal(handle) for handle, data in transaction[(obj_type, trans_type)]] else: - handles = [handle for handle, data in + handles = [handle2internal(handle) for handle, data in transaction[(obj_type, trans_type)] if (handle, None) not in transaction[(obj_type, TXNDEL)]] diff --git a/gramps/gen/filters/rules/_matchesfilterbase.py b/gramps/gen/filters/rules/_matchesfilterbase.py index d8b874b2e..8a6a149ad 100644 --- a/gramps/gen/filters/rules/_matchesfilterbase.py +++ b/gramps/gen/filters/rules/_matchesfilterbase.py @@ -26,6 +26,8 @@ # #------------------------------------------------------------------------- from ...ggettext import gettext as _ +import logging +LOG = logging.getLogger(".filter") #------------------------------------------------------------------------- # @@ -61,6 +63,12 @@ class MatchesFilterBase(Rule): filt = filters[self.list[0]] for rule in filt.flist: rule.requestprepare(db) + else: + LOG.warning(_("Can't find filter %s in the defined custom filters") + % self.list[0]) + else: + LOG.warning(_("Can't find filter %s in the defined custom filters") + % self.list[0]) def reset(self): if CustomFilters: diff --git a/gramps/gen/utils/db.py b/gramps/gen/utils/db.py index c28ec1773..c3864c553 100644 --- a/gramps/gen/utils/db.py +++ b/gramps/gen/utils/db.py @@ -558,7 +558,6 @@ def get_referents(handle, db, primary_objects): for primary in primary_objects: primary_list = [item[1] for item in object_list if item[0] == primary] the_lists = the_lists + (primary_list, ) - return the_lists def get_source_referents(source_handle, db): diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py index 1a85cd1b3..fb4c72178 100644 --- a/gramps/gui/views/treemodels/flatbasemodel.py +++ b/gramps/gui/views/treemodels/flatbasemodel.py @@ -76,8 +76,8 @@ from gi.repository import Gtk #------------------------------------------------------------------------- from gramps.gen.filters import SearchFilter, ExactSearchFilter from gramps.gen.utils.cast import conv_unicode_tosrtkey, conv_tosrtkey -from gramps.gen.constfunc import cuni, UNITYPE - +from gramps.gen.constfunc import cuni, UNITYPE, conv_to_unicode, handle2internal + #------------------------------------------------------------------------- # # FlatNodeMap @@ -252,8 +252,6 @@ class FlatNodeMap(object): :param type: an object handle :Returns: the path, or None if handle does not link to a path """ - if isinstance(handle, UNITYPE): - handle = handle.encode('utf-8') index = self._hndl2index.get(handle) if index is None: return None @@ -593,7 +591,8 @@ class FlatBaseModel(GObject.Object, Gtk.TreeModel): #loop over database and store the sort field, and the handle, and #allow for a third iter return sorted((list(map(conv_tosrtkey, - self.sort_func(data))), key) for key, data in cursor) + self.sort_func(data))), handle2internal(key)) + for key, data in cursor) def _rebuild_search(self, ignore=None): """ function called when view must be build, given a search text @@ -659,6 +658,8 @@ class FlatBaseModel(GObject.Object, Gtk.TreeModel): Add a row. This is called after object with handle is created. Row is only added if search/filter data is such that it must be shown """ + if sys.version_info[0] >= 3: + assert isinstance(handle, str) if self.node_map.get_path_from_handle(handle) is not None: return # row is already displayed data = self.map(handle) @@ -679,6 +680,8 @@ class FlatBaseModel(GObject.Object, Gtk.TreeModel): """ Delete a row, called after the object with handle is deleted """ + if sys.version_info[0] >= 3: + assert isinstance(handle, str) if self.node_map.get_path_from_handle(handle) is None: return # row is not currently displayed self.clear_cache(handle) diff --git a/gramps/gui/views/treemodels/treebasemodel.py b/gramps/gui/views/treemodels/treebasemodel.py index b9413c5fa..5293961b0 100644 --- a/gramps/gui/views/treemodels/treebasemodel.py +++ b/gramps/gui/views/treemodels/treebasemodel.py @@ -536,6 +536,9 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): pmon.add_op(status) with gen_cursor() as cursor: for handle, data in cursor: + # for python3 this returns a byte object, so conversion needed + if not isinstance(handle, UNITYPE): + handle = handle.decode('utf-8') status.heartbeat() if status.should_cancel(): break @@ -581,6 +584,9 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): def beat(key): status_ppl.heartbeat() + # for python3 this returns a byte object, so conversion needed + if not isinstance(key, UNITYPE): + key = key.decode('utf-8') return key with gen_cursor() as cursor: @@ -749,6 +755,8 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): """ Add a row to the model. """ + if sys.version_info[0] >= 3: + assert isinstance(handle, str) if self.get_node(handle) is not None: return # row already exists cput = time.clock() @@ -770,6 +778,8 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): """ Delete a row from the model. """ + if sys.version_info[0] >= 3: + assert isinstance(handle, str) cput = time.clock() node = self.get_node(handle) if node is None: @@ -800,6 +810,8 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): """ Update a row in the model. """ + if sys.version_info[0] >= 3: + assert isinstance(handle, str) if self.get_node(handle) is None: return # row not currently displayed @@ -848,8 +860,6 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): """ Get the node for a handle. """ - if isinstance(handle, UNITYPE): - handle = handle.encode('utf-8') return self.handle2node.get(handle) def handle2path(self, handle): @@ -905,7 +915,7 @@ class TreeBaseModel(GObject.Object, Gtk.TreeModel): # according to column_defs table val = self._get_value(node.handle, col, node.secondary) #GTK 3 should convert unicode objects automatically, but this - # gives wrong column values, so we convert, so we convert for python 2.7 + # gives wrong column values, so convert for python 2.7 if not isinstance(val, str): return val.encode('utf-8') else: