Problems with str and unicode conversion after GTK3 use.

These changes set the encoding explicitly, so as to avoid ascii being used


svn: r19950

src/gen/db/read.py

@@ -58,6 +58,7 @@ from gen.lib import (MediaObject, Person, Family, Source, Citation, Event,
                      NameOriginType)
 from gen.db.dbconst import *
 from gen.utils.callback import Callback
+from gen.utils.cast import conv_dbstr_to_unicode, conv_unicode_tosrtkey
 from gen.db import (BsddbBaseCursor, DbReadBase)
 from gen.utils.id import create_id
 from gen.errors import DbError
@@ -101,11 +102,11 @@ def __index_surname(surn_list):
     pa/matronymic not as they change for every generation!
     """
     if surn_list:
-        surn = " ".join([x[0] for x in surn_list if not (x[3][0] in [
+        surn = u" ".join([x[0] for x in surn_list if not (x[3][0] in [
                     NameOriginType.PATRONYMIC, NameOriginType.MATRONYMIC]) ])
     else:
-        surn = ""
+        surn = u""
-    return str(surn)
+    return surn.encode('utf-8')
     
 
 #-------------------------------------------------------------------------
@@ -822,14 +823,19 @@ class DbBsddbRead(DbReadBase, Callback):
     def get_name_group_mapping(self, surname):
         """
         Return the default grouping name for a surname.
+        Return type is a unicode object
         """
-        return unicode(self.name_group.get(str(surname), surname))
+        if isinstance(surname, unicode):
+            ssurname = conv_unicode_tosrtkey(surname)
+            return unicode(self.name_group.get(ssurname, ssurname), 'utf-8')
+        else:
+            return unicode(self.name_group.get(surname, surname), 'utf-8')
 
     def get_name_group_keys(self):
         """
         Return the defined names that have been assigned to a default grouping.
         """
-        return map(unicode, self.name_group.keys())
+        return map(conv_dbstr_to_unicode, self.name_group.keys())
 
     def has_name_group_key(self, name):
         """
@@ -837,7 +843,10 @@ class DbBsddbRead(DbReadBase, Callback):
         """
         # The use of has_key seems allright because there is no write lock
         # on the name_group table when this is called.
-        return self.name_group.has_key(str(name))
+        if isinstance(name, unicode):
+            return self.name_group.has_key(conv_unicode_tosrtkey(name))
+        else:
+            return self.name_group.has_key(name)
 
     def get_number_of_records(self, table):
         if not self.db_is_open:

src/gen/db/write.py

@@ -62,6 +62,8 @@ from gen.db import (DbBsddbRead, DbWriteBase, BSDDBTxn,
                     find_surname_name, DbUndoBSDDB as DbUndo)
 from gen.db.dbconst import *
 from gen.utils.callback import Callback
+from gen.utils.cast import (conv_unicode_tosrtkey_ongtk, conv_dbstr_to_unicode,
+                            conv_unicode_tosrtkey)
 from gen.updatecallback import UpdateCallback
 from gen.errors import DbError
 from gen.constfunc import win
@@ -1414,7 +1416,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         if not self.readonly:
             # Start transaction
             with BSDDBTxn(self.env, self.name_group) as txn:
-                sname = str(name)
+                sname = conv_unicode_tosrtkey(name)
                 data = txn.get(sname)
                 if data is not None:
                     txn.delete(sname)
@@ -1427,15 +1429,19 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
             self.emit('person-groupname-rebuild', (name, grouppar))
 
     def sort_surname_list(self):
-        self.surname_list.sort(key=locale.strxfrm)
+        self.surname_list.sort(key=conv_unicode_tosrtkey_ongtk)
 
     @catch_db_error
     def build_surname_list(self):
         """
         Build surname list for use in autocompletion
+        This is a list of unicode objects, which are decoded from the utf-8 in 
+        bsddb
         """
-        self.surname_list = sorted(map(unicode, set(self.surnames.keys())), 
+        #TODO GTK3: Why double conversion? Convert to a list of str objects!
-                                   key=locale.strxfrm)
+        self.surname_list = sorted(
+                        map(conv_dbstr_to_unicode, set(self.surnames.keys())), 
+                        key=conv_unicode_tosrtkey_ongtk)
 
     def add_to_surname_list(self, person, batch_transaction):
         """

src/gen/utils/cast.py

@@ -26,6 +26,30 @@
 Utility functions to cast types
 """
 
+#-------------------------------------------------------------------------
+#
+# Python modules
+#
+#-------------------------------------------------------------------------
+import locale
+
+#-------------------------------------------------------------------------
+#
+# Gramps modules
+#
+#-------------------------------------------------------------------------
+from gen.datehandler import codeset
+
+"""
+strxfrm needs it's unicode argument correctly cast before used.
+"""
+conv_unicode_tosrtkey = lambda x: locale.strxfrm(x.encode('utf-8', 'replace'))
+
+conv_unicode_tosrtkey_ongtk = lambda x: locale.strxfrm(x.encode(
+                                                           codeset, 'replace'))
+
+conv_dbstr_to_unicode = lambda x: unicode(x, 'utf-8')
+
 def cast_to_bool(val):
     if val == str(True):
         return True

src/gui/views/treemodels/Makefile.am

@@ -8,7 +8,6 @@ pkgpythondir = $(datadir)/@PACKAGE@/gui/views/treemodels
 
 pkgpython_PYTHON = \
 	__init__.py \
-	bugfix.py \
 	eventmodel.py \
 	familymodel.py \
 	flatbasemodel.py \

src/gui/views/treemodels/bugfix.py

@@ -1,51 +0,0 @@
-#
-# Gramps - a GTK+/GNOME based genealogy program
-#
-# Copyright (C) 2000-2007  Donald N. Allingham
-# Copyright (C) 2009       Gary Burton
-# Copyright (C) 2011       Tim G L Lyons
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, 
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-
-# $Id$
-
-"""
-Bug fix for strxfrm.
-
-strxfrm is apparently broken in Win ?? --> they should fix base lib, 
-we need strxfrm, fix it in this module.
-"""
-
-#-------------------------------------------------------------------------
-#
-# Python modules
-#
-#-------------------------------------------------------------------------
-import locale
-
-#-------------------------------------------------------------------------
-#
-# Gramps modules
-#
-#-------------------------------------------------------------------------
-from gen.datehandler import codeset
-from gen.constfunc import win
-
-if win():
-    conv_unicode_tosrtkey_ongtk = lambda x: locale.strxfrm(x.encode(
-                                                           codeset,'replace'))
-else:
-    conv_unicode_tosrtkey_ongtk = lambda x: locale.strxfrm(x)

src/gui/views/treemodels/flatbasemodel.py

@@ -74,7 +74,7 @@ from gi.repository import Gtk
 #
 #-------------------------------------------------------------------------
 from gen.filters import SearchFilter, ExactSearchFilter
-from bugfix import conv_unicode_tosrtkey_ongtk
+from gen.utils.cast import conv_unicode_tosrtkey_ongtk
 
 #-------------------------------------------------------------------------
 #
@@ -561,9 +561,7 @@ class FlatBaseModel(GObject.Object, Gtk.TreeModel):
         Return the (sort_key, handle) list of all data that can maximally 
         be shown. 
         This list is sorted ascending, via localized string sort. 
-        conv_unicode_tosrtkey_ongtk which uses strxfrm, which is apparently 
+        conv_unicode_tosrtkey_ongtk which uses strxfrm
-        broken in Win ?? --> they should fix base lib, we need strxfrm, fix it 
-        in the bugfix module.
         """
         # use cursor as a context manager
         with self.gen_cursor() as cursor:   
@@ -758,7 +756,12 @@ class FlatBaseModel(GObject.Object, Gtk.TreeModel):
         if col == self._tooltip_column:
             return val
         else:
-            return str(val)
+            #GTK 3 should convert unicode objects automatically, but this
+            # gives wrong column values, so we convert
+            if isinstance(val, unicode):
+                return val.encode('utf-8')
+            else:
+                return val
 
     def do_iter_previous(self, iter):
         #print 'do_iter_previous'

src/gui/views/treemodels/treebasemodel.py

@@ -55,7 +55,7 @@ from gi.repository import Gtk
 # GRAMPS modules
 #
 #-------------------------------------------------------------------------
-from bugfix import conv_unicode_tosrtkey_ongtk
+from gen.utils.cast import conv_unicode_tosrtkey_ongtk
 import gui.widgets.progressdialog as progressdlg
 from lru import LRU
 from bisect import bisect_right