0006529: Cancelling database upgrade can corrupt the database. Check whether the bsddb version has changed (or was originally unknown). If it is an upgrade, ask the user whether he has made a backup, and is ready to upgrade. If so, made a zip backup and open the database with the new bsddb version. Make messages on the dialogues more explicit with the version numbers mentioned.

svn: r21852

gramps/cli/arghandler.py

@@ -376,12 +376,8 @@ class ArgHandler(object):
                 if not self.check_db(db_path, self.force_unlock):
                     sys.exit(0)
                 # Add the file to the recent items
-                path = os.path.join(db_path, "name.txt")
+                title = self.dbstate.db.get_dbname()
-                try:
+                if not title:
-                    ifile = open(path)
-                    title = ifile.readline().strip()
-                    ifile.close()
-                except:
                     title = db_path
                 recent_files(db_path, title)
                 self.open = db_path

gramps/cli/clidbman.py

@@ -51,6 +51,8 @@ import tempfile
 #-------------------------------------------------------------------------
 import logging
 LOG = logging.getLogger(".clidbman")
+from gramps.gen.db.dbconst import DBLOGNAME
+_LOG = logging.getLogger(DBLOGNAME)
 
 #-------------------------------------------------------------------------
 #
@@ -350,6 +352,11 @@ class CLIDbManager(object):
                 # write locally:
                 temp_fp.write(data)
                 url_fp.close()
+                from  gen.db.dbconst import BDBVERSFN
+                versionpath = os.path.join(name, BDBVERSFN)
+                _LOG.debug("Write bsddb version %s" % str(dbase.version()))
+                with open(versionpath, "w") as version_file:
+                    version_file.write(str(dbase.version()))
                 temp_fp.close()
 
         (name, ext) = os.path.splitext(os.path.basename(filename))

gramps/cli/grampscli.py

@@ -158,6 +158,15 @@ class CLIDbLoader(object):
         try:
             self.dbstate.db.load(filename, self._pulse_progress, mode)
             self.dbstate.db.set_save_path(filename)
+        except gen.db.exceptions.DbEnvironmentError as msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except gen.db.exceptions.BsddbUpgradeRequiredError as msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except gen.db.exceptions.BsddbDowngradeError as msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
         except gen.db.exceptions.DbUpgradeRequiredError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))

gramps/gen/db/base.py

@@ -1030,7 +1030,8 @@ class DbReadBase(object):
         """
         raise NotImplementedError
 
-    def load(self, name, callback, mode=None, upgrade=False):
+    def load(self, name, callback, mode=None, force_schema_upgrade=False,
+             force_bsddb_upgrade=False):
         """
         Open the specified database.
         """
@@ -1415,7 +1416,7 @@ class DbWriteBase(DbReadBase):
         """
         raise NotImplementedError
 
-    def need_upgrade(self):
+    def need_schema_upgrade(self):
         """
         Return True if database needs to be upgraded
         """

gramps/gen/db/exceptions.py

@@ -72,13 +72,23 @@ class DbVersionError(Exception):
     Error used to report that a file could not be read because it is written 
     in an unsupported version of the file format.
     """
-    def __init__(self):
+    def __init__(self, tree_vers, min_vers, max_vers):
         Exception.__init__(self)
+        self.tree_vers = tree_vers
+        self.min_vers = min_vers
+        self.max_vers = max_vers
 
     def __str__(self):
-        return _("The database version is not supported by this version of "
+        return _("The schema version is not supported by this version of "
-                 "Gramps.\nPlease upgrade to the corresponding version or use "
+                 "Gramps.\n\n"
-                 "XML for porting data between different database versions.")
+                 "This Family tree is schema version %(tree_vers)s, and this "
+                 "version of Gramps supports versions %(min_vers)s to "
+                 "%(max_vers)s\n\n"
+                 "Please upgrade to the corresponding version or use "
+                 "XML for porting data between different database versions.") %\
+                 {'tree_vers': self.tree_vers,
+                  'min_vers': self.min_vers,
+                  'max_vers': self.max_vers}
     
 class BsddbDowngradeError(Exception):
     """
@@ -104,6 +114,30 @@ class BsddbDowngradeError(Exception):
                  'intend to use.') % {'env_version': self.env_version,
                  'bdb_version': self.bdb_version}
 
+class BsddbUpgradeRequiredError(Exception):
+    """
+    Error used to report that the Berkeley database used to create the family
+    tree is of a version that is too new to be supported by the current version.
+    """
+    def __init__(self, env_version, bsddb_version):
+        Exception.__init__(self)
+        self.env_version = str(env_version)
+        self.bsddb_version = str(bsddb_version)
+
+    def __str__(self):
+        return _('The BSDDB version of the Family Tree you are trying to open '
+              'needs to be upgraded from %(env_version)s to %(bdb_version)s.\n\n'
+              'This probably means that the Family Tree was created with '
+              'an old version of Gramps. Opening the tree with this version '
+              'of Gramps may irretrievably corrupt your tree. You are '
+              'strongly advised to backup your tree before proceeding, '
+              'see: \n'
+              'http://www.gramps-project.org/wiki/index.php?title=How_to_make_a_backup\n\n'
+              'If you have made a backup, then you can get Gramps to try '
+              'to open the tree and upgrade it') % \
+              {'env_version': self.env_version,
+               'bdb_version': self.bsddb_version}
+
 class DbEnvironmentError(Exception):
     """
     Error used to report that the database 'environment' could not be opened.

gramps/gen/db/read.py

@@ -1900,7 +1900,7 @@ class DbBsddbRead(DbReadBase, Callback):
         filepath = os.path.join(self.path, "name.txt")
         try:
             name_file = open(filepath, "r")
-            name = name_file.read()
+            name = name_file.readline().strip()
             name_file.close()
         except (OSError, IOError) as msg:
             self.__log_error()

gramps/gen/db/test/db_test.py

@@ -213,7 +213,7 @@ class DbTest(object):
         "commit_source",
         "commit_tag",
         "delete_primary_from_reference_map",
-        "need_upgrade",
+        "need_schema_upgrade",
         "rebuild_secondary",
         "reindex_reference_map",
         "remove_event",

gramps/gen/db/write.py

@@ -43,7 +43,7 @@ import time
 import bisect
 from functools import wraps
 import logging
-from sys import maxsize
+from sys import getfilesystemencoding
 
 from ..config import config
 if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3:
@@ -72,7 +72,8 @@ from ..lib.researcher import Researcher
 from . import (DbBsddbRead, DbWriteBase, BSDDBTxn, 
                     DbTxn, BsddbBaseCursor, BsddbDowngradeError, DbVersionError,
                     DbEnvironmentError, DbUpgradeRequiredError, find_surname,
-                    find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo)
+                    find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo,
+                    exceptions)
 from .dbconst import *
 from ..utils.callback import Callback
 from ..utils.cast import conv_dbstr_to_unicode
@@ -397,29 +398,67 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
             with BSDDBTxn(self.env, self.metadata) as txn:
                 txn.put(b'mediapath', path)            
 
-    def __check_bdb_version(self, name):
+    def __make_zip_backup(self, dirname):
+        import zipfile
+        title = self.get_dbname()
+        
+        if not os.access(dirname, os.W_OK):
+            _LOG.warning("Can't write technical DB backup for %s" % title)
+            return
+        (grampsdb_path, db_code) = os.path.split(dirname)
+        dotgramps_path = os.path.dirname(grampsdb_path)
+        zipname = title + time.strftime("%Y-%m-%d %H-%M-%S") + ".zip"
+        if sys.version_info[0] < 3:
+            zipname = zipname.encode(glocale.getfilesystemencoding())
+        zippath = os.path.join(dotgramps_path, zipname)
+        myzip = zipfile.ZipFile(zippath, 'w')
+        for filename in os.listdir(dirname):
+            pathname = os.path.join(dirname, filename)
+            myzip.write(pathname, os.path.join(db_code, filename))
+        myzip.close()
+        _LOG.warning("If upgrade and loading the Family Tree works, you can "
+                     "delete the zip file at %s" %
+                     zippath)
+    
+    def __check_bdb_version(self, name, force_bsddb_upgrade=False):
         """Older version of Berkeley DB can't read data created by a newer
         version."""
         bdb_version = db.version()
-        env_version = (0, 0, 0)
         versionpath = os.path.join(self.path, BDBVERSFN)
-        try:
+        # Compare the current version of the database (bsddb_version) with the
+        # version of the database code (env_version). If it is a downgrade,
+        # raise an exception because we can't do anything. If they are the same,
+        # return. If it is an upgrade, raise an exception unless  the user has
+        # already told us we can upgrade.
+        if os.path.isfile(versionpath):
             with open(versionpath, "r") as version_file:
-                env_version = version_file.read().strip()
+                bsddb_version = version_file.read().strip()
-                env_version = tuple(map(int, env_version[1:-1].split(', ')))
+                env_version = tuple(map(int, bsddb_version[1:-1].split(', ')))
-        except:
-            # Just assume that the Berkeley DB version is OK.
-            pass
-        if not env_version:
-            #empty file, assume it is ok to open
-            env_version = (0, 0, 0)
             if (env_version[0] > bdb_version[0]) or \
                 (env_version[0] == bdb_version[0] and
                  env_version[1] > bdb_version[1]):
                 clear_lock_file(name)
                 raise BsddbDowngradeError(env_version, bdb_version)
-        elif env_version != bdb_version and not self.readonly:
+            elif env_version == bdb_version:
+                return
+        else:
+            # bsddb version is unknown
+            bsddb_version = "Unknown"
+
+        # An upgrade is needed, raise an exception unless the user has allowed
+        # an upgrade
+        if not force_bsddb_upgrade:
+            _LOG.debug("Bsddb upgrade required from %s to %s" %
+                       (bsddb_version, str(bdb_version)))
+            raise exceptions.BsddbUpgradeRequiredError(bsddb_version,
+                                                       str(bdb_version))
+        
+        if not self.readonly:
+            _LOG.warning("Bsddb upgrade requested from %s to %s" %
+                         (bsddb_version, str(bdb_version)))
             self.update_env_version = True
+        # Make a backup of the database files anyway 
+        self.__make_zip_backup(name)   
 
     @catch_db_error
     def version_supported(self):
@@ -427,7 +466,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         return ((dbversion <= _DBVERSION) and (dbversion >= _MINVERSION))
 
     @catch_db_error
-    def need_upgrade(self):
+    def need_schema_upgrade(self):
         dbversion = self.metadata.get(b'version', default=0)
         return not self.readonly and dbversion < _DBVERSION
 
@@ -453,7 +492,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         return False
 
     @catch_db_error
-    def load(self, name, callback, mode=DBMODE_W, upgrade=False):
+    def load(self, name, callback, mode=DBMODE_W, force_schema_upgrade=False,
+             force_bsddb_upgrade=False):
 
         if self.__check_readonly(name):
             mode = DBMODE_R
@@ -473,7 +513,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.path = self.full_name
         self.brief_name = os.path.basename(name)
 
-        self.__check_bdb_version(name)
+        self.__check_bdb_version(name, force_bsddb_upgrade)
 
         # Set up database environment
         self.env = db.DBEnv()
@@ -525,8 +565,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         # If we cannot work with this DB version,
         # it makes no sense to go further
         if not self.version_supported():
+            tree_vers = self.metadata.get(b'version', default=0)
             self.__close_early()
-            raise DbVersionError()
+            raise DbVersionError(tree_vers, _MINVERSION, _DBVERSION)
         
         self.__load_metadata()
         gstats = self.metadata.get(b'gender_stats', default=None)
@@ -573,12 +614,27 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.name_group = self.__open_db(self.full_name, NAME_GROUP,
                               db.DB_HASH, db.DB_DUP)
 
+        # We have now successfully opened the database, so if the BSDDB version
+        # has changed, we update the DBSDB version file. 
+        
+        if self.update_env_version:
+            versionpath = os.path.join(name, BDBVERSFN)
+            with open(versionpath, "w") as version_file:
+                version = str(db.version())
+                if sys.version_info[0] < 3:
+                    if isinstance(version, UNITYPE):
+                        version = version.encode('utf-8')
+                version_file.write(version)
+            _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version()))
+
         # Here we take care of any changes in the tables related to new code.
         # If secondary indices change, then they should removed
         # or rebuilt by upgrade as well. In any case, the
         # self.secondary_connected flag should be set accordingly.
-        if self.need_upgrade():
+        if self.need_schema_upgrade():
-            if upgrade == True:
+            _LOG.debug("Schema upgrade required from %s to %s" %
+                       (self.metadata.get('version', default=0), _DBVERSION))
+            if force_schema_upgrade == True or force_bsddb_upgrade == True:
                 self.gramps_upgrade(callback)
             else:
                 self.__close_early()
@@ -1210,20 +1266,6 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.undo_history_callback = None
         self.undodb = None
 
-        if self.update_env_version:
-            versionpath = os.path.join(self.path, BDBVERSFN)
-            try:
-                with open(versionpath, "w") as version_file:
-                    version = str(db.version())
-                    if sys.version_info[0] < 3:
-                        if isinstance(version, UNITYPE):
-                            version = version.encode('utf-8')
-                    version_file.write(version)
-            except:
-                # Storing the version of Berkeley Db is not really vital.
-                print ("Error storing berkeley db version")
-                pass
-
         try:
             clear_lock_file(self.get_save_path())
         except IOError:
@@ -2076,9 +2118,19 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
 
         self.metadata  = self.__open_shelf(full_name, META)
         
+        _LOG.debug("Write schema version %s" % _DBVERSION)
         with BSDDBTxn(self.env, self.metadata) as txn:
             txn.put(b'version', _DBVERSION)
         
+        versionpath = os.path.join(name, BDBVERSFN)
+        version = str(db.version())
+        if sys.version_info[0] < 3:
+            if isinstance(version, UNITYPE):
+                version = version.encode('utf-8')
+        _LOG.debug("Write bsddb version %s" % version)
+        with open(versionpath, "w") as version_file:
+            version_file.write(version)
+
         self.metadata.close()
         self.env.close()
   

gramps/gui/dbloader.py

@@ -61,7 +61,8 @@ from gramps.gen.db import DbBsddb
 from gramps.gen.db.exceptions import (DbUpgradeRequiredError, 
                                       BsddbDowngradeError, 
                                       DbVersionError, 
-                                      DbEnvironmentError)
+                                      DbEnvironmentError,
+                                      BsddbUpgradeRequiredError)
 from gramps.gen.constfunc import STRTYPE
 from gramps.gen.utils.file import get_unicode_path_from_file_chooser
 from .pluginmanager import GuiPluginManager
@@ -304,24 +305,39 @@ class DbLoader(CLIDbLoader):
 
         self._begin_progress()
         
+        force_schema_upgrade = False
+        force_bsddb_upgrade = False
         try:
+            while True:
                 try:
                     db.load(filename, self._pulse_progress, 
-                                     mode, upgrade=False)
+                            mode, force_schema_upgrade,
+                            force_bsddb_upgrade)
+                    db.set_save_path(filename)
                     self.dbstate.change_database(db)
+                    break
                 except DbUpgradeRequiredError as msg:
                     if QuestionDialog2(_("Need to upgrade database!"), 
                                        str(msg), 
                                        _("Upgrade now"), 
                                        _("Cancel")).run():
-                    db = DbBsddb()
+                        force_schema_upgrade = True
-                    db.disable_signals()
+                        force_bsddb_upgrade = False
-                    db.load(filename, self._pulse_progress, 
-                                         mode, upgrade=True)
-                    db.set_save_path(filename)
-                    self.dbstate.change_database(db)
                     else:
                         self.dbstate.no_database()
+                        break
+                except BsddbUpgradeRequiredError as msg:
+                    if QuestionDialog2(_("Need to upgrade BSDDB database!"), 
+                                       str(msg), 
+                                       _("I have made a backup, "
+                                         "please upgrade my tree"), 
+                                       _("Cancel")).run():
+                        force_schema_upgrade = False
+                        force_bsddb_upgrade = True
+                    else:
+                        self.dbstate.no_database()
+                        break
+        # Get here is there is an exception the while loop does not handle
         except BsddbDowngradeError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))