From 9c9dc2c8e22c86ee2c67d8e20a144e4441c9e98e Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 26 May 2015 13:19:03 -0400 Subject: [PATCH] 7143: Database Differences Report fails on gpkg with media Added return values on imports, so we can tell the difference between failure and success. Return value of None means failed. Now, all importers should return an ImportInfo object. The Differences will now stop if there is an error (such as existing media directory.) --- gramps/gen/merge/diff.py | 11 +++-- gramps/gen/utils/libformatting.py | 28 +++++++++++ gramps/plugins/importer/importcsv.py | 59 ++---------------------- gramps/plugins/importer/importgedcom.py | 2 + gramps/plugins/importer/importgeneweb.py | 2 + gramps/plugins/importer/importprogen.py | 2 + gramps/plugins/importer/importvcard.py | 3 +- 7 files changed, 48 insertions(+), 59 deletions(-) diff --git a/gramps/gen/merge/diff.py b/gramps/gen/merge/diff.py index 1152fac51..1ba5a45f7 100644 --- a/gramps/gen/merge/diff.py +++ b/gramps/gen/merge/diff.py @@ -195,7 +195,9 @@ def import_as_dict(filename, user=None): #print("ERROR:", name, exception) return False import_function = getattr(mod, pdata.import_function) - import_function(db, filename, user) + results = import_function(db, filename, user) + if results is None: + return None return db return None @@ -311,9 +313,10 @@ def diff_db_to_file(old_db, filename, user=None): user = User() # First, get data as a DictionaryDb new_db = import_as_dict(filename, user, user) - # Next get differences: - diffs, m_old, m_new = diff_dbs(old_db, new_db, user) - return diffs, m_old, m_new + if new_db is not None: + # Next get differences: + diffs, m_old, m_new = diff_dbs(old_db, new_db, user) + return diffs, m_old, m_new def from_struct(struct): """ diff --git a/gramps/gen/utils/libformatting.py b/gramps/gen/utils/libformatting.py index 2f4e29287..2ce8d77d7 100644 --- a/gramps/gen/utils/libformatting.py +++ b/gramps/gen/utils/libformatting.py @@ -197,3 +197,31 @@ class FormattingHelper(object): self._text_cache = {} self._markup_cache = {} + +class ImportInfo(object): + """ + Class object that can hold information about the import + """ + def __init__(self, default_info=None): + """ + Init of the import class. + + This creates the datastructures to hold info + """ + if default_info is None: + self.info = {} + else: + self.info = default_info + + def info_text(self): + """ + Construct an info message from the data in the class. + """ + text = "" + for key in self.info: + # Do not fail; can this fail? + try: + text += "%s: %s\n" % (key, self.info[key]) + except: + pass + return text diff --git a/gramps/plugins/importer/importcsv.py b/gramps/plugins/importer/importcsv.py index 2b25a588f..4c1e61764 100644 --- a/gramps/plugins/importer/importcsv.py +++ b/gramps/plugins/importer/importcsv.py @@ -51,7 +51,6 @@ _ = glocale.translation.sgettext ngettext = glocale.translation.ngettext # else "nearby" comments are ignored from gramps.gen.lib import ChildRef, Citation, Event, EventRef, EventType, Family, FamilyRelType, Name, NameType, Note, NoteType, Person, Place, Source, Surname, Tag from gramps.gen.db import DbTxn -from gramps.gen.plug.utils import OpenFileOrStdin from gramps.gen.datehandler import parser as _dp from gramps.gen.utils.string import gender as gender_map from gramps.gen.utils.id import create_id @@ -59,6 +58,7 @@ from gramps.gen.lib.eventroletype import EventRoleType from gramps.gen.constfunc import conv_to_unicode from gramps.gen.config import config from gramps.gen.display.place import displayer as place_displayer +from gramps.gen.utils.libformatting import ImportInfo #------------------------------------------------------------------------- # @@ -76,54 +76,6 @@ def get_primary_event_ref_from_type(dbase, person, event_name): return ref return None -#------------------------------------------------------------------------- -# -# Encoding support for CSV, from http://docs.python.org/lib/csv-examples.html -# -#------------------------------------------------------------------------- -class UTF8Recoder(object): - """ - Iterator that reads an encoded stream and reencodes the input to UTF-8 - """ - def __init__(self, stream, encoding): - self.reader = codecs.getreader(encoding)(stream) - - def __iter__(self): - return self - - def __next__(self): - "Encode the next line of the file." - return self.reader.next().encode("utf-8") - - next = __next__ - -class UnicodeReader(object): - """ - A CSV reader which will iterate over lines in the CSV file, - which is encoded in the given encoding. - """ - - def __init__(self, csvfile, encoding="utf-8", **kwds): - self.first_row = True - csvfile = UTF8Recoder(csvfile, encoding) - self.reader = csv.reader(csvfile, **kwds) - - def __next__(self): - "Read the next line of the file." - row = next(self.reader) - rowlist = [conv_to_unicode(s, "utf-8") for s in row] - # Add check for Byte Order Mark (Windows, Notepad probably): - if self.first_row: - if len(rowlist) > 0 and rowlist[0].startswith("\ufeff"): - rowlist[0] = rowlist[0][1:] - self.first_row = False - return rowlist - - def __iter__(self): - return self - - next = __next__ - #------------------------------------------------------------------------- # # Support and main functions @@ -151,12 +103,12 @@ def importData(dbase, filename, user): parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if config.get('preferences.tag-on-import') else None)) try: - with OpenFileOrStdin(filename, 'b') as filehandle: + with open(filename, 'r') as filehandle: parser.parse(filehandle) except EnvironmentError as err: user.notify_error(_("%s could not be opened\n") % filename, str(err)) return - return None # This module doesn't provide info about what got imported. + return ImportInfo({_("Results"): _("done")}) #------------------------------------------------------------------------- # @@ -260,12 +212,11 @@ class CSVParser(object): def read_csv(self, filehandle): "Read the data from the file and return it as a list." - reader = UnicodeReader(filehandle) try: - data = [[r.strip() for r in row] for row in reader] + data = [[r.strip() for r in row] for row in csv.reader(filehandle)] except csv.Error as err: self.user.notify_error(_('format error: line %(line)d: %(zero)s') % { - 'line' : reader.reader.line_num, 'zero' : err } ) + 'line' : reader.line_num, 'zero' : err } ) return None return data diff --git a/gramps/plugins/importer/importgedcom.py b/gramps/plugins/importer/importgedcom.py index da257a764..5f095674a 100644 --- a/gramps/plugins/importer/importgedcom.py +++ b/gramps/plugins/importer/importgedcom.py @@ -40,6 +40,7 @@ from gramps.gen.errors import DbError, GedcomError from gramps.gui.glade import Glade from gramps.plugins.lib.libmixin import DbMixin from gramps.plugins.lib import libgedcom +from gramps.gen.utils.libformatting import ImportInfo # The following code is necessary to ensure that when Help->Plugin # Manager->Reload is executed, not only is the top-level exportgedcom file # reloaded, but also the dependent libgedcom. This ensures that testing can have @@ -143,3 +144,4 @@ def importData(database, filename, user): except GedcomError as msg: user.notify_error(_('Error reading GEDCOM file'), str(msg)) return + return ImportInfo({_("Results"): _("done")}) diff --git a/gramps/plugins/importer/importgeneweb.py b/gramps/plugins/importer/importgeneweb.py index 7d9192f2c..4613bcf62 100644 --- a/gramps/plugins/importer/importgeneweb.py +++ b/gramps/plugins/importer/importgeneweb.py @@ -44,6 +44,7 @@ LOG = logging.getLogger(".ImportGeneWeb") # #------------------------------------------------------------------------- from gramps.gen.const import GRAMPS_LOCALE as glocale +from gramps.gen.utils.libformatting import ImportInfo _ = glocale.translation.gettext ngettext = glocale.translation.ngettext # else "nearby" comments are ignored from gramps.gen.errors import GedcomError @@ -91,6 +92,7 @@ def importData(database, filename, user): errmsg = _("%s could not be opened\n") % filename user.notify_error(errmsg,str(msg)) return + return ImportInfo({_("Results"): _("done")}) #------------------------------------------------------------------------- # For a description of the file format see diff --git a/gramps/plugins/importer/importprogen.py b/gramps/plugins/importer/importprogen.py index 584a3281e..16b72eeea 100644 --- a/gramps/plugins/importer/importprogen.py +++ b/gramps/plugins/importer/importprogen.py @@ -54,6 +54,7 @@ from gramps.gen.lib import (Attribute, AttributeType, ChildRef, Date, Event, NameType, Note, NoteType, Person, Place, Source, Surname, Citation, Location, NameOriginType) from gramps.gen.db import DbTxn +from gramps.gen.utils.libformatting import ImportInfo class ProgenError(Exception): """Error used to report Progen errors.""" @@ -81,6 +82,7 @@ def _importData(database, filename, user): except IOError as msg: user.notify_error(_("%s could not be opened") % filename, str(msg)) return + return ImportInfo({_("Results"): _("done")}) def _find_from_handle(progen_id, table): diff --git a/gramps/plugins/importer/importvcard.py b/gramps/plugins/importer/importvcard.py index 15cf804e5..9495afb5f 100644 --- a/gramps/plugins/importer/importvcard.py +++ b/gramps/plugins/importer/importvcard.py @@ -52,6 +52,7 @@ from gramps.gen.lib import (Address, Date, DateError, Event, EventRef, EventType, Name, NameType, Person, Surname, Url, UrlType) from gramps.gen.db import DbTxn from gramps.gen.plug.utils import OpenFileOrStdin +from gramps.gen.utils.libformatting import ImportInfo #------------------------------------------------------------------------- # @@ -70,7 +71,7 @@ def importData(database, filename, user): except GrampsImportError as msg: user.notify_error(_("%s could not be opened\n") % filename, str(msg)) return - return None # This module doesn't provide info about what got imported. + return ImportInfo({_("Results"): _("done")}) def splitof_nameprefix(name):