Align interface with Xml import

svn: r17521

src/plugins/import/ImportCsv.py

@@ -32,7 +32,6 @@
 import time
 import csv
 import codecs
-import cStringIO
 
 #------------------------------------------------------------------------
 #
@@ -51,6 +50,7 @@ from gen.ggettext import sgettext as _
 from gen.ggettext import ngettext
 import gen.lib
 from gen.db import DbTxn
+from gen.plug.utils import OpenFileOrStdin
 from QuestionDialog import ErrorDialog
 from DateHandler import parser as _dp
 from Utils import gender as gender_map
@@ -63,13 +63,13 @@ from gen.lib.eventroletype import EventRoleType
 # Support Functions
 #
 #-------------------------------------------------------------------------
-def get_primary_event_ref_from_type(db, person, event_name):
+def get_primary_event_ref_from_type(dbase, person, event_name):
     """
-    >>> get_primary_event_ref_from_type(db, Person(), "Baptism"):
+    >>> get_primary_event_ref_from_type(dbase, Person(), "Baptism"):
     """
     for ref in person.event_ref_list:
         if ref.get_role() == EventRoleType.PRIMARY:
-            event = db.get_event_from_handle(ref.ref)
+            event = dbase.get_event_from_handle(ref.ref)
             if event and event.type.is_type(event_name):
                 return ref
     return None
@@ -83,27 +83,29 @@ class UTF8Recoder(object):
     """
     Iterator that reads an encoded stream and reencodes the input to UTF-8
     """
-    def __init__(self, f, encoding):
+    def __init__(self, stream, encoding):
-        self.reader = codecs.getreader(encoding)(f)
+        self.reader = codecs.getreader(encoding)(stream)
 
     def __iter__(self):
         return self
 
     def next(self):
+        "Encode the next line of the file."
         return self.reader.next().encode("utf-8")
 
 class UnicodeReader(object):
     """
-    A CSV reader which will iterate over lines in the CSV file "f",
+    A CSV reader which will iterate over lines in the CSV file,
     which is encoded in the given encoding.
     """
 
-    def __init__(self, f, encoding="utf-8", **kwds):
+    def __init__(self, csvfile, encoding="utf-8", **kwds):
         self.first_row = True
-        f = UTF8Recoder(f, encoding)
+        csvfile = UTF8Recoder(csvfile, encoding)
-        self.reader = csv.reader(f, **kwds)
+        self.reader = csv.reader(csvfile, **kwds)
 
     def next(self):
+        "Read the next line of the file."
         row = self.reader.next()
         rowlist = [unicode(s, "utf-8") for s in row]
         # Add check for Byte Order Mark (Windows, Notepad probably):
@@ -116,37 +118,6 @@ class UnicodeReader(object):
     def __iter__(self):
         return self
 
-class UnicodeWriter(object):
-    """
-    A CSV writer which will write rows to CSV file "f",
-    which is encoded in the given encoding.
-    """
-
-    def __init__(self, f, encoding="utf-8", **kwds):
-        # Redirect output to a queue
-        self.queue = cStringIO.StringIO()
-        self.writer = csv.writer(self.queue, **kwds)
-        self.stream = f
-        self.encoder = codecs.getencoder(encoding)
-
-    def writerow(self, row):
-        self.writer.writerow([s.encode('utf-8') for s in row])
-        # Fetch UTF-8 output from the queue ...
-        data = self.queue.getvalue()
-        data = data.decode('utf-8')
-        # ... and reencode it into the target encoding
-        data, length = self.encoder(data)
-        # write to the target stream
-        self.stream.write(data)
-        # empty queue
-        self.queue.truncate(0)
-
-    def writerows(self, rows):
-        map(self.writerow, rows)
-
-    def close(self):
-        self.stream.close()
-
 #-------------------------------------------------------------------------
 #
 # Support and main functions
@@ -166,195 +137,16 @@ def rd(line_number, row, col, key, default = None):
     else:
         return default
 
-def cleanup_column_name(column):
+def importData(dbase, filename, callback=None):
-    """ Handle column aliases for CSV spreadsheet import and SQL """
+    """Function called by Gramps to import data on persons in CSV format."""
-    retval = column
+    parser = CSVParser(dbase, callback)
-    # Title case:
+    try:
-    if retval in ["Lastname", 
+        with OpenFileOrStdin(filename, 'b') as filehandle:
-                  "Surname", _("Surname")]:
+            parser.parse(filehandle)
-        return "surname"
+    except EnvironmentError, err:
-    elif retval in ["Firstname", 
+        ErrorDialog(_("%s could not be opened\n") % filename, str(err))
-                    "Given name", _("Given name"), 
+        return
-                    "Given", _("Given")]:
+    return None # This module doesn't provide info about what got imported.
-        return "firstname"
-    elif retval in ["Callname", 
-                    "Call name", _("Call name"),
-                    "Call", _("Call")]:
-        return "callname"
-    elif retval in ["Title", _("Person|Title")]:
-        return "title"
-    elif retval in ["Prefix", _("Prefix")]:
-        return "prefix"
-    elif retval in ["Suffix", _("Suffix")]:
-        return "suffix"
-    elif retval in ["Gender", _("Gender")]:
-        return "gender"
-    elif retval in ["Source", _("Source")]:
-        return "source"
-    elif retval in ["Note", _("Note")]:
-        return "note"
-    elif retval in ["Birthplace", 
-                    "Birth place", _("Birth place")]:
-        return "birthplace"
-    elif retval in ["Birthdate", 
-                    "Birth date", _("Birth date")]:
-        return "birthdate"
-    elif retval in ["Birthsource", 
-                    "Birth source", _("Birth source")]:
-        return "birthsource"
-    elif retval in ["Baptismplace", 
-                    "Baptism place", _("Baptism place")]:
-        return "baptismplace"
-    elif retval in ["Baptismdate", 
-                    "Baptism date", _("Baptism date")]:
-        return "baptismdate"
-    elif retval in ["Baptismsource", 
-                    "Baptism source", _("Baptism source")]:
-        return "baptismsource"
-    elif retval in ["Burialplace", 
-                    "Burial place", _("Burial place")]:
-        return "burialplace"
-    elif retval in ["Burialdate", 
-                    "Burial date", _("Burial date")]:
-        return "burialdate"
-    elif retval in ["Burialsource", 
-                    "Burial source", _("Burial source")]:
-        return "burialsource"
-    elif retval in ["Deathplace", 
-                    "Death place", _("Death place")]:
-        return "deathplace"
-    elif retval in ["Deathdate", 
-                    "Death date", _("Death date")]:
-        return "deathdate"
-    elif retval in ["Deathsource", 
-                    "Death source", _("Death source")]:
-        return "deathsource"
-    elif retval in ["Deathcause", 
-                    "Death cause", _("Death cause")]:
-        return "deathcause"
-    elif retval in ["Grampsid", "ID",
-                    "Gramps id", _("Gramps ID")]:
-        return "grampsid"
-    elif retval in ["Person", _("Person")]:
-        return "person"
-    # ----------------------------------
-    elif retval in ["Child", _("Child")]:
-        return "child"
-    elif retval in ["Source", _("Source")]:
-        return "source"
-    elif retval in ["Family", _("Family")]:
-        return "family"
-    # ----------------------------------
-    elif retval in ["Mother", _("Mother"), 
-                    "Wife", _("Wife"),
-                    "Parent2", _("Parent2")]:
-        return "wife"
-    elif retval in ["Father", _("Father"), 
-                    "Husband", _("Husband"),
-                    "Parent1", _("Parent1")]:
-        return "husband"
-    elif retval in ["Marriage", _("Marriage")]:
-        return "marriage"
-    elif retval in ["Date", _("Date")]:
-        return "date"
-    elif retval in ["Place", _("Place")]:
-        return "place"
-    # lowercase
-    elif retval in ["lastname", "last_name", 
-                  "surname", _("surname")]:
-        return "surname"
-    elif retval in ["firstname", "first_name", "given_name",
-                    "given name", _("given name"), 
-                    "given", _("given")]:
-        return "firstname"
-    elif retval in ["callname", "call_name",
-                    "call name", 
-                    "call", _("call")]:
-        return "callname"
-    elif retval in ["title", _("Person|title")]:
-        return "title"
-    elif retval in ["prefix", _("prefix")]:
-        return "prefix"
-    elif retval in ["suffix", _("suffix")]:
-        return "suffix"
-    elif retval in ["gender", _("gender")]:
-        return "gender"
-    elif retval in ["source", _("source")]:
-        return "source"
-    elif retval in ["note", _("note")]:
-        return "note"
-    elif retval in ["birthplace", "birth_place",
-                    "birth place", _("birth place")]:
-        return "birthplace"
-    elif retval in ["birthdate", "birth_date",
-                    "birth date", _("birth date")]:
-        return "birthdate"
-    elif retval in ["birthsource", "birth_source",
-                    "birth source", _("birth source")]:
-        return "birthsource"
-    elif retval in ["baptismplace", 
-                    "baptism place", _("baptism place")]:
-        return "baptismplace"
-    elif retval in ["baptismdate", 
-                    "baptism date", _("baptism date")]:
-        return "baptismdate"
-    elif retval in ["baptismsource", 
-                    "baptism source", _("baptism source")]:
-        return "baptismsource"
-    elif retval in ["burialplace", 
-                    "burial place", _("burial place")]:
-        return "burialplace"
-    elif retval in ["burialdate", 
-                    "burial date", _("burial date")]:
-        return "burialdate"
-    elif retval in ["burialsource", 
-                    "burial source", _("burial source")]:
-        return "burialsource"
-    elif retval in ["deathplace", "death_place",
-                    "death place", _("death place")]:
-        return "deathplace"
-    elif retval in ["deathdate", "death_date",
-                    "death date", _("death date")]:
-        return "deathdate"
-    elif retval in ["deathsource", "death_source",
-                    "death source", _("death source")]:
-        return "deathsource"
-    elif retval in ["deathcause", "death_cause",
-                    "death cause", _("death cause")]:
-        return "deathcause"
-    elif retval in ["grampsid", "id", "gramps_id", 
-                    "gramps id", _("Gramps ID")]:
-        return "grampsid"
-    elif retval in ["person", _("person")]:
-        return "person"
-    # ----------------------------------
-    elif retval in ["child", _("child")]:
-        return "child"
-    elif retval in ["source", _("source")]:
-        return "source"
-    elif retval in ["family", _("family")]:
-        return "family"
-    # ----------------------------------
-    elif retval in ["mother", _("mother"), 
-                    "wife", _("wife"),
-                    "parent2", _("parent2")]:
-        return "wife"
-    elif retval in ["father", _("father"), 
-                    "husband", _("husband"),
-                    "parent1", _("parent1")]:
-        return "husband"
-    elif retval in ["marriage", _("marriage")]:
-        return "marriage"
-    elif retval in ["date", _("date")]:
-        return "date"
-    elif retval in ["place", _("place")]:
-        return "place"
-    #----------------------------------------------------
-    return retval
-
-def importData(db, filename, callback=None):
-    g = CSVParser(db, filename, callback)
-    g.process()
 
 #-------------------------------------------------------------------------
 #
@@ -362,88 +154,176 @@ def importData(db, filename, callback=None):
 #
 #-------------------------------------------------------------------------
 class CSVParser(object):
-    def __init__(self, db, filename, callback):
+    """Class to read data in CSV format from a file object."""
-        self.db = db
+    def __init__(self, dbase, callback):
-        self.filename = filename
+        self.db = dbase
         self.callback = callback
+        self.trans = None
+        self.lineno = 0
+        self.index = 0
+        self.fam_count = 0
+        self.indi_count = 0
+        self.pref  = {} # person ref, internal to this sheet
+        self.fref  = {} # family ref, internal to this sheet        
+        column2label = {
+            "surname": ("Lastname", "Surname", _("Surname"), "lastname",
+                "last_name", "surname", _("surname")),
+            "firstname": ("Firstname", "Given name", _("Given name"), "Given",
+                _("Given"), "firstname", "first_name", "given_name",
+                "given name", _("given name"), "given", _("given")),
+            "callname": ("Callname", "Call name", _("Call name"), "Call",
+                _("Call"), "callname", "call_name", "call name", "call",
+                _("call")),
+            "title": ("Title", _("Person|Title"), "title", _("Person|title")),
+            "prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
+            "suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
+            "gender": ("Gender", _("Gender"), "gender", _("gender")),
+            "source": ("Source", _("Source"), "source", _("source")),
+            "note": ("Note", _("Note"), "note", _("note")),
+            "birthplace": ("Birthplace", "Birth place", _("Birth place"),
+                "birthplace", "birth_place", "birth place", _("birth place")),
+            "birthdate": ("Birthdate", "Birth date", _("Birth date"),
+                "birthdate", "birth_date", "birth date", _("birth date")),
+            "birthsource": ("Birthsource", "Birth source", _("Birth source"),
+                "birthsource", "birth_source", "birth source",
+                _("birth source")),
+            "baptismplace": ("Baptismplace", "Baptism place",
+                _("Baptism place"), "baptismplace", "baptism place",
+                _("baptism place")),
+            "baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
+                "baptismdate", "baptism date", _("baptism date")),
+            "baptismsource": ("Baptismsource", "Baptism source",
+                _("Baptism source"), "baptismsource", "baptism source",
+                _("baptism source")),
+            "burialplace": ("Burialplace", "Burial place", _("Burial place"),
+                "burialplace", "burial place", _("burial place")),
+            "burialdate": ("Burialdate", "Burial date", _("Burial date"),
+                "burialdate", "burial date", _("burial date")),
+            "burialsource": ("Burialsource", "Burial source",
+                _("Burial source"), "burialsource", "burial source",
+                _("burial source")),
+            "deathplace": ("Deathplace", "Death place", _("Death place"),
+                "deathplace", "death_place", "death place", _("death place")),
+            "deathdate": ("Deathdate", "Death date", _("Death date"),
+                "deathdate", "death_date", "death date", _("death date")),
+            "deathsource": ("Deathsource", "Death source", _("Death source"),
+                "deathsource", "death_source", "death source",
+                _("death source")),
+            "deathcause": ("Deathcause", "Death cause", _("Death cause"),
+                "deathcause", "death_cause", "death cause", _("death cause")),
+            "grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
+                "grampsid", "id", "gramps_id", "gramps id", _("Gramps id")),
+            "person": ("Person", _("Person"), "person", _("person")),
+            # ----------------------------------
+            "child": ("Child", _("Child"), "child", _("child")),
+            "family": ("Family", _("Family"), "family", _("family")),
+            # ----------------------------------
+            "wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
+                _("Parent2"), "mother", _("mother"), "wife", _("wife"),
+                "parent2", _("parent2")),
+            "husband": ("Father", _("Father"), "Husband", _("Husband"),
+                "Parent1", _("Parent1"), "father", _("father"), "husband",
+                _("husband"), "parent1", _("parent1")),
+            "marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
+            "date": ("Date", _("Date"), "date", _("date")),
+            "place": ("Place", _("Place"), "place", _("place")),
+            }
+        lab2col_dict = []
+        for key in column2label.keys():
+            for val in column2label[key]:
+                lab2col_dict.append((val, key))
+        self.label2column = dict(lab2col_dict)
 
-    def readCSV(self):
+    def cleanup_column_name(self, column):
-        fp = None
+        """Handle column aliases for CSV spreadsheet import and SQL."""
-        reader = []
+        return self.label2column.get(column, column)
-        try:
+
-            fp = open(self.filename, "rb")
+    def read_csv(self, filehandle):
-            reader = UnicodeReader(fp) 
+        "Read the data from the file and return it as a list."
-        except IOError, msg:
+        reader = UnicodeReader(filehandle)
-            errmsg = _("%s could not be opened\n") % self.filename
-            ErrorDialog(errmsg,str(msg))
-            try:
-                fp.close()
-            except:
-                pass
-            return None
         try:
             data = [[r.strip() for r in row] for row in reader]
-        except csv.Error, e:
+        except csv.Error, err:
-            ErrorDialog(_('format error: file %(fname)s, line %(line)d: %(zero)s') % {
+            ErrorDialog(_('format error: line %(line)d: %(zero)s') % {
-                        'fname' : self.filename, 'line' : reader.line_num, 'zero' : e } )
+                        'line' : reader.reader.line_num, 'zero' : err } )
-            try:
-                fp.close()
-            except:
-                pass
             return None
         return data
 
-    def lookup(self, type, id):
+    def lookup(self, type_, id_):
-        if id is None: return None
+        """
-        if type == "family":
+        Return the object of type type_ with id id_ from db or previously
-            if id.startswith("[") and id.endswith("]"):
+        stored value.
-                id = self.db.fid2user_format(id[1:-1])
+        """
-                db_lookup = self.db.get_family_from_gramps_id(id)
+        if id_ is None:
+            return None
+        if type_ == "family":
+            if id_.startswith("[") and id_.endswith("]"):
+                id_ = self.db.fid2user_format(id_[1:-1])
+                db_lookup = self.db.get_family_from_gramps_id(id_)
                 if db_lookup is None:
-                    return self.lookup(type, id)
+                    return self.lookup(type_, id_)
                 else:
                     return db_lookup
-            elif id.lower() in self.fref:
+            elif id_.lower() in self.fref:
-                return self.fref[id.lower()]
+                return self.fref[id_.lower()]
             else:
                 return None
-        elif type == "person":
+        elif type_ == "person":
-            if id.startswith("[") and id.endswith("]"):
+            if id_.startswith("[") and id_.endswith("]"):
-                id = self.db.id2user_format(id[1:-1])
+                id_ = self.db.id2user_format(id_[1:-1])
-                db_lookup = self.db.get_person_from_gramps_id(id)
+                db_lookup = self.db.get_person_from_gramps_id(id_)
                 if db_lookup is None:
-                    return self.lookup(type, id)
+                    return self.lookup(type_, id_)
                 else:
                     return db_lookup
-            elif id.lower() in self.pref:
+            elif id_.lower() in self.pref:
-                return self.pref[id.lower()]
+                return self.pref[id_.lower()]
             else:
                 return None
         else:
-            LOG.warn("invalid lookup type in CSV import: '%s'" % type)
+            LOG.warn("invalid lookup type in CSV import: '%s'" % type_)
             return None
 
-    def storeup(self, type, id, object):
+    def storeup(self, type_, id_, object_):
-        if id.startswith("[") and id.endswith("]"):
+        "Store object object_ of type type_ in a dictionary under key id_."
-            id = id[1:-1]
+        if id_.startswith("[") and id_.endswith("]"):
+            id_ = id_[1:-1]
             #return # do not store gramps people; go look them up
-        if type == "person":
+        if type_ == "person":
-            id = self.db.id2user_format(id)
+            id_ = self.db.id2user_format(id_)
-            self.pref[id.lower()] = object
+            self.pref[id_.lower()] = object_
-        elif type == "family":
+        elif type_ == "family":
-            id = self.db.fid2user_format(id)
+            id_ = self.db.fid2user_format(id_)
-            self.fref[id.lower()] = object
+            self.fref[id_.lower()] = object_
         else:
-            LOG.warn("invalid storeup type in CSV import: '%s'" % type)
+            LOG.warn("invalid storeup type in CSV import: '%s'" % type_)
 
-    def process(self):
+    def parse(self, filehandle):
+        """
+        Prepare the database and parse the input file.
+
+        :param filehandle: open file handle positioned at start of the file
+        """
+        data = self.read_csv(filehandle)
         progress = ProgressMeter(_('CSV Import'))
         progress.set_pass(_('Reading data...'), 1)
-        data = self.readCSV() 
         progress.set_pass(_('Importing data...'), len(data))
-        with DbTxn(_("CSV import"), self.db, batch=True) as self.trans:
+        tym = time.time()
         self.db.disable_signals()
-            t = time.time()
+        with DbTxn(_("CSV import"), self.db, batch=True) as self.trans:
+            self._parse_csv_data(data, progress)
+        self.db.enable_signals()
+        self.db.request_rebuild()
+        tym = time.time() - tym
+        msg = ngettext('Import Complete: %d second',
+                'Import Complete: %d seconds', tym ) % tym
+        LOG.debug(msg)
+        LOG.debug("New Families: %d" % self.fam_count)
+        LOG.debug("New Individuals: %d" % self.indi_count)
+        progress.close()
+
+    def _parse_csv_data(self, data, progress=None):
+        """Parse each line of the input data and act accordingly."""
         self.lineno = 0
         self.index = 0
         self.fam_count = 0
@@ -453,6 +333,7 @@ class CSVParser(object):
         header = None
         line_number = 0
         for row in data:
+            if progress is not None:
                 progress.step()
             line_number += 1
             if "".join(row) == "": # no blanks are allowed inside a table
@@ -460,7 +341,7 @@ class CSVParser(object):
                 continue
             ######################################
             if header is None:
-                    header = [cleanup_column_name(r) for r in row]
+                header = [self.cleanup_column_name(r) for r in row]
                 col = {}
                 count = 0
                 for key in header:
@@ -471,7 +352,17 @@ class CSVParser(object):
             if (("marriage" in header) or
                 ("husband" in header) or
                 ("wife" in header)):
-                    # marriage, husband, wife
+                self._parse_marriage(line_number, row, col)
+            elif "family" in header:
+                self._parse_family(line_number, row, col)
+            elif "surname" in header:
+                self._parse_person(line_number, row, col)
+            else:
+                LOG.warn("ignoring line %d" % line_number)
+        return None
+
+    def _parse_marriage(self, line_number, row, col):
+        "Parse the content of a Marriage,Husband,Wife line."
         marriage_ref   = rd(line_number, row, col, "marriage")
         husband  = rd(line_number, row, col, "husband")
         wife     = rd(line_number, row, col, "wife")
@@ -483,7 +374,8 @@ class CSVParser(object):
         husband = self.lookup("person", husband)
         if husband is None and wife is None:
             # might have children, so go ahead and add
-                        LOG.warn("no parents on line %d; adding family anyway" % line_number)
+            LOG.warn("no parents on line %d; adding family anyway" %
+                     line_number)
         family = self.get_or_create_family(marriage_ref, husband, wife)
         # adjust gender, if not already provided
         if husband:
@@ -508,8 +400,9 @@ class CSVParser(object):
             marriagedate = _dp.parse(marriagedate)
         if marriagedate or marriageplace or marriagesource or note:
             # add, if new; replace, if different
-                        new, marriage = self.get_or_create_event(family, gen.lib.EventType.MARRIAGE, 
+            new, marriage = self.get_or_create_event(family,
-                                                                 marriagedate, marriageplace, marriagesource)
+                    gen.lib.EventType.MARRIAGE, marriagedate,
+                    marriageplace, marriagesource)
             if new:
                 mar_ref = gen.lib.EventRef()
                 mar_ref.set_reference_handle(marriage.get_handle())
@@ -521,7 +414,8 @@ class CSVParser(object):
                 previous_notes_list = marriage.get_note_list()
                 updated_note = False
                 for note_handle in previous_notes_list:
-                                previous_note = self.db.get_note_from_handle(note_handle)
+                    previous_note = self.db.get_note_from_handle(
+                            note_handle)
                     if previous_note.type == gen.lib.NoteType.EVENT:
                         previous_text = previous_note.get()
                         if note not in previous_text:
@@ -539,12 +433,14 @@ class CSVParser(object):
                     self.db.add_note(new_note, self.trans)
                     marriage.add_note(new_note.handle)
                 self.db.commit_event(marriage, self.trans)
-                elif "family" in header:
+
-                    # family, child
+    def _parse_family(self, line_number, row, col):
+        "Parse the content of a family line"
         family_ref   = rd(line_number, row, col, "family")
         if family_ref is None:
-                        LOG.warn("no family reference found for family on line %d" % line_number)
+            LOG.warn("no family reference found for family on line %d" %
-                        continue # required
+                     line_number)
+            return # required
         child   = rd(line_number, row, col, "child")
         source  = rd(line_number, row, col, "source")
         note  = rd(line_number, row, col, "note")
@@ -552,18 +448,22 @@ class CSVParser(object):
         child = self.lookup("person", child)
         family = self.lookup("family", family_ref)
         if family is None:
-                        LOG.warn("no matching family reference found for family on line %d" % line_number)
+            LOG.warn("no matching family reference found for family "
-                        continue
+                     "on line %d" % line_number)
+            return
         if child is None:
-                        LOG.warn("no matching child reference found for family on line %d" % line_number)
+            LOG.warn("no matching child reference found for family "
-                        continue
+                     "on line %d" % line_number)
+            return
         # is this child already in this family? If so, don't add
-                    LOG.debug("children: %s", [ref.ref for ref in family.get_child_ref_list()])
+        LOG.debug("children: %s", [ref.ref for ref in
+                                   family.get_child_ref_list()])
         LOG.debug("looking for: %s", child.get_handle())
-                    if child.get_handle() not in [ref.ref for ref in family.get_child_ref_list()]:
+        if child.get_handle() not in [ref.ref for ref in
+                                      family.get_child_ref_list()]:
             # add child to family
-                        LOG.debug("   adding child [%s] to family [%s]", child.get_gramps_id(), 
+            LOG.debug("   adding child [%s] to family [%s]",
-                                  family.get_gramps_id())
+                      child.get_gramps_id(), family.get_gramps_id())
             childref = gen.lib.ChildRef()
             childref.set_reference_handle(child.get_handle())
             family.add_child_ref( childref)
@@ -581,11 +481,12 @@ class CSVParser(object):
             child.set_gender(gender)
         if source:
             # add, if new
-                        new, source = self.get_or_create_source(source)
+            dummy_new, source = self.get_or_create_source(source)
             source_refs = child.get_source_references()
             found = 0
             for ref in source_refs:
-                            LOG.debug("child: %s looking for ref: %s", ref.ref, source.get_handle())
+                LOG.debug("child: %s looking for ref: %s", ref.ref,
+                          source.get_handle())
                 if ref.ref == source.get_handle():
                     found = 1
             if not found:
@@ -616,8 +517,9 @@ class CSVParser(object):
                 self.db.add_note(new_note, self.trans)
                 child.add_note(new_note.handle)
         self.db.commit_person(child, self.trans)
-                elif "surname" in header:              # person data
+
-                    # surname, and any of the following
+    def _parse_person(self, line_number, row, col):
+        "Parse the content of a Person line."
         surname   = rd(line_number, row, col, "surname")
         firstname = rd(line_number, row, col, "firstname", "")
         callname  = rd(line_number, row, col, "callname")
@@ -647,7 +549,8 @@ class CSVParser(object):
         person = self.lookup("person", person_ref)
         if person is None:
             if surname is None:
-                            LOG.warn("empty surname for new person on line %d" % line_number)
+                LOG.warn("empty surname for new person on line %d" %
+                         line_number)
                 surname = ""
             # new person
             person = self.create_person()
@@ -704,7 +607,8 @@ class CSVParser(object):
         elif person_ref is not None:
             if person_ref.startswith("[") and person_ref.endswith("]"):
                 person.gramps_id = self.db.id2user_format(person_ref[1:-1])
-                    if person.get_gender() == gen.lib.Person.UNKNOWN and gender is not None:
+        if (person.get_gender() == gen.lib.Person.UNKNOWN and
+                gender is not None):
             gender = gender.lower()
             if gender == gender_map[gen.lib.Person.MALE].lower():
                 gender = gen.lib.Person.MALE
@@ -743,7 +647,8 @@ class CSVParser(object):
             new, baptism = self.get_or_create_event(person, 
                  gen.lib.EventType.BAPTISM, baptismdate, 
                  baptismplace, baptismsource)
-                        baptism_ref = get_primary_event_ref_from_type(self.db, person, "Baptism")
+            baptism_ref = get_primary_event_ref_from_type(self.db, person,
+                                                          "Baptism")
             if baptism_ref is None:
                 # new
                 baptism_ref = gen.lib.EventRef()
@@ -757,8 +662,9 @@ class CSVParser(object):
         if deathsource is not None:
             new, deathsource = self.get_or_create_source(deathsource)
         if deathdate or deathplace or deathsource or deathcause:
-                        new, death = self.get_or_create_event(person, gen.lib.EventType.DEATH, 
+            new, death = self.get_or_create_event(person,
-                                                              deathdate, deathplace, deathsource)
+                    gen.lib.EventType.DEATH, deathdate, deathplace,
+                    deathsource)
             if deathcause:
                 death.set_description(deathcause)
                 self.db.commit_event(death, self.trans)
@@ -779,7 +685,8 @@ class CSVParser(object):
             new, burial = self.get_or_create_event(person, 
                  gen.lib.EventType.BURIAL, burialdate, 
                  burialplace, burialsource)
-                        burial_ref = get_primary_event_ref_from_type(self.db, person, "Burial")
+            burial_ref = get_primary_event_ref_from_type(self.db, person,
+                                                         "Burial")
             if burial_ref is None:
                 # new
                 burial_ref = gen.lib.EventRef()
@@ -791,7 +698,8 @@ class CSVParser(object):
             source_refs = person.get_source_references()
             found = 0
             for ref in source_refs:
-                            LOG.debug("person: %s looking for ref: %s", ref.ref, source.get_handle())
+                LOG.debug("person: %s looking for ref: %s", ref.ref,
+                          source.get_handle())
                 if ref.ref == source.get_handle():
                     found = 1
             if not found:
@@ -799,25 +707,14 @@ class CSVParser(object):
                 sref.set_reference_handle(source.get_handle())
                 person.add_source_reference(sref)
         self.db.commit_person(person, self.trans)
-                else:
-                    LOG.warn("ignoring line %d" % line_number)
-            t = time.time() - t
-            msg = ngettext('Import Complete: %d second','Import Complete: %d seconds', t ) % t
-
-        self.db.enable_signals()
-        self.db.request_rebuild()
-        LOG.debug(msg)
-        LOG.debug("New Families: %d" % self.fam_count)
-        LOG.debug("New Individuals: %d" % self.indi_count)
-        progress.close()
-        return None
 
     def get_or_create_family(self, family_ref, husband, wife):
+        "Return the family object for the give family ID."
         # if a gramps_id and exists:
         LOG.debug("get_or_create_family")
         if family_ref.startswith("[") and family_ref.endswith("]"):
-            id = self.db.fid2user_format(family_ref[1:-1])
+            id_ = self.db.fid2user_format(family_ref[1:-1])
-            family = self.db.get_family_from_gramps_id(id)
+            family = self.db.get_family_from_gramps_id(id_)
             if family:
                 # don't delete, only add
                 fam_husband_handle = family.get_father_handle()
@@ -836,8 +733,8 @@ class CSVParser(object):
         family = gen.lib.Family()
         # was marked with a gramps_id, but didn't exist, so we'll use it:
         if family_ref.startswith("[") and family_ref.endswith("]"):
-            id = self.db.fid2user_format(family_ref[1:-1])
+            id_ = self.db.fid2user_format(family_ref[1:-1])
-            family.set_gramps_id(id)
+            family.set_gramps_id(id_)
         # add it:
         family.set_handle(self.db.create_id())
         if husband:
@@ -856,17 +753,19 @@ class CSVParser(object):
         self.fam_count += 1
         return family
         
-    def get_or_create_event(self, object, type, date=None, place=None, source=None):
+    def get_or_create_event(self, object_, type_, date=None, place=None,
+                            source=None):
         """ Add or find a type event on object """
         # first, see if it exists
         LOG.debug("get_or_create_event")
-        ref_list = object.get_event_ref_list()
+        ref_list = object_.get_event_ref_list()
         LOG.debug("refs: %s", ref_list)
         # look for a match, and possible correction
         for ref in ref_list:
             event = self.db.get_event_from_handle(ref.ref)
-            LOG.debug("   compare event type %s == %s", int(event.get_type()), type)
+            LOG.debug("   compare event type %s == %s", int(event.get_type()),
-            if int(event.get_type()) == type:
+                      type_)
+            if int(event.get_type()) == type_:
                 # Match! Let's update
                 if date:
                     event.set_date_object(date)
@@ -876,21 +775,22 @@ class CSVParser(object):
                     source_refs = event.get_source_references()
                     found = 0
                     for ref in source_refs:
-                        LOG.debug("get_or_create_event: %s looking for ref: %s", ref.ref, source.get_handle())
+                        LOG.debug("get_or_create_event: %s looking for ref: %s",
+                                  ref.ref, source.get_handle())
                         if ref.ref == source.get_handle():
                             found = 1
                     if not found:
                         sref = gen.lib.SourceRef()
                         sref.set_reference_handle(source.get_handle())
                         event.add_source_reference(sref)
-                self.db.commit_event(event,self.trans)
+                self.db.commit_event(event, self.trans)
                 LOG.debug("   returning existing event")
                 return (0, event)
         # else create it:
         LOG.debug("   creating event")
         event = gen.lib.Event()
-        if type:
+        if type_:
-            event.set_type(gen.lib.EventType(type))
+            event.set_type(gen.lib.EventType(type_))
         if date:
             event.set_date_object(date)
         if place:
@@ -906,17 +806,18 @@ class CSVParser(object):
                 sref = gen.lib.SourceRef()
                 sref.set_reference_handle(source.get_handle())
                 event.add_source_reference(sref)
-        self.db.add_event(event,self.trans)
+        self.db.add_event(event, self.trans)
         return (1, event)
     
     def create_person(self):
         """ Used to create a new person we know doesn't exist """
         person = gen.lib.Person()
-        self.db.add_person(person,self.trans)
+        self.db.add_person(person, self.trans)
         self.indi_count += 1
         return person
 
-    def get_or_create_place(self,place_name):
+    def get_or_create_place(self, place_name):
+        "Return the requested place object tuple-packed with a new indicator."
         LOG.debug("get_or_create_place: looking for: %s", place_name)
         for place_handle in self.db.iter_place_handles():
             place = self.db.get_place_from_handle(place_handle)
@@ -924,11 +825,12 @@ class CSVParser(object):
                 return (0, place)
         place = gen.lib.Place()
         place.set_title(place_name)
-        self.db.add_place(place,self.trans)
+        self.db.add_place(place, self.trans)
         return (1, place)
 
     def get_or_create_source(self, source_text):
-        source_list = self.db.get_source_handles()
+        "Return the requested source object tuple-packed with a new indicator."
+        source_list = self.db.get_source_handles(sort_handles=False)
         LOG.debug("get_or_create_source: list: %s", source_list)
         LOG.debug("get_or_create_source: looking for: %s", source_text)
         for source_handle in source_list: