From 50e4fc1222f24e5624350c71165b157edc5f5b21 Mon Sep 17 00:00:00 2001 From: prculley Date: Mon, 29 Aug 2016 11:04:34 -0500 Subject: [PATCH] Bug 9665 Gedcom import exception processing fixes --- data/tests/imp_notetest_dfs.ged | 227 +++++++++++++ data/tests/imp_notetest_dfs.gramps | 489 +++++++++++++++++++++++++++++ gramps/plugins/lib/libgedcom.py | 42 ++- 3 files changed, 735 insertions(+), 23 deletions(-) create mode 100644 data/tests/imp_notetest_dfs.ged create mode 100644 data/tests/imp_notetest_dfs.gramps diff --git a/data/tests/imp_notetest_dfs.ged b/data/tests/imp_notetest_dfs.ged new file mode 100644 index 000000000..08d8612d2 --- /dev/null +++ b/data/tests/imp_notetest_dfs.ged @@ -0,0 +1,227 @@ +0 HEAD +1 SOUR GEDitCOM +2 NAME GEDitCOM +2 VERS 2.9.4 +2 CORP RSAC Software +1 SUBM @SUBMITTER@ +1 SUBN @SUBMISSION@ +1 DEST ANSTFILE +1 DATE 1 JAN 1998 +2 TIME 13:57:24.80 +1 FILE imp_notetest.ged +1 COPR Tom Tester 2016 +1 GEDC +2 VERS 5.5 +2 FORM LINEAGE-LINKED +1 LANG English +1 NOTE Header note +2 TEST Header Note +1 NOTE +2 TEST Empty Note +1 CHAR UTF8 +0 @SUBMISSION@ SUBN +1 SUBM @SUBMITTER@ +1 FAMF NameOfFamilyFile +1 NOTE Submission Note +2 TEST Submission Note +1 NOTE +2 TEST Empty Note +1 TEST submission +1 NOTE @N2@ +0 @N2@ NOTE Submission xref note +1 RIN Submission Note RIN +1 REFN Submission Note REFN +2 TYPE Submission Note REFN TYPE +1 SOUR Submission note source +1 CHAN +2 DATE 11 Jan 2001 +3 TIME 16:00:06 +1 TEST on XREF Note +0 @SUBMITTER@ SUBM +1 NAME John A. Tester +1 NOTE Submitter Note +2 TEST Submitter Note +1 NOTE +2 TEST Empty Note +1 TEST Submitter +1 NOTE @N3@ +0 @N3@ NOTE Submitter xref note +0 @I1@ INDI +1 NAME Tom /Tester/ +1 TEST person +1 SEX M +1 BIRT +2 NOTE +3 NOTE Empty note subordinate (Should be skipped) +3 TEST Empty Note +2 NOTE Birth Event note +3 TEST 123456 Event Note +2 DATE 15 JUN 1900 +2 ADDR 123 main, Norwalk, Ohio, USA +3 NOTE Location Note +4 TEST Location Note +3 TEST Location +3 NOTE Location Note 2 +1 DEAT +2 NOTE +3 TEST Empty Note +2 NOTE +3 TEST Empty Note +2 NOTE @N5@ +1 FAMS @F1@ +2 NOTE FAMS Note +3 TEST FAMS Note +4 TEST skip on FAMS Note +2 TEST FAMS +2 NOTE FAMS Note 2 +1 NOTE @N4@ +1 NOTE Tom Tester Note +2 TEST Person Note +1 OBJE +2 FORM URL +2 TEST media +2 TITL GEDCOM 5.5 documentation web site +2 FILE http://homepages.rootsweb.com/~pmcbride/gedcom/55gctoc.htm +2 NOTE +3 TEST Empty Note +2 NOTE Media Note +3 TEST 123456 Note +1 RIN 123456 Person +1 REFN 98765 for PERSON +2 TEST REFN +2 TYPE Who knows OBJE REFN TYPE +1 CHAN +2 DATE 11 Jan 2001 +3 TIME 16:00:06 +2 NOTE @N6@ +2 NOTE +3 TEST Empty Note +1 OBJE @M1@ +0 @N4@ NOTE Tom Tester xref note +0 @N5@ NOTE Death Event xref note +0 @N6@ NOTE Media xref note +1 RIN 123456 +1 REFN 98765 +2 TYPE Who knows REFN TYPE +1 CHAN +2 DATE 11 Jan 2001 +3 TIME 16:00:06 +2 NOTE Note on a change on a note!!! +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +0 @M1@ OBJE +1 FILE photo.jpg +2 FORM jpeg +3 TYPE Film +2 TITL Tom Tester's photo +1 RIN 123456 +1 REFN 98765 +2 TYPE Who knows REFN TYPE +1 CHAN +2 DATE 11 Jan 2001 +3 TIME 16:00:06 +0 @I2@ INDI +1 NAME Mrs /Tester/ +1 SEX F +1 BIRT +2 DATE 15 JUN 1901 +1 DEAT +2 DATE 5 JUL 1975 +1 FAMS @F1@ +2 NOTE +3 TEST Empty Note +2 NOTE Family Spouse reference Note +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +0 @I3@ INDI +1 NAME Ed /Tester/ +2 NICK Eddie +2 NOTE +3 TEST Empty Note +2 NOTE Name note +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +1 SEX M +1 BIRT +2 DATE 15 JUN 1922 +1 DEAT +2 DATE 5 JUL 1994 +1 FAMC @F1@ +1 BAPL +2 DATE 5 MAY 0005 B.C. +2 PLAC Salt Lake City +3 NOTE Place note +2 STAT Cleared +2 TEMP SLAKE +2 SOUR @SOURCE1@ +2 NOTE @N8@ +1 ASSO @I4@ +2 NOTE Association link note +3 TEST Accociation note +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +2 NOTE +3 TEST Empty Note +0 @I4@ INDI +1 NAME George /Testee/ +2 NOTE Just for association +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +0 @F1@ FAM +1 HUSB @I1@ +1 TEST family +1 WIFE @I2@ +1 CHIL @I3@ +1 SOUR @S1@ +1 NOTE @N7@ +1 NOTE Family note +2 TEST Family Note +2 CHAN +3 DATE 11 Jan 2001 +4 TIME 16:00:06 +1 NOTE +2 TEST Empty Note +1 SOUR @S1@ +2 TEST citation +2 PAGE 42 +2 DATA +3 DATE 31 DEC 1900 +3 TEST Citation Data +3 NOTE Citation Data Note +4 TEST Citation Data Note +3 TEXT A sample text from a source of this family +2 QUAY 0 +2 NOTE A note this citation is on the FAMILY record. +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +0 @N7@ NOTE Family xref note +0 @S1@ SOUR +1 TITL Note Test file Source: Tester +1 TEST source +1 REPO @R1@ +2 NOTE +3 TEST Empty Note +2 NOTE A short note about the repository link. +3 CHAN +4 DATE 11 Jan 2001 +5 TIME 16:00:06 +1 NOTE +2 TEST Empty Note +1 NOTE note embedded in the SOURCE Record +2 CHAN +3 DATE 11 Jan 2001 +4 TIME 16:00:06 +0 @N8@ NOTE LDS xref note +0 @R1@ REPO +1 NAME The Testers repository +1 TEST Repo +1 NOTE +2 TEST Empty Note +1 NOTE Repository Note +0 TRLR diff --git a/data/tests/imp_notetest_dfs.gramps b/data/tests/imp_notetest_dfs.gramps new file mode 100644 index 000000000..0fea2a86b --- /dev/null +++ b/data/tests/imp_notetest_dfs.gramps @@ -0,0 +1,489 @@ + + + +
+ + + John A. Tester + +
+ + + TEST + person + + + Birth + + + + + + Death + + + + Who knows OBJE REFN TYPE + REFN + + + Birth + + + + Death + + + + Birth + + + + Death + + + + TEST + family + + + + + M + + Tom + Tester + + + + + + + + + + + + + + + + + + + F + + Mrs + Tester + + + + + + + + + + M + + Ed + Tester + Eddie + + + + + + + + + + + + + + + + + + + + + U + + George + Testee + + + + + + + + + + + + + + + + + + + + + + 2 + + + + 2 + + + + 2 + + + + 2 + + + + 2 + + + + 2 + + + + 2 + + + + + 42 + 0 + + + + + + + + + Import from imp_notetest.ged + Tom Tester 2016 + + + + + + + + + + + + + + + + + + @SOURCE1@ + + + Note Test file Source: Tester + + + + + + + + + + 123 main, Norwalk, Ohio, USA + + + + + + + Salt Lake City + + + + + + + + + + + + + + + + + + Business that produced the product: RSAC Software + GEDCOM data + + + SUBM (Submitter): (@SUBMITTER@) John A. Tester + GEDCOM data +
+
+ + + + The Testers repository + Library + + + + + + + Header note + + + Records not imported into HEAD (header): + +Line ignored as not understood Line 18: 2 TEST Header Note +Empty note ignored Line 19: 1 NOTE +Skipped subordinate line Line 20: 2 TEST Empty Note + + + + Records not imported into Top Level: + +Line ignored as not understood Line 25: 1 NOTE Submission Note +Skipped subordinate line Line 26: 2 TEST Submission Note +Line ignored as not understood Line 27: 1 NOTE +Skipped subordinate line Line 28: 2 TEST Empty Note +Line ignored as not understood Line 29: 1 TEST submission +Line ignored as not understood Line 30: 1 NOTE N2 + + + + Submission xref note + + + Records not imported into NOTE Gramps ID N0003: + +Tag recognized but not supported Line 32: 1 RIN Submission Note RIN +Tag recognized but not supported Line 33: 1 REFN Submission Note REFN +Skipped subordinate line Line 34: 2 TYPE Submission Note REFN TYPE +Tag recognized but not supported Line 35: 1 SOUR Submission note source +Line ignored as not understood Line 39: 1 TEST on XREF Note + + + + Records not imported into SUBM (Submitter): (@SUBMITTER@) John A. Tester: + +Line ignored as not understood Line 42: 1 NOTE Submitter Note +Skipped subordinate line Line 43: 2 TEST Submitter Note +Line ignored as not understood Line 44: 1 NOTE +Skipped subordinate line Line 45: 2 TEST Empty Note +Line ignored as not understood Line 46: 1 TEST Submitter +Line ignored as not understood Line 47: 1 NOTE N3 + + + + Submitter xref note + + + Birth Event note + + + Location Note + + + Location Note 2 + + + Death Event xref note + + + FAMS Note + + + FAMS Note 2 + + + Tom Tester xref note + + + Tom Tester Note + + + Media Note + + + Records not imported into INDI (individual) Gramps ID I0001: + +Empty event note ignored Line 54: 2 NOTE +Skipped subordinate line Line 55: 3 NOTE Empty note subordinate (Should be skipped) +Skipped subordinate line Line 56: 3 TEST Empty Note +Skipped subordinate line Line 58: 3 TEST 123456 Event Note +Line ignored as not understood Line 62: 4 TEST Location Note +Line ignored as not understood Line 63: 3 TEST Location +Empty event note ignored Line 66: 2 NOTE +Skipped subordinate line Line 67: 3 TEST Empty Note +Empty event note ignored Line 68: 2 NOTE +Skipped subordinate line Line 69: 3 TEST Empty Note +Line ignored as not understood Line 73: 3 TEST FAMS Note +Skipped subordinate line Line 74: 4 TEST skip on FAMS Note +Line ignored as not understood Line 75: 2 TEST FAMS +Line ignored as not understood Line 79: 2 TEST Person Note +Line ignored as not understood Line 82: 2 TEST media +Empty note ignored Line 85: 2 NOTE +Skipped subordinate line Line 86: 3 TEST Empty Note +Line ignored as not understood Line 88: 3 TEST 123456 Note +Line ignored as not understood Line 96: 2 NOTE N6 +Skipped subordinate line Line 98: 3 TEST Empty Note + + + + Media xref note + + + Records not imported into NOTE Gramps ID N0017: + +Tag recognized but not supported Line 103: 1 RIN 123456 +Tag recognized but not supported Line 104: 1 REFN 98765 +Skipped subordinate line Line 105: 2 TYPE Who knows REFN TYPE +Skipped subordinate line Line 110: 3 CHAN +Skipped subordinate line Line 111: 4 DATE 2001-01-11 +Skipped subordinate line Line 112: 5 TIME 16:00:06 + + + + Who knows REFN TYPE + + + Records not imported into OBJE (multi-media object) Gramps ID M1: + +Could not import photo.jpg Line 114: 1 FILE photo.jpg + + + + Family Spouse reference Note + + + Records not imported into INDI (individual) Gramps ID I0002: + +Empty note ignored Line 132: 2 NOTE +Skipped subordinate line Line 133: 3 TEST Empty Note + + + + Name note + + + Place note + + + LDS xref note + + + Association link note + + + Records not imported into INDI (individual) Gramps ID I0003: + +Empty note ignored Line 141: 2 NOTE +Skipped subordinate line Line 142: 3 TEST Empty Note +Line ignored as not understood Line 163: 3 TEST Accociation note +Empty note ignored Line 167: 2 NOTE +Skipped subordinate line Line 168: 3 TEST Empty Note + + + + Just for association + + + Family xref note + + + Family note + + + Citation Data Note + + + A sample text from a source of this family + + + A note this citation is on the FAMILY record. + + + Records not imported into FAM (family) Gramps ID F0001: + +Line ignored as not understood Line 183: 2 TEST Family Note +Empty note ignored Line 187: 1 NOTE +Skipped subordinate line Line 188: 2 TEST Empty Note +Line ignored as not understood Line 190: 2 TEST citation +Line ignored as not understood Line 194: 3 TEST Citation Data +Line ignored as not understood Line 196: 4 TEST Citation Data Note + + + + A short note about the repository link. + + + note embedded in the SOURCE Record + + + Records not imported into SOUR (source) Gramps ID S0001: + +Line ignored as not understood Line 206: 1 TEST source +Empty note ignored Line 208: 2 NOTE +Skipped subordinate line Line 209: 3 TEST Empty Note +Empty note ignored Line 214: 1 NOTE +Skipped subordinate line Line 215: 2 TEST Empty Note + + + + Repository Note + + + Records not imported into REPO (repository) Gramps ID R0002: + +Line ignored as not understood Line 223: 1 TEST Repo +Empty note ignored Line 224: 1 NOTE +Skipped subordinate line Line 225: 2 TEST Empty Note + + + + diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index fb74beade..e39a6e016 100644 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -3147,21 +3147,26 @@ class GedcomParser(UpdateCallback): @param state: The current state @type state: CurrentState """ - self.__not_recognized(line, state.level+1, state) + self.__not_recognized(line, state) def __ignore(self, line, state): """ - Ignores an unsupported tag + Prints a message when an unexpected token is found. If the token is + known, then the line is considered "not supported", otherwise the line + is "not understood". @param line: The current line in GedLine format @type line: GedLine @param state: The current state @type state: CurrentState """ - self.__add_msg(_("Tag recognized but not supported"), line, state) - self.__skip_subordinate_levels(state.level+1, state) + if line.token == TOKEN_UNKNOWN: + self.__add_msg(_("Line ignored as not understood"), line, state) + else: + self.__add_msg(_("Tag recognized but not supported"), line, state) + self.__skip_subordinate_levels(line.level+1, state) - def __not_recognized(self, line, level, state): + def __not_recognized(self, line, state): """ Prints a message when an undefined token is found. All subordinate items to the current item are ignored. @@ -3170,7 +3175,7 @@ class GedcomParser(UpdateCallback): @type level: int """ self.__add_msg(_("Line ignored as not understood"), line, state) - self.__skip_subordinate_levels(level, state) + self.__skip_subordinate_levels(line.level+1, state) def __skip_record(self, line, state): """ @@ -3481,7 +3486,7 @@ class GedcomParser(UpdateCallback): line = self.__get_next_line() if line and line.token != TOKEN_TRLR: state = CurrentState() - self.__not_recognized(line, 0, state) + self.__not_recognized(line, state) self.__check_msgs(_("TRLR (trailer)"), state, None) except TypeError: return @@ -3621,7 +3626,7 @@ class GedcomParser(UpdateCallback): self.__parse_inline_note(line, 1) else: state = CurrentState() - self.__not_recognized(line, 1, state) + self.__not_recognized(line, state) self.__check_msgs(_("Top Level"), state, None) def __parse_level(self, state, __map, default): @@ -5854,12 +5859,12 @@ class GedcomParser(UpdateCallback): # empty: discard, with warning and skip subs # Note: level+2 self.__add_msg(_("Empty event note ignored"), line, state) - self.__skip_subordinate_levels(state.level+2, state) + self.__skip_subordinate_levels(line.level+1, state) else: new_note = Note(line.data) new_note.set_handle(create_id()) self.dbase.add_note(new_note, self.trans) - self.__skip_subordinate_levels(state.level+2, state) + self.__skip_subordinate_levels(line.level+1, state) state.event.add_note(new_note.get_handle()) def __event_source(self, line, state): @@ -6528,15 +6533,6 @@ class GedcomParser(UpdateCallback): """ self.__parse_change(line, state.source, state.level+1, state) - def __source_undef(self, line, state): - """ - @param line: The current line in GedLine format - @type line: GedLine - @param state: The current state - @type state: CurrentState - """ - self.__not_recognized(line, state.level+1, state) - def __source_repo(self, line, state): """ @param line: The current line in GedLine format @@ -7177,7 +7173,7 @@ class GedcomParser(UpdateCallback): self.__parse_note(line, state.place, state.level, state) else: # This causes notes below SUBMitter to be ignored - self.__not_recognized(line, state.level, state) + self.__not_recognized(line, state) def __optional_note(self, line, state): """ @@ -7621,7 +7617,7 @@ class GedcomParser(UpdateCallback): # +1 SOUR @@ {0:M} if not line.data: self.__add_msg(_("Empty note ignored"), line, state) - self.__skip_subordinate_levels(level+1, state) + self.__skip_subordinate_levels(line.level+1, state) else: new_note = Note(line.data) new_note.set_gramps_id(self.nid_map[""]) @@ -7737,7 +7733,7 @@ class GedcomParser(UpdateCallback): elif line.token == TOKEN_UNKNOWN and line.token_text == "ORDI": msg = _("Submission: Ordinance process flag") else: - self.__not_recognized(line, state.level+1, state) + self.__not_recognized(line, state) continue if self.use_def_src and msg != "": @@ -7807,7 +7803,7 @@ class GedcomParser(UpdateCallback): elif line.token == TOKEN_NOTE: self.__skip_subordinate_levels(level+1, state) else: - self.__not_recognized(line, level+1, state) + self.__not_recognized(line, state) # Attempt to convert the values to a valid change time if dobj: