From 8a385af882868712ef5aef741034a2eac6f3ceab Mon Sep 17 00:00:00 2001 From: prculley Date: Wed, 19 Oct 2016 10:09:30 -0500 Subject: [PATCH] bug 9755, fix duplicated Gramps IDs on Gedcom import --- data/tests/imp_sample.ged | 79 ++- data/tests/imp_sample.gramps | 1185 ++++++++++++++++++------------- gramps/plugins/lib/libgedcom.py | 25 +- 3 files changed, 776 insertions(+), 513 deletions(-) diff --git a/data/tests/imp_sample.ged b/data/tests/imp_sample.ged index 6c939a488..459f08241 100644 --- a/data/tests/imp_sample.ged +++ b/data/tests/imp_sample.ged @@ -21,6 +21,7 @@ 1 NAME Anna /Hansdotter/ 2 GIVN Anna 2 SURN Hansdotter +2 NOTE Hans daughter? N0000 1 SEX F 1 BIRT 2 TYPE Birth of Anna Hansdotter @@ -30,14 +31,17 @@ 2 TYPE Death of Anna Hansdotter 2 DATE 29 SEP 1945 2 PLAC Sparks, Washoe Co., NV +2 NOTE Her eulogy was great! N0001 1 FAMS @F3@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Inline note should get ID N0002 0 @I1@ INDI 1 NAME Keith Lloyd /Smith/ 2 GIVN Keith Lloyd 2 SURN Smith +2 NOTE @N7@ 1 SEX M 1 BIRT 2 TYPE Birth of Keith Lloyd Smith @@ -47,6 +51,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Keith Lloyd Smith inline N0003 0 @I10@ INDI 1 NAME Hans Peter /Smith/ 2 GIVN Hans Peter @@ -66,6 +71,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Hans Peter Smith inline N0004 0 @I11@ INDI 1 NAME Hanna /Smith/ 2 GIVN Hanna @@ -79,6 +85,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Hanna Smith inline N0005 0 @I12@ INDI 1 NAME Herman Julius /Nielsen/ 2 GIVN Herman Julius @@ -95,6 +102,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Herman Julius Nielsen N0006 0 @I13@ INDI 1 NAME Evelyn /Michaels/ 2 GIVN Evelyn @@ -107,6 +115,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 NOTE Evelyn Michaels N0007 0 @I14@ INDI 1 NAME Marjorie Lee /Smith/ 2 GIVN Marjorie Lee @@ -246,18 +255,26 @@ 2 TYPE Birth of Hjalmar Smith 2 DATE 31 JAN 1893 2 PLAC Rønne, Bornholm, Denmark +1 BIRT Y 1 DEAT 2 TYPE Death of Hjalmar Smith 2 DATE 25 SEP 1894 2 PLAC Rønne, Bornholm, Denmark +1 DEAT Y 1 FAMC @F3@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 ALIA @I22@ +1 ALIA James /Smith/ +1 NAME Jimmy Smith +1 TITL Sir Jimmy Smith +2 DATE 5 OCT 1874 +2 PLAC Rønne, Bornholm, Denmark 0 @I22@ INDI 1 NAME Martin /Smith/ -2 GIVN Martin 2 SURN Smith +2 GIVN Martin 1 SEX M 1 BIRT 2 TYPE Birth of Martin Smith @@ -275,17 +292,29 @@ 1 FAMS @F2@ 1 NOTE @N0002@ 1 CHAN +1 RESN locked 2 DATE 21 DEC 2007 3 TIME 01:35:26 0 @I23@ INDI +1 NAME Astrid Shermanna /Augusta/ +2 GIVN Astrid Shermanna +2 SURN Augusta +1 NAME Star /Augusta/ +2 GIVN Star +2 SURN Augusta +2 TYPE AKA +2 DATE FROM 1889 TO 1898 1 NAME Astrid Shermanna Augusta /Smith/ 2 GIVN Astrid Shermanna Augusta 2 SURN Smith +2 TYPE MARRIED +2 NPFX Dr. 1 SEX F 1 BIRT 2 TYPE Birth of Astrid Shermanna Augusta Smith 2 DATE 31 JAN 1889 2 PLAC Rønne, Bornholm, Denmark +2 FAMC @F3@ 1 DEAT 2 TYPE Death of Astrid Shermanna Augusta Smith 2 DATE 21 DEC 1963 @@ -318,6 +347,9 @@ 2 DATE 7 DEC 1862 2 PLAC Gladsax, Kristianstad Län, Sweden 1 FAMC @F2@ +2 NOTE A FAMC note +2 PEDI birth +2 SOUR @S2@ 1 FAMS @F3@ 1 CHAN 2 DATE 21 DEC 2007 @@ -326,6 +358,7 @@ 1 NAME Marta /Ericsdotter/ 2 GIVN Marta 2 SURN Ericsdotter +2 _MARN Smith 1 SEX F 1 BIRT 2 TYPE Birth of Marta Ericsdotter @@ -335,6 +368,17 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 FACT Housekeeper +2 TYPE Skills +1 CONL +2 DATE 1790 +1 ENDL +2 DATE 1795 +1 SLGC +2 DATE 1796 +2 PLAC Sweden +3 FORM Country +2 FAMC @F1@ 0 @I26@ INDI 1 NAME Kirsti Marie /Smith/ 2 GIVN Kirsti Marie @@ -532,6 +576,8 @@ 2 GIVN Edwin Michael 2 SURN Smith 2 SOUR @S1@ +3 DATA +4 TEXT Record for Edwin Michael Smith 1 SEX M 1 BIRT 2 TYPE Birth of Edwin Michael Smith @@ -742,6 +788,7 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 _STAT 0 @F11@ FAM 1 HUSB @I3@ 1 WIFE @I28@ @@ -749,9 +796,20 @@ 2 TYPE Marriage of Magnes Smith and Anna Streiffert 2 DATE 24 AUG 1884 2 PLAC Rønne, Bornholm, Denmark +3 MAP +4 LATI N55.1010 +4 LONG E14.7083 +3 OBJE +4 FILE Magnes&Anna_smiths_marr_cert.jpg +3 SOUR @S0@ +2 OBJE +3 FILE Magnes&Anna_smiths_marr_cert.jpg 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 EVEN Celebration +2 TYPE MARB +2 DATE 24 AUG 1883 0 @F12@ FAM 1 HUSB @I18@ 1 WIFE @I34@ @@ -765,6 +823,11 @@ 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 SLGS +2 PLAC Sparks, Washoe Co., NV +1 NCHI 2 +1 OBJE +2 FILE John&Alice_smiths_marr_cert.jpg 0 @F13@ FAM 1 HUSB @I37@ 1 WIFE @I30@ @@ -772,10 +835,18 @@ 2 TYPE Marriage of Edwin Michael Smith and Janice Ann Adams 2 DATE 27 MAY 1995 2 PLAC San Ramon, Conta Costa Co., CA +2 _WITN @I18@ +3 TYPE WITNESS_OF_MARRIAGE 1 ENGA 2 TYPE Engagement of Edwin Michael Smith and Janice Ann Adams 2 DATE 5 OCT 1994 +2 TIME 2:00pm +2 AGNC Lovely ring presentation +2 _WITN George Smorge +2 _PRIV +2 NOTE Description: Engagement of Edwin Michael Smith and Janice Ann Adams 2 PLAC San Francisco, San Francisco Co., CA +2 _UID 123456 1 CHIL @I5@ 1 CHIL @I2@ 1 CHAN @@ -887,11 +958,16 @@ 3 TIME 01:35:26 0 @S1@ SOUR 1 TITL Birth Certificate +1 REPO Invalid REPO Name +2 NOTE Invalid REPO (Name instead of xref) 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 +1 TEXT Source text of Birth cert 0 @S2@ SOUR 1 TITL Birth Records +1 REPO +2 NOTE Invalid REPO (no name) 1 CHAN 2 DATE 21 DEC 2007 3 TIME 01:35:26 @@ -943,4 +1019,5 @@ 0 @N0004@ NOTE But Aunt Martha still keeps the original! 0 @N0005@ NOTE The repository reference from the source is important 0 @N0006@ NOTE Some note on the repo +0 @N7@ NOTE 'Smith': a very common name 0 TRLR diff --git a/data/tests/imp_sample.gramps b/data/tests/imp_sample.gramps index f02190cea..da8dbd302 100644 --- a/data/tests/imp_sample.gramps +++ b/data/tests/imp_sample.gramps @@ -3,536 +3,564 @@ "http://gramps-project.org/xml/1.7.1/grampsxml.dtd">
- + Alex Roitman,,, Not Provided
- + Birth - + Birth of Anna Hansdotter - + Death - + Death of Anna Hansdotter + - + Birth - + Birth of Keith Lloyd Smith - + Birth - + Birth of Hans Peter Smith - + Death - + Death of Hans Peter Smith - + Birth - + Birth of Hanna Smith - + Birth - + Birth of Herman Julius Nielsen - + Death Death of Herman Julius Nielsen - + Birth Birth of Evelyn Michaels - + Birth - + Birth of Marjorie Lee Smith - + Birth - + Birth of Gus Smith - + Death - + Death of Gus Smith - + Birth - + Birth of Jennifer Anderson - + Death - + Death of Jennifer Anderson - + Birth - + Birth of Lillie Harriet Jones - + Death Death of Lillie Harriet Jones - + Birth - + Birth of John Hjalmar Smith - + Birth - + Birth of Eric Lloyd Smith - + Adopted - + Birth - + Birth of Amber Marie Smith - + Christening - + Christening of Amber Marie Smith - + Birth - + Birth of Carl Emil Smith - + Death - + Death of Carl Emil Smith - + Birth - + Birth of Hjalmar Smith - + + Birth + + Death - + Death of Hjalmar Smith - + + Death + + + Nobility Title + + + Sir Jimmy Smith + + Birth - + Birth of Martin Smith - + Death - + Death of Martin Smith - + Baptism - + Baptism of Martin Smith - + + DATE + 2007-12-21 + + + Birth - + Birth of Astrid Shermanna Augusta Smith - + Death - + Death of Astrid Shermanna Augusta Smith - + Birth - + Birth of Gustaf Smith, Sr. - + Death - + Death of Gustaf Smith, Sr. - + Immi - + - + Christening - + Christening of Gustaf Smith, Sr. - + Birth - + Birth of Marta Ericsdotter - + Birth - + Birth of Kirsti Marie Smith - + Death - + Death of Kirsti Marie Smith - + Birth - + Birth of Ingeman Smith - + Birth - + Birth of Anna Streiffert - + Death - + Death of Anna Streiffert - + Birth - + Birth of Craig Peter Smith - + Census Census of Craig Peter Smith - + - + Birth - + Birth of Magnes Smith - + Death - + Death of Magnes Smith - + Birth - + Birth of Janice Ann Adams - + Occupation Retail Manager - + Degree Business Management - + Birth - + Birth of Marjorie Ohman - + Death - + Death of Marjorie Ohman - + Birth - + Birth of Darcy Horne - + Birth - + Birth of Lloyd Smith - + Birth - + Birth of Alice Paula Perkins - + Birth - + Birth of Lars Peter Smith - + Adopted - + Birth - + Birth of Elna Jefferson - + Death - + Death of Elna Jefferson - + Christening - + Christening of Elna Jefferson - + Birth - + Birth of Edwin Michael Smith - + - + Occupation Software Engineer - + - + Education - + Education of Edwin Michael Smith - + Degree B.S.E.E. - + Birth - + Birth of Kerstina Hansdotter - + Death - + Death of Kerstina Hansdotter - + Birth - + Birth of Martin Smith - + Death - + Death of Martin Smith - + Birth - + Birth of Ingeman Smith - + Birth - + Birth of Marjorie Alice Smith - + Birth Birth of Janis Elaine Green - + Birth - + Birth of Mason Michael Smith - + Christening - + Christening of Mason Michael Smith - + Birth Birth of Edwin Willard - + Birth - + Birth of Ingar Smith - + Birth - + Birth of Hjalmar Smith - + Death - + Death of Hjalmar Smith - + Baptism - + Baptism of Hjalmar Smith - + Immi - + - + Birth - + Birth of Emil Smith - + Marriage - + Marriage of Martin Smith and Elna Jefferson - + Marriage - + Marriage of Ingeman Smith and Marta Ericsdotter - + Marriage - + Marriage of Eric Lloyd Smith and Darcy Horne - + Marriage - + Marriage of Magnes Smith and Anna Streiffert + - + + Marriage Banns + + Celebration + + Marriage - + Marriage of John Hjalmar Smith and Alice Paula Perkins - + - + Marriage - + Marriage of Edwin Michael Smith and Janice Ann Adams - + Engagement - + Engagement of Edwin Michael Smith and Janice Ann Adams + + + + - + Marriage Marriage of Martin Smith and Kerstina Hansdotter - + Marriage - + Marriage of Gustaf Smith, Sr. and Anna Hansdotter - + Marriage Marriage of Edwin Willard and Kirsti Marie Smith - + Marriage - + Marriage of Herman Julius Nielsen and Astrid Shermanna Augusta Smith - + Marriage - + Marriage of Hjalmar Smith and Marjorie Ohman - + Marriage Marriage of Gus Smith and Evelyn Michaels - + Marriage - + Marriage of Lloyd Smith and Janis Elaine Green @@ -542,684 +570,777 @@ Anna Hansdotter + - - - + + + + - + M Keith Lloyd Smith + - - + + + - + M Hans Peter Smith - - - - - + + + + + + - + F Hanna Smith - - + + + - + M Herman Julius Nielsen - - - + + + + - + F Evelyn Michaels - - + + + - + F Marjorie Lee Smith - - + + - + M Gus Smith - - - - + + + + - + F Jennifer Anderson - - - + + + - + F Lillie Harriet Jones - - - + + + - + M John Hjalmar Smith - - - + + + + - + M Eric Lloyd Smith - - - - + + + + - + F Amber Marie Smith - - - + + + - + M Carl Emil Smith - - - + + + - + M Hjalmar Smith - - - + + James + Smith + + + Jimmy Smith + + + + + + + + - + M Martin Smith - - - - - - + + + + + + + + - + F + Astrid Shermanna + Augusta + + + Star + Augusta + + + Astrid Shermanna Augusta Smith + Dr. - - - - + + + + - + M Gustaf Smith Sr. - - - - - - + + + + + + + + - + F Marta Ericsdotter - - + + Marta + Smith + + + + + + + + + + + + + + + - + F Kirsti Marie Smith - - - - + + + + - + M Ingeman Smith - - + + - + F Anna Streiffert - - - + + + - + M Craig Peter Smith - - - + + + - + M Magnes Smith - - - - + + + + - + F Janice Ann Adams - - - - + + + + - + F Marjorie Ohman - - - + + + - + F Darcy Horne - - + + - + M Lloyd Smith - - - + + + - + F Alice Paula Perkins - - + + - + M Lars Peter Smith - - - + + + - + F Elna Jefferson - - - - + + + + - + M Edwin Michael Smith - + - - + + - - - - + + + + - + F Kerstina Hansdotter - - - + + + - + M Martin Smith - - - - + + + + - + M Ingeman Smith - - + + - + F Marjorie Alice Smith - - + + - + F Janis Elaine Green - - + + - + M Mason Michael Smith - - - + + + - + M Edwin Willard - - + + - + F Ingar Smith - - + + - + M Hjalmar Smith - - - - - - - + + + + + + + - + M Emil Smith - - + + - + - + - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + - + - - - - + + + + + - + - - - - - - - - - - - - - + + + + + + + - + + + - + - - - - + + + + + - + - - - + + + + + + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + 2 - + - + 2 - + + - + 2 - + + + + 2 + + + + 2 + - - Birth Certificate - - - Birth, Death and Marriage Records - - - - - Marriage Certificae - - - - + Birth Records + + + + + + Birth Certificate + + + + + + + Birth, Death and Marriage Records + + + + + Marriage Certificae + + - + Löderup, Malmöhus Län, Sweden - + Sparks, Washoe Co., NV - + San Francisco, San Francisco Co., CA - + Rønne, Bornholm, Denmark + + - + Gladsax, Kristianstad Län, Sweden - + Reno, Washoe Co., NV - + Hayward, Alameda Co., CA - + Community Presbyterian Church, Danville, CA - + Sweden - + Grostorp, Kristianstad Län, Sweden - + Copenhagen, Denmark - + + Sweden + + + Hoya/Jona/Hoia, Sweden - + Simrishamn, Kristianstad Län, Sweden - + Fremont, Alameda Co., CA - + Denver, Denver Co., CO - + Sacramento, Sacramento Co., CA - + Santa Rosa, Sonoma Co., CA - + San Jose, Santa Clara Co., CA - + UC Berkeley - + Smestorp, Kristianstad Län, Sweden - + Tommarp, Kristianstad Län, Sweden - + Rønne Bornholm, Denmark - + Woodland, Yolo Co., CA - + + Sparks, Washoe Co., NV + + + San Ramon, Conta Costa Co., CA + + + + + + + + - + New York Public Library Library
@@ -1230,7 +1351,14 @@ 11111
- + + Invalid REPO Name + Library + + + Library + + Aunt Martha's Attic Library
@@ -1240,34 +1368,93 @@ USA
- + - + + Hans daughter? N0000 + + + Her eulogy was great! N0001 + + + Inline note should get ID N0002 + + + 'Smith': a very common name + + + Keith Lloyd Smith inline N0003 + + + Hans Peter Smith inline N0004 + + + Hanna Smith inline N0005 + + + Herman Julius Nielsen N0006 + + + Evelyn Michaels N0007 + + BIOGRAPHY Martin was listed as being a Husman, (owning a house as opposed to a farm) in the house records of Gladsax. - + + A FAMC note + + Witness name: John Doe Witness comment: This is a simple test. - + + Record for Edwin Michael Smith + + Witness name: No Name - + BIOGRAPHY Hjalmar sailed from Copenhagen, Denmark on the OSCAR II, 14 November 1912 arriving in New York 27 November 1912. He was seventeen years old. On the ship passenger list his trade was listed as a Blacksmith. He came to Reno, Nevada and lived with his sister Marie for a time before settling in Sparks. He worked for Southern Pacific Railroad as a car inspector for a time, then went to work for Standard Oil Company. He enlisted in the army at Sparks 7 December 1917 and served as a Corporal in the Medical Corp until his discharge 12 August 1919 at the Presidio in San Francisco, California. Both he and Marjorie are buried in the Masonic Memorial Gardens Mausoleum in Reno, he the 30th June 1975, and she the 25th of June 1980. - + + Records not imported into FAM (family) Gramps ID F0011: + +Could not import Magnes&Anna_smiths_marr_cert.jpg Line 802: 3 OBJE +Could not import Magnes&Anna_smiths_marr_cert.jpg Line 805: 2 OBJE + + + + Records not imported into FAM (family) Gramps ID F0012: + +Could not import John&Alice_smiths_marr_cert.jpg Line 829: 1 OBJE + + + But Aunt Martha still keeps the original! - + + Invalid REPO (Name instead of xref) + + + Source text of Birth cert + + + Invalid REPO (no name) + + The repository reference from the source is important - + Some note on the repo diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index 60c4b8089..c83f72793 100644 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -2778,12 +2778,11 @@ class GedcomParser(UpdateCallback): def __find_or_create_note(self, gramps_id): """ - Finds or creates a repository based on the GRAMPS ID. If the ID is + Finds or creates a note based on the GRAMPS ID. If the ID is already used (is in the db), we return the item in the db. Otherwise, - we create a new repository, assign the handle and GRAMPS ID. - - Some GEDCOM "flavors" destroy the specification, and declare the - repository inline instead of in a object. + we create a new note, assign the handle and GRAMPS ID. + If no GRAMPS ID is passed in, we not only make a Note with GID, we + commit it. """ note = Note() if not gramps_id: @@ -3347,7 +3346,7 @@ class GedcomParser(UpdateCallback): if self.use_def_src: repo.set_name(submitter_name) repo.set_handle(create_id()) - repo.set_gramps_id(self.dbase.find_next_repository_gramps_id()) + repo.set_gramps_id(self.rid_map[""]) addr = Address() addr.set_street(state.res.get_address()) @@ -6096,7 +6095,7 @@ class GedcomParser(UpdateCallback): def __source_text(self, line, state): note = Note() note.set(line.data) - gramps_id = self.dbase.find_next_note_gramps_id() + gramps_id = self.nid_map[""] note.set_gramps_id(gramps_id) note.set_type(NoteType.SOURCE_TEXT) self.dbase.add_note(note, self.trans) @@ -6106,7 +6105,7 @@ class GedcomParser(UpdateCallback): def __citation_data_text(self, line, state): note = Note() note.set(line.data) - gramps_id = self.dbase.find_next_note_gramps_id() + gramps_id = self.nid_map[""] note.set_gramps_id(gramps_id) note.set_type(NoteType.SOURCE_TEXT) self.dbase.add_note(note, self.trans) @@ -6123,7 +6122,7 @@ class GedcomParser(UpdateCallback): line.data, [(0, len(line.data))]) note.set_styledtext(StyledText(line.data, [tags])) - gramps_id = self.dbase.find_next_note_gramps_id() + gramps_id = self.nid_map[""] note.set_gramps_id(gramps_id) note.set_type(NoteType.CITATION) self.dbase.add_note(note, self.trans) @@ -6136,7 +6135,7 @@ class GedcomParser(UpdateCallback): """ note = Note() note.set(line.data) - gramps_id = self.dbase.find_next_note_gramps_id() + gramps_id = self.nid_map[""] note.set_gramps_id(gramps_id) note.set_type(_("Citation Justification")) self.dbase.add_note(note, self.trans) @@ -6333,7 +6332,7 @@ class GedcomParser(UpdateCallback): # This format has no repository name. See http://west- # penwith.org.uk/misc/ftmged.htm which points out this is # incorrect - gid = self.dbase.find_next_repository_gramps_id() + gid = self.rid_map[""] repo = self.__find_or_create_repository(gid) self.dbase.commit_repository(repo, self.trans) else: @@ -6347,7 +6346,7 @@ class GedcomParser(UpdateCallback): # non-standard GEDCOM. gid = self.repo2id.get(line.data) if gid is None: - gid = self.dbase.find_next_repository_gramps_id() + gid = self.rid_map[""] repo = self.__find_or_create_repository(gid) self.repo2id[line.data] = repo.get_gramps_id() repo.set_name(line.data) @@ -7524,7 +7523,7 @@ class GedcomParser(UpdateCallback): handle = self.inline_srcs.get(title, create_id()) src = Source() src.handle = handle - src.gramps_id = self.dbase.find_next_source_gramps_id() + src.gramps_id = self.sid_map[""] self.inline_srcs[title] = handle else: src = self.__find_or_create_source(self.sid_map[line.data])