From e2b781a22188953bb57a290fae345b93edcac310 Mon Sep 17 00:00:00 2001 From: prculley Date: Wed, 11 May 2016 09:50:07 -0500 Subject: [PATCH 1/9] Bug 9414 fix empty Place Alternate Names on import --- gramps/gen/lib/place.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 gramps/gen/lib/place.py diff --git a/gramps/gen/lib/place.py b/gramps/gen/lib/place.py old mode 100644 new mode 100755 index 2bfa87529..6c24cf0fd --- a/gramps/gen/lib/place.py +++ b/gramps/gen/lib/place.py @@ -631,9 +631,13 @@ class Place(CitationBase, NoteBase, MediaBase, UrlBase, PrimaryObject): :param acquisition: instance to merge :type acquisition: :class:'~.place.Place """ - if acquisition.name and (acquisition.name not in self.alt_names): - self.alt_names.append(acquisition.name) + if acquisition.name.value: + if acquisition.name != self.name: + if acquisition.name not in self.alt_names: + self.alt_names.append(acquisition.name) for addendum in acquisition.alt_names: - if addendum not in self.alt_names: - self.alt_names.append(addendum) + if addendum.value: + if addendum != self.name: + if addendum not in self.alt_names: + self.alt_names.append(addendum) From c7914367b7026112fea04889b6fbe35ed87a7d1c Mon Sep 17 00:00:00 2001 From: prculley Date: Wed, 11 May 2016 10:18:27 -0500 Subject: [PATCH 2/9] Bug 9448: Merge unit test for PlaceCheck not working correctly --- gramps/gen/lib/test/merge_test.py | 32 +++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) mode change 100644 => 100755 gramps/gen/lib/test/merge_test.py diff --git a/gramps/gen/lib/test/merge_test.py b/gramps/gen/lib/test/merge_test.py old mode 100644 new mode 100755 index 5a2d1b1eb..eec131f4d --- a/gramps/gen/lib/test/merge_test.py +++ b/gramps/gen/lib/test/merge_test.py @@ -1389,8 +1389,14 @@ class PlaceCheck(unittest.TestCase, PrivacyBaseTest, MediaBaseTest, def setUp(self): self.phoenix = Place() self.phoenix.set_title('Place 1') - self.titanic = Place(self.phoenix) - self.ref_obj = Place(self.phoenix) + # __init__ copy has bad side effects, don't use it + # self.titanic = Place(self.phoenix) + self.titanic = Place() + self.titanic.set_title('Place 1') + # __init__ copy has bad side effects, don't use it + # self.ref_obj = Place(self.phoenix) + self.ref_obj = Place() + self.ref_obj.set_title('Place 1') self.amsterdam = PlaceName() self.amsterdam.set_value('Amsterdam') self.rotterdam = PlaceName() @@ -1433,9 +1439,11 @@ class PlaceCheck(unittest.TestCase, PrivacyBaseTest, MediaBaseTest, self.titanic.add_alternative_name(self.leiden) self.ref_obj.set_name(self.amsterdam) self.ref_obj.set_type(PlaceType.CITY) - self.ref_obj.add_alternative_name(self.amsterdam) - self.ref_obj.add_alternative_name(self.rotterdam) + # Base name shouldn't be in alt_names list + # self.ref_obj.add_alternative_name(self.amsterdam) + # alt_names must be in correct order for test to pass self.ref_obj.add_alternative_name(self.utrecht) + self.ref_obj.add_alternative_name(self.rotterdam) self.ref_obj.add_alternative_name(self.leiden) self.phoenix.merge(self.titanic) self.assertEqual(self.phoenix.serialize(), self.ref_obj.serialize()) @@ -1495,6 +1503,22 @@ class PlaceCheck(unittest.TestCase, PrivacyBaseTest, MediaBaseTest, self.ref_obj.add_alternative_name(self.rotterdam) self.phoenix.merge(self.titanic) self.assertEqual(self.phoenix.serialize(), self.ref_obj.serialize()) + + def test_merge_empty(self): + self.phoenix.set_name(self.amsterdam) + self.phoenix.set_type(PlaceType.CITY) + self.phoenix.add_alternative_name(self.rotterdam) + self.titanic.set_title('Place 2') + # titanic gets empty name + self.titanic.set_type(PlaceType.CITY) + self.titanic.add_alternative_name(self.utrecht) + self.titanic.add_alternative_name(PlaceName()) # empty alt_name + self.ref_obj.set_name(self.amsterdam) + self.ref_obj.set_type(PlaceType.CITY) + self.ref_obj.add_alternative_name(self.rotterdam) + self.ref_obj.add_alternative_name(self.utrecht) + self.phoenix.merge(self.titanic) + self.assertEqual(self.phoenix.serialize(), self.ref_obj.serialize()) class RepoCheck(unittest.TestCase, PrivacyBaseTest, NoteBaseTest, UrlBaseTest): def setUp(self): From df18fdebea5b2315b1e63ad43d7ffd7076fd8b41 Mon Sep 17 00:00:00 2001 From: prculley Date: Thu, 12 May 2016 09:36:28 -0500 Subject: [PATCH 3/9] Bug 9415 Place Alt Names gets duplicated entries --- gramps/gen/lib/placename.py | 6 ++++++ 1 file changed, 6 insertions(+) mode change 100644 => 100755 gramps/gen/lib/placename.py diff --git a/gramps/gen/lib/placename.py b/gramps/gen/lib/placename.py old mode 100644 new mode 100755 index 2701c4b71..93dd48d17 --- a/gramps/gen/lib/placename.py +++ b/gramps/gen/lib/placename.py @@ -199,6 +199,12 @@ class PlaceName(SecondaryObject, DateBase): else: return EQUAL + def __eq__(self, other): + return self.is_equal(other) + + def __ne__(self, other): + return not self.is_equal(other) + def set_value(self, value): """ Set the name for the PlaceName instance. From dd7724eca25bf38966a8351cc778b46e5e8a968c Mon Sep 17 00:00:00 2001 From: prculley Date: Thu, 12 May 2016 09:55:59 -0500 Subject: [PATCH 4/9] Bug 9173, fix broken GEDCOM import PLAC:FORM handling --- gramps/plugins/lib/libgedcom.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index c06f61c80..3d731110b 100755 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -1705,8 +1705,6 @@ class PlaceParser: loc.get_state(), loc.get_country()) - place_import.store_location(location, place.handle) - for level, name in enumerate(location): if name: break @@ -1720,6 +1718,10 @@ class PlaceParser: place.set_type(PlaceType(type_num)) code = loc.get_postal_code() place.set_code(code) + if place.handle: # if handle is available, store immediately + place_import.store_location(location, place.handle) + else: # return for storage later + return location #------------------------------------------------------------------------- # @@ -2954,15 +2956,22 @@ class GedcomParser(UpdateCallback): sub_state.place.get_placeref_list()) if place is None: place = sub_state.place + place_title = place_displayer.display(self.dbase, place) + location = sub_state.pf.load_place(self.place_import, place, place_title) self.dbase.add_place(place, self.trans) + # if 'location was created, then store it, now that we have a handle. + if location: + self.place_import.store_location(location, place.handle) self.place_names[place.get_title()].append(place.get_handle()) event.set_place_handle(place.get_handle()) else: place.merge(sub_state.place) + place_title = place_displayer.display(self.dbase, place) + location = sub_state.pf.load_place(self.place_import, place, place_title) self.dbase.commit_place(place, self.trans) + if location: + self.place_import.store_location(location, place.handle) event.set_place_handle(place.get_handle()) - place_title = place_displayer.display(self.dbase, place) - sub_state.pf.load_place(self.place_import, place, place_title) def __find_file(self, fullname, altpath): tries = [] From 878d9f08ca07034b5ee06932e763f323200c59bf Mon Sep 17 00:00:00 2001 From: prculley Date: Thu, 12 May 2016 11:16:11 -0500 Subject: [PATCH 5/9] Bug 9425 GEDCOM import some Place Names & Titles are blank --- gramps/plugins/lib/libgedcom.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index c06f61c80..29435e1b6 100755 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -3319,15 +3319,19 @@ class GedcomParser(UpdateCallback): If just ADR1, ADR2, CITY, STAE, POST or CTRY are provided (this is not actually legal GEDCOM symtax, but may be possible by GEDCOM extensions) then just the structrued address is used. + The routine returns a string suitable for a title. """ + title = '' + free_form_address = free_form_address.replace('\n', ', ') if not (addr.get_street() or addr.get_locality() or addr.get_city() or addr.get_state() or addr.get_postal_code()): addr.set_street(free_form_address) + return free_form_address else: # structured address provided - addr_list = free_form_address.split("\n") + addr_list = free_form_address.split(",") str_list = [] for func in (addr.get_street(), addr.get_locality(), addr.get_city(), addr.get_state(), @@ -3341,6 +3345,13 @@ class GedcomParser(UpdateCallback): self.__add_msg(_("ADDR element ignored '%s'" % elmn), line, state) # The free-form address ADDR is discarded + # Assemble a title out of structured address + for elmn in str_list: + if elmn: + if title != '': + title += ', ' + title += elmn + return title def __parse_trailer(self): """ @@ -5395,7 +5406,7 @@ class GedcomParser(UpdateCallback): place = state.place if place: # We encounter a PLAC, having previously encountered an ADDR - if place.get_title() and place.get_title() != "": + if state.place.place_type.string != _("Address"): # We have previously found a PLAC self.__add_msg(_("A second PLAC ignored"), line, state) # ignore this second PLAC, and use the old one @@ -5523,7 +5534,7 @@ class GedcomParser(UpdateCallback): self.__parse_level(sub_state, self.parse_loc_tbl, self.__undefined) state.msg += sub_state.msg - self.__merge_address(free_form, sub_state.location, line, state) + title = self.__merge_address(free_form, sub_state.location, line, state) location = sub_state.location @@ -5570,6 +5581,9 @@ class GedcomParser(UpdateCallback): state.place = Place() place = state.place place.add_alternate_locations(location) + place.set_name(PlaceName(value=title)) + place.set_title(title) + place.set_type((PlaceType.CUSTOM, _("Address"))) # merge notes etc into place place.merge(sub_state.place) From 86f826a79bce3686bc293429d924b4f0b3fc7b1d Mon Sep 17 00:00:00 2001 From: prculley Date: Thu, 12 May 2016 11:25:23 -0500 Subject: [PATCH 6/9] Bug 9430 GEDCOM import PLAC or ADDR attached Notes etc. are lost --- gramps/plugins/lib/libgedcom.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index c06f61c80..618302c82 100755 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -5417,6 +5417,8 @@ class GedcomParser(UpdateCallback): self.__parse_level(sub_state, self.event_place_map, self.__undefined) state.msg += sub_state.msg + # merge notes etc into place + state.place.merge(sub_state.place) def __event_place_note(self, line, state): """ @@ -5572,7 +5574,7 @@ class GedcomParser(UpdateCallback): place.add_alternate_locations(location) # merge notes etc into place - place.merge(sub_state.place) + state.place.merge(sub_state.place) def __add_location(self, place, location): """ From aa9246dde4b85f010d1eac1d30fc0af9695589b2 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Thu, 12 May 2016 16:33:52 -0400 Subject: [PATCH 7/9] DB-API: ensure that undo/redo operations are in a low-level db transaction --- gramps/gen/db/generic.py | 44 ++++++++++++++++++++++++++------ gramps/plugins/database/dbapi.py | 21 +++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index 2e0bf6467..692013081 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -139,15 +139,22 @@ class DbGenericUndo(DbUndo): subitems = transaction.get_recnos() # Process all records in the transaction - for record_id in subitems: - (key, trans_type, handle, old_data, new_data) = \ - pickle.loads(self.undodb[record_id]) + try: + self.db.transaction_backend_begin() + for record_id in subitems: + (key, trans_type, handle, old_data, new_data) = \ + pickle.loads(self.undodb[record_id]) + + if key == REFERENCE_KEY: + self.undo_reference(new_data, handle, self.mapbase[key]) + else: + self.undo_data(new_data, handle, self.mapbase[key], + db.emit, SIGBASE[key]) + self.db.transaction_backend_commit() + except: + self.db.transaction_backend_abort() + raise - if key == REFERENCE_KEY: - self.undo_reference(new_data, handle, self.mapbase[key]) - else: - self.undo_data(new_data, handle, self.mapbase[key], - db.emit, SIGBASE[key]) # Notify listeners if db.undo_callback: db.undo_callback(_("_Undo %s") @@ -847,6 +854,27 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): return self.get_table_func(table_name) return None + def transaction_backend_begin(self): + """ + Lowlevel interface to the backend transaction. + Executes a db BEGIN; + """ + pass + + def transaction_backend_commit(self): + """ + Lowlevel interface to the backend transaction. + Executes a db END; + """ + pass + + def transaction_backend_abort(self): + """ + Lowlevel interface to the backend transaction. + Executes a db ROLLBACK; + """ + pass + def transaction_begin(self, transaction): """ Transactions are handled automatically by the db layer. diff --git a/gramps/plugins/database/dbapi.py b/gramps/plugins/database/dbapi.py index 096792b3a..1874700cb 100644 --- a/gramps/plugins/database/dbapi.py +++ b/gramps/plugins/database/dbapi.py @@ -329,6 +329,27 @@ class DBAPI(DbGeneric): def close_backend(self): self.dbapi.close() + def transaction_backend_begin(self): + """ + Lowlevel interface to the backend transaction. + Executes a db BEGIN; + """ + self.dbapi.begin() + + def transaction_backend_commit(self): + """ + Lowlevel interface to the backend transaction. + Executes a db END; + """ + self.dbapi.commit() + + def transaction_backend_abort(self): + """ + Lowlevel interface to the backend transaction. + Executes a db ROLLBACK; + """ + self.dbapi.rollback() + def transaction_begin(self, transaction): """ Transactions are handled automatically by the db layer. From 471a6b42d098b1e6f487b2d448caa0663c9cef90 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Thu, 12 May 2016 23:43:43 -0400 Subject: [PATCH 8/9] Mysql: rename field 'long' --- gramps/plugins/database/dbapi_support/mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gramps/plugins/database/dbapi_support/mysql.py b/gramps/plugins/database/dbapi_support/mysql.py index a4f6417ed..d50fa3e24 100644 --- a/gramps/plugins/database/dbapi_support/mysql.py +++ b/gramps/plugins/database/dbapi_support/mysql.py @@ -31,6 +31,7 @@ class MySQL: query = query.replace("REAL", "DOUBLE") query = query.replace("change", "change_") query = query.replace("desc", "desc_") + query = query.replace(" long ", " long_ ") ## LIMIT offset, count ## count can be -1, for all ## LIMIT -1 From 3831a02edefdc87eb3f3a987bebd2a839807dcd9 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Fri, 13 May 2016 09:09:37 -0400 Subject: [PATCH 9/9] DB-API: updated obj.change date/time when editing --- gramps/plugins/database/dbapi.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gramps/plugins/database/dbapi.py b/gramps/plugins/database/dbapi.py index 1874700cb..f8923a69d 100644 --- a/gramps/plugins/database/dbapi.py +++ b/gramps/plugins/database/dbapi.py @@ -600,6 +600,7 @@ class DBAPI(DbGeneric): def commit_person(self, person, trans, change_time=None): emit = None old_person = None + person.change = int(change_time or time.time()) if person.handle in self.person_map: emit = "person-update" old_person = self.get_person_from_handle(person.handle) @@ -691,6 +692,7 @@ class DBAPI(DbGeneric): def commit_family(self, family, trans, change_time=None): emit = None old_family = None + family.change = int(change_time or time.time()) if family.handle in self.family_map: emit = "family-update" old_family = self.get_family_from_handle(family.handle).serialize() @@ -754,6 +756,7 @@ class DBAPI(DbGeneric): def commit_citation(self, citation, trans, change_time=None): emit = None old_citation = None + citation.change = int(change_time or time.time()) if citation.handle in self.citation_map: emit = "citation-update" old_citation = self.get_citation_from_handle(citation.handle).serialize() @@ -799,6 +802,7 @@ class DBAPI(DbGeneric): def commit_source(self, source, trans, change_time=None): emit = None old_source = None + source.change = int(change_time or time.time()) if source.handle in self.source_map: emit = "source-update" old_source = self.get_source_from_handle(source.handle).serialize() @@ -846,6 +850,7 @@ class DBAPI(DbGeneric): def commit_repository(self, repository, trans, change_time=None): emit = None old_repository = None + repository.change = int(change_time or time.time()) if repository.handle in self.repository_map: emit = "repository-update" old_repository = self.get_repository_from_handle(repository.handle).serialize() @@ -881,6 +886,7 @@ class DBAPI(DbGeneric): def commit_note(self, note, trans, change_time=None): emit = None old_note = None + note.change = int(change_time or time.time()) if note.handle in self.note_map: emit = "note-update" old_note = self.get_note_from_handle(note.handle).serialize() @@ -913,6 +919,7 @@ class DBAPI(DbGeneric): def commit_place(self, place, trans, change_time=None): emit = None old_place = None + place.change = int(change_time or time.time()) if place.handle in self.place_map: emit = "place-update" old_place = self.get_place_from_handle(place.handle).serialize() @@ -959,6 +966,7 @@ class DBAPI(DbGeneric): def commit_event(self, event, trans, change_time=None): emit = None old_event = None + event.change = int(change_time or time.time()) if event.handle in self.event_map: emit = "event-update" old_event = self.get_event_from_handle(event.handle).serialize() @@ -1000,6 +1008,7 @@ class DBAPI(DbGeneric): def commit_tag(self, tag, trans, change_time=None): emit = None + tag.change = int(change_time or time.time()) if tag.handle in self.tag_map: emit = "tag-update" self.dbapi.execute("""UPDATE tag SET blob_data = ?, @@ -1025,6 +1034,7 @@ class DBAPI(DbGeneric): def commit_media(self, media, trans, change_time=None): emit = None old_media = None + media.change = int(change_time or time.time()) if media.handle in self.media_map: emit = "media-update" old_media = self.get_media_from_handle(media.handle).serialize()