From 7dfe9ebe05ea816ed28bca570527b5ab51c6c581 Mon Sep 17 00:00:00 2001 From: Richard Taylor Date: Fri, 16 Dec 2005 11:59:13 +0000 Subject: [PATCH] further work to complete the reference_map implementation. svn: r5563 --- ChangeLog | 7 +++ src/GrampsBSDDB.py | 63 ++++++++++++++++++++ src/GrampsDbBase.py | 2 + test/GrampsDbBase_Test.py | 118 ++++++++++++++++++++++++++++++++++++++ test/RunAllTests.py | 54 +++++++++++++++++ 5 files changed, 244 insertions(+) create mode 100644 test/GrampsDbBase_Test.py create mode 100644 test/RunAllTests.py diff --git a/ChangeLog b/ChangeLog index 5fcdaec0d..9e96561f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2005-12-16 Richard Taylor + * src/GrampsBSDDB.py: reindex_reference_map added to rebuild the + reference_map when upgrading database + * test/RunAllTests.py: script to run multiple unittests + * test/GrampsDbBase_Test.py: unittest to test reference_map table + implementation. + 2005-12-15 Don Allingham * src/DisplayState.py: Window management completed * src/ViewManger.py: progress bar added back in diff --git a/src/GrampsBSDDB.py b/src/GrampsBSDDB.py index f8f1763bf..9209209d6 100644 --- a/src/GrampsBSDDB.py +++ b/src/GrampsBSDDB.py @@ -554,7 +554,63 @@ class GrampsBSDDB(GrampsDbBase): for (ref_class_name,ref_handle) in no_longer_required_references: self.reference_map.delete(str((handle,ref_handle),)) + def reindex_reference_map(self): + """Reindex all primary records in the database. This will be a + slow process for large databases. + At present this method does not clear the reference_map before it + reindexes. This is fine when if reindex is run to index new content or + when upgrading from a non-reference_map version of the database. But it + might be a problem if reindex is used to repair a broken index because any + references to primary objects that are no longer in the database will + remain in the reference_map index. So if you want to reindex for repair + purposes you need to clear the reference_map first. + """ + + + # Make a dictionary of the functions and classes that we need for + # each of the primary object tables. + primary_tables = {'Person': {'cursor_func': self.get_person_cursor, + 'class_func': Person}, + 'Family': {'cursor_func': self.get_family_cursor, + 'class_func': Family}, + 'Event': {'cursor_func': self.get_event_cursor, + 'class_func': Event}, + 'Place': {'cursor_func': self.get_place_cursor, + 'class_func': Place}, + 'Source': {'cursor_func': self.get_source_cursor, + 'class_func': Source}, + 'MediaObject': {'cursor_func': self.get_media_cursor, + 'class_func': MediaObject}, + 'Repository': {'cursor_func': self.get_repository_cursor, + 'class_func': Repository}, + } + + # Now we use the functions and classes defined above to loop through each of the + # primary object tables. + for primary_table_name in primary_tables.keys(): + + cursor = primary_tables[primary_table_name]['cursor_func']() + data = cursor.first() + + # Grap the real object class here so that the lookup does + # not happen inside the main loop. + class_func = primary_tables[primary_table_name]['class_func'] + + while data: + found_handle,val = data + obj = class_func() + obj.unserialize(val) + + self._update_reference_map(obj,primary_table_name) + + data = cursor.next() + + cursor.close() + + return + + def abort_changes(self): while self.undo(): pass @@ -682,6 +738,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(PERSON_KEY,handle,person.serialize()) self.emit('person-delete',([str(handle)],)) self.person_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_source(self,handle,transaction): if not self.readonly and handle and str(handle) in self.source_map: @@ -690,6 +747,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(SOURCE_KEY,handle,old_data) self.emit('source-delete',([handle],)) self.source_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_repository(self,handle,transaction): if not self.readonly and handle and str(handle) in self.repository_map: @@ -698,6 +756,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(REPOSITORY_KEY,handle,old_data) self.emit('repository-delete',([handle],)) self.repository_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_family(self,handle,transaction): if not self.readonly and handle and str(handle) in self.family_map: @@ -706,6 +765,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(FAMILY_KEY,handle,old_data) self.emit('family-delete',([str(handle)],)) self.family_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_event(self,handle,transaction): if not self.readonly and handle and str(handle) in self.event_map: @@ -714,6 +774,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(EVENT_KEY,handle,old_data) self.emit('event-delete',([str(handle)],)) self.event_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_place(self,handle,transaction): if not self.readonly and handle and str(handle) in self.place_map: @@ -722,6 +783,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(PLACE_KEY,handle,old_data) self.emit('place-delete',([handle],)) self.place_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def remove_object(self,handle,transaction): if not self.readonly and handle and str(handle) in self.media_map: @@ -730,6 +792,7 @@ class GrampsBSDDB(GrampsDbBase): transaction.add(MEDIA_KEY,handle,old_data) self.emit('media-delete',([handle],)) self.media_map.delete(str(handle)) + self._delete_primary_from_reference_map(handle) def get_person_from_gramps_id(self,val): """finds a Person in the database from the passed gramps' ID. diff --git a/src/GrampsDbBase.py b/src/GrampsDbBase.py index b717ba9da..a57e3bf55 100644 --- a/src/GrampsDbBase.py +++ b/src/GrampsDbBase.py @@ -1020,6 +1020,7 @@ class GrampsDbBase(GrampsDBCallback.GrampsDBCallback): """ Commits the transaction to the assocated UNDO database. """ + if self.__LOG_ALL: log("%s: Transaction commit '%s'\n" % (self.__class__.__name__, str(msg))) if not len(transaction) or self.readonly: @@ -1238,6 +1239,7 @@ class GrampsDbBase(GrampsDBCallback.GrampsDBCallback): database, preserving the change in the passed transaction. This method must be overridden in the derived class. """ + if not self.readonly: person = self.get_person_from_handle(handle) self.genderStats.uncount_person (person) diff --git a/test/GrampsDbBase_Test.py b/test/GrampsDbBase_Test.py new file mode 100644 index 000000000..ae82a83d2 --- /dev/null +++ b/test/GrampsDbBase_Test.py @@ -0,0 +1,118 @@ +import unittest +import logging +import os +import tempfile +import shutil + +import sys +sys.path.append('../src') + +import GrampsBSDDB +import RelLib + +logger = logging.getLogger('Gramps.GrampsDbBase_Test') + +class ReferenceMapTest (unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._filename = os.path.join(self._tmpdir,'test.grdb') + + self._db = GrampsBSDDB.GrampsBSDDB() + self._db.load(self._filename, + None, # callback + "w") + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + + def _add_person_and_source(self): + # Add a Source + + tran = self._db.transaction_begin() + source = RelLib.Source() + self._db.add_source(source,tran) + self._db.commit_source(source,tran) + self._db.transaction_commit(tran, "Add Source") + + src_ref = RelLib.SourceRef() + src_ref.set_base_handle(source.get_handle()) + + # Add Person with reference to the Source + + tran = self._db.transaction_begin() + person = RelLib.Person() + + person.add_source_reference(src_ref) + self._db.add_person(person,tran) + self._db.commit_person(person,tran) + self._db.transaction_commit(tran, "Add Person") + + return (person,source) + + def test_simple_lookup(self): + """insert a record and a reference and check that + a lookup for the reference returns the original + record.""" + + person,source = self._add_person_and_source() + + references = [ ref for ref in self._db.find_backlink_handles(source.get_handle()) ] + + assert len(references) == 1 + assert references[0] == ('Person',person.get_handle()) + + def test_delete_primary(self): + """check that deleting a primary will remove the backreferences + from the reference_map""" + + person,source = self._add_person_and_source() + + assert self._db.get_person_from_handle(person.get_handle()) is not None + + tran = self._db.transaction_begin() + self._db.remove_person(person.get_handle(),tran) + self._db.transaction_commit(tran, "Del Person") + + assert self._db.get_person_from_handle(person.get_handle()) == None + + references = [ ref for ref in self._db.find_backlink_handles(source.get_handle()) ] + + assert len(references) == 0, "len(references) == %s " % str(len(references)) + + + def test_reindex_reference_map(self): + """Test that the reindex function works.""" + + # unhook the reference_map update function so that we + # can insert some records without the reference_map being updated. + update_method = self._db._update_reference_map + self._db._update_reference_map = lambda x,y: 1 + + # Insert a person/source pair. + person,source = self._add_person_and_source() + + # Check that the reference map does not contain the reference. + references = [ ref for ref in self._db.find_backlink_handles(source.get_handle()) ] + + assert len(references) == 0, "len(references) == %s " % str(len(references)) + + # Reinstate the reference_map method and reindex the database + self._db._update_reference_map = update_method + self._db.reindex_reference_map() + + # Check that the reference now appears in the reference_map + references = [ ref for ref in self._db.find_backlink_handles(source.get_handle()) ] + + assert len(references) == 1, "len(references) == %s " % str(len(references)) + + + + + +def testSuite(): + return unittest.makeSuite(ReferenceMapTest,'test') + +if __name__ == '__main__': + unittest.TextTestRunner().run(testSuite()) diff --git a/test/RunAllTests.py b/test/RunAllTests.py new file mode 100644 index 000000000..ffd0c3ab7 --- /dev/null +++ b/test/RunAllTests.py @@ -0,0 +1,54 @@ +"""Copyright QinetiQ Ltd (2005) - See LICENSE.TXT for licensing information. + +""" + +import logging + +import os +import unittest +from optparse import OptionParser + +def make_parser(): + usage = "usage: %prog [options]" + parser = OptionParser(usage) + parser.add_option("-v", "--verbosity", type="int", dest="verbose_level", default=0, + help="Level of verboseness") + return parser + + +def getTestSuites(): + + test_modules = [ i for i in os.listdir('.') if i[-8:] == "_Test.py" ] + + test_suites = [] + for module in test_modules: + mod = __import__(module[:-3]) + test_suites.append(mod.testSuite()) + + return test_suites + +def allTheTests(): + return unittest.TestSuite(getTestSuites()) + +if __name__ == '__main__': + + console = logging.StreamHandler() + console.setLevel(logging.INFO) + console.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) + + logger = logging.getLogger('Gramps') + logger.addHandler(console) + + (options,args) = make_parser().parse_args() + + if options.verbose_level == 1: + logger.setLevel(logging.INFO) + elif options.verbose_level >= 2: + logger.setLevel(logging.DEBUG) + os.environ['GRAMPS_SIGNAL'] = "1" + elif options.verbose_level >= 3: + logger.setLevel(logging.NOTSET) + else: + logger.setLevel(logging.ERROR) + + unittest.TextTestRunner(verbosity=options.verbose_level).run(allTheTests())