From 975f4dc1180ccdc3e0e2d1fd36a8a6794653b0bc Mon Sep 17 00:00:00 2001 From: Tim G L Lyons Date: Fri, 6 Jan 2012 22:56:08 +0000 Subject: [PATCH] GEPS 023: Updates to Check and Repair, Test Case Generator and repo and Utils Check and Repair tool - new check source and citation reference function, with extensive changes from previous (temporary) check citation references function, - low level in-table duplicate handle check removed as this is superfluous because the main database tables do not use the DB_DUP flag, - Progress meter added for cross table duplicate checking and fixed for all checks, - diagnostic prints added for all checks to indicate success, and for many to indicate details of failures detected. These prints could be simply changed to Log messages if necessary. Comments added to show how checks relate to Test Case Generator test data, - order of checks revised so empty objects are removed first, - fix_encoding of media paths and descriptions modified to remove invalid characters (see change to Utils), - check and remove empty citations added. repo.py remove_citation_references added to fix removal of citations on addresses in Repository records. Utils.py fix_encoding modified to add an option to ignore characters that can't be unicode encoded. TestCaseGenerator - generate families extended to generate family events, - options dialogue updated to reflect available features and to make it clearer, - block transactions options removed as they were by-passed in the code. - progress meter updated to use current function, and to properly update for all options, - signal testing code (that wasn't functional anyway) removed, - tag generating code called when it needed to be, - data error generating code broken up into functions to reflect the functions in the Check and Repair tool that they are designed to test, - various test data functions added for testing cross table duplicates, media encoding errors, missing photos, control characters in notes, empty objects, source and citation references and a few missing broken family links tests, - some fixes for some test cases (check events, person events were marked as birth events) - fix random text so tags don't contain invalid characters and add a styled text option, - ensure that some citations are shared, - remove old redundant commit_transaction function, - media linked to pictures that exist in the Gramps code so that they don't appear as missing images. svn: r18713 --- src/Utils.py | 9 +- src/gen/lib/repo.py | 14 + src/plugins/tool/Check.py | 601 +++++++++++------ src/plugins/tool/TestcaseGenerator.py | 921 ++++++++++++++++++++------ 4 files changed, 1110 insertions(+), 435 deletions(-) diff --git a/src/Utils.py b/src/Utils.py index 63e5ce3be..777803452 100644 --- a/src/Utils.py +++ b/src/Utils.py @@ -124,7 +124,12 @@ def history_broken(): data_recover_msg = _('The data can only be recovered by Undo operation ' 'or by quitting with abandoning changes.') -def fix_encoding(value): +def fix_encoding(value, errors='strict'): + # The errors argument specifies the response when the input string can't be + # converted according to the encoding's rules. Legal values for this + # argument are 'strict' (raise a UnicodeDecodeError exception), 'replace' + # (add U+FFFD, 'REPLACEMENT CHARACTER'), or 'ignore' (just leave the + # character out of the Unicode result). if not isinstance(value, unicode): try: return unicode(value) @@ -136,7 +141,7 @@ def fix_encoding(value): codeset = locale.getpreferredencoding() except: codeset = "UTF-8" - return unicode(value, codeset) + return unicode(value, codeset, errors) else: return value diff --git a/src/gen/lib/repo.py b/src/gen/lib/repo.py index 1125f1803..a684d2dbe 100644 --- a/src/gen/lib/repo.py +++ b/src/gen/lib/repo.py @@ -163,6 +163,20 @@ class Repository(NoteBase, AddressBase, UrlBase, PrimaryObject): return False + def remove_citation_references(self, citation_handle_list): + """ + Remove references to all citation handles in the list in all child + objects. + + Note: the same comment about citationbase in has_citation_reference + applies here too. + + :param citation_handle_list: The list of citation handles to be removed. + :type citation_handle_list: list + """ + for item in self.get_citation_child_list(): + item.remove_citation_references(citation_handle_list) + def replace_citation_references(self, old_handle, new_handle): """ Replace references to citation handles in the list in this object and diff --git a/src/plugins/tool/Check.py b/src/plugins/tool/Check.py index e3363346d..c8f87d062 100644 --- a/src/plugins/tool/Check.py +++ b/src/plugins/tool/Check.py @@ -30,6 +30,7 @@ # python modules # #------------------------------------------------------------------------- +from __future__ import print_function from __future__ import with_statement import os import sys @@ -45,7 +46,7 @@ from collections import defaultdict # #------------------------------------------------------------------------ import logging -log = logging.getLogger(".CheckRepair") +LOG_OBJ = logging.getLogger(".CheckRepair") #------------------------------------------------------------------------- # @@ -74,83 +75,15 @@ from glade import Glade # All except 09, 0A, 0D are replaced with space. strip_dict = dict.fromkeys(range(9)+range(11,13)+range(14, 32), u" ") +# This is assigned so that it can be either LOG_OBJ.debug or print, depending on +# which is deemed to better interface for Gramps +LOG = print + #------------------------------------------------------------------------- # # Low Level repair # #------------------------------------------------------------------------- -def low_level(db): - """ - This is a low-level repair routine. - - It is fixing DB inconsistencies such as duplicates. - Returns a (status, name) tuple. - The boolean status indicates the success of the procedure. - The name indicates the problematic table (empty if status is True). - """ - - for the_map in [('Person', db.person_map), - ('Family', db.family_map), - ('Event', db.event_map), - ('Place', db.place_map), - ('Source', db.source_map), - ('Media', db.media_map), - ('Repository', db.repository_map), - ('Note', db.note_map)]: - - print "Low-level repair: table: %s" % the_map[0] - if _table_low_level(db, the_map[1]): - print "Done." - else: - print "Low-level repair: Problem with table: %s" % the_map[0] - return (False, the_map[0]) - return (True, '') - - -def _table_low_level(db,table): - """ - Low level repair for a given db table. - """ - - handle_list = table.keys() - handle_dict = defaultdict(int) - for key in handle_list: - handle_dict[key] += 1 - - if len(handle_dict) == len(handle_list): - print " No duplicates found for this table" - return True - dup_handles = [handle for handle, count in handle_dict.items() - if count > 1] -# import gen.db - from gen.db import DbBsddbAssocCursor - table_cursor = DbBsddbAssocCursor(table) - for handle in dup_handles: - print " Duplicates found for handle: %s" % handle - try: - ret = table_cursor.set(handle) - except: - print " Failed setting initial cursor." - return False - - for count in range(handle_list.count(handle)-1): - try: - table_cursor.delete() - print " Successfully deleted duplicate #%d" % (count+1) - except: - print " Failed deleting duplicate." - return False - - try: - ret = table_cursor.next_dup() - except: - print " Failed moving the cursor." - return False - - table_cursor.close() - table.sync() - return True - def cross_table_duplicates(db): """ Function to find the presence of identical handles that occur in different @@ -163,13 +96,22 @@ def cross_table_duplicates(db): :returns: the presence of cross table duplicate handles :rtype: bool """ + progress = ProgressMeter(_('Checking Database'),'') + progress.set_pass(_('Looking for cross table duplicates'), 9) + LOG('Looking for cross table duplicates') total_nr_handles = 0 all_handles = set([]) for the_map in [db.person_map, db.family_map, db.event_map, db.place_map, - db.source_map, db.media_map, db.repository_map, db.note_map]: + db.source_map, db.citation_map, db.media_map, db.repository_map, + db.note_map]: handle_list = the_map.keys() total_nr_handles += len(handle_list) all_handles.update(handle_list) + progress.step() + progress.close() + num_errors = total_nr_handles - len(all_handles) + LOG(' OK: No cross table duplicates' if num_errors == 0 else + ' FAIL: Found %d cross table duplicates' % num_errors) return total_nr_handles > len(all_handles) #------------------------------------------------------------------------- @@ -195,19 +137,23 @@ class Check(tool.BatchTool): # As such, we run it before starting the transaction. # We only do this for the dbdir backend. if self.db.__class__.__name__ == 'DbBsddb': - low_level(self.db) if cross_table_duplicates(self.db): Report(uistate, _( "Your family tree contains cross table duplicate handles.\n " "This is bad and can be fixed by making a backup of your\n" "family tree and importing that backup in an empty family\n" "tree. The rest of the checking is skipped, the Check and\n" - "Repair tool should be run anew on this new family tree."), cli) + "Repair tool should be run anew on this new family tree."), + cli) return - with DbTxn(_("Check Integrity"), self.db, batch=True) as trans: self.db.disable_signals() checker = CheckIntegrity(dbstate, uistate, trans) + # start with empty objects, broken links can be corrected below + # then. This is done before fixing encoding and missing photos, + # since otherwise we will be trying to fix empty records which are + # then going to be deleted. + checker.cleanup_empty_objects() checker.fix_encoding() checker.fix_ctrlchars_in_notes() checker.cleanup_missing_photos(cli) @@ -216,8 +162,6 @@ class Check(tool.BatchTool): prev_total = -1 total = 0 - #start with empty objects, broken links can be corrected below then - checker.cleanup_empty_objects() while prev_total != total: prev_total = total @@ -232,8 +176,7 @@ class Check(tool.BatchTool): checker.check_person_references() checker.check_family_references() checker.check_place_references() - checker.check_citation_references() - # FIXME: CITATION should also check source references + checker.check_source_and_citation_references() checker.check_media_references() checker.check_repo_references() checker.check_note_references() @@ -268,6 +211,7 @@ class CheckIntegrity(object): self.invalid_person_references = [] self.invalid_family_references = [] self.invalid_place_references = [] + self.invalid_source_references = [] self.invalid_citation_references = [] self.invalid_repo_references = [] self.invalid_media_references = [] @@ -295,6 +239,7 @@ class CheckIntegrity(object): """ self.progress.set_pass(_('Looking for invalid name format references'), self.db.get_number_of_people()) + LOG('Looking for invalid name format references') deleted_name_formats = [number for (number, name, fmt_str,act) in self.db.name_formats if not act] @@ -339,10 +284,15 @@ class CheckIntegrity(object): self.db.name_formats = _nd.get_name_format(only_custom=True, only_active=False) + if len(self.removed_name_format) == 0: + LOG(' OK: no invalid name formats found found') + def cleanup_duplicate_spouses(self): self.progress.set_pass(_('Looking for duplicate spouses'), self.db.get_number_of_people()) + LOG('Looking for duplicate spouses') + previous_errors = len(self.duplicate_links) for handle in self.db.person_map.keys(): value = self.db.person_map[handle] @@ -358,16 +308,28 @@ class CheckIntegrity(object): self.db.commit_person(p, self.trans) self.progress.step() + if previous_errors == len(self.duplicate_links): + LOG(' OK: no duplicate spouses found') + def fix_encoding(self): self.progress.set_pass(_('Looking for character encoding errors'), self.db.get_number_of_media_objects()) + LOG('Looking for character encoding errors') + error_count = 0 for handle in self.db.media_map.keys(): data = self.db.media_map[handle] if not isinstance(data[2], unicode) or not isinstance(data[4], unicode): obj = self.db.get_object_from_handle(handle) - obj.path = Utils.fix_encoding( obj.path) - obj.desc = Utils.fix_encoding( obj.desc) + obj.path = Utils.fix_encoding( obj.path, errors='ignore') + obj.desc = Utils.fix_encoding( obj.desc, errors='ignore') self.db.commit_media_object(obj, self.trans) + if not isinstance(data[2], unicode): + LOG(' FAIL: encoding error on media object "%s"' + ' path "%s"' % (obj.gramps_id, obj.path)) + if not isinstance(data[2], unicode): + LOG(' FAIL: encoding error on media object "%s"' + ' description "%s"' % (obj.gramps_id, obj.desc)) + error_count += 1 # Once we are here, fix the mime string if not str if not isinstance(data[3], str): obj = self.db.get_object_from_handle(handle) @@ -379,22 +341,34 @@ class CheckIntegrity(object): except: obj.mime = "" self.db.commit_media_object(obj, self.trans) + LOG(' FAIL: encoding error on media object "%s"' + ' mime "%s"' % (obj.desc, obj.mime)) + error_count += 1 self.progress.step() + if error_count == 0: + LOG(' OK: no encoding errors found') def fix_ctrlchars_in_notes(self): self.progress.set_pass(_('Looking for ctrl characters in notes'), self.db.get_number_of_notes()) + LOG('Looking for ctrl characters in notes') + error_count = 0 for handle in self.db.note_map.keys(): note = self.db.get_note_from_handle(handle) stext = note.get_styledtext() old_text = unicode(stext) new_text = old_text.translate(strip_dict) if old_text != new_text: - print "Note=", self.db.note_map[handle][1], " had control character" - # Commit only if ctrl char found. - note.set_styledtext(gen.lib.StyledText(text=new_text, tags=stext.get_tags())) + LOG(' FAIL: control characters found in note "%s"' % + self.db.note_map[handle][1]) + error_count += 1 + # Commit only if ctrl char found. + note.set_styledtext(gen.lib.StyledText(text=new_text, + tags=stext.get_tags())) self.db.commit_note(note, self.trans) self.progress.step() + if error_count == 0: + LOG(' OK: no ctrl characters in notes found') def check_for_broken_family_links(self): # Check persons referenced by the family objects @@ -402,8 +376,9 @@ class CheckIntegrity(object): fhandle_list = self.db.get_family_handles() self.progress.set_pass(_('Looking for broken family links'), len(fhandle_list) + - self.db.get_number_of_people() - ) + self.db.get_number_of_people()) + LOG('Looking for broken family links') + previous_errors = len(self.broken_parent_links + self.broken_links) for family_handle in fhandle_list: family = self.db.get_family_from_handle(family_handle) @@ -412,40 +387,66 @@ class CheckIntegrity(object): if father_handle: father = self.db.get_person_from_handle(father_handle) if not father: - # The person referenced by the father handle does not exist in the database + # The person referenced by the father handle does not exist + # in the database + # This is tested by TestcaseGenerator where the mother is + # "Broken6" family.set_father_handle(None) self.db.commit_family(family, self.trans) self.broken_parent_links.append((father_handle, family_handle)) + LOG(" FAIL: family '%s' father handle '%s'" + " does not exist" % (family.gramps_id, father_handle)) father_handle = None if mother_handle: mother = self.db.get_person_from_handle(mother_handle) if not mother: - # The person referenced by the mother handle does not exist in the database + # The person referenced by the mother handle does not exist + # in the database + # This is tested by TestcaseGenerator where the mother is + # "Broken7" family.set_mother_handle(None) self.db.commit_family(family, self.trans) self.broken_parent_links.append((mother_handle, family_handle)) + LOG(" FAIL: family '%s' mother handle '%s'" + " does not exist" % (family.gramps_id, mother_handle)) mother_handle = None if father_handle and father and \ family_handle not in father.get_family_handle_list(): # The referenced father has no reference back to the family + # This is tested by TestcaseGenerator where the father is + # "Broken1" self.broken_parent_links.append((father_handle, family_handle)) father.add_family_handle(family_handle) self.db.commit_person(father, self.trans) - + LOG(" FAIL: family '%s' father '%s'" + " does not refer back to the family" % + (family.gramps_id, father_handle)) + if mother_handle and mother and \ family_handle not in mother.get_family_handle_list(): - # The referenced mother has no reference back to the family + # The referenced mother has no reference back to the family. + # This is tested by TestcaseGenerator where the father is + # "Broken4" self.broken_parent_links.append((mother_handle, family_handle)) mother.add_family_handle(family_handle) self.db.commit_person(mother, self.trans) + LOG(" FAIL: family '%s' mother '%s'" + " does not refer back to the family" % + (family.gramps_id, mother_handle)) + for child_ref in family.get_child_ref_list(): child_handle = child_ref.ref child = self.db.get_person_from_handle(child_handle) if child: if child_handle in [father_handle, mother_handle]: - # The child is one of the parents: impossible - # Remove such child from the family + # The child is one of the parents: impossible Remove + # such child from the family + # This is tested by TestcaseGenerator where the father + # is "Broken19" + LOG(" FAIL: family '%s' child '%s'" + " is one of the parents" % + (family.gramps_id, child.gramps_id)) family.remove_child_ref(child_ref) self.db.commit_family(family, self.trans) self.broken_links.append((child_handle, family_handle)) @@ -455,12 +456,22 @@ class CheckIntegrity(object): if family_handle not in \ child.get_parent_family_handle_list(): # The referenced child has no reference to the family + # This is tested by TestcaseGenerator where the father + # is "Broken8" + LOG(" FAIL: family '%s' child '%s'" + " has no reference to the family" % + (family.gramps_id, child.gramps_id)) family.remove_child_ref(child_ref) self.db.commit_family(family, self.trans) self.broken_links.append((child_handle, family_handle)) else: # The person referenced by the child handle # does not exist in the database + # This is tested by TestcaseGenerator where the father + # is "Broken20" + LOG(" FAIL: family '%s' child '%s'" + " does not exist in the database" % + (family.gramps_id, child_handle)) family.remove_child_ref(child_ref) self.db.commit_family(family, self.trans) self.broken_links.append((child_handle, family_handle)) @@ -504,6 +515,11 @@ class CheckIntegrity(object): break else: # Person is not a child in the referenced parent family + # This is tested by TestcaseGenerator where the father + # is "Broken9" + LOG(" FAIL: family '%s' person '%s'" + " is not a child in the referenced parent family" % + (family.gramps_id, person.gramps_id)) person.remove_parent_family_handle(par_family_handle) self.db.commit_person(person, self.trans) self.broken_links.append((person_handle,family_handle)) @@ -511,6 +527,11 @@ class CheckIntegrity(object): family = self.db.get_family_from_handle(family_handle) if not family: # The referenced family does not exist in database + # This is tested by TestcaseGenerator where the father + # is "Broken20" + LOG(" FAIL: person '%s' refers to " + "family '%s' which is not in the database" % + (person.gramps_id, family_handle)) person.remove_family_handle(family_handle) self.db.commit_person(person, self.trans) self.broken_links.append((person_handle, family_handle)) @@ -520,15 +541,26 @@ class CheckIntegrity(object): if family.get_mother_handle() == person_handle: continue # The person is not a member of the referenced family + # This is tested by TestcaseGenerator where the father is + # "Broken2" and the family misses the link to the father, and + # where the mother is "Broken3" and the family misses the link + # to the mother + LOG(" FAIL: family '%s' person '%s'" + " is not member of the referenced family" % + (family.gramps_id, person.gramps_id)) person.remove_family_handle(family_handle) self.db.commit_person(person, self.trans) self.broken_links.append((person_handle, family_handle)) self.progress.step() + if previous_errors == len(self.broken_parent_links + self.broken_links): + LOG(' OK: no broken family links found') + def cleanup_missing_photos(self, cl=0): self.progress.set_pass(_('Looking for unused objects'), len(self.db.get_media_object_handles())) + LOG('Looking for missing photos') missmedia_action = 0 #------------------------------------------------------------------------- @@ -559,6 +591,12 @@ class CheckIntegrity(object): source.remove_media_references([ObjectId]) self.db.commit_source(source, self.trans) + for handle in self.db.get_citation_handles(): + citation = self.db.get_citation_from_handle(handle) + if citation.has_media_reference(ObjectId): + citation.remove_media_references([ObjectId]) + self.db.commit_citation(citation, self.trans) + for handle in self.db.get_place_handles(): place = self.db.get_place_from_handle(handle) if place.has_media_reference(ObjectId): @@ -567,14 +605,17 @@ class CheckIntegrity(object): self.removed_photo.append(ObjectId) self.db.remove_object(ObjectId,self.trans) - + LOG(' FAIL: media object and all references to it removed') + def leave_clicked(): self.bad_photo.append(ObjectId) + LOG(' FAIL: references to missing file kept') def select_clicked(): # File is lost => select a file to replace the lost one def fs_close_window(obj): self.bad_photo.append(ObjectId) + LOG(' FAIL: references to missing file kept') def fs_ok_clicked(obj): name = Utils.get_unicode_path_from_file_chooser(fs_top.get_filename()) @@ -583,8 +624,10 @@ class CheckIntegrity(object): obj.set_path(name) self.db.commit_media_object(obj, self.trans) self.replaced_photo.append(ObjectId) + LOG(' FAIL: media object reselected to "%s"' % name) else: self.bad_photo.append(ObjectId) + LOG(' FAIL: references to missing file kept') fs_top = gtk.FileChooserDialog("%s - Gramps" % _("Select file"), buttons=(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, @@ -602,14 +645,17 @@ class CheckIntegrity(object): for ObjectId in self.db.get_media_object_handles(): obj = self.db.get_object_from_handle(ObjectId) photo_name = Utils.media_path_full(self.db, obj.get_path()) + photo_desc = obj.get_description() if photo_name is not None and photo_name != "" and not Utils.find_file(photo_name): if cl: # Convert to file system encoding before prining fn = os.path.basename(photo_name).encode(sys.getfilesystemencoding()) - print "Warning: media file %s was not found." % fn + print ("Warning: media file %s was not found." % fn) self.bad_photo.append(ObjectId) else: if missmedia_action == 0: + LOG(' FAIL: media object "%s" reference to missing ' + 'file "%s" found' % (photo_desc, photo_name)) mmd = MissingMediaDialog(_("Media object could not be found"), _("The file:\n %(file_name)s \nis referenced in the database, but no longer exists. " "The file may have been deleted or moved to a different location. " @@ -619,12 +665,20 @@ class CheckIntegrity(object): remove_clicked, leave_clicked, select_clicked) missmedia_action = mmd.default_action elif missmedia_action == 1: + LOG(' FAIL: media object "%s" reference to missing ' + 'file "%s" found' % (photo_desc, photo_name)) remove_clicked() elif missmedia_action == 2: + LOG(' FAIL: media object "%s" reference to missing ' + 'file "%s" found' % (photo_desc, photo_name)) leave_clicked() elif missmedia_action == 3: + LOG(' FAIL: media object "%s" reference to missing ' + 'file "%s" found' % (photo_desc, photo_name)) select_clicked() self.progress.step() + if len(self.bad_photo + self.removed_photo) == 0: + LOG(' OK: no missing photos found') def cleanup_empty_objects(self): #the position of the change column in the primary objects @@ -632,6 +686,7 @@ class CheckIntegrity(object): CHANGE_FAMILY = 12 CHANGE_EVENT = 10 CHANGE_SOURCE = 8 + CHANGE_CITATION = 9 CHANGE_PLACE = 11 CHANGE_MEDIA = 8 CHANGE_REPOS = 7 @@ -641,6 +696,7 @@ class CheckIntegrity(object): empty_family_data = gen.lib.Family().serialize() empty_event_data = gen.lib.Event().serialize() empty_source_data = gen.lib.Source().serialize() + empty_citation_data = gen.lib.Citation().serialize() empty_place_data = gen.lib.Place().serialize() empty_media_data = gen.lib.MediaObject().serialize() empty_repos_data = gen.lib.Repository().serialize() @@ -697,6 +753,14 @@ class CheckIntegrity(object): _empty(empty_source_data, CHANGE_SOURCE), _db.remove_source, ), + ('citations', + _db.get_citation_from_handle, + _db.get_citation_cursor, + _db.get_number_of_citations, + _('Looking for empty citation records'), + _empty(empty_citation_data, CHANGE_CITATION), + _db.remove_citation, + ), ('places', _db.get_place_from_handle, _db.get_place_cursor, @@ -739,17 +803,22 @@ class CheckIntegrity(object): with cursor_func() as cursor: total = total_func() self.progress.set_pass(text, total) + LOG(text) for handle, data in cursor: self.progress.step() if check_func(data): # we cannot remove here as that would destroy cursor # so save the handles for later removal + LOG(' FAIL: empty %s record with handle "%s" was found' + % (the_type, handle)) self.empty_objects[the_type].append(handle) #now remove for handle in self.empty_objects[the_type]: remove_func(handle, self.trans) + if len(self.empty_objects[the_type]) == 0: + LOG(' OK: no empty %s found' % the_type) def _check_empty(self, data, empty_data, changepos): """compare the data with the data of an empty object @@ -767,6 +836,8 @@ class CheckIntegrity(object): self.progress.set_pass(_('Looking for empty families'), len(fhandle_list)) + LOG('Looking for empty families') + previous_errors = len(self.empty_family) for family_handle in fhandle_list: self.progress.step() @@ -780,6 +851,9 @@ class CheckIntegrity(object): self.empty_family.append(family_id) self.delete_empty_family(family_handle) + if previous_errors == len(self.empty_family): + LOG(' OK: no empty families found') + def delete_empty_family(self, family_handle): for key in self.db.get_person_handles(sort_handles=False): child = self.db.get_person_from_handle(key) @@ -797,6 +871,8 @@ class CheckIntegrity(object): fhandle_list = self.db.get_family_handles() self.progress.set_pass(_('Looking for broken parent relationships'), len(fhandle_list)) + LOG('Looking for broken parent relationships') + previous_errors = len(self.fam_rel) for family_handle in fhandle_list: self.progress.step() @@ -819,15 +895,21 @@ class CheckIntegrity(object): if (fgender == gen.lib.Person.FEMALE or mgender == gen.lib.Person.MALE) and fgender != mgender: # swap. note: (at most) one handle may be None + LOG(' FAIL: the family "%s" has a father=female or ' + ' mother=male in a different sex family' % family.gramps_id) family.set_father_handle(mother_handle) family.set_mother_handle(father_handle) self.db.commit_family(family, self.trans) self.fam_rel.append(family_handle) + if previous_errors == len(self.fam_rel): + LOG(' OK: no broken parent relationships found') + def check_events(self): self.progress.set_pass(_('Looking for event problems'), self.db.get_number_of_people() +self.db.get_number_of_families()) + LOG('Looking for event problems') for key in self.db.get_person_handles(sort_handles=False): self.progress.step() @@ -840,12 +922,20 @@ class CheckIntegrity(object): if not birth: # The birth event referenced by the birth handle # does not exist in the database + # This is tested by TestcaseGenerator person "Broken11" person.set_birth_ref(None) + LOG(' FAIL: the person "%s" refers to a birth event' + ' "%s" which does not exist in the database' % + (person.gramps_id, birth_handle)) self.db.commit_person(person, self.trans) self.invalid_events.append(key) else: if int(birth.get_type()) != gen.lib.EventType.BIRTH: # Birth event was not of the type "Birth" + # This is tested by TestcaseGenerator person "Broken14" + LOG(' FAIL: the person "%s" refers to a birth event' + ' which is of type "%s" instead of Birth' % + (person.gramps_id, int(birth.get_type()))) birth.set_type(gen.lib.EventType(gen.lib.EventType.BIRTH)) self.db.commit_event(birth, self.trans) self.invalid_birth_events.append(key) @@ -856,12 +946,20 @@ class CheckIntegrity(object): if not death: # The death event referenced by the death handle # does not exist in the database + # This is tested by TestcaseGenerator person "Broken12" + LOG(' FAIL: the person "%s" refers to a death event' + ' "%s" which does not exist in the database' % + (person.gramps_id, death_handle)) person.set_death_ref(None) self.db.commit_person(person, self.trans) self.invalid_events.append(key) else: if int(death.get_type()) != gen.lib.EventType.DEATH: # Death event was not of the type "Death" + # This is tested by TestcaseGenerator person "Broken15" + LOG(' FAIL: the person "%s" refers to a death event' + ' which is of type "%s" instead of Death' % + (person.gramps_id, int(death.get_type()))) death.set_type(gen.lib.EventType(gen.lib.EventType.DEATH)) self.db.commit_event(death, self.trans) self.invalid_death_events.append(key) @@ -874,11 +972,19 @@ class CheckIntegrity(object): # The event referenced by the person # does not exist in the database #TODO: There is no better way? + # This is tested by TestcaseGenerator person "Broken11" + # This is tested by TestcaseGenerator person "Broken12" + # This is tested by TestcaseGenerator person "Broken13" + LOG(' FAIL: the person "%s" refers to an event' + ' "%s" which does not exist in the database' % + (person.gramps_id, event_handle)) person.get_event_ref_list().remove(event_ref) self.db.commit_person(person,self.trans) self.invalid_events.append(key) elif not isinstance(person.get_event_ref_list(), list): # event_list is None or other garbage + LOG(' FAIL: the person "%s" has an event ref list' + ' which is invalid' % (person.gramps_id)) person.set_event_ref_list([]) self.db.commit_person(person, self.trans) self.invalid_events.append(key) @@ -893,7 +999,9 @@ class CheckIntegrity(object): if not event: # The event referenced by the family # does not exist in the database - print family.gramps_id + LOG(' FAIL: the family "%s" refers to an event' + ' "%s" which does not exist in the database' % + (family.gramps_id, event_handle)) nlist = [x for x in family.get_event_ref_list() if x.ref != event_handle] family.set_event_ref_list(nlist) @@ -901,17 +1009,25 @@ class CheckIntegrity(object): self.invalid_events.append(key) elif not isinstance(family.get_event_ref_list(), list): # event_list is None or other garbage + LOG(' FAIL: the family "%s" has an event ref list' + ' which is invalid' % (family.gramps_id)) family.set_event_ref_list([]) self.db.commit_family(family, self.trans) self.invalid_events.append(key) + if len (self.invalid_birth_events) + len(self.invalid_death_events) +\ + len(self.invalid_events) == 0: + LOG(' OK: no event problems found') + def check_person_references(self): plist = self.db.get_person_handles() self.progress.set_pass(_('Looking for person reference problems'), len(plist)) + LOG('Looking for person reference problems') for key in plist: + self.progress.step() person = self.db.get_person_from_handle(key) for pref in person.get_person_ref_list(): p = self.db.get_person_from_handle( pref.ref) @@ -921,13 +1037,18 @@ class CheckIntegrity(object): self.db.commit_person(person, self.trans) self.invalid_person_references.append(key) + if len (self.invalid_person_references) == 0: + LOG(' OK: no event problems found') + def check_family_references(self): plist = self.db.get_person_handles() self.progress.set_pass(_('Looking for family reference problems'), len(plist)) + LOG('Looking for family reference problems') for key in plist: + self.progress.step() person = self.db.get_person_from_handle(key) for ordinance in person.get_lds_ord_list(): family_handle = ordinance.get_family_handle() @@ -939,13 +1060,18 @@ class CheckIntegrity(object): self.db.commit_person(person, self.trans) self.invalid_family_references.append(key) + if len (self.invalid_family_references) == 0: + LOG(' OK: no event problems found') + def check_repo_references(self): slist = self.db.get_source_handles() self.progress.set_pass(_('Looking for repository reference problems'), len(slist)) + LOG('Looking for repository reference problems') for key in slist: + self.progress.step() source = self.db.get_source_from_handle(key) for reporef in source.get_reporef_list(): r = self.db.get_repository_from_handle(reporef.ref) @@ -955,14 +1081,20 @@ class CheckIntegrity(object): self.db.commit_source(source, self.trans) self.invalid_repo_references.append(key) + if len (self.invalid_repo_references) == 0: + LOG(' OK: no repository reference problems found') + def check_place_references(self): plist = self.db.get_person_handles() flist = self.db.get_family_handles() elist = self.db.get_event_handles() self.progress.set_pass(_('Looking for place reference problems'), len(elist)+len(plist)+len(flist)) + LOG('Looking for place reference problems') + # check persons -> the LdsOrd references a place for key in plist: + self.progress.step() person = self.db.get_person_from_handle(key) for ordinance in person.lds_ord_list: place_handle = ordinance.get_place_handle() @@ -970,11 +1102,17 @@ class CheckIntegrity(object): place = self.db.get_place_from_handle(place_handle) if not place: # The referenced place does not exist in the database + # This is tested by TestcaseGenerator person "Broken17" + # This is tested by TestcaseGenerator person "Broken18" ordinance.set_place_handle("") + LOG(' FAIL: the person "%s" refers to an LdsOrd' + ' place "%s" which does not exist in the database' % + (person.gramps_id, place_handle)) self.db.commit_person(person, self.trans) self.invalid_place_references.append(key) # check families -> the LdsOrd references a place for key in flist: + self.progress.step() family = self.db.get_family_from_handle(key) for ordinance in family.lds_ord_list: place_handle = ordinance.get_place_handle() @@ -983,10 +1121,14 @@ class CheckIntegrity(object): if not place: # The referenced place does not exist in the database ordinance.set_place_handle("") + LOG(' FAIL: the family "%s" refers to an LdsOrd' + ' place "%s" which does not exist in the database' % + (family.gramps_id, place_handle)) self.db.commit_family(family, self.trans) self.invalid_place_references.append(key) # check events for key in elist: + self.progress.step() event = self.db.get_event_from_handle(key) place_handle = event.get_place_handle() if place_handle: @@ -994,11 +1136,39 @@ class CheckIntegrity(object): if not place: # The referenced place does not exist in the database event.set_place_handle("") + LOG(' FAIL: the event "%s" refers to an LdsOrd place' + ' "%s" which does not exist in the database' % + (event.gramps_id, place_handle)) self.db.commit_event(event, self.trans) self.invalid_place_references.append(key) - def check_citation_references(self): - known_handles = self.db.get_citation_handles() + if len (self.invalid_place_references) == 0: + LOG(' OK: no place reference problems found') + + def check_source_and_citation_references(self): + # We check both source and citations in one pass. If there is a problem + # with a citation reference from an object to a citation, then we need + # to remove the reference from the object. This is the same as any other + # reference check. However, if there is a problem with a source + # reference from a citation to a source, we can't just remove the source + # reference from the citation object (as we would in other cases), + # because all citations must have exactly one source. Therefore we must + # remove the citation as a whole, and also remove the reference to the + # citation from the object. Hence the reason why we do this while we are + # processing the object. + + # bad_citation_handles and invalid_citation_references are citation + # handles which occur in objects and we need to remove these citation + # references from the object. The citation reference needs to be removed + # either because there is no such citation, or because the citation will + # be removed as it doesn't point validly to a source object. + + # invalid_source_references are also citation handles, but these refer + # to real citation object which need to be deleted because the citations + # don't refer to valid sources. + + known_source_handles = self.db.get_source_handles() + good_citation_handles = set() total = ( self.db.get_number_of_people() + @@ -1010,121 +1180,103 @@ class CheckIntegrity(object): self.db.get_number_of_repositories() ) - self.progress.set_pass(_('Looking for citation reference problems'), - total) + self.progress.set_pass(_('Looking for source and citation reference' + ' problems'), total) + LOG('Looking for source and citation reference problems') - for handle in self.db.person_map.keys(): - self.progress.step() - info = self.db.person_map[handle] - person = gen.lib.Person() - person.unserialize(info) - handle_list = person.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - person.remove_citation_references(bad_handles) - self.db.commit_person(person,self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - for handle in self.db.family_map.keys(): - self.progress.step() - info = self.db.family_map[handle] - family = gen.lib.Family() - family.unserialize(info) - handle_list = family.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - family.remove_citation_references(bad_handles) - self.db.commit_family(family, self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - for handle in self.db.place_map.keys(): - self.progress.step() - info = self.db.place_map[handle] - place = gen.lib.Place() - place.unserialize(info) - handle_list = place.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - place.remove_citation_references(bad_handles) - self.db.commit_place(place,self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - for handle in self.db.repository_map.keys(): - self.progress.step() - info = self.db.repository_map[handle] - repo = gen.lib.Repository() - repo.unserialize(info) - handle_list = repo.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - repo.remove_citation_references(bad_handles) - self.db.commit_repository(repo, self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - #I think this for loop is useless! citations in citations map exist... - for handle in known_handles: - self.progress.step() - info = self.db.citation_map[handle] - citation = gen.lib.Citation() - citation.unserialize(info) - handle_list = citation.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - citation.remove_citation_references(bad_handles) - self.db.commit_citation(citation, self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - for handle in self.db.media_map.keys(): - self.progress.step() - info = self.db.media_map[handle] - obj = gen.lib.MediaObject() - obj.unserialize(info) - handle_list = obj.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - obj.remove_citation_references(bad_handles) - self.db.commit_media_object(obj, self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles - - for handle in self.db.event_map.keys(): - self.progress.step() - info = self.db.event_map[handle] - event = gen.lib.Event() - event.unserialize(info) - handle_list = event.get_referenced_handles_recursively() - bad_handles = [ item[1] for item in handle_list - if item[0] == 'Citation' and - item[1] not in known_handles ] - if bad_handles: - event.remove_citation_references(bad_handles) - self.db.commit_event(event, self.trans) - new_bad_handles = [handle for handle in bad_handles if handle - not in self.invalid_citation_references] - self.invalid_citation_references += new_bad_handles + def check(name, map_func, class_func, commit_func): + for handle in map_func.keys(): + self.progress.step() + info = map_func[handle] + obj = class_func() + obj.unserialize(info) + handle_list = obj.get_referenced_handles_recursively() + + bad_citation_handles = set() + bad_citation_text = set() + for classn, handle in handle_list: + if classn == 'Citation': + if not handle: + bad_citation_handles.add(handle) + bad_citation_text.add("None") + else: + citation = self.db.get_citation_from_handle(handle) + if not citation: + bad_citation_handles.add(handle) + bad_citation_text.add(handle) + else: + # The citation is good, check whether the + # source_handle is OK + source_handle = citation.source_handle + if not source_handle or \ + source_handle not in known_source_handles: + bad_citation_handles.add(handle) + bad_citation_text.add(citation.gramps_id + + ": " + citation.page) + if handle not in \ + self.invalid_source_references: + self.invalid_source_references.append( + handle) + else: + good_citation_handles.add(handle) + if bad_citation_handles: + LOG(' FAIL: the %s "%s" refers to citation(s) "%s"' + ' which do not exist in the database' + ' or where the referenced source does not exist' % + (name, obj.gramps_id, + " ".join(h for h in bad_citation_text))) + obj.remove_citation_references(list(bad_citation_handles)) + commit_func(obj,self.trans) + new_bad_handles = [handle for handle in bad_citation_handles + if handle + not in self.invalid_citation_references] + self.invalid_citation_references += new_bad_handles + + check("person", self.db.person_map, gen.lib.Person, + self.db.commit_person) + check("family", self.db.family_map, gen.lib.Family, + self.db.commit_family) + check("event", self.db.event_map, gen.lib.Event, + self.db.commit_event) + check("media object", self.db.media_map, gen.lib.MediaObject, + self.db.commit_media_object) + check("place", self.db.place_map, gen.lib.Place, self.db.commit_place) + check("repository", self.db.repository_map, gen.lib.Repository, + self.db.commit_repository) + # There is no point in checking sources, because they don't have + # citations. + # check("source", self.db.source_map, gen.lib.Source, + # self.db.commit_source) + + # Now we need to check any citations that are not referenced from other + # objects, in case they too have invalid source references. + for handle in self.db.citation_map: + if handle not in good_citation_handles: + citation = self.db.get_citation_from_handle(handle) + source_handle = citation.source_handle + if not source_handle or \ + source_handle not in known_source_handles: + LOG(' FAIL: the citation "%s" refers to a source' + ' "%s" which does not exist in the database' % + (citation.gramps_id, citation.source_handle)) + if handle not in \ + self.invalid_source_references: + self.invalid_source_references.append( + handle) + + # bad citation references in objects have already been removed. Now + # remove any bad citations that were detected. + for citation_handle in self.invalid_source_references: + LOG(' FAIL: the citation "%s" which refers to source handle "%s"' + ' has been removed' % + (self.db.get_citation_from_handle(citation_handle).gramps_id, + self.db.get_citation_from_handle(citation_handle).source_handle)) + self.db.remove_citation(citation_handle, self.trans) + + if len(self.invalid_source_references) + \ + len(self.invalid_citation_references) == 0: + LOG(' OK: no invalid source or citation references found') + return def check_media_references(self): known_handles = self.db.get_media_object_handles(False) @@ -1139,6 +1291,7 @@ class CheckIntegrity(object): self.progress.set_pass(_('Looking for media object reference problems'), total) + LOG('Looking for media object reference problems') for handle in self.db.person_map.keys(): self.progress.step() @@ -1220,6 +1373,9 @@ class CheckIntegrity(object): not in self.invalid_media_references] self.invalid_media_references += new_bad_handles + if len (self.invalid_media_references) == 0: + LOG(' OK: no media reference problems found') + def check_note_references(self): known_handles = self.db.get_note_handles() @@ -1235,6 +1391,7 @@ class CheckIntegrity(object): self.progress.set_pass(_('Looking for note reference problems'), total) + LOG('Looking for note reference problems') for handle in self.db.person_map.keys(): self.progress.step() @@ -1350,6 +1507,9 @@ class CheckIntegrity(object): not in self.invalid_note_references] self.invalid_note_references += new_bad_handles + if len (self.invalid_note_references) == 0: + LOG(' OK: no note reference problems found') + def build_report(self, uistate=None): self.progress.close() bad_photos = len(self.bad_photo) @@ -1370,6 +1530,7 @@ class CheckIntegrity(object): invalid_dates = len(self.invalid_dates) place_references = len(self.invalid_place_references) citation_references = len(self.invalid_citation_references) + source_references = len(self.invalid_source_references) repo_references = len(self.invalid_repo_references) media_references = len(self.invalid_media_references) note_references = len(self.invalid_note_references) @@ -1380,7 +1541,8 @@ class CheckIntegrity(object): event_invalid + person + person_references + family_references + place_references + citation_references + repo_references + media_references + - note_references + name_format + empty_objs + invalid_dates + note_references + name_format + empty_objs + invalid_dates + + source_references ) if errors == 0: @@ -1568,6 +1730,13 @@ class CheckIntegrity(object): citation_references) % {'quantity': citation_references} ) + if source_references: + self.text.write( + ngettext("%(quantity)d source was referenced but not found\n", + "%(quantity)d sources were referenced, but not found\n", + source_references) % {'quantity': source_references} + ) + if media_references: self.text.write( ngettext("%(quantity)d media object was referenced but not found\n", @@ -1621,8 +1790,8 @@ class Report(ManagedWindow.ManagedWindow): def __init__(self, uistate, text, cl=0): if cl: - # Convert to file system encoding before prining - print text.encode(sys.getfilesystemencoding()) + # Convert to file system encoding before printing + print (text.encode(sys.getfilesystemencoding())) return ManagedWindow.ManagedWindow.__init__(self, uistate, [], self) diff --git a/src/plugins/tool/TestcaseGenerator.py b/src/plugins/tool/TestcaseGenerator.py index 8cf5b1c8a..291318117 100644 --- a/src/plugins/tool/TestcaseGenerator.py +++ b/src/plugins/tool/TestcaseGenerator.py @@ -33,6 +33,7 @@ #------------------------------------------------------------------------- from random import randint,choice,random from gen.ggettext import gettext as _ +import time #------------------------------------------------------------------------- # @@ -47,10 +48,15 @@ import gtk # #------------------------------------------------------------------------- import gen.lib +from gen.lib import StyledText, StyledTextTag, StyledTextTagType from gen.db import DbTxn +import gen.mime from gui.plug import tool import Utils +from gui.utils import ProgressMeter import LdsUtils +from gen.db.dbconst import * +import const #------------------------------------------------------------------------- # @@ -66,6 +72,38 @@ class TestcaseGenerator(tool.BatchTool): NOTE = 5 SHORT = 6 LONG = 7 + TAG = 8 + STYLED_TEXT = 9 + +# GEDCON definition: +# +# FAMILY_EVENT_STRUCTURE:= +# [ +# n [ ANUL | CENS | DIV | DIVF ] [Y|] {1:1} +# +1 <> {0:1} p.29 +# | +# n [ ENGA | MARR | MARB | MARC ] [Y|] {1:1} +# +1 <> {0:1} p.29 +# | +# n [ MARL | MARS ] [Y|] {1:1} +# +1 <> {0:1} p.29 +# | +# n EVEN {1:1} +# +1 <> {0:1} p.29 +# ] + + FAMILY_EVENTS = set([ + gen.lib.EventType.ANNULMENT, + gen.lib.EventType.CENSUS, + gen.lib.EventType.DIVORCE, + gen.lib.EventType.DIV_FILING, + gen.lib.EventType.ENGAGEMENT, + gen.lib.EventType.MARRIAGE, + gen.lib.EventType.MARR_BANNS, + gen.lib.EventType.MARR_CONTR, + gen.lib.EventType.MARR_LIC, + gen.lib.EventType.MARR_SETTL, + gen.lib.EventType.CUSTOM ]) def __init__(self, dbstate, uistate, options_class, name, callback=None): self.person = None @@ -83,6 +121,7 @@ class TestcaseGenerator(tool.BatchTool): self.person_dates = {} self.generated_repos = [] self.generated_sources = [] + self.generated_citations = [] self.generated_media = [] self.generated_places = [] self.generated_events = [] @@ -134,21 +173,20 @@ class TestcaseGenerator(tool.BatchTool): label.set_use_markup(True) self.top.vbox.pack_start(label,0,0,5) - self.check_bugs = gtk.CheckButton( _("Generate Database errors")) + self.check_lowlevel = gtk.CheckButton( _("Generate low level database " + "errors\nCorrection needs database reload")) + self.check_lowlevel.set_active( self.options.handler.options_dict['lowlevel']) + self.top.vbox.pack_start(self.check_lowlevel,0,0,5) + + self.check_bugs = gtk.CheckButton( _("Generate database errors")) self.check_bugs.set_active( self.options.handler.options_dict['bugs']) self.top.vbox.pack_start(self.check_bugs,0,0,5) - self.check_persons = gtk.CheckButton( _("Generate dummy families")) + self.check_persons = gtk.CheckButton( _("Generate dummy data")) self.check_persons.set_active( self.options.handler.options_dict['persons']) + self.check_persons.connect('clicked', self.on_dummy_data_clicked) self.top.vbox.pack_start(self.check_persons,0,0,5) - # doesn't work any more since revision ... (earlier than version 3.3) - self.check_trans = gtk.CheckButton( _("Don't block transactions")) - #self.check_trans.set_active( self.options.handler.options_dict['no_trans']) - self.check_trans.set_active(False) - self.check_trans.set_sensitive(False) - self.top.vbox.pack_start(self.check_trans,0,0,5) - self.check_longnames = gtk.CheckButton( _("Generate long names")) self.check_longnames.set_active( self.options.handler.options_dict['long_names']) self.top.vbox.pack_start(self.check_longnames,0,0,5) @@ -165,8 +203,15 @@ class TestcaseGenerator(tool.BatchTool): self.check_linebreak.set_active( self.options.handler.options_dict['add_linebreak']) self.top.vbox.pack_start(self.check_linebreak,0,0,5) + self.label = gtk.Label(_("Number of people to generate\n" + "(Number is approximate because families " + "are generated)")) + self.label.set_alignment(0.0, 0.5) + self.top.vbox.pack_start(self.label,0,0,5) + self.entry_count = gtk.Entry() self.entry_count.set_text( unicode( self.options.handler.options_dict['person_count'])) + self.on_dummy_data_clicked(self.check_persons) self.top.vbox.pack_start(self.entry_count,0,0,5) self.top.add_button(gtk.STOCK_CANCEL,gtk.RESPONSE_CANCEL) @@ -175,12 +220,12 @@ class TestcaseGenerator(tool.BatchTool): self.top.show_all() response = self.top.run() + self.options.handler.options_dict['lowlevel'] = int( + self.check_lowlevel.get_active()) self.options.handler.options_dict['bugs'] = int( self.check_bugs.get_active()) self.options.handler.options_dict['persons'] = int( self.check_persons.get_active()) - self.options.handler.options_dict['no_trans'] = int( - self.check_trans.get_active()) self.options.handler.options_dict['long_names'] = int( self.check_longnames.get_active()) self.options.handler.options_dict['specialchars'] = int( @@ -197,175 +242,38 @@ class TestcaseGenerator(tool.BatchTool): self.run_tool( cli=False) # Save options self.options.handler.save_options() + + def on_dummy_data_clicked(self, obj): + self.label.set_sensitive(obj.get_active()) + self.entry_count.set_sensitive(obj.get_active()) def run_tool(self, cli=False): self.cli = cli if( not cli): - title = "%s - Gramps" % _("Generate testcases") - self.top = gtk.Window() - self.top.set_title(title) - self.top.set_position(gtk.WIN_POS_MOUSE) - self.top.set_modal(True) - self.top.set_default_size(400,150) - vbox = gtk.VBox() - self.top.add(vbox) - label = gtk.Label(_("Generating persons and families.\nPlease wait.")) - vbox.pack_start(label,0,0,5) - self.progress = gtk.ProgressBar() - self.progress.set_fraction(0.0) - vbox.pack_end(self.progress,0,0,5) - self.top.show_all() while gtk.events_pending(): gtk.main_iteration() + self.progress = ProgressMeter(_('Generating testcases'),'') self.transaction_count = 0; - if not self.options.handler.options_dict['no_trans']: - batch = False - self.db.disable_signals() - else: - batch = False - with DbTxn(_("Testcase generator"), self.db) as self.trans: + if self.options.handler.options_dict['lowlevel']: + self.progress.set_pass(_('Generating low level database errors'), + 1) + self.test_low_level(); self.progress.step() - if False and self.options.handler.options_dict['no_trans']: - - print "TESTING SIGNALS..." - - print "\nCREATE PERSON" - p = gen.lib.Person() - h = self.db.add_person( p, self.trans) - print "\nUPDATE PERSON" - self.db.commit_person( p, self.trans) - print "\nDELETE PERSON" - self.db.remove_person( h, self.trans) - - print "\nCREATE FAMILY" - f = gen.lib.Family() - h = self.db.add_family( f, self.trans) - print "\nUPDATE FAMILY" - self.db.commit_family( f, self.trans) - print "\nDELETE FAMILY" - self.db.remove_family( h, self.trans) - - print "\nCREATE EVENT" - e = gen.lib.Event() - h = self.db.add_event( e, self.trans) - print "\nUPDATE EVENT" - self.db.commit_event( e, self.trans) - print "\nDELETE EVENT" - self.db.remove_event( h, self.trans) - - print "\nCREATE PLACE" - p = gen.lib.Place() - h = self.db.add_place( p, self.trans) - print "\nUPDATE PLACE" - self.db.commit_place( p, self.trans) - print "\nDELETE PLACE" - self.db.remove_place( h, self.trans) - - print "\nCREATE SOURCE" - s = gen.lib.Source() - h = self.db.add_source( s, self.trans) - print "\nUPDATE SOURCE" - self.db.commit_source( s, self.trans) - print "\nDELETE SOURCE" - self.db.remove_source( h, self.trans) - - print "\nCREATE MEDIA" - m = gen.lib.MediaObject() - h = self.db.add_object( m, self.trans) - print "\nUPDATE MEDIA" - self.db.commit_media_object( m, self.trans) - print "\nDELETE MEDIA" - self.db.remove_object( h, self.trans) - - print "DONE." - - - print "TESTING DB..." - - print "\nCREATE PERSON None" - self.db.add_person( None, self.trans) - print "\nUPDATE PERSON None" - self.db.commit_person( None, self.trans) - print "\nDELETE PERSON Invalid Handle" - self.db.remove_person( "Invalid Handle", self.trans) - - print "\nCREATE FAMILY None" - self.db.add_family( None, self.trans) - print "\nUPDATE FAMILY None" - self.db.commit_family( None, self.trans) - print "\nDELETE FAMILY Invalid Handle" - self.db.remove_family( "Invalid Handle", self.trans) - - print "\nCREATE EVENT None" - self.db.add_event( None, self.trans) - print "\nUPDATE EVENT None" - self.db.commit_event( None, self.trans) - print "\nDELETE EVENT Invalid Handle" - self.db.remove_event( "Invalid Handle", self.trans) - - print "\nCREATE PLACE None" - self.db.add_place( None, self.trans) - print "\nUPDATE PLACE None" - self.db.commit_place( None, self.trans) - print "\nDELETE PLACE Invalid Handle" - self.db.remove_place( "Invalid Handle", self.trans) - - print "\nCREATE SOURCE None" - self.db.add_source( None, self.trans) - print "\nUPDATE SOURCE None" - self.db.commit_source( None, self.trans) - print "\nDELETE SOURCE Invalid Handle" - self.db.remove_source( "Invalid Handle", self.trans) - - print "\nCREATE MEDIA None" - self.db.add_object( None, self.trans) - print "\nUPDATE MEDIA None" - self.db.commit_media_object( None, self.trans) - print "\nDELETE MEDIA Invalid Handle" - self.db.remove_object( "Invalid Handle", self.trans) - - print "DONE." - - - # if self.options.handler.options_dict['bugs']\ - # or self.options.handler.options_dict['dates']\ - # or self.options.handler.options_dict['persons']: - # # bootstrap random source and media - # self.rand_source() - # self.rand_media() - - - # FIXME: generate_tags needs to be run before generate_broken_relations - # otherwise you get - -# File "/Users/tim/gramps/gramps33/src/plugins/tool/TestcaseGenerator.py", line 1404, in rand_tags -# tag = choice(self.generated_tags) -# File "/opt/local/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/random.py", line 261, in choice -# return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty -#IndexError: list index out of range - - # FIXME: If tags have arbitrary forms, then you can get errors because - # add_ui_from_string parses the tag as part of parsing - - -# Traceback (most recent call last): -# File "/Users/tim/gramps/gramps33/src/gui/viewmanager.py", line 1265, in view_changed -# self.__change_page(page_num) -# File "/Users/tim/gramps/gramps33/src/gui/viewmanager.py", line 1278, in __change_page -# self.active_page.set_active() -# File "/Users/tim/gramps/gramps33/src/plugins/lib/libpersonview.py", line 399, in set_active -# self.uistate.viewmanager.tags.tag_enable() -# File "/Users/tim/gramps/gramps33/src/gui/views/tags.py", line 122, in tag_enable -# self.tag_id = self.uistate.uimanager.add_ui_from_string(self.tag_ui) -#GError: Error on line 6 char 470: '#+#000001#-#' is not a valid name + if self.options.handler.options_dict['bugs'] or \ + self.options.handler.options_dict['persons']: + self.generate_tags() if self.options.handler.options_dict['bugs']: - self.generate_broken_relations() + self.generate_data_errors() if self.options.handler.options_dict['persons']: - self.generate_tags() + self.progress.set_pass(_('Generating families'), + self.options.handler.options_dict['person_count']) + self.person_count = 0 + self.progress_step = self.progress.step + while True: if not self.persons_todo: ph = self.generate_person(0) @@ -383,16 +291,174 @@ class TestcaseGenerator(tool.BatchTool): self.generate_parents(child_h) if self.person_count > self.options.handler.options_dict['person_count']: break + self.progress.close() - if not self.options.handler.options_dict['no_trans']: - self.db.enable_signals() - self.db.request_rebuild() if( not cli): self.top.destroy() + def generate_data_errors(self): + """This generates errors in the database to test src/plugins/tool/Check + The module names correspond to the checking methods in + src/plugins/tool/Check.CheckIntegrity """ + self.progress.set_pass(_('Generating database errors'), + 18) + # The progress meter is normally stepped every time a person is + # generated by generate_person. However in this case, generate_person is + # called by some of the constituent functions, but we only want the + # meter to be stepped every time a test function has been completed. + self.progress_step = lambda: None - def generate_broken_relations(self): - # Create a family, that links to father and mother, but father does not link back + self.test_fix_encoding(); self.progress.step() + self.test_fix_ctrlchars_in_notes(); self.progress.step() + self.test_cleanup_missing_photos(); self.progress.step() + self.test_cleanup_deleted_name_formats(); self.progress.step() + self.test_cleanup_empty_objects(); self.progress.step() + self.test_check_for_broken_family_links(); self.progress.step() + self.test_check_parent_relationships(); self.progress.step() + self.test_cleanup_empty_families(); self.progress.step() + self.test_cleanup_duplicate_spouses(); self.progress.step() + self.test_check_events(); self.progress.step() + self.test_check_person_references(); self.progress.step() + self.test_check_family_references(); self.progress.step() + self.test_check_place_references(); self.progress.step() + self.test_check_source_references(); self.progress.step() + self.test_check_citation_references(); self.progress.step() + self.test_check_media_references(); self.progress.step() + self.test_check_repo_references(); self.progress.step() + self.test_check_note_references(); self.progress.step() + self.progress.close() + + def test_low_level(self): + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + o = gen.lib.Note() + o.set("dup 1" + self.rand_text(self.NOTE)) + o.set_format( choice( (gen.lib.Note.FLOWED,gen.lib.Note.FORMATTED))) + o.set_type( self.rand_type(gen.lib.NoteType())) + h = self.db.add_note(o, self.trans) + print "object %s, handle %s, Gramps_Id %s" % (o, o.handle, + o.gramps_id) + + handle = o.get_handle() + + o = gen.lib.Source() + o.set_title("dup 2" + self.rand_text(self.SHORT)) + if randint(0,1) == 1: + o.set_author( self.rand_text(self.SHORT)) + if randint(0,1) == 1: + o.set_publication_info( self.rand_text(self.LONG)) + if randint(0,1) == 1: + o.set_abbreviation( self.rand_text(self.SHORT)) + while randint(0,1) == 1: + o.set_data_item( self.rand_text(self.SHORT), self.rand_text(self.SHORT)) + o.set_handle(handle) + self.db.add_source(o, self.trans) + print "object %s, handle %s, Gramps_Id %s" % (o, o.handle, + o.gramps_id) + + def test_fix_encoding(self): + # Creates a media object with character encoding errors. This tests + # Check.fix_encoding() and also cleanup_missing_photos + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + m = gen.lib.MediaObject() + self.fill_object(m) + m.set_description("leave this media object invalid description\x9f") + m.set_path("/tmp/click_on_keep_reference.png\x9f") + m.set_mime_type("image/png\x9f") + self.db.add_object(m, self.trans) + + m = gen.lib.MediaObject() + self.fill_object(m) + m.set_description("reselect this media object invalid description\x9f") + m.set_path("/tmp/click_on_select_file.png\x9f") + m.set_mime_type("image/png\x9f") + self.db.add_object(m, self.trans) + + # setup media attached to Source and Citation to be removed + + m = gen.lib.MediaObject() + self.fill_object(m) + m.set_description(u'remove this media object') + m.set_path(u"/tmp/click_on_remove_object.png") + m.set_mime_type("image/png") + self.db.add_object(m, self.trans) + + s = gen.lib.Source() + s.set_title(u'media should be removed from this source') + r = gen.lib.MediaRef() + r.set_reference_handle(m.handle) + s.add_media_reference(r) + self.db.add_source( s, self.trans) + + c = gen.lib.Citation() + self.fill_object(c) + c.set_reference_handle(s.handle) + c.set_page(u'media should be removed from this citation') + r = gen.lib.MediaRef() + r.set_reference_handle(m.handle) + c.add_media_reference(r) + self.db.add_citation(c, self.trans) + + def test_fix_ctrlchars_in_notes(self): + # Creates a note with control characters. This tests + # Check.fix_ctrlchars_in_notes() + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + o = gen.lib.Note() + o.set("This is a text note with a \x03 control character") + o.set_format(choice( (gen.lib.Note.FLOWED,gen.lib.Note.FORMATTED))) + o.set_type(self.rand_type(gen.lib.NoteType())) + self.db.add_note(o, self.trans) + + def test_cleanup_missing_photos(self): + pass + + def test_cleanup_deleted_name_formats(self): + pass + + def test_cleanup_empty_objects(self): + # Generate empty objects to test their deletion + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + p = gen.lib.Person() + self.db.add_person( p, self.trans) + + f = gen.lib.Family() + self.db.add_family( f, self.trans) + + e = gen.lib.Event() + self.db.add_event( e, self.trans) + + p = gen.lib.Place() + self.db.add_place( p, self.trans) + + s = gen.lib.Source() + self.db.add_source( s, self.trans) + + c = gen.lib.Citation() + self.db.add_citation( c, self.trans) + + m = gen.lib.MediaObject() + self.db.add_object( m, self.trans) + + r = gen.lib.Repository() + self.db.add_repository( r, self.trans) + + n = gen.lib.Note() + self.db.add_note( n, self.trans) + + def test_check_for_broken_family_links(self): + # Create a family, that links to father and mother, but father does not + # link back with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: self.transaction_count += 1 @@ -446,7 +512,8 @@ class TestcaseGenerator(tool.BatchTool): person2.add_family_handle(fam_h) self.db.commit_person(person2,self.trans) - # Create a family, that links to father and mother, but father does not link back + # Create a family, that links to father and mother, but mother does not + # link back with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: self.transaction_count += 1 @@ -465,6 +532,7 @@ class TestcaseGenerator(tool.BatchTool): #self.db.commit_person(person2,self.trans) # Create two married people of same sex. + # This is NOT detected as an error by plugins/tool/Check.py with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: self.transaction_count += 1 @@ -555,7 +623,32 @@ class TestcaseGenerator(tool.BatchTool): fam.set_father_handle(person1_h) fam.set_mother_handle(person2_h) fam.set_relationship((gen.lib.FamilyRelType.MARRIED,'')) - #fam.add_child_handle(child_h) + # child_ref = gen.lib.ChildRef() + # child_ref.set_reference_handle(child_h) + # self.fill_object(child_ref) + # fam.add_child_ref(child_ref) + fam_h = self.db.add_family(fam,self.trans) + person1 = self.db.get_person_from_handle(person1_h) + person1.add_family_handle(fam_h) + self.db.commit_person(person1,self.trans) + person2 = self.db.get_person_from_handle(person2_h) + person2.add_family_handle(fam_h) + self.db.commit_person(person2,self.trans) + child = self.db.get_person_from_handle(child_h) + child.add_parent_family_handle(fam_h) + self.db.commit_person(child,self.trans) + + # Creates a family where the child is one of the parents + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + person1_h = self.generate_person(gen.lib.Person.MALE,"Broken19",None) + person2_h = self.generate_person(gen.lib.Person.FEMALE,"Broken19",None) + child_h = person2_h + fam = gen.lib.Family() + fam.set_father_handle(person1_h) + fam.set_mother_handle(person2_h) + fam.set_relationship((gen.lib.FamilyRelType.MARRIED,'')) child_ref = gen.lib.ChildRef() child_ref.set_reference_handle(child_h) self.fill_object(child_ref) @@ -571,6 +664,42 @@ class TestcaseGenerator(tool.BatchTool): child.add_parent_family_handle(fam_h) self.db.commit_person(child,self.trans) + # Creates a couple that refer to a family that does not exist in the + # database. + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + person1_h = self.generate_person(gen.lib.Person.MALE,"Broken20",None) + person2_h = self.generate_person(gen.lib.Person.FEMALE,"Broken20",None) +# fam = gen.lib.Family() +# fam.set_father_handle(person1_h) +# fam.set_mother_handle(person2_h) +# fam.set_relationship((gen.lib.FamilyRelType.MARRIED,'')) +# child_ref = gen.lib.ChildRef() +# # child_ref.set_reference_handle(child_h) +# # self.fill_object(child_ref) +# # fam.add_child_ref(child_ref) +# fam_h = self.db.add_family(fam,self.trans) + person1 = self.db.get_person_from_handle(person1_h) + person1.add_family_handle("InvalidHandle3") + self.db.commit_person(person1,self.trans) + person2 = self.db.get_person_from_handle(person2_h) + person2.add_family_handle("InvalidHandle3") + self.db.commit_person(person2,self.trans) +# child = self.db.get_person_from_handle(child_h) +# child.add_parent_family_handle(fam_h) +# self.db.commit_person(child,self.trans) + + def test_check_parent_relationships(self): + pass + + def test_cleanup_empty_families(self): + pass + + def test_cleanup_duplicate_spouses(self): + pass + + def test_check_events(self): # Creates a person having a non existing birth event handle set with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: @@ -610,6 +739,7 @@ class TestcaseGenerator(tool.BatchTool): self.transaction_count += 1 person_h = self.generate_person(None,"Broken14",None) event = gen.lib.Event() + # The default type _DEFAULT = BIRTH is set in eventtype event.set_type('') event.set_description("Test for Broken14") event_h = self.db.add_event(event,self.trans) @@ -625,6 +755,8 @@ class TestcaseGenerator(tool.BatchTool): self.transaction_count += 1 person_h = self.generate_person(None,"Broken15",None) event = gen.lib.Event() + # The default type _DEFAULT = BIRTH is set in eventtype + event.set_type('') event.set_description("Test for Broken15") event_h = self.db.add_event(event,self.trans) event_ref = gen.lib.EventRef() @@ -634,11 +766,14 @@ class TestcaseGenerator(tool.BatchTool): self.db.commit_person(person,self.trans) # Creates a person with an event having an empty type + # This is NOT detected as an error by plugins/tool/Check.py with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: self.transaction_count += 1 person_h = self.generate_person(None,"Broken16",None) event = gen.lib.Event() + # The default type _DEFAULT = BIRTH is set in eventtype + event.set_type('') event.set_description("Test for Broken16") event_h = self.db.add_event(event,self.trans) event_ref = gen.lib.EventRef() @@ -647,6 +782,13 @@ class TestcaseGenerator(tool.BatchTool): person.add_event_ref(event_ref) self.db.commit_person(person,self.trans) + def test_check_person_references(self): + pass + + def test_check_family_references(self): + pass + + def test_check_place_references(self): # Creates a person with a birth event pointing to nonexisting place with DbTxn(_("Testcase generator step %d") % self.transaction_count, self.db) as self.trans: @@ -679,10 +821,289 @@ class TestcaseGenerator(tool.BatchTool): person.add_event_ref(event_ref) self.db.commit_person(person,self.trans) + def test_check_source_references(self): + + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + c = gen.lib.Citation() + self.fill_object(c) + c.set_reference_handle("unknownsourcehandle") + c.set_page(u'unreferenced citation with invalid source ref') + self.db.add_citation(c, self.trans) + + c = gen.lib.Citation() + self.fill_object(c) + c.set_reference_handle(None) + c.set_page(u'unreferenced citation with invalid source ref') + self.db.add_citation(c, self.trans) + + c = gen.lib.Citation() + self.fill_object(c) + c.set_reference_handle("unknownsourcehandle") + c.set_page(u'citation and references to it should be removed') + c_h1 = self.db.add_citation(c, self.trans) + + c = gen.lib.Citation() + self.fill_object(c) + c.set_reference_handle(None) + c.set_page(u'citation and references to it should be removed') + c_h2 = self.db.add_citation(c, self.trans) + + self.create_all_possible_citations([c_h1, c_h2], "Broken21", + u'non-existent source') + + def test_check_citation_references(self): + # Generate objects that refer to non-existant citations + with DbTxn(_("Testcase generator step %d") % self.transaction_count, + self.db) as self.trans: + self.transaction_count += 1 + + c_h = "unknowncitationhandle" + self.create_all_possible_citations([c_h, None], "Broken22", + u'non-existent citation') + + def create_all_possible_citations(self, c_h_list, name, message): + # Create citations attached to each of the following objects: + # Person + # Name + # Address + # Attribute + # PersonRef + # MediaRef + # Attribute + # LdsOrd + # + # Family + # Attribute + # ChildRef + # MediaRef + # Attribute + # LdsOrd + # + # Event + # Attribute + # MediaRef + # Attribute + # + # MediaObject + # Attribute + # + # Place + # MediaRef + # Attribute + # + # Repository (Repositories themselves do not have SourceRefs) + # Address + m = gen.lib.MediaObject() + m.set_description(message) + m.set_path(unicode(const.ICON)) + m.set_mime_type(gen.mime.get_type(m.get_path())) + m.add_citation(choice(c_h_list)) + # MediaObject : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + m.add_attribute(a) + self.db.add_object(m, self.trans) + + person1_h = self.generate_person(gen.lib.Person.MALE,name,None) + person2_h = self.generate_person(gen.lib.Person.FEMALE,name,None) + child_h = self.generate_person(None,name,None) + fam = gen.lib.Family() + fam.set_father_handle(person1_h) + fam.set_mother_handle(person2_h) + fam.set_relationship((gen.lib.FamilyRelType.MARRIED,'')) + # Family + fam.add_citation(choice(c_h_list)) + # Family : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + fam.add_attribute(a) + # Family : ChildRef + child_ref = gen.lib.ChildRef() + child_ref.set_reference_handle(child_h) + self.fill_object(child_ref) + child_ref.add_citation(choice(c_h_list)) + fam.add_child_ref(child_ref) + # Family : MediaRef + mr = gen.lib.MediaRef() + mr.set_reference_handle(m.handle) + mr.add_citation(choice(c_h_list)) + # Family : MediaRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + mr.add_attribute(a) + fam.add_media_reference(mr) + # Family : LDSORD + ldsord = gen.lib.LdsOrd() + self.fill_object( ldsord) + # TODO: adapt type and status to family/person + #if isinstance(o,gen.lib.Person): + #if isinstance(o,gen.lib.Family): + ldsord.set_type( choice( + [item[0] for item in gen.lib.LdsOrd._TYPE_MAP] )) + ldsord.set_status( randint(0,len(gen.lib.LdsOrd._STATUS_MAP)-1)) + ldsord.add_citation(choice(c_h_list)) + fam.add_lds_ord(ldsord) + # Family : EventRef + e = gen.lib.Event() + e.set_type(gen.lib.EventType.MARRIAGE) + (year, d) = self.rand_date() + e.set_date_object(d) + e.set_description(message) + event_h = self.db.add_event(e, self.trans) + er = gen.lib.EventRef() + er.set_reference_handle(event_h) + er.set_role(self.rand_type(gen.lib.EventRoleType())) + # Family : EventRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + er.add_attribute(a) + fam.add_event_ref(er) + fam_h = self.db.add_family(fam,self.trans) + person1 = self.db.get_person_from_handle(person1_h) + person1.add_family_handle(fam_h) + # Person + person1.add_citation(choice(c_h_list)) + # Person : Name + alt_name = gen.lib.Name(person1.get_primary_name()) + alt_name.set_first_name(message) + alt_name.add_citation(choice(c_h_list)) + person1.add_alternate_name(alt_name) + # Person : Address + a = gen.lib.Address() + a.set_street(message) + a.add_citation(choice(c_h_list)) + person1.add_address(a) + # Person : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + person1.add_attribute(a) + # Person : PersonRef + asso_h = self.generate_person() + asso = gen.lib.PersonRef() + asso.set_reference_handle(asso_h) + asso.set_relation(self.rand_text(self.SHORT)) + self.fill_object(asso) + asso.add_citation(choice(c_h_list)) + person1.add_person_ref(asso) + # Person : MediaRef + mr = gen.lib.MediaRef() + mr.set_reference_handle(m.handle) + mr.add_citation(choice(c_h_list)) + # Person : MediaRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(self.rand_text(self.SHORT)) + a.add_citation(choice(c_h_list)) + mr.add_attribute(a) + person1.add_media_reference(mr) + # Person : LDSORD + ldsord = gen.lib.LdsOrd() + self.fill_object( ldsord) + # TODO: adapt type and status to family/person + #if isinstance(o,gen.lib.Person): + #if isinstance(o,gen.lib.Family): + ldsord.set_type( choice( + [item[0] for item in gen.lib.LdsOrd._TYPE_MAP] )) + ldsord.set_status( randint(0,len(gen.lib.LdsOrd._STATUS_MAP)-1)) + ldsord.add_citation(choice(c_h_list)) + person1.add_lds_ord(ldsord) + # Person : EventRef + e = gen.lib.Event() + e.set_type(gen.lib.EventType.ELECTED) + (year, d) = self.rand_date() + e.set_date_object(d) + e.set_description(message) + event_h = self.db.add_event(e, self.trans) + er = gen.lib.EventRef() + er.set_reference_handle(event_h) + er.set_role(self.rand_type(gen.lib.EventRoleType())) + # Person : EventRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + er.add_attribute(a) + person1.add_event_ref(er) + self.db.commit_person(person1,self.trans) + person2 = self.db.get_person_from_handle(person2_h) + person2.add_family_handle(fam_h) + self.db.commit_person(person2,self.trans) + + e = gen.lib.Event() + e.set_description(message) + e.set_type(gen.lib.EventType.MARRIAGE) + # Event + e.add_citation(choice(c_h_list)) + # Event : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(message) + a.add_citation(choice(c_h_list)) + e.add_attribute(a) + # Event : MediaRef + mr = gen.lib.MediaRef() + mr.set_reference_handle(m.handle) + mr.add_citation(choice(c_h_list)) + # Event : MediaRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(self.rand_text(self.SHORT)) + a.add_citation(choice(c_h_list)) + mr.add_attribute(a) + e.add_media_reference(mr) + self.db.add_event(e, self.trans) + + p = gen.lib.Place() + p.set_title(message) + p.add_citation(choice(c_h_list)) + # Place : MediaRef + mr = gen.lib.MediaRef() + mr.set_reference_handle(m.handle) + mr.add_citation(choice(c_h_list)) + # Place : MediaRef : Attribute + a = gen.lib.Attribute() + a.set_type(self.rand_type(gen.lib.AttributeType())) + a.set_value(self.rand_text(self.SHORT)) + a.add_citation(choice(c_h_list)) + mr.add_attribute(a) + p.add_media_reference(mr) + self.db.add_place(p, self.trans) + + r = gen.lib.Repository() + r.set_name(message) + r.set_type(gen.lib.RepositoryType.LIBRARY) + # Repository : Address + a = gen.lib.Address() + a.set_street(message) + a.add_citation(choice(c_h_list)) + r.add_address(a) + self.db.add_repository(r, self.trans) + + def test_check_media_references(self): + pass + + def test_check_repo_references(self): + pass + + def test_check_note_references(self): + pass + def generate_person(self,gender=None,lastname=None, note=None, alive_in_year=None): if not self.cli: - self.progress.set_fraction(min(1.0,max(0.0, 1.0*self.person_count/self.options.handler.options_dict['person_count']))) if self.person_count % 10 == 0: while gtk.events_pending(): gtk.main_iteration() @@ -798,7 +1219,7 @@ class TestcaseGenerator(tool.BatchTool): # some other events while randint(0,5) == 1: (birth_year, eref) = self.rand_personal_event( None, by,dy) - np.set_birth_ref(eref) + np.add_event_ref(eref) # some shared events if self.generated_events: @@ -829,6 +1250,9 @@ class TestcaseGenerator(tool.BatchTool): person_handle = self.db.add_person(np,self.trans) self.person_count = self.person_count+1 + self.progress_step() + if self.person_count % 10 == 1: + print "person count", self.person_count self.person_dates[person_handle] = (by,dy) return( person_handle) @@ -874,6 +1298,39 @@ class TestcaseGenerator(tool.BatchTool): fam.set_father_handle(person1_h) if person2_h: fam.set_mother_handle(person2_h) + + # Avoid adding the same event more than once to the same family + event_set = set() + + # Generate at least one family event with a probability of 75% + if randint(0, 3) > 0: + (birth_year, eref) = self.rand_family_event(None) + fam.add_event_ref(eref) + event_set.add(eref.get_reference_handle()) + + # generate some more events with a lower probability + while randint(0, 3) == 1: + (birth_year, eref) = self.rand_family_event(None) + if eref.get_reference_handle() in event_set: + continue + fam.add_event_ref(eref) + event_set.add(eref.get_reference_handle()) + + # some shared events + if self.generated_events: + while randint(0, 5) == 1: + typeval = gen.lib.EventType.UNKNOWN + while int(typeval) not in self.FAMILY_EVENTS: + e_h = choice(self.generated_events) + typeval = self.db.get_event_from_handle(e_h).get_type() + if e_h in event_set: + break + eref = gen.lib.EventRef() + self.fill_object( eref) + eref.set_reference_handle(e_h) + fam.add_event_ref(eref) + event_set.add(e_h) + fam_h = self.db.add_family(fam,self.trans) self.generated_families.append(fam_h) fam = self.db.commit_family(fam,self.trans) @@ -962,7 +1419,7 @@ class TestcaseGenerator(tool.BatchTool): self.transaction_count += 1 for counter in range(10): tag = gen.lib.Tag() - tag.set_name(self.rand_text(self.SHORT)) + tag.set_name(self.rand_text(self.TAG)) tag.set_color(self.rand_color()) tag.set_priority(self.db.get_number_of_tags()) tag_handle = self.db.add_tag(tag, self.trans) @@ -1115,15 +1572,20 @@ class TestcaseGenerator(tool.BatchTool): o.set_county( self.rand_text(self.SHORT)) if issubclass(o.__class__,gen.lib.mediabase.MediaBase): - while randint(0,1) == 1: + # FIXME: frequency changed to prevent recursion + while randint(0,10) == 1: o.add_media_reference( self.fill_object( gen.lib.MediaRef())) if isinstance(o,gen.lib.MediaObject): if randint(0,3) == 1: - o.set_description( self.rand_text(self.LONG)) + o.set_description(unicode(self.rand_text(self.LONG))) + path = choice((const.ICON, const.LOGO, const.SPLASH)) + o.set_path(unicode(path)) + mime = gen.mime.get_type(path) + o.set_mime_type(mime) else: - o.set_description( self.rand_text(self.SHORT)) - o.set_path("/tmp/TestcaseGenerator.png") + o.set_description(unicode(self.rand_text(self.SHORT))) + o.set_path(unicode(const.ICON)) o.set_mime_type("image/png") if isinstance(o,gen.lib.MediaRef): @@ -1157,9 +1619,13 @@ class TestcaseGenerator(tool.BatchTool): # o.set_sort_as() if isinstance(o,gen.lib.Note): - o.set( self.rand_text(self.NOTE)) + type = self.rand_type(gen.lib.NoteType()) + if type == gen.lib.NoteType.HTML_CODE: + o.set( self.rand_text(self.NOTE)) + else: + o.set_styledtext(self.rand_text(self.STYLED_TEXT)) o.set_format( choice( (gen.lib.Note.FLOWED,gen.lib.Note.FORMATTED))) - o.set_type( self.rand_type(gen.lib.NoteType())) + o.set_type(type) if issubclass(o.__class__,gen.lib.notebase.NoteBase): while randint(0,1) == 1: @@ -1228,13 +1694,17 @@ class TestcaseGenerator(tool.BatchTool): self.fill_object(r) o.add_repo_reference( r) - if issubclass(o.__class__,gen.lib.srcbase.SourceBase): + if issubclass(o.__class__,gen.lib.citationbase.CitationBase): while randint(0,1) == 1: - s = gen.lib.SourceRef() - self.fill_object(s) - o.add_source_reference( s) + if not self.generated_citations or randint(1,10) == 1: + s = gen.lib.Citation() + self.fill_object(s) + self.db.add_citation( s, self.trans) + self.generated_citations.append(s.get_handle()) + s_h = choice(self.generated_citations) + o.add_citation(s_h) - if isinstance(o,gen.lib.SourceRef): + if isinstance(o,gen.lib.Citation): if not self.generated_sources or randint(0,10) == 1: s = gen.lib.Source() self.fill_object(s) @@ -1278,7 +1748,9 @@ class TestcaseGenerator(tool.BatchTool): if type: typeval = gen.lib.EventType(type) else: - typeval = self.rand_type(gen.lib.EventType()) + typeval = gen.lib.EventType.UNKNOWN + while int(typeval) not in self.FAMILY_EVENTS: + typeval = self.rand_type(gen.lib.EventType()) return self._rand_event( typeval, start, end) def _rand_event( self, type, start, end): @@ -1327,17 +1799,22 @@ class TestcaseGenerator(tool.BatchTool): minsyllables = 2 maxsyllables = 5 - result = "" - if self.options.handler.options_dict['specialchars']: - result = result + u"ä<ö&ü%ß'\"" - if self.options.handler.options_dict['add_serial']: - result = result + "#+#%06d#-#" % self.text_serial_number - self.text_serial_number = self.text_serial_number + 1 + if type == self.STYLED_TEXT: + result = StyledText("") + else: + result = "" + + if type <> self.TAG: + if self.options.handler.options_dict['specialchars']: + result = result + u"ä<ö&ü%ß'\"" + if self.options.handler.options_dict['add_serial']: + result = result + "#+#%06d#-#" % self.text_serial_number + self.text_serial_number = self.text_serial_number + 1 if not type: type = self.SHORT - if type == self.SHORT: + if type == self.SHORT or type == self.TAG: minwords = 1 maxwords = 3 minsyllables = 2 @@ -1371,7 +1848,7 @@ class TestcaseGenerator(tool.BatchTool): if not self.options.handler.options_dict['long_names']: maxsyllables = 3 - if type == self.NOTE: + if type == self.NOTE or type == self.STYLED_TEXT: result = result + "Generated by TestcaseGenerator." minwords = 20 maxwords = 100 @@ -1405,7 +1882,26 @@ class TestcaseGenerator(tool.BatchTool): word = word + "." elif randint(0,30) == 1: word = word + ".\n" - result = result + word + if type == self.STYLED_TEXT: + tags = [] + if randint(0,10) == 1: + tags += [StyledTextTag(StyledTextTagType.BOLD, True, + [(0, len(word))])] + elif randint(0,10) == 1: + tags += [StyledTextTag(StyledTextTagType.ITALIC, True, + [(0, len(word))])] + elif randint(0,10) == 1: + tags += [StyledTextTag(StyledTextTagType.UNDERLINE, True, + [(0, len(word))])] + word = StyledText(word, tags) + if randint(0,20) == 1: + word = word + "." + elif randint(0,30) == 1: + word = word + ".\n" + if type == self.STYLED_TEXT: + result = StyledText("").join((result, word)) + else: + result += word if type == self.LASTNAME: n = randint(0,2) @@ -1414,7 +1910,8 @@ class TestcaseGenerator(tool.BatchTool): elif n == 1: result = result.upper() - if self.options.handler.options_dict['add_linebreak']: + if self.options.handler.options_dict['add_linebreak'] and \ + type <> self.TAG: result = result + u"\nNEWLINE" return result @@ -1431,16 +1928,6 @@ class TestcaseGenerator(tool.BatchTool): taglist.append(tag) return taglist - def commit_transaction(self): - # The way transactions are used in this file is outdated; use a with - # statement so that transaction abort is called on failure. It is too - # much effort to update this file. - #if self.options.handler.options_dict['no_trans']: - self.db.transaction_commit(self.trans,_("Testcase generator step %d") % self.transaction_count) - self.transaction_count += 1 - self.trans = self.db.transaction_begin() - - #------------------------------------------------------------------------ # # @@ -1456,16 +1943,20 @@ class TestcaseGeneratorOptions(tool.ToolOptions): # Options specific for this report self.options_dict = { + 'lowlevel' : 0, 'bugs' : 0, 'persons' : 1, 'person_count' : 2000, - 'no_trans' : 0, 'long_names' : 0, 'specialchars' : 0, 'add_serial' : 0, 'add_linebreak' : 0, } self.options_help = { + 'lowlevel' : ("=0/1", + "Whether to create low level database errors.", + ["Skip test","Create low level database errors"], + True), 'bugs' : ("=0/1", "Whether to create invalid database references.", ["Skip test","Create invalid Database references"], @@ -1477,24 +1968,20 @@ class TestcaseGeneratorOptions(tool.ToolOptions): 'person_count' : ("=int", "Number of dummy persons to generate", "Number of persons"), - 'no_trans' : ("=0/1", - "Wheter to use one transaction or multiple small ones", - ["One transaction","Multiple transactions"], - True), 'long_names' : ("=0/1", - "Wheter to create short or long names", + "Whether to create short or long names", ["Short names","Long names"], True), 'specialchars' : ("=0/1", - "Wheter to ass some special characters to every text field", + "Whether to ass some special characters to every text field", ["No special characters","Add special characters"], True), 'add_serial' : ("=0/1", - "Wheter to add a serial number to every text field", + "Whether to add a serial number to every text field", ["No serial","Add serial number"], True), 'add_linebreak' : ("=0/1", - "Wheter to add a line break to every text field", + "Whether to add a line break to every text field", ["No linebreak","Add line break"], True), }