Merge pull request #237 from prculley/check

Update Check & Repair tool & Test Case Generator tool to fix bad alternative Place names

gramps/plugins/test/test_tools.py

@@ -0,0 +1,168 @@
+#! /usr/bin/env python3
+"""
+Gramps - a GTK+/GNOME based genealogy program
+
+Copyright (c) 2016 Gramps Development Team
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+"""
+
+import os
+import sys
+import codecs
+import unittest
+import random
+
+from gramps.test.test_util import Gramps
+from gramps.gen.const import DATA_DIR
+from gramps.cli.user import User
+from gramps.gen.utils.id import set_det_id
+from gramps.gen import const
+from gramps.gen.utils.config import config
+
+TREE_NAME = "Test_tooltest"
+TEST_DIR = os.path.abspath(os.path.join(DATA_DIR, "tests"))
+const.myrand = random.Random()
+
+
+def call(*args):
+    """ Call Gramps to perform the action with out and err captured """
+    if __debug__:
+        print ("call: %s", args)
+    gramps = Gramps(user=User(auto_accept=True, quiet=True))
+    # gramps = Gramps(user=User(auto_accept=True))
+    out, err = gramps.run(*args)
+    # print("out:", out, "err:", err)
+    return out, err
+
+
+def check_res(out, err, expected, do_out=True):
+    """
+    compare the output with the expected
+    """
+    # return all([(line in out) for line in expect])
+    retval = True
+    for line in expected:
+        if line in (out if do_out else err):
+            continue
+        else:
+            print("*** Expected: '%s', not found" % line)
+            retval = False
+    if not retval:
+        if do_out:
+            print("*** The following was the actual stdout:\n%s"
+                  "*** The following was the stderr:\n%s"
+                  "*** End of stderr" % (out, err))
+        else:
+            print("*** The following was the actual stderr:\n%s"
+                  "*** The following was the stdout:\n%s"
+                  "*** End of stdout" % (err, out))
+    return retval
+
+
+class ToolControl(unittest.TestCase):
+    """
+    These tests run some of the tools used to maintain Gramps db
+    """
+    def setUp(self):
+        self.db_backend = config.get('database.backend')
+        call("--config=database.backend:bsddb", "-y", "-q", "--remove", TREE_NAME)
+
+    def tearDown(self):
+        config.set('database.backend', self.db_backend)
+        call("-y", "-q", "--remove", TREE_NAME)
+
+    def test_tcg_and_check_and_repair(self):
+        """
+        Run a 'Test Case Generator' and 'Check & Repair Database' test.
+        Note that the 'Test Case Generator" uses a lot of random numbers to
+        generate its test cases.  This makes it less than ideal for a
+        predictable unit test.  Still it contains good code for testing the
+        'Check and Repair' tool.  So I force the random functions to be
+        predictable by seeding it with a fixed number.  I also used the
+        'Deterministic ID' function to make the usual db handle generation
+        stop using random numbers and potentially reduce Gramps version to
+        version issues.
+        """
+        # the TCG creates bad strings with illegal characters, so we need to
+        # ignore them when we print the results
+        try:
+            sys.stderr = codecs.getwriter(sys.getdefaultencoding()) \
+                (sys.stderr.buffer, 'replace')
+            sys.stdout = codecs.getwriter(sys.getdefaultencoding()) \
+                (sys.stdout.buffer, 'replace')
+        except:
+            pass
+        tst_file = os.path.join(TEST_DIR, "data.gramps")
+        set_det_id(True)
+        # the following line assumes that TCG has run through init code, where
+        # it puts 'myrand', a 'Random' class object, into the 'const' module so
+        # we can access it here.
+        const.myrand.seed(1234, version=1)
+        # print(const.myrand.random())
+#         out, err = call("-s")
+#         expect = ["bsddb"]
+#         check_res(out, err, expect, do_out=True)
+        out, err = call("-C", TREE_NAME, "-q",
+                        "--import", tst_file)
+        expect = ["Opened successfully!",
+                  "data.gramps, format gramps.",
+                  "Cleaning up."]
+        self.assertTrue(check_res(out, err, expect, do_out=False))
+        self.assertEqual(out, "")
+        out, err = call("-O", TREE_NAME,
+                        "-y", "-q", "-a", "tool", "-p",
+                        "name=testcasegenerator,bugs=1,persons=0,"
+                        "add_linebreak=0,add_serial=0,"
+                        "long_names=0,lowlevel=0,person_count=20,"
+                        "specialchars=0")
+        expect = ["Opened successfully!",
+                  "Performing action: tool.",
+                  "Using options string: name=testcasegenerator,bugs=1",
+                  "Cleaning up."]
+        self.assertTrue(check_res(out, err, expect, do_out=False))
+        expect = ["person count 41"]
+        self.assertTrue(check_res(out, err, expect, do_out=True))
+        out, err = call("-O", TREE_NAME,
+                        "-y", "-a", "tool", "-p", "name=check")
+        expect = ["7 broken child/family links were fixed",
+                  "4 broken spouse/family links were fixed",
+                  "1 place alternate names fixed",
+                  "10 media objects were referenced, but not found",
+                  "References to 10 media objects were kept",
+                  "3 events were referenced, but not found",
+                  "1 invalid birth event name was fixed",
+                  "1 invalid death event name was fixed",
+                  "2 places were referenced, but not found",
+                  "13 citations were referenced, but not found",
+                  "16 sources were referenced, but not found",
+                  "7 empty objects removed",
+                  "1 person objects",
+                  "1 family objects",
+                  "1 event objects",
+                  "1 source objects",
+                  "0 media objects",
+                  "0 place objects",
+                  "1 repository objects",
+                  "1 note objects"]
+        self.assertTrue(check_res(out, err, expect, do_out=True))
+        expect = ["Opened successfully!",
+                  "Performing action: tool.",
+                  "Using options string: name=check",
+                  "Cleaning up."]
+        self.assertTrue(check_res(out, err, expect, do_out=False))
+
+if __name__ == "__main__":
+    unittest.main()

gramps/plugins/tool/check.py

@@ -171,6 +171,7 @@ class Check(tool.BatchTool):
             # then going to be deleted.
             checker.cleanup_empty_objects()
             checker.fix_encoding()
+            checker.fix_alt_place_names()
             checker.fix_ctrlchars_in_notes()
             checker.cleanup_missing_photos(cli)
             checker.cleanup_deleted_name_formats()
@@ -246,6 +247,7 @@ class CheckIntegrity:
         self.removed_name_format = []
         self.empty_objects = defaultdict(list)
         self.replaced_sourceref = []
+        self.place_errors = 0
         self.last_img_dir = config.get('behavior.addmedia-image-dir')
         self.progress = ProgressMeter(_('Checking Database'), '',
                                       parent=self.parent_window)
@@ -410,6 +412,38 @@ class CheckIntegrity:
         if error_count == 0:
             logging.info('    OK: no ctrl characters in notes found')
 
+    def fix_alt_place_names(self):
+        """
+        This scans all places and cleans up alternative names.  It removes
+        Blank names, names that are duplicates of the primary name, and
+        duplicates in the alt_names list.
+        """
+        self.progress.set_pass(_('Looking for bad alternate place names'),
+                               self.db.get_number_of_places())
+        logging.info('Looking for bad alternate place names')
+        for bhandle in self.db.place_map.keys():
+            handle = bhandle.decode('utf-8')
+            place = self.db.get_place_from_handle(handle)
+            fixed_alt_names = []
+            fixup = False
+            for name in place.get_alternative_names():
+                if not name.value or \
+                        name == place.name or \
+                        name in fixed_alt_names:
+                    fixup = True
+                    continue
+                fixed_alt_names.append(name)
+            if fixup:
+                place.set_alternative_names(fixed_alt_names)
+                self.db.commit_place(place, self.trans)
+                self.place_errors += 1
+            self.progress.step()
+        if self.place_errors == 0:
+            logging.info('    OK: no bad alternate places found')
+        else:
+            logging.info('    %d bad alternate places found and fixed',
+                         self.place_errors)
+
     def check_for_broken_family_links(self):
         # Check persons referenced by the family objects
 
@@ -717,9 +751,8 @@ class CheckIntegrity:
             photo_desc = obj.get_description()
             if photo_name is not None and photo_name != "" and not find_file(photo_name):
                 if cl:
-                    fn = os.path.basename(photo_name)
                     logging.warning("    FAIL: media file %s was not found." %
-                                    fn)
+                                    photo_name)
                     self.bad_photo.append(ObjectId)
                 else:
                     if missmedia_action == 0:
@@ -2116,7 +2149,7 @@ class CheckIntegrity:
         empty_objs = sum(len(obj) for obj in self.empty_objects.values())
 
         errors = (photos + efam + blink + plink + slink + rel +
-                  event_invalid + person +
+                  event_invalid + person + self.place_errors +
                   person_references  + family_references + place_references +
                   citation_references + repo_references + media_references  +
                   note_references + tag_references + name_format + empty_objs +
@@ -2233,6 +2266,14 @@ class CheckIntegrity:
                          rel).format(quantity=rel)
                 )
 
+        if self.place_errors:
+            self.text.write(
+                # translators: leave all/any {...} untranslated
+                ngettext("{quantity} place alternate name fixed\n",
+                         "{quantity} place alternate names fixed\n",
+                         rel).format(quantity=self.place_errors)
+                )
+
         if person_references:
             self.text.write(
                 # translators: leave all/any {...} untranslated

gramps/test/test_util.py

@@ -260,6 +260,8 @@ class Gramps:
             args = [sys.executable] + list(args)
             argparser = ArgParser(args)
             argparser.need_gui() # initializes some variables
+            if argparser.errors:
+                print(argparser.errors, file=sys.stderr)
             argparser.print_help()
             argparser.print_usage()
             handler = ArgHandler(self.dbstate, argparser, self.climanager)