Update Merge tests to actually run

So this was a lot of work...

Updated to use lxml, steep learning curve for someone who never examined XML before...

The merge test code needed some updates because it was last used with an older version of the Gramps XML.

Found another bug in the mergefamilyquery code when test started running; another nonetype issue.

Found another bug in the mergepersonquery code when test started running; another nonetype issue.

Couldn't get the subprocess stuff to work right, so changed code to just call Gramps with capture, similar to export_tests. This in turn required that importxml and exportxml be slightly changed to support StringIO redirection of stdin and stdout.

And test_util needed a change to allow stdout to accept an unencoded stream, so I added an option to use BytesIO for this test as well.

Added some code to save input, result, and expected data to files in Gramps temp directory for debugging whenever an error occurred. Easier to use my editor in diff mode than look at the outputs.

.travis.yml

@@ -89,8 +89,8 @@ before_script:
  - export PYTHONPATH=meta
 # set module exclusions. --exclude=TestUser because of older version of mock
 # without configure_mock
- - export EXCLUDE="--exclude=TestcaseGenerator --exclude=vcard
+ - export EXCLUDE="--exclude=TestcaseGenerator --exclude=vcard"
-   --exclude=merge_ref_test"
+#   --exclude=merge_ref_test"
 # set GRAMPS_RESOURCES for locale, data,image and documentation
  - export GRAMPS_RESOURCES=.
 

gramps/gen/merge/mergefamilyquery.py

@@ -134,15 +134,35 @@ class MergeFamilyQuery:
 
         with DbTxn(_('Merge Family'), self.database) as trans:
 
-            phoenix_father = self.database.get_person_from_handle(self.phoenix_fh)
+            if self.phoenix_fh != self.titanic_fh:
-            titanic_father = self.database.get_person_from_handle(self.titanic_fh)
+                if self.phoenix_fh:
-            self.merge_person(phoenix_father, titanic_father, 'father', trans)
+                    phoenix_father = self.database.get_person_from_handle(
+                        self.phoenix_fh)
+                else:
+                    phoenix_father = None
+                if self.titanic_fh:
+                    titanic_father = self.database.get_person_from_handle(
+                        self.titanic_fh)
+                else:
+                    titanic_father = None
+                self.merge_person(phoenix_father, titanic_father,
+                                  'father', trans)
 
-            phoenix_mother = self.database.get_person_from_handle(self.phoenix_mh)
+            if self.phoenix_mh != self.titanic_mh:
-            titanic_mother = self.database.get_person_from_handle(self.titanic_mh)
+                if self.phoenix_mh:
+                    phoenix_mother = self.database.get_person_from_handle(
+                        self.phoenix_mh)
+                else:
+                    phoenix_mother = None
+                if self.titanic_mh:
+                    titanic_mother = self.database.get_person_from_handle(
+                        self.titanic_mh)
+                else:
+                    titanic_mother = None
+                self.merge_person(phoenix_mother, titanic_mother,
+                                  'mother', trans)
             self.phoenix = self.database.get_family_from_handle(new_handle)
             self.titanic = self.database.get_family_from_handle(old_handle)
-            self.merge_person(phoenix_mother, titanic_mother, 'mother', trans)
 
             phoenix_father = self.database.get_person_from_handle(self.phoenix_fh)
             phoenix_mother = self.database.get_person_from_handle(self.phoenix_mh)

gramps/gen/merge/mergepersonquery.py

@@ -93,10 +93,11 @@ class MergePersonQuery:
         family_father_handle = family.get_father_handle()
         spouse_handle = family.get_mother_handle() if \
                 new_handle == family_father_handle else family_father_handle
-        spouse = self.database.get_person_from_handle(spouse_handle)
+        if spouse_handle:
-        if spouse:
+            spouse = self.database.get_person_from_handle(spouse_handle)
-            spouse.remove_family_handle(family_handle)
+            if spouse:
-            self.database.commit_person(spouse, trans)
+                spouse.remove_family_handle(family_handle)
+                self.database.commit_person(spouse, trans)
         # replace the family in lds ordinances
         for (dummy, person_handle) in self.database.find_backlink_handles(
                 family_handle, ['Person']):

gramps/plugins/export/exportxml.py

@@ -126,7 +126,10 @@ class GrampsXmlWriter(UpdateCallback):
         """
         if filename == '-':
             import sys
-            g = sys.stdout.buffer
+            try:
+                g = sys.stdout.buffer
+            except:
+                g = sys.stdout
             self.compress = False
         else:
             base = os.path.dirname(filename)

gramps/plugins/importer/importxml.py

@@ -399,7 +399,10 @@ class ImportOpenFileContextManager:
 
     def __enter__(self):
         if self.filename == '-':
-            self.filehandle = sys.stdin.buffer
+            try:
+                self.filehandle = sys.stdin.buffer
+            except:
+                self.filehandle = sys.stdin
         else:
             self.filehandle = self.open_file(self.filename)
         return self.filehandle

gramps/test/test_util.py

@@ -221,14 +221,14 @@ def new_exit(edit_code=None):
     raise SystemExit()
 
 @contextlib.contextmanager
-def capture(stdin):
+def capture(stdin, bytesio=False):
     oldout, olderr = sys.stdout, sys.stderr
     oldexit = sys.exit
     if stdin:
         oldin = sys.stdin
         sys.stdin = stdin
     try:
-        output = [StringIO(), StringIO()]
+        output = [BytesIO() if bytesio else StringIO(), StringIO()]
         sys.stdout, sys.stderr = output
         sys.exit = new_exit
         yield output
@@ -252,8 +252,8 @@ class Gramps:
         self.climanager = CLIManager(self.dbstate, setloader=True, user=self.user)
         self.clidbmanager = CLIDbManager(self.dbstate)
 
-    def run(self, *args, stdin=None):
+    def run(self, *args, stdin=None, bytesio=False):
-        with capture(stdin) as output:
+        with capture(stdin, bytesio=bytesio) as output:
             #load the plugins
             self.climanager.do_reg_plugins(self.dbstate, uistate=None)
             # handle the arguments