Merge from 2.0.1

svn: r4666

src/plugins/ImportGeneWeb.py

@@ -52,6 +52,7 @@ import Utils
 import const
 from QuestionDialog import ErrorDialog
 from DateHandler import parser as _dp
+from htmlentitydefs import name2codepoint
 
 #-------------------------------------------------------------------------
 #
@@ -419,6 +420,8 @@ class GeneWebParser:
         person.set_primary_name(name)
         if gender != None:
             person.set_gender(gender)
+        else:
+            person.set_gender(RelLib.Person.UNKNOWN)
         self.db.commit_person(person,self.trans)
         personDataRe = re.compile("^[0-9<>~#\[({!].*$")
         dateRe = re.compile("^[0-9~<>?]+.*$")
@@ -660,7 +663,7 @@ class GeneWebParser:
     def get_or_create_person(self,firstname,lastname):
         person = None
         mykey = firstname+lastname
-        if mykey in self.ikeys:
+        if mykey in self.ikeys and firstname != "?" and lastname != "?":
             person = self.db.get_person_from_handle(self.ikeys[mykey])
         else:
             person = RelLib.Person()
@@ -696,7 +699,29 @@ class GeneWebParser:
         return sref
 
     def decode(self,s):
-        return( latin_utf8.latin_to_utf8( s.replace('_',' ')))
+        s = latin_utf8.latin_to_utf8( s.replace('_',' '))
+        charref_re = re.compile('(&#)(x?)([0-9a-zA-Z]+)(;)')
+        for match in charref_re.finditer(s):
+            try:
+                if match.group(2):  # HEX
+                    nchar = unichr(int(match.group(3),16))
+                else:   # Decimal
+                    nchar = unichr(int(match.group(3)))
+                s = s.replace(match.group(0),nchar)
+            except UnicodeDecodeError:
+                pass
+        
+        # replace named entities
+        entref_re = re.compile('(&)([a-zA-Z]+)(;)')
+        for match in entref_re.finditer(s):
+            try:
+                if match.group(2) in name2codepoint:
+                    nchar = unichr(name2codepoint[match.group(2)])
+                s = s.replace(match.group(0),nchar)
+            except UnicodeDecodeError:
+                pass
+        
+        return( s)
 
     def cnv(seld,s):
         return( latin_utf8.latin_to_utf8(s))