svn: r8090

This commit is contained in:
Don Allingham 2007-02-12 05:45:34 +00:00
parent 2a6e0cb197
commit 9c64624930
2 changed files with 267 additions and 262 deletions

2
TODO
View File

@ -14,7 +14,7 @@
* Split views
* Export to spreadsheet, print, CSV of views
* Export to spreadsheet, print, CSV of views - DONE
* Date calculator. See
http://sourceforge.net/mailarchive/forum.php?thread_id=3252078&forum_id=1993

View File

@ -22,6 +22,9 @@
"Import from GEDCOM"
__revision__ = "$Revision: $"
__author__ = "Don Allingham"
#-------------------------------------------------------------------------
#
# standard python modules
@ -76,12 +79,12 @@ except:
# Address/Place constants
#
#-------------------------------------------------------------------------
addr_re = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)\s+(\d+)\s*(.*)')
addr2_re = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)\s+(\d+)')
addr3_re = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)')
ADDR_RE = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)\s+(\d+)\s*(.*)')
ADDR2_RE = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)\s+(\d+)')
ADDR3_RE = re.compile('(.+)([\n\r]+)(.+)\s*,(.+)')
_place_field = []
_place_match = {
PLACE_FIELD = []
PLACE_MATCH = {
'addr' : RelLib.Location.set_street,
'subdivision' : RelLib.Location.set_street,
'addr1' : RelLib.Location.set_street,
@ -104,20 +107,20 @@ _place_match = {
#
#-------------------------------------------------------------------------
def _empty_func(a,b):
def _empty_func(first, second):
return
def utf8_to_latin(s):
return s.encode('iso-8859-1','replace')
def utf8_to_latin(msg):
return msg.encode('iso-8859-1', 'replace')
def latin_to_utf8(s):
if type(s) == unicode:
return s
def latin_to_utf8(msg):
if type(msg) == unicode:
return msg
else:
return unicode(s,'iso-8859-1')
return unicode(msg, 'iso-8859-1')
def nocnv(s):
return unicode(s)
def nocnv(msg):
return unicode(msg)
#-------------------------------------------------------------------------
#
@ -128,24 +131,24 @@ ANSEL = 1
UNICODE = 2
UPDATE = 25
_TYPE_BIRTH = RelLib.ChildRefType()
_TYPE_ADOPT = RelLib.ChildRefType(RelLib.ChildRefType.ADOPTED)
_TYPE_FOSTER = RelLib.ChildRefType(RelLib.ChildRefType.FOSTER)
TYPE_BIRTH = RelLib.ChildRefType()
TYPE_ADOPT = RelLib.ChildRefType(RelLib.ChildRefType.ADOPTED)
TYPE_FOSTER = RelLib.ChildRefType(RelLib.ChildRefType.FOSTER)
rel_types = (RelLib.ChildRefType.BIRTH,
RELATION_TYPES = (RelLib.ChildRefType.BIRTH,
RelLib.ChildRefType.UNKNOWN,
RelLib.ChildRefType.NONE,
)
pedi_type = {
PEDIGREE_TYPES = {
'birth' : RelLib.ChildRefType(),
'natural': RelLib.ChildRefType(),
'step' : _TYPE_ADOPT,
'adopted': _TYPE_ADOPT,
'foster' : _TYPE_FOSTER,
'step' : TYPE_ADOPT,
'adopted': TYPE_ADOPT,
'foster' : TYPE_FOSTER,
}
mime_map = {
MIME_MAP = {
'jpeg' : 'image/jpeg', 'jpg' : 'image/jpeg',
'rtf' : 'text/rtf', 'pdf' : 'application/pdf',
'mpeg' : 'video/mpeg', 'mpg' : 'video/mpeg',
@ -156,45 +159,44 @@ mime_map = {
}
_event_family_str = _("%(event_name)s of %(family)s")
_event_person_str = _("%(event_name)s of %(person)s")
EVENT_FAMILY_STR = _("%(event_name)s of %(family)s")
EVENT_PERSON_STR = _("%(event_name)s of %(person)s")
_transtable = string.maketrans('','')
_delc = _transtable[0:8] + _transtable[10:31]
_transtable2 = _transtable[0:128] + ('?' * 128)
TRANS_TABLE = string.maketrans('', '')
DEL_CHARS = TRANS_TABLE[0:8] + TRANS_TABLE[10:31]
TRANS_TABLE2 = TRANS_TABLE[0:128] + ('?' * 128)
#-------------------------------------------------------------------------
#
# GEDCOM events to GRAMPS events conversion
#
#-------------------------------------------------------------------------
ged2gramps = {}
GED_2_GRAMPS = {}
for _val in personalConstantEvents.keys():
_key = personalConstantEvents[_val]
if _key != "":
ged2gramps[_key] = _val
GED_2_GRAMPS[_key] = _val
ged2fam = {}
GED_2_FAMILY = {}
for _val in familyConstantEvents.keys():
_key = familyConstantEvents[_val]
if _key != "":
ged2fam[_key] = _val
GED_2_FAMILY[_key] = _val
ged2fam_custom = {}
GED_2_FAMILY_CUSTOM = {}
#-------------------------------------------------------------------------
#
# regular expressions
#
#-------------------------------------------------------------------------
intRE = re.compile(r"\s*(\d+)\s*$")
nameRegexp = re.compile(r"/?([^/]*)(/([^/]*)(/([^/]*))?)?")
modRegexp = re.compile(r"\s*(EST|CAL)\s+(.*)$")
calRegexp = re.compile(r"\s*(ABT|BEF|AFT)?\s*@#D([^@]+)@\s*(.*)$")
rangeRegexp = re.compile(r"\s*BET\s+@#D([^@]+)@\s*(.*)\s+AND\s+@#D([^@]+)@\s*(.*)$")
spanRegexp = re.compile(r"\s*FROM\s+@#D([^@]+)@\s*(.*)\s+TO\s+@#D([^@]+)@\s*(.*)$")
intRegexp = re.compile(r"\s*INT\s+([^(]+)\((.*)\)$")
snameRegexp = re.compile(r"/([^/]*)/([^/]*)")
INT_RE = re.compile(r"\s*(\d+)\s*$")
NAME_RE = re.compile(r"/?([^/]*)(/([^/]*)(/([^/]*))?)?")
SURNAME_RE = re.compile(r"/([^/]*)/([^/]*)")
NOTE_RE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+NOTE(.*)$")
CONT_RE = re.compile(r"\s*\d+\s+CONT\s?(.*)$")
CONC_RE = re.compile(r"\s*\d+\s+CONC\s?(.*)$")
PERSON_RE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+INDI(.*)$")
#-------------------------------------------------------------------------
#
@ -284,7 +286,7 @@ def import_from_string(database, text, callback, codeset, use_trans):
g = GedcomParser(database, f, "inline-string", callback, codeset, np.get_map(),
np.get_lines(),np.get_persons())
except IOError, msg:
ErrorDialog(_("%s could not be opened\n") % filename,str(msg))
ErrorDialog(_("%s could not be opened\n") % "inline-string", str(msg))
return
if database.get_number_of_people() == 0:
@ -297,7 +299,7 @@ def import_from_string(database, text, callback, codeset, use_trans):
database.readonly = ro
f.close()
except IOError, msg:
errmsg = _("%s could not be opened\n") % filename
errmsg = _("%s could not be opened\n") % 'inline-string'
ErrorDialog(errmsg, str(msg))
return
except db.DBSecondaryBadError, msg:
@ -344,10 +346,6 @@ class IdFinder:
#
#
#-------------------------------------------------------------------------
noteRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+NOTE(.*)$")
contRE = re.compile(r"\s*\d+\s+CONT\s?(.*)$")
concRE = re.compile(r"\s*\d+\s+CONC\s?(.*)$")
personRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+INDI(.*)$")
#-------------------------------------------------------------------------
#
@ -365,6 +363,12 @@ class CurrentState:
self.event_ref = event_ref
self.source_ref = None
def __getattr__(self, name):
return self.__dict__.get(name)
def __setattr__(self, name, value):
self.__dict__[name] = value
def add_to_note(self, text):
self.note += text
@ -407,24 +411,24 @@ class NoteParser:
for line in ifile:
try:
text = string.translate(line,_transtable,_delc)
text = string.translate(line, TRANS_TABLE, DEL_CHARS)
except:
text = line
try:
text = self.cnv(text)
except:
text = string.translate(text,_transtable2)
text = string.translate(text, TRANS_TABLE2)
self.count += 1
if innote:
match = contRE.match(text)
match = CONT_RE.match(text)
if match:
noteobj.append("\n" + match.groups()[0])
continue
match = concRE.match(text)
match = CONC_RE.match(text)
if match:
if broken:
noteobj.append(" " + match.groups()[0])
@ -435,14 +439,14 @@ class NoteParser:
# Here we have finished parsing CONT/CONC tags for the NOTE
# and ignored the rest of the tags (SOUR,CHAN,REFN,RIN).
innote = False
match = noteRE.match(text)
match = NOTE_RE.match(text)
if match:
data = match.groups()[0]
noteobj = RelLib.Note()
self.name_map["@%s@" % data] = noteobj
noteobj.append(match.groups()[1])
innote = True
elif personRE.match(line):
elif PERSON_RE.match(line):
self.person_count += 1
def get_map(self):
@ -870,14 +874,14 @@ class GedcomParser(UpdateCallback):
def parse_name_personal(self, text):
name = RelLib.Name()
m = snameRegexp.match(text)
m = SURNAME_RE.match(text)
if m:
names = m.groups()
name.set_first_name(names[1].strip())
name.set_surname(names[0].strip())
else:
try:
names = nameRegexp.match(text).groups()
names = NAME_RE.match(text).groups()
name.set_first_name(names[0].strip())
name.set_surname(names[2].strip())
name.set_suffix(names[4].strip())
@ -1061,9 +1065,7 @@ class GedcomParser(UpdateCallback):
state = CurrentState(person=self.person, level=1)
# do the actual parsing
self.debug=True
self.parse_person_level(state, self.indi_parse_tbl, self.func_person_event)
self.debug=False
# Add the default reference if no source has found
if self.use_def_src and len(self.person.get_source_references()) == 0:
@ -1308,7 +1310,7 @@ class GedcomParser(UpdateCallback):
sub_state.lds_ord = RelLib.LdsOrd()
sub_state.lds_ord.set_type(lds_type)
sub_state.place = None
sub_state.place_fields = _place_field
sub_state.place_fields = PLACE_FIELD
state.person.lds_ord_list.append(sub_state.lds_ord)
self.parse_person_level(sub_state, self.lds_parse_tbl, self.func_person_ignore)
@ -1336,8 +1338,8 @@ class GedcomParser(UpdateCallback):
try:
state.place = self.find_or_create_place(line.data)
state.place.set_title(line.data)
state.place_handle = place.handle
state.lds_ord.set_place_handle(place_handle)
state.place.handle = state.place.handle
state.lds_ord.set_place_handle(state.place.handle)
except NameError:
pass
@ -1365,7 +1367,7 @@ class GedcomParser(UpdateCallback):
sub_state = CurrentState()
sub_state.person = state.person
sub_state.level = state.level + 1
sub_state.ftype = _TYPE_BIRTH
sub_state.ftype = TYPE_BIRTH
sub_state.primary = False
notelist = []
@ -1379,7 +1381,7 @@ class GedcomParser(UpdateCallback):
# need to add it to thie list.
if not handle in [f[0] for f in self.person.get_parent_family_handle_list()]:
if int(sub_state.ftype) in rel_types:
if int(sub_state.ftype) in RELATION_TYPES:
state.person.add_parent_family_handle(handle)
else:
if state.person.get_main_parents_family_handle() == handle:
@ -1399,12 +1401,12 @@ class GedcomParser(UpdateCallback):
ref = RelLib.ChildRef()
ref.ref = state.person.handle
ref.set_mother_relation(sub_state.ftype)
ref.set_father_relation(sub_stateftype)
ref.set_father_relation(sub_state.ftype)
family.add_child_ref(ref)
self.db.commit_family(family, self.trans)
def func_person_famc_pedi(self, line, state):
state.ftype = pedi_type.get(line.data.lower(),RelLib.ChildRefType.UNKNOWN)
state.ftype = PEDIGREE_TYPES.get(line.data.lower(), RelLib.ChildRefType.UNKNOWN)
def func_person_famc_note(self, line, state):
if not line.data.strip() or line.data and line.data[0] != "@":
@ -1466,7 +1468,8 @@ class GedcomParser(UpdateCallback):
sub_state.ref.ref = handle
sub_state.ignore = False
self.parse_person_level(sub_state, self.asso_parse_tbl, self.func_person_ignore)
self.parse_person_level(sub_state, self.asso_parse_tbl,
self.func_person_ignore)
if not sub_state.ignore:
state.person.add_person_ref(sub_state.ref)
@ -1618,7 +1621,7 @@ class GedcomParser(UpdateCallback):
intid = create_id()
place.set_handle(intid)
place.set_title(title)
load_place_values(place,title,_place_field)
load_place_values(place,title,PLACE_FIELD)
place.set_gramps_id(new_id)
self.db.add_place(place,self.trans)
self.lid2id[title] = intid
@ -1695,17 +1698,17 @@ class GedcomParser(UpdateCallback):
break
# FTW
elif line.token == TOKEN__FREL:
frel = pedi_type.get(line.data.lower(),_TYPE_BIRTH)
frel = PEDIGREE_TYPES.get(line.data.lower(),TYPE_BIRTH)
# FTW
elif line.token == TOKEN__MREL:
mrel = pedi_type.get(line.data.lower(),_TYPE_BIRTH)
mrel = PEDIGREE_TYPES.get(line.data.lower(),TYPE_BIRTH)
elif line.token == TOKEN_ADOP:
mrel = _TYPE_ADOPT
frel = _TYPE_ADOPT
mrel = TYPE_ADOPT
frel = TYPE_ADOPT
# Legacy
elif line.token == TOKEN__STAT:
mrel = _TYPE_BIRTH
frel = _TYPE_BIRTH
mrel = TYPE_BIRTH
frel = TYPE_BIRTH
# Legacy _PREF
elif line.token == TOKEN__PRIMARY:
pass
@ -1926,7 +1929,7 @@ class GedcomParser(UpdateCallback):
if int(event.get_type()) != RelLib.EventType.CUSTOM:
if not event.get_description():
text = _event_family_str % {
text = EVENT_FAMILY_STR % {
'event_name' : str(event.get_type()),
'family' : Utils.family_name(self.family,self.db),
}
@ -2145,7 +2148,7 @@ class GedcomParser(UpdateCallback):
def parse_ord(self,lds_ord,level):
note = ""
pf = _place_field
pf = PLACE_FIELD
place = None
while True:
@ -2215,8 +2218,8 @@ class GedcomParser(UpdateCallback):
def func_event_type(self, line, event_ref, event, level):
if event.get_type().is_custom():
if ged2gramps.has_key(line.data):
name = RelLib.EventType(ged2gramps[line.data])
if GED_2_GRAMPS.has_key(line.data):
name = RelLib.EventType(GED_2_GRAMPS[line.data])
else:
val = self.gedsource.tag2gramps(line.data)
if val:
@ -2226,8 +2229,8 @@ class GedcomParser(UpdateCallback):
event.set_type(name)
else:
try:
if not ged2gramps.has_key(line.data) and \
not ged2fam.has_key(line.data) and \
if not GED_2_GRAMPS.has_key(line.data) and \
not GED_2_FAMILY.has_key(line.data) and \
line.data[0] != 'Y':
event.set_description(line.data)
except IndexError:
@ -2342,7 +2345,7 @@ class GedcomParser(UpdateCallback):
place_handle = place.handle
place.set_title(val)
event.set_place_handle(place_handle)
pf = _place_field
pf = PLACE_FIELD
while True:
line = self.get_next()
@ -2403,17 +2406,17 @@ class GedcomParser(UpdateCallback):
event.add_attribute(a)
def parse_adopt_famc(self,level):
mrel = _TYPE_BIRTH
frel = _TYPE_BIRTH
mrel = TYPE_BIRTH
frel = TYPE_BIRTH
while True:
line = self.get_next()
if self.level_is_finished(line, level):
break
elif line.token == TOKEN_ADOP:
if line.data == "HUSB":
frel = _TYPE_ADOPT
frel = TYPE_ADOPT
elif line.data == "WIFE":
mrel = _TYPE_ADOPT
mrel = TYPE_ADOPT
else:
self.not_recognized(level+1)
return (mrel,frel)
@ -2448,8 +2451,8 @@ class GedcomParser(UpdateCallback):
def func_person_attr_type(self, attr, line, level):
if attr.get_type() == "":
if ged2gramps.has_key(line.data):
name = ged2gramps[line.data]
if GED_2_GRAMPS.has_key(line.data):
name = GED_2_GRAMPS[line.data]
else:
val = self.gedsource.tag2gramps(line.data)
if val:
@ -2488,7 +2491,8 @@ class GedcomParser(UpdateCallback):
def func_srcref_data(self, line, state):
date,text = self.parse_source_data(state.level+1)
if date:
d = self.dp.parse(date)
import DateHandler
d = DateHandler.parser.parse(date)
state.src_ref.set_date_object(d)
state.src_ref.set_text(text)
@ -2641,7 +2645,7 @@ class GedcomParser(UpdateCallback):
if self.level_is_finished(line, level):
break
else:
ged2gramps[line.token] = self.parse_label(level+1)
GED_2_GRAMPS[line.token] = self.parse_label(level+1)
def parse_label(self,level):
value = None
@ -2664,7 +2668,7 @@ class GedcomParser(UpdateCallback):
if self.level_is_finished(line, level):
break
else:
ged2fam_custom[line.token_text] = self.parse_label(level+1)
GED_2_FAMILY_CUSTOM[line.token_text] = self.parse_label(level+1)
def ignore_sub_junk(self, level):
while True:
@ -2686,8 +2690,8 @@ class GedcomParser(UpdateCallback):
if self.level_is_finished(line, level):
break
elif line.token == TOKEN_FORM:
global _place_field
_place_field = self.parse_place_form_line(line)
global PLACE_FIELD
PLACE_FIELD = self.parse_place_form_line(line)
else:
self.not_recognized(level+1)
@ -2695,7 +2699,7 @@ class GedcomParser(UpdateCallback):
pf = []
for item in line.data.split(','):
item = item.lower().strip()
fcn = _place_match.get(item,_empty_func)
fcn = PLACE_MATCH.get(item,_empty_func)
pf.append(fcn)
return pf
@ -2890,7 +2894,7 @@ class GedcomParser(UpdateCallback):
if os.path.isfile(full_path):
photo.set_mime_type(Mime.get_type(full_path))
else:
photo.set_mime_type(mime_map.get(form.lower(),'unknown'))
photo.set_mime_type(MIME_MAP.get(form.lower(),'unknown'))
self.db.add_object(photo, self.trans)
self.media_map[path] = photo.handle
else:
@ -2931,7 +2935,7 @@ class GedcomParser(UpdateCallback):
person_event_name(event,state.person)
self.db.add_event(event, self.trans)
def func_person_attr_plac(self, line, level):
def func_person_attr_plac(self, line, state):
if state.attr.get_value() == "":
state.attr.set_value(line.data)
@ -2955,7 +2959,7 @@ class GedcomParser(UpdateCallback):
state.person.add_source_reference(source_ref)
def func_person_refn(self, line, state):
if intRE.match(line.data):
if INT_RE.match(line.data):
try:
self.refn[self.person.handle] = int(line.data)
except:
@ -2995,7 +2999,7 @@ class GedcomParser(UpdateCallback):
if line.data[0] == '@':
aka = RelLib.Name()
try:
names = nameRegexp.match(line.data).groups()
names = NAME_RE.match(line.data).groups()
except:
names = (line.data,"","","","")
if names[0]:
@ -3024,7 +3028,7 @@ class GedcomParser(UpdateCallback):
text = line.data.strip()
data = text.split()
if len(data) == 1:
name = RelLib.Name(person.primary_name)
name = RelLib.Name(state.person.primary_name)
name.set_surname(data[0].strip())
name.set_type(RelLib.NameType.MARRIED)
state.person.add_alternate_name(name)
@ -3107,7 +3111,7 @@ class GedcomParser(UpdateCallback):
if not addr.get_city() and not addr.get_state() and \
not addr.get_postal_code() and not addr.get_country():
match = addr_re.match(text)
match = ADDR_RE.match(text)
if match:
groups = match.groups()
addr.set_street(groups[0].strip())
@ -3117,7 +3121,7 @@ class GedcomParser(UpdateCallback):
addr.set_country(groups[5].strip())
matched = True
match = addr2_re.match(text)
match = ADDR2_RE.match(text)
if match:
groups = match.groups()
addr.set_street(groups[0].strip())
@ -3126,7 +3130,7 @@ class GedcomParser(UpdateCallback):
addr.set_postal_code(groups[4].strip())
matched = True
match = addr3_re.match(text)
match = ADDR3_RE.match(text)
if match:
groups = match.groups()
addr.set_street(groups[0].strip())
@ -3287,7 +3291,7 @@ class GedcomParser(UpdateCallback):
def person_event_name(event,person):
if event.get_type().is_custom():
if not event.get_description():
text = _event_person_str % {
text = EVENT_PERSON_STR % {
'event_name' : str(event.get_type()),
'person' : NameDisplay.displayer.display(person),
}
@ -3296,7 +3300,7 @@ def person_event_name(event,person):
def load_place_values(place,text,pf=None):
items = [item.strip() for item in text.split(',')]
if not pf:
pf = _place_field
pf = PLACE_FIELD
if len(items) != len(pf):
return
@ -3342,8 +3346,9 @@ if __name__ == "__main__":
db_class = gramps_db_factory(const.app_gramps)
database = db_class()
database.load("test.grdb",lambda x: None, mode="w")
np = NoteParser(sys.argv[1], False, 0)
g = GedcomParser(database,sys.argv[1],callback, codeset, np.get_map(),
f = open(sys.argv[1],"rU")
np = NoteParser(f, False, 0)
g = GedcomParser(database, f, sys.argv[1], callback, codeset, np.get_map(),
np.get_lines(),np.get_persons())
if False:
pr = hotshot.Profile('mystats.profile')