From f3a7e259e5afa1f51b69e58c95756aae70c65d77 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Fri, 7 Aug 2009 22:32:30 +0000 Subject: [PATCH] SQL Export/Import about 30% done: Notes are complete. People and Events close. The rest still to check. svn: r12917 --- src/plugins/export/ExportSql.py | 151 +++++++++------ src/plugins/import/ImportSql.py | 318 +++++++++++++++++++++++++++++++- 2 files changed, 401 insertions(+), 68 deletions(-) diff --git a/src/plugins/export/ExportSql.py b/src/plugins/export/ExportSql.py index ec4fcb4bc..6ceb90629 100644 --- a/src/plugins/export/ExportSql.py +++ b/src/plugins/export/ExportSql.py @@ -39,6 +39,18 @@ from gui.utils import ProgressMeter import ExportOptions from Utils import create_id +def lookup(index, event_ref_list): + if index < 0: + return None + else: + count = 0 + for event_ref in event_ref_list: + (private, note_list, attribute_list, ref, role) = event_ref + if index == count: + return ref + count += 1 + return None + def makeDB(db): db.query("""drop table note;""") db.query("""drop table person;""") @@ -59,37 +71,40 @@ def makeDB(db): db.query("""drop table media_ref;""") db.query("""drop table address;""") db.query("""drop table attribute;""") - + # Completed db.query("""CREATE TABLE note ( handle CHARACTER(25), gid CHARACTER(25), text TEXT, - format TEXT, + format INTEGER, note_type1 INTEGER, note_type2 TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") - + db.query("""CREATE TABLE name ( + from_handle CHARACTER(25), handle CHARACTER(25), + primary_name BOOLEAN, private BOOLEAN, first_name TEXT, surname TEXT, suffix TEXT, title TEXT, - name_type0 TEXT, + name_type0 INTEGER, name_type1 TEXT, prefix TEXT, patronymic TEXT, group_as TEXT, - sort_as TEXT, - display_as TEXT, + sort_as INTEGER, + display_as INTEGER, call TEXT);""") db.query("""CREATE TABLE date ( - type CHARACTER(10), + from_type CHARACTER(25), + from_handle CHARACTER(25), calendar INTEGER, modifier INTEGER, quality INTEGER, @@ -108,11 +123,11 @@ def makeDB(db): db.query("""CREATE TABLE person ( handle CHARACTER(25), gid CHARACTER(25), - gender CHAR(1), - death_ref_index TEXT, - birth_ref_index TEXT, - change TEXT, - marker0 TEXT, + gender INTEGER, + death_ref_handle TEXT, + birth_ref_handle TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") @@ -121,10 +136,10 @@ def makeDB(db): gid CHARACTER(25), father_handle CHARACTER(25), mother_handle CHARACTER(25), - the_type0 TEXT, + the_type0 INTEGER, the_type1 TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") @@ -134,19 +149,19 @@ def makeDB(db): title TEXT, long FLOAT, lat FLOAT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") db.query("""CREATE TABLE event ( handle CHARACTER(25), gid CHARACTER(25), - the_type0 TEXT, + the_type0 INTEGER, the_type1 TEXT, description TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") @@ -157,8 +172,8 @@ def makeDB(db): author TEXT, pubinfo TEXT, abbrev TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") @@ -168,26 +183,26 @@ def makeDB(db): path TEXT, mime TEXT, desc TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") db.query("""CREATE TABLE repository ( handle CHARACTER(25), gid CHARACTER(25), - the_type0 TEXT, + the_type0 INTEGER, the_type1 TEXT, name TEXT, - change TEXT, - marker0 TEXT, + change INTEGER, + marker0 INTEGER, marker1 TEXT, private BOOLEAN);""") db.query("""CREATE TABLE link ( - from_type CHARACTER(10), + from_type CHARACTER(25), from_handle CHARACTER(25), - to_type CHARACTER(10), + to_type CHARACTER(25), to_handle CHARACTER(25));""") db.query("""CREATE TABLE markup ( @@ -232,7 +247,7 @@ def makeDB(db): from_type CHARACTER(25), from_handle CHARACTER(25), handle CHARACTER(25), - type CHARACTER(10), + type CHARACTER(25), place TEXT, famc CHARACTER(25), temple TEXT, @@ -261,7 +276,7 @@ def makeDB(db): db.query("""CREATE TABLE attribute ( handle CHARACTER(25), - from_type CHARACTER(10), + from_type CHARACTER(25), from_handle CHARACTER(25), the_type0 INTEGER, the_type1 TEXT, @@ -440,7 +455,7 @@ def export_note(db, handle, gid, text, format, note_type0, handle, gid, text, format, note_type0, note_type1, change, marker0, marker1, private) -def export_name(db, handle, data): +def export_name(db, from_handle, handle, primary, data): if data: (private, source_list, note_list, date, first_name, surname, suffix, title, @@ -448,7 +463,9 @@ def export_name(db, handle, data): group_as, sort_as, display_as, call) = data db.query("""INSERT into name ( + from_handle, handle, + primary_name, private, first_name, surname, @@ -462,19 +479,20 @@ def export_name(db, handle, data): sort_as, display_as, call - ) values (?, ?, ?, ?, ?, ?, ?, + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""", - handle, private, first_name, surname, suffix, title, + from_handle, handle, primary, private, first_name, surname, suffix, title, name_type[0], name_type[1], prefix, patronymic, group_as, sort_as, display_as, call) - export_date(db, "name", handle, date) + if date: + export_date(db, "name", handle, date) export_list(db, "name", handle, "note", note_list) export_source_list(db, "name", handle, source_list) -def export_date(db, date_type, handle, data): - if data: - (calendar, modifier, quality, dateval, text, sortval, newyear) = data +def export_date(db, date_type, handle, date): + if True: + (calendar, modifier, quality, dateval, text, sortval, newyear) = date if len(dateval) == 4: day1, month1, year1, slash1 = dateval day2, month2, year2, slash2 = 0, 0, 0, 0 @@ -483,7 +501,8 @@ def export_date(db, date_type, handle, data): else: raise ("ERROR:", dateval) db.query("""INSERT INTO date ( - type, + from_type, + from_handle, calendar, modifier, quality, @@ -497,9 +516,9 @@ def export_date(db, date_type, handle, data): slash2, text, sortval, - newyear) VALUES (?, ?, ?, ?, ?, ?, ?, ?, + newyear) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""", - date_type, calendar, modifier, quality, + date_type, handle, calendar, modifier, quality, day1, month1, year1, slash1, day2, month2, year2, slash2, text, sortval, newyear) @@ -520,7 +539,8 @@ def export_source_list(db, from_type, handle, source_list): for source in source_list: (date, private, note_list, confidence, ref, page) = source export_source_ref(db, from_type, handle, ref, private, confidence, page) - export_date(db, "source", ref, date) + if date: + export_date(db, "source", ref, date) export_list(db, "source", ref, "note", note_list) def export_media_list(db, from_type, from_handle, media_list): @@ -588,11 +608,9 @@ def exportData(database, filename, option_box=None, callback=None): len(database.family_map) + len(database.repository_map) + len(database.place_map) + - len(database.source_map) + len(database.media_map) + - len(database.place_map) + len(database.source_map)) - count = 0 + count = 0.0 db = Database(filename) makeDB(db) @@ -622,7 +640,8 @@ def exportData(database, filename, option_box=None, callback=None): (handle, gid, the_type, date, description, place, source_list, note_list, media_list, attribute_list, change, marker, private) = event - export_date(db, "event", event_handle, date) + if date: + export_date(db, "event", event_handle, date) db.query("""INSERT INTO event ( handle, gid, @@ -680,13 +699,12 @@ def exportData(database, filename, option_box=None, callback=None): private, # 19 person_ref_list, # 20 ) = person - db.query("""INSERT INTO person ( handle, gid, gender, - death_ref_index, - birth_ref_index, + death_ref_handle, + birth_ref_handle, change, marker0, marker1, @@ -694,8 +712,8 @@ def exportData(database, filename, option_box=None, callback=None): handle, gid, gender, - death_ref_index, - birth_ref_index, + lookup(death_ref_index, event_ref_list), + lookup(birth_ref_index, event_ref_list), change, marker[0], marker[1], @@ -724,7 +742,8 @@ def exportData(database, filename, option_box=None, callback=None): (street, city, county, state, country, postal, phone) = location addr_handle = create_id() export_address(db, "person", handle, addr_handle, street, city, county, state, country, postal, phone, private) - export_date(db, "address", addr_handle, date) + if date: + export_date(db, "address", addr_handle, date) export_list(db, "address", addr_handle, "note", anote_list) export_source_list(db, "address", addr_handle, source_list) @@ -736,15 +755,17 @@ def exportData(database, filename, option_box=None, callback=None): famc, temple, status, lprivate) = ldsord lds_handle = create_id() export_lds(db, "person", handle, lds_handle, type, place, famc, temple, status, lprivate) - export_date(db, "lds", lds_handle, date) + if date: + export_date(db, "lds", lds_handle, date) export_list(db, "lds", lds_handle, "note", lnote_list) export_source_list(db, "lds", lds_handle, lsource_list) # ------------------------------------- # Names # ------------------------------------- - export_name(db, handle, primary_name) - map(lambda name: export_name(db, handle, name), alternate_names) + export_name(db, handle, create_id(), True, primary_name) + map(lambda name: export_name(db, handle, create_id(), False, name), + alternate_names) count += 1 callback(100 * count/total) @@ -794,7 +815,8 @@ def exportData(database, filename, option_box=None, callback=None): famc, temple, status, lprivate) = ldsord lds_handle = create_id() export_lds(db, "family", handle, lds_handle, type, place, famc, temple, status, lprivate) - export_date(db, "lds", lds_handle, date) + if date: + export_date(db, "lds", lds_handle, date) export_list(db, "lds", lds_handle, "note", lnote_list) export_source_list(db, "lds", lds_handle, lsource_list) @@ -830,7 +852,8 @@ def exportData(database, filename, option_box=None, callback=None): addr_handle = create_id() export_address(db, "repository", handle, addr_handle, street, city, county, state, country, postal, phone, private) - export_date(db, "address", addr_handle, date) + if date: + export_date(db, "address", addr_handle, date) export_list(db, "address", addr_handle, "note", anote_list) export_source_list(db, "address", addr_handle, asource_list) @@ -896,6 +919,12 @@ def exportData(database, filename, option_box=None, callback=None): export_list(db, "source", handle, "note", note_list) export_media_list(db, "source", handle, media_list) # FIXME: reporef_list, datamap + #print "FIXME: reporef_list", reporef_list + #print "FIXME: datamap", datamap +#FIXME: reporef_list [] +#FIXME: datamap {} +#FIXME: reporef_list [([], u'b2cfa6e37654b308559', '', (2, u''), False)] +#FIXME: datamap {} count += 1 callback(100 * count/total) @@ -926,8 +955,8 @@ def exportData(database, filename, option_box=None, callback=None): private) VALUES (?,?,?,?,?,?,?,?,?);""", handle, gid, path, mime, desc, change, marker[0], marker[1], private) - - export_date(db, "media", handle, date) + if date: + export_date(db, "media", handle, date) export_list(db, "media", handle, "note", note_list) export_source_list(db, "media", handle, source_list) export_attribute_list(db, "media", handle, attribute_list) diff --git a/src/plugins/import/ImportSql.py b/src/plugins/import/ImportSql.py index c719f8b40..f5f2e2ed7 100644 --- a/src/plugins/import/ImportSql.py +++ b/src/plugins/import/ImportSql.py @@ -53,6 +53,19 @@ from Utils import gender as gender_map from gui.utils import ProgressMeter from Utils import create_id +def lookup(handle, event_ref_list): + if handle is None: + return -1 + else: + count = 0 + for event_ref in event_ref_list: + (private, note_list, attribute_list, ref, role) = event_ref + if handle == ref: + return count + count += 1 + return -1 + + #------------------------------------------------------------------------- # # SQL Reader @@ -91,7 +104,8 @@ class Database(object): class SQLReader(object): def __init__(self, db, filename, callback): - print filename + if not callable(callback): + callback = lambda (percent): None # dummy self.db = db self.filename = filename self.callback = callback @@ -107,15 +121,162 @@ class SQLReader(object): return None return sql + def get_event_refs(self, sql, from_type, from_handle): + results = sql.query("select * from event_ref where from_type = ? and from_handle = ?;", + from_type, + from_handle) + return [self.pack_event_ref(sql, result) for result in results] + + def pack_event_ref(self, sql, data): + (from_type, + from_handle, + ref, + role0, + role1, + private) = data + note_list = self.get_links(sql, "event_ref", from_handle, "note") + attribute_list = self.get_attribute_list(sql, "event_ref", from_handle) + return (private, note_list, attribute_list, ref, (role0, role1)) + + def get_url_list(self, sql, from_type, from_handle): + # FIXME + return [] + + def get_attribute_list(self, sql, from_type, from_handle): + # FIXME + return [] + + def get_media_list(self, sql, from_type, from_handle): + # FIXME + return [] + + def get_sources(self, sql, handle): + results = sql.query("""select * from source where handle = ?;""", + handle) + return [self.pack_source(sql, result) for result in results] + + def pack_source(self, sql, data): + (handle, + gid, + title, + author, + pubinfo, + abbrev, + change, + marker0, + marker1, + private) = data + + # FIXME + note_list = self.get_links(sql, "source", handle, "note") + media_list = self.get_media_list(sql, "source", handle) + reporef_list = [] + datamap = None + + return (handle, gid, title, + author, pubinfo, + note_list, + media_list, + abbrev, + change, datamap, + reporef_list, + (marker0, marker1), private) + + def get_links(self, sql, from_type, from_handle, to_type): + results = sql.query("""select to_handle from link where from_type = ? and from_handle = ? and to_type = ?;""", + from_type, from_handle, to_type) + return [str(result) for result in results] + + def get_names(self, sql, handle, primary): + result = sql.query("""select * from name where from_handle = ? and primary_name = ?;""", + handle, primary) + return result + + def package_name(self, sql, handle, primary): + names = self.get_names(sql, handle, primary) + result = [self.pack(sql, name) for name in names] + if primary: + if len(result) == 1: + return result[0] + elif len(result) == 0: + return None + else: + raise Exception("too many primary names") + else: + return result + + def pack(self, sql, data): + # unpack name from SQL table: + (from_handle, + handle, + primary_name, + private, + first_name, + surname, + suffix, + title, + name_type0, + name_type1, + prefix, + patronymic, + group_as, + sort_as, + display_as, + call) = data + # FIXME + source_list = self.get_links(sql, "name", from_handle, "source") + note_list = self.get_links(sql, "name", from_handle, "note") + date = self.get_date(sql, "name", from_handle) + return (private, source_list, note_list, date, + first_name, surname, suffix, title, + (name_type0, name_type1), prefix, patronymic, + group_as, sort_as, display_as, call) + + def get_date(self, sql, from_type, from_handle): + rows = sql.query("select * from date where from_type = ? and from_handle = ?;", + from_type, from_handle) + if len(rows) == 1: + (from_type, + from_handle, + calendar, + modifier, + quality, + day1, + month1, + year1, + slash1, + day2, + month2, + year2, + slash2, + text, + sortval, + newyear) = rows[0] + dateval = day1, month1, year1, slash1, day2, month2, year2, slash2 + return (calendar, modifier, quality, dateval, text, sortval, newyear) + elif len(rows) == 0: + return None + else: + raise Exception("ERROR, wrong number of dates: %s" % rows) + def process(self): - progress = ProgressMeter(_('SQLite Import')) - progress.set_pass(_('Reading data...'), 1) sql = self.openSQL() - progress.set_pass(_('Importing data...'), 100) + total = (sql.query("select count(*) from note;")[0][0] + + sql.query("select count(*) from person;")[0][0] + + sql.query("select count(*) from event;")[0][0] + + sql.query("select count(*) from family;")[0][0] + + sql.query("select count(*) from repository;")[0][0] + + sql.query("select count(*) from place;")[0][0] + + sql.query("select count(*) from media;")[0][0] + + sql.query("select count(*) from source;")[0][0]) self.trans = self.db.transaction_begin("",batch=True) self.db.disable_signals() + count = 0.0 t = time.time() - + # --------------------------------- + # Process note + # --------------------------------- + # FIXME: 11 note objects were referenced but not found notes = sql.query("""select * from note;""") for note in notes: (handle, @@ -128,6 +289,7 @@ class SQLReader(object): marker0, marker1, private) = note + styled_text = [text, []] markups = sql.query("""select * from markup where handle = ?""", handle) for markup in markups: (mhandle, @@ -135,14 +297,156 @@ class SQLReader(object): markup1, value, start_stop_list) = markup + ss_list = eval(start_stop_list) + styled_text[1] += [((markup0, markup1), value, ss_list)] + self.db.note_map[str(handle)] = (str(handle), gid, styled_text, + format, (note_type1, note_type2), change, + (marker0, marker1), private) + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process person + # --------------------------------- + people = sql.query("""select * from person;""") + for person in people: + if person is None: + continue + (handle, # 0 + gid, # 1 + gender, # 2 + death_ref_handle, # 5 + birth_ref_handle, # 6 + change, # 17 + marker0, # 18 + marker1, # 18 + private, # 19 + ) = person + # primary_name, # 3 + primary_name = self.package_name(sql, handle, True) + # alternate_names, # 4 + alternate_names = self.package_name(sql, handle, False) # list + # event_ref_list, # 7 + event_ref_list = self.get_event_refs(sql, "person", handle) + # family_list, # 8 + family_list = [] + # parent_family_list, # 9 + parent_family_list = [] + # media_list, # 10 + media_list = [] + # address_list, # 11 + address_list = [] + # attribute_list, # 12 + attribute_list = [] + # urls, # 13 + urls = self.get_url_list(sql, "person", handle) + # lds_ord_list, # 14 + lds_ord_list = [] + # psource_list, # 15 + psource_list = self.get_links(sql, "person", handle, "source") + # pnote_list, # 16 + pnote_list = [] + # person_ref_list, # 20 + person_ref_list = [] + death_ref_index = lookup(death_ref_handle, event_ref_list) + birth_ref_index = lookup(birth_ref_handle, event_ref_list) + self.db.person_map[str(handle)] = (str(handle), # 0 + gid, # 1 + gender, # 2 + primary_name, # 3 + alternate_names, # 4 + death_ref_index, # 5 + birth_ref_index, # 6 + event_ref_list, # 7 + family_list, # 8 + parent_family_list, # 9 + media_list, # 10 + address_list, # 11 + attribute_list, # 12 + urls, # 13 + lds_ord_list, # 14 + psource_list, # 15 + pnote_list, # 16 + change, # 17 + (marker0, marker1), # 18 + private, # 19 + person_ref_list, # 20 + ) + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process event + # --------------------------------- + events = sql.query("""select * from event;""") + for event in events: + (handle, + gid, + the_type0, + the_type1, + description, + change, + marker0, + marker1, + private) = event + + # FIXME: + source_list = self.get_links(sql, "event", handle, "source") + note_list = self.get_links(sql, "event", handle, "note") + media_list = self.get_links(sql, "event", handle, "media") + attribute_list = self.get_links(sql, "event", handle, "attribute") + + date = self.get_date(sql, "event", handle) + place = None # self.get_place() + + data = (str(handle), gid, (the_type0, the_type1), date, description, place, + source_list, note_list, media_list, attribute_list, + change, (marker0, marker1), private) + + self.db.event_map[str(handle)] = data + + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process family + # --------------------------------- + people = sql.query("""select * from family;""") + for person in people: + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process repository + # --------------------------------- + people = sql.query("""select * from repository;""") + for person in people: + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process place + # --------------------------------- + people = sql.query("""select * from place;""") + for person in people: + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process source + # --------------------------------- + people = sql.query("""select * from source;""") + for person in people: + count += 1 + self.callback(100 * count/total) + # --------------------------------- + # Process media + # --------------------------------- + people = sql.query("""select * from media;""") + for person in people: + count += 1 + self.callback(100 * count/total) t = time.time() - t msg = ngettext('Import Complete: %d second','Import Complete: %d seconds', t ) % t - self.db.transaction_commit(self.trans,_("CSV import")) + self.db.transaction_commit(self.trans,_("SQL import")) self.db.enable_signals() self.db.request_rebuild() print msg - progress.close() return None def importData(db, filename, callback=None):