Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean up of transactions; fixed bug on reference to events; removed clean_cache; GEDCOM import works

Browse Source 
svn: r18320
This commit is contained in:
Doug Blank 2011-10-14 14:43:59 +00:00
parent 6693cbe240
commit 8fd1b37d86
2 changed files with 72 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/web/dbdjango.py
View File

@ -224,23 +224,28 @@ class DbDjango(DbWriteBase, DbReadBase):
self.undo_history_callback = None self.undo_history_callback = None
self.modified = 0 self.modified = 0
self.txn = DjangoTxn("DbDjango Transaction", self) self.txn = DjangoTxn("DbDjango Transaction", self)
self.cache = {} # Import cache for gedcom import, uses transactions, and
self.use_cache = False # two step adding of objects.
self.import_cache = {}
self.use_import_cache = False
self.use_db_cache = True
def prepare_import(self): def prepare_import(self):
""" """
DbDjango does not commit data, but saves them for later DbDjango does not commit data on gedcom import, but saves them
commit. for later commit.
""" """
self.use_cache = True self.use_import_cache = True
self.cache = {} self.import_cache = {}
def commit_import(self): def commit_import(self):
""" """
Commits the items that were queued up during the last import. Commits the items that were queued up during the last gedcom
import for two step adding.
""" """
for key in self.cache.keys(): # First we add the primary objects:
obj = self.cache[key] for key in self.import_cache.keys():
obj = self.import_cache[key]
if isinstance(obj, gen.lib.Person): if isinstance(obj, gen.lib.Person):
self.dji.add_person(obj.serialize()) self.dji.add_person(obj.serialize())
elif isinstance(obj, gen.lib.Family): elif isinstance(obj, gen.lib.Family):
@ -255,8 +260,9 @@ class DbDjango(DbWriteBase, DbReadBase):
self.dji.add_source(obj.serialize()) self.dji.add_source(obj.serialize())
elif isinstance(obj, gen.lib.Note): elif isinstance(obj, gen.lib.Note):
self.dji.add_note(obj.serialize()) self.dji.add_note(obj.serialize())
for key in self.cache.keys(): # Next we add the links:
obj = self.cache[key] for key in self.import_cache.keys():
obj = self.import_cache[key]
if isinstance(obj, gen.lib.Person): if isinstance(obj, gen.lib.Person):
self.dji.add_person_detail(obj.serialize()) self.dji.add_person_detail(obj.serialize())
elif isinstance(obj, gen.lib.Family): elif isinstance(obj, gen.lib.Family):
@ -271,8 +277,8 @@ class DbDjango(DbWriteBase, DbReadBase):
self.dji.add_source_detail(obj.serialize()) self.dji.add_source_detail(obj.serialize())
elif isinstance(obj, gen.lib.Note): elif isinstance(obj, gen.lib.Note):
self.dji.add_note_detail(obj.serialize()) self.dji.add_note_detail(obj.serialize())
self.use_cache = False self.use_import_cache = False
self.cache = {} self.import_cache = {}
def transaction_commit(self, txn): def transaction_commit(self, txn):
pass pass
@ -547,17 +553,17 @@ class DbDjango(DbWriteBase, DbReadBase):
return [] return []
def get_event_from_handle(self, handle): def get_event_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
event = self.dji.Event.get(handle=handle) event = self.dji.Event.get(handle=handle)
except: except:
return None return None
self.make_event(event) return self.make_event(event)
def get_family_from_handle(self, handle): def get_family_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
family = self.dji.Family.get(handle=handle) family = self.dji.Family.get(handle=handle)
except: except:
@ -565,10 +571,10 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_family(family) return self.make_family(family)
def get_family_from_gramps_id(self, gramps_id): def get_family_from_gramps_id(self, gramps_id):
if self.cache: if self.import_cache:
for handle in self.cache: for handle in self.import_cache:
if self.cache[handle].gramps_id == gramps_id: if self.import_cache[handle].gramps_id == gramps_id:
return self.cache[handle] return self.import_cache[handle]
try: try:
family = self.dji.Family.get(gramps_id=gramps_id) family = self.dji.Family.get(gramps_id=gramps_id)
except: except:
@ -576,8 +582,8 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_family(family) return self.make_family(family)
def get_repository_from_handle(self, handle): def get_repository_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
repository = self.dji.Repository.get(handle=handle) repository = self.dji.Repository.get(handle=handle)
except: except:
@ -585,8 +591,8 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_repository(repository) return self.make_repository(repository)
def get_person_from_handle(self, handle): def get_person_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
#person = self.dji.Person.select_related().get(handle=handle) #person = self.dji.Person.select_related().get(handle=handle)
person = self.dji.Person.get(handle=handle) person = self.dji.Person.get(handle=handle)
@ -595,64 +601,64 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_person(person) return self.make_person(person)
def make_repository(self, repository): def make_repository(self, repository):
if repository.cache: if self.use_db_cache and repository.cache:
data = cPickle.loads(base64.decodestring(repository.cache)) data = cPickle.loads(base64.decodestring(repository.cache))
else: else:
data = self.dji.get_family(family) data = self.dji.get_family(family)
return gen.lib.Repository.create(data) return gen.lib.Repository.create(data)
def make_source(self, source): def make_source(self, source):
if source.cache: if self.use_db_cache and source.cache:
data = cPickle.loads(base64.decodestring(source.cache)) data = cPickle.loads(base64.decodestring(source.cache))
else: else:
data = self.dji.get_source(source) data = self.dji.get_source(source)
return gen.lib.Source.create(data) return gen.lib.Source.create(data)
def make_family(self, family): def make_family(self, family):
if family.cache: if self.use_db_cache and family.cache:
data = cPickle.loads(base64.decodestring(family.cache)) data = cPickle.loads(base64.decodestring(family.cache))
else: else:
data = self.dji.get_family(family) data = self.dji.get_family(family)
return gen.lib.Family.create(data) return gen.lib.Family.create(data)
def make_person(self, person): def make_person(self, person):
if person.cache: if self.use_db_cache and person.cache:
data = cPickle.loads(base64.decodestring(person.cache)) data = cPickle.loads(base64.decodestring(person.cache))
else: else:
data = self.dji.get_person(person) data = self.dji.get_person(person)
return gen.lib.Person.create(data) return gen.lib.Person.create(data)
def make_event(self, event): def make_event(self, event):
if event.cache: if self.use_db_cache and event.cache:
data = cPickle.loads(base64.decodestring(event.cache)) data = cPickle.loads(base64.decodestring(event.cache))
else: else:
data = self.dji.get_event(event) data = self.dji.get_event(event)
return gen.lib.Event.create(data) return gen.lib.Event.create(data)
def make_note(self, note): def make_note(self, note):
if note.cache: if self.use_db_cache and note.cache:
data = cPickle.loads(base64.decodestring(note.cache)) data = cPickle.loads(base64.decodestring(note.cache))
else: else:
data = self.dji.get_note(note) data = self.dji.get_note(note)
return gen.lib.Note.create(data) return gen.lib.Note.create(data)
def make_place(self, place): def make_place(self, place):
if place.cache: if self.use_db_cache and place.cache:
data = cPickle.loads(base64.decodestring(place.cache)) data = cPickle.loads(base64.decodestring(place.cache))
else: else:
data = self.dji.get_place(place) data = self.dji.get_place(place)
return gen.lib.Place.create(data) return gen.lib.Place.create(data)
def make_media(self, media): def make_media(self, media):
if media.cache: if self.use_db_cache and media.cache:
data = cPickle.loads(base64.decodestring(media.cache)) data = cPickle.loads(base64.decodestring(media.cache))
else: else:
data = self.dji.get_media(media) data = self.dji.get_media(media)
return gen.lib.Media.create(data) return gen.lib.Media.create(data)
def get_place_from_handle(self, handle): def get_place_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
# FIXME: use object cache # FIXME: use object cache
try: try:
dji_obj = self.dji.Place.get(handle=handle) dji_obj = self.dji.Place.get(handle=handle)
@ -667,8 +673,8 @@ class DbDjango(DbWriteBase, DbReadBase):
return None return None
def get_source_from_handle(self, handle): def get_source_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
source = self.dji.Source.get(handle=handle) source = self.dji.Source.get(handle=handle)
except: except:
@ -676,8 +682,8 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_source(source) return self.make_source(source)
def get_note_from_handle(self, handle): def get_note_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
note = self.dji.Note.get(handle=handle) note = self.dji.Note.get(handle=handle)
except: except:
@ -685,8 +691,8 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.make_note(note) return self.make_note(note)
def get_object_from_handle(self, handle): def get_object_from_handle(self, handle):
if handle in self.cache: if handle in self.import_cache:
return self.cache[handle] return self.import_cache[handle]
try: try:
media = self.dji.Media.get(handle=handle) media = self.dji.Media.get(handle=handle)
except: except:
@ -717,10 +723,10 @@ class DbDjango(DbWriteBase, DbReadBase):
return (family.handle for family in self.dji.Family.all()) return (family.handle for family in self.dji.Family.all())
def get_person_from_gramps_id(self, gramps_id): def get_person_from_gramps_id(self, gramps_id):
if self.cache: if self.import_cache:
for handle in self.cache: for handle in self.import_cache:
if self.cache[handle].gramps_id == gramps_id: if self.import_cache[handle].gramps_id == gramps_id:
return self.cache[handle] return self.import_cache[handle]
match_list = self.dji.Person.filter(gramps_id=gramps_id) match_list = self.dji.Person.filter(gramps_id=gramps_id)
if match_list.count() > 0: if match_list.count() > 0:
return self.make_person(match_list[0]) return self.make_person(match_list[0])
@ -749,19 +755,19 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.dji.Repository.count() return self.dji.Repository.count()
def get_place_cursor(self): def get_place_cursor(self):
return Cursor(self.dji.Place, self.dji.get_place).iter() return Cursor(self.dji.Place, self.make_place).iter()
def get_person_cursor(self): def get_person_cursor(self):
return Cursor(self.dji.Person, self.dji.get_person).iter() return Cursor(self.dji.Person, self.make_person).iter()
def get_family_cursor(self): def get_family_cursor(self):
return Cursor(self.dji.Family, self.dji.get_family).iter() return Cursor(self.dji.Family, self.make_family).iter()
def get_events_cursor(self): def get_events_cursor(self):
return Cursor(self.dji.Event, self.dji.get_event).iter() return Cursor(self.dji.Event, self.make_event).iter()
def get_source_cursor(self): def get_source_cursor(self):
return Cursor(self.dji.Source, self.dji.get_source).iter() return Cursor(self.dji.Source, self.make_source).iter()
def has_person_handle(self, handle): def has_person_handle(self, handle):
return self.dji.Person.filter(handle=handle).count() == 1 return self.dji.Person.filter(handle=handle).count() == 1
@ -782,6 +788,7 @@ class DbDjango(DbWriteBase, DbReadBase):
return self.dji.Place.filter(handle=handle).count() == 1 return self.dji.Place.filter(handle=handle).count() == 1
def get_raw_person_data(self, handle): def get_raw_person_data(self, handle):
# FIXME?: not cached
try: try:
return self.dji.get_person(self.dji.Person.get(handle=handle)) return self.dji.get_person(self.dji.Person.get(handle=handle))
except: except:
@ -820,7 +827,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_person(self, person, trans, set_gid=True): def add_person(self, person, trans, set_gid=True):
if not person.handle: if not person.handle:
person.handle = Utils.create_id() person.handle = Utils.create_id()
if not person.gramps_id: if not person.gramps_id or set_gid:
person.gramps_id = self.find_next_person_gramps_id() person.gramps_id = self.find_next_person_gramps_id()
self.commit_person(person, trans) self.commit_person(person, trans)
return person.handle return person.handle
@ -828,7 +835,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_family(self, family, trans, set_gid=True): def add_family(self, family, trans, set_gid=True):
if not family.handle: if not family.handle:
family.handle = Utils.create_id() family.handle = Utils.create_id()
if not family.gramps_id: if not family.gramps_id or set_gid:
family.gramps_id = self.find_next_family_gramps_id() family.gramps_id = self.find_next_family_gramps_id()
self.commit_family(family, trans) self.commit_family(family, trans)
return family.handle return family.handle
@ -836,7 +843,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_source(self, source, trans, set_gid=True): def add_source(self, source, trans, set_gid=True):
if not source.handle: if not source.handle:
source.handle = Utils.create_id() source.handle = Utils.create_id()
if not source.gramps_id: if not source.gramps_id or set_gid:
source.gramps_id = self.find_next_source_gramps_id() source.gramps_id = self.find_next_source_gramps_id()
self.commit_source(source, trans) self.commit_source(source, trans)
return source.handle return source.handle
@ -844,7 +851,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_repository(self, repository, trans, set_gid=True): def add_repository(self, repository, trans, set_gid=True):
if not repository.handle: if not repository.handle:
repository.handle = Utils.create_id() repository.handle = Utils.create_id()
if not repository.gramps_id: if not repository.gramps_id or set_gid:
repository.gramps_id = self.find_next_repository_gramps_id() repository.gramps_id = self.find_next_repository_gramps_id()
self.commit_repository(repository, trans) self.commit_repository(repository, trans)
return repository.handle return repository.handle
@ -852,7 +859,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_note(self, note, trans, set_gid=True): def add_note(self, note, trans, set_gid=True):
if not note.handle: if not note.handle:
note.handle = Utils.create_id() note.handle = Utils.create_id()
if not note.gramps_id: if not note.gramps_id or set_gid:
note.gramps_id = self.find_next_note_gramps_id() note.gramps_id = self.find_next_note_gramps_id()
self.commit_note(note, trans) self.commit_note(note, trans)
return note.handle return note.handle
@ -860,7 +867,7 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_place(self, place, trans, set_gid=True): def add_place(self, place, trans, set_gid=True):
if not place.handle: if not place.handle:
place.handle = Utils.create_id() place.handle = Utils.create_id()
if not place.gramps_id: if not place.gramps_id or set_gid:
place.gramps_id = self.find_next_place_gramps_id() place.gramps_id = self.find_next_place_gramps_id()
self.commit_place(place, trans) self.commit_place(place, trans)
return place.handle return place.handle
@ -868,31 +875,31 @@ class DbDjango(DbWriteBase, DbReadBase):
def add_event(self, event, trans, set_gid=True): def add_event(self, event, trans, set_gid=True):
if not event.handle: if not event.handle:
event.handle = Utils.create_id() event.handle = Utils.create_id()
if not event.gramps_id: if not event.gramps_id or set_gid:
event.gramps_id = self.find_next_event_gramps_id() event.gramps_id = self.find_next_event_gramps_id()
self.commit_event(event, trans) self.commit_event(event, trans)
return event.handle return event.handle
def commit_person(self, person, trans, change_time=None): def commit_person(self, person, trans, change_time=None):
self.cache[person.handle] = person self.import_cache[person.handle] = person
def commit_family(self, family, trans, change_time=None): def commit_family(self, family, trans, change_time=None):
self.cache[family.handle] = family self.import_cache[family.handle] = family
def commit_source(self, source, trans, change_time=None): def commit_source(self, source, trans, change_time=None):
self.cache[source.handle] = source self.import_cache[source.handle] = source
def commit_repository(self, repository, trans, change_time=None): def commit_repository(self, repository, trans, change_time=None):
self.cache[repository.handle] = repository self.import_cache[repository.handle] = repository
def commit_note(self, note, trans, change_time=None): def commit_note(self, note, trans, change_time=None):
self.cache[note.handle] = note self.import_cache[note.handle] = note
def commit_place(self, place, trans, change_time=None): def commit_place(self, place, trans, change_time=None):
self.cache[place.handle] = place self.import_cache[place.handle] = place
def commit_event(self, event, trans, change_time=None): def commit_event(self, event, trans, change_time=None):
self.cache[event.handle] = event self.import_cache[event.handle] = event
def get_gramps_ids(self, obj_key): def get_gramps_ids(self, obj_key):
key2table = { key2table = {

67
src/web/libdjango.py
View File

@ -1482,70 +1482,3 @@ class DjangoInterface(object):
callback(100) callback(100)
def clear_caches(self, callback=None):
"""
Call this to clear the caches for all primary models.
"""
if not callable(callback):
callback = lambda (percent): None # dummy
callback(0)
count = 0.0
total = (self.Note.all().count() +
self.Person.all().count() +
self.Event.all().count() +
self.Family.all().count() +
self.Repository.all().count() +
self.Place.all().count() +
self.Media.all().count() +
self.Source.all().count())
for item in self.Person.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Family.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Source.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Event.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Repository.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Place.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Media.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
for item in self.Note.all():
item.cache = None
item.save()
callback(100 * count/total)
count += 1
callback(100)