added clear_cache; removed cache set from inside for now; bugs introduced (eg, people events wrong handles to events)---why?

svn: r18319

src/web/dbdjango.py

@@ -21,6 +21,14 @@
 
 """ Implements a Db interface """
 
+## Major issues to fix:
+
+## 1) import from GEDCOM is not associating proper gramps handles between objects
+##    suspect that something not working correct in adding with transactions
+##    BUG: People event_ref are no longer pointing to correct event, even
+##    previously-working export to django from gramps gtk
+## 2) did export from gramps gtk break with the changes for importing gedcom?
+
 #------------------------------------------------------------------------
 #
 # Gramps Modules
@@ -68,7 +76,6 @@ def import_file(db, filename, callback):
 
     >>> import_file(DbDjango(), "/home/user/Untitled_1.ged", lambda a: a)
     """
-    global count
     dbstate = DbState.DbState()
     climanager = CLIManager(dbstate, False) # do not load db_loader
     climanager.do_reg_plugins(dbstate, None)
@@ -217,12 +224,15 @@ class DbDjango(DbWriteBase, DbReadBase):
         self.undo_history_callback = None
         self.modified   = 0
         self.txn = DjangoTxn("DbDjango Transaction", self)
+        self.cache = {}
+        self.use_cache = False
 
     def prepare_import(self):
         """
         DbDjango does not commit data, but saves them for later
         commit.
         """
+        self.use_cache = True
         self.cache = {}
 
     def commit_import(self):
@@ -261,6 +271,8 @@ class DbDjango(DbWriteBase, DbReadBase):
                 self.dji.add_source_detail(obj.serialize())
             elif isinstance(obj, gen.lib.Note):
                 self.dji.add_note_detail(obj.serialize())
+        self.use_cache = False
+        self.cache = {}
 
     def transaction_commit(self, txn):
         pass
@@ -269,8 +281,7 @@ class DbDjango(DbWriteBase, DbReadBase):
         pass
 
     def request_rebuild(self):
-        # FIXME: rebuild cache
+        self.dji.rebuild_caches()
-        pass
 
     def get_undodb(self):
         return None
@@ -536,6 +547,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return []
 
     def get_event_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             event = self.dji.Event.get(handle=handle)
         except:
@@ -543,6 +556,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         self.make_event(event)
 
     def get_family_from_handle(self, handle): 
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             family = self.dji.Family.get(handle=handle)
         except:
@@ -550,6 +565,10 @@ class DbDjango(DbWriteBase, DbReadBase):
         return self.make_family(family)
 
     def get_family_from_gramps_id(self, gramps_id):
+        if self.cache:
+            for handle in self.cache:
+                if self.cache[handle].gramps_id == gramps_id:
+                    return self.cache[handle]
         try:
             family = self.dji.Family.get(gramps_id=gramps_id)
         except:
@@ -557,6 +576,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return self.make_family(family)
 
     def get_repository_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             repository = self.dji.Repository.get(handle=handle)
         except:
@@ -564,6 +585,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return self.make_repository(repository)
 
     def get_person_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             #person = self.dji.Person.select_related().get(handle=handle)
             person = self.dji.Person.get(handle=handle)
@@ -578,6 +601,13 @@ class DbDjango(DbWriteBase, DbReadBase):
             data = self.dji.get_family(family)
         return gen.lib.Repository.create(data)
 
+    def make_source(self, source):
+        if source.cache:
+            data = cPickle.loads(base64.decodestring(source.cache))
+        else:
+            data = self.dji.get_source(source)
+        return gen.lib.Source.create(data)
+
     def make_family(self, family):
         if family.cache:
             data = cPickle.loads(base64.decodestring(family.cache))
@@ -599,7 +629,7 @@ class DbDjango(DbWriteBase, DbReadBase):
             data = self.dji.get_event(event)
         return gen.lib.Event.create(data)
 
-    def make_note(self, event):
+    def make_note(self, note):
         if note.cache:
             data = cPickle.loads(base64.decodestring(note.cache))
         else:
@@ -621,7 +651,9 @@ class DbDjango(DbWriteBase, DbReadBase):
         return gen.lib.Media.create(data)
 
     def get_place_from_handle(self, handle):
-        # FIXME: use cache
+        if handle in self.cache:
+            return self.cache[handle]
+        # FIXME: use object cache
         try:
             dji_obj = self.dji.Place.get(handle=handle)
         except:
@@ -635,6 +667,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return None
 
     def get_source_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             source = self.dji.Source.get(handle=handle)
         except:
@@ -642,6 +676,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return self.make_source(source)
 
     def get_note_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             note = self.dji.Note.get(handle=handle)
         except:
@@ -649,6 +685,8 @@ class DbDjango(DbWriteBase, DbReadBase):
         return self.make_note(note)
 
     def get_object_from_handle(self, handle):
+        if handle in self.cache:
+            return self.cache[handle]
         try:
             media = self.dji.Media.get(handle=handle)
         except:
@@ -679,6 +717,10 @@ class DbDjango(DbWriteBase, DbReadBase):
         return (family.handle for family in self.dji.Family.all())
 
     def get_person_from_gramps_id(self, gramps_id):
+        if self.cache:
+            for handle in self.cache:
+                if self.cache[handle].gramps_id == gramps_id:
+                    return self.cache[handle]
         match_list = self.dji.Person.filter(gramps_id=gramps_id)
         if match_list.count() > 0:
             return self.make_person(match_list[0])

src/web/libdjango.py

@@ -1051,7 +1051,7 @@ class DjangoInterface(object):
                             last_changed=todate(change),
                             private=private,
                             gender_type=models.get_type(models.GenderType, gender))
-        person.cache = base64.encodestring(cPickle.dumps(data))
+        #person.cache = base64.encodestring(cPickle.dumps(data))
         person.save()
 
     def add_person_detail(self, data):
@@ -1140,7 +1140,7 @@ class DjangoInterface(object):
                         preformatted=format,
                         text=text,
                         note_type=models.get_type(models.NoteType, note_type))
-        n.cache = base64.encodestring(cPickle.dumps(data))
+        #n.cache = base64.encodestring(cPickle.dumps(data))
         n.save()
         count = 1
         for markup in markup_list:
@@ -1164,7 +1164,7 @@ class DjangoInterface(object):
                                family_rel_type = models.get_type(models.FamilyRelType, the_type),
                                last_changed=todate(change), 
                                private=private)
-        family.cache = base64.encodestring(cPickle.dumps(data))
+        #family.cache = base64.encodestring(cPickle.dumps(data))
         family.save()
 
     def add_family_detail(self, data):
@@ -1216,7 +1216,7 @@ class DjangoInterface(object):
         source = models.Source(handle=handle, gramps_id=gid, title=title,
                                author=author, pubinfo=pubinfo, abbrev=abbrev,
                                last_changed=todate(change), private=private)
-        source.cache = base64.encodestring(cPickle.dumps(data))
+        #source.cache = base64.encodestring(cPickle.dumps(data))
         source.save()
 
     def add_source_detail(self, data):
@@ -1249,7 +1249,7 @@ class DjangoInterface(object):
                                        private=private,
                                        repository_type=models.get_type(models.RepositoryType, the_type),
                                        name=name)
-        repository.cache = base64.encodestring(cPickle.dumps(data))
+        #repository.cache = base64.encodestring(cPickle.dumps(data))
         repository.save()
 
     def add_repository_detail(self, data):
@@ -1313,7 +1313,7 @@ class DjangoInterface(object):
         place = models.Place(handle=handle, gramps_id=gid, title=title,
                              long=long, lat=lat, last_changed=todate(change),
                              private=private)
-        place.cache = base64.encodestring(cPickle.dumps(data))
+        #place.cache = base64.encodestring(cPickle.dumps(data))
         place.save()
 
     def add_place_detail(self, data):
@@ -1353,7 +1353,7 @@ class DjangoInterface(object):
                              path=path, mime=mime, 
                              desc=desc, last_changed=todate(change),
                              private=private)
-        media.cache = base64.encodestring(cPickle.dumps(data))
+        #media.cache = base64.encodestring(cPickle.dumps(data))
         self.add_date(media, date)
         media.save()
     
@@ -1386,7 +1386,7 @@ class DjangoInterface(object):
                              private=private,
                              description=description,
                              last_changed=todate(change))
-        event.cache = base64.encodestring(cPickle.dumps(data))
+        #event.cache = base64.encodestring(cPickle.dumps(data))
         self.add_date(event, date)
         event.save()
 
@@ -1481,3 +1481,71 @@ class DjangoInterface(object):
             count += 1
 
         callback(100)
+
+    def clear_caches(self, callback=None):
+        """
+        Call this to clear the caches for all primary models.
+        """
+        if not callable(callback): 
+            callback = lambda (percent): None # dummy
+
+        callback(0)
+        count = 0.0
+        total = (self.Note.all().count() + 
+                 self.Person.all().count() +
+                 self.Event.all().count() + 
+                 self.Family.all().count() +
+                 self.Repository.all().count() +
+                 self.Place.all().count() +
+                 self.Media.all().count() +
+                 self.Source.all().count())
+
+        for item in self.Person.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Family.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Source.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Event.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Repository.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Place.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Media.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        for item in self.Note.all():
+            item.cache = None
+            item.save()
+            callback(100 * count/total)
+            count += 1
+
+        callback(100)