diff --git a/gramps/gui/editors/displaytabs/backrefmodel.py b/gramps/gui/editors/displaytabs/backrefmodel.py
index 0aeb2a2a2..e8b6c584b 100644
--- a/gramps/gui/editors/displaytabs/backrefmodel.py
+++ b/gramps/gui/editors/displaytabs/backrefmodel.py
@@ -56,13 +56,15 @@ class BackRefModel(Gtk.ListStore):
         self.db = db
         self.sref_list = sref_list
         self.count = 0
+        self.loading = False
         if sys.version_info[0] < 3:
             self.idle = GObject.idle_add(self.load_model().next)
         else:
             self.idle = GObject.idle_add(self.load_model().__next__)
 
     def destroy(self):
-        GObject.source_remove(self.idle)
+        if self.loading:
+            GObject.source_remove(self.idle)
 
     def load_model(self):
         """
@@ -73,6 +75,7 @@ class BackRefModel(Gtk.ListStore):
             while the GUI using this model is no longer used. Disconnect any
             methods before closing the GUI.
         """
+        self.loading = True
         self.count = 0
         for ref in self.sref_list:
             self.count += 1
@@ -149,4 +152,5 @@ class BackRefModel(Gtk.ListStore):
             # but we don't need to show that in the view.
             self.append(row=[_(dtype), gid, name, handle, dtype])
             yield True
+        self.loading = False
         yield False