Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added initial fast model implementation

Browse Source 
svn: r5872
This commit is contained in:
Richard Taylor
2006-02-03 15:49:59 +00:00
parent 64ba0101d4
commit 754231f151
14 changed files with 1181 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
src/Models/_FastFilterModel.py Normal file
View File

@@ -0,0 +1,153 @@
import gtk
import time
import logging
log = logging.getLogger(".")
class FastFilterModel(gtk.GenericTreeModel):
"""A I{gtk.GenericTreeModel} that links to a BSDB cursor to provide
fast access to large tables. This is a pure virtual class, it must be
subclassed and the subclass must implement L{_get_table}, L{_get_cursor} and
L{_get_object_class}.
The primary trick is to use the path specification as the tree iter.
This means that when the TreeView asks for the iter for path=[1,2]
we just echo it straight back. The onlt hard part is making sure that
on_iter_next can do something sensible. It needs to know how many
non duplicate records are in the table and then it can just accept the
iter from the TreeView and increment it until it reaches the total
length.
The record itself is only fetched when its value is requested from
on_get_value() and when the number of childen need to calculated. The
cursor looks after the number of children calculation but it does require
walking the list of duplicate keys, usually this is quite short.
@ivar _db: handle of the Gramps DB
@ivar _table: main table to be displayed
@ivar _cursor: cursor for accessing the table.
@ivar _obj_class: the class of the object that is being pulled from
the database. This should probably be one of the primary RelLib
classes.
@ivar _num_children_cache: dictionary to hold the number of
children for each primary record so that we don't have to look
it up every time.
@ivar _length: the number of primary (non duplicate) records.
"""
column_types = (object,)
def __init__(self,db,data_filter):
gtk.GenericTreeModel.__init__(self)
self._db = db
self._data_filter = data_filter
self._fetch_func = self._get_fetch_func(db)
self._build_data()
def _build_data(self):
if not self._data_filter.is_empty():
self._keys = self._data_filter.apply(self._db)
else:
return
self._length = len(self._keys)
# Methods that must be implemented by subclasses.
def _get_fetch_func(self,db):
raise NotImplementedError("subclass of FastModel must implement _get_fetch_func")
# GenericTreeModel methods
def on_get_flags(self):
return gtk.TREE_MODEL_LIST_ONLY|gtk.TREE_MODEL_ITERS_PERSIST
def on_get_n_columns(self):
return len(self.__class__.column_types)
def on_get_column_type(self, index):
return self.column_types[index]
def on_get_iter(self, path):
return list(path)
def on_get_path(self, rowref):
return list(rowref)
def on_get_value(self, rowref, column):
"""
Fetch the real object from the database.
"""
# We only have one column
if column is 0:
record = self._fetch_func(self._keys[rowref[0]])
# This should never return none, but there is a subtle bug
# somewhere that I can't find and sometimes it does.
if record is None:
log.warn("Failed to fetch a record from the cursor rowref = %s" % (str(rowref)))
return (record,rowref)
def on_iter_next(self, rowref):
"""
Calculate the next iter at the same level in the tree.
"""
# The length of the rowref (i.e. the number of elements in the path)
# tells us the level in the tree.
if len(rowref) == 1:
# If we are at the top of the tree we just increment the
# first element in the iter until we reach the total length.
if rowref[0]+1 >= self._length:
ret = None
else:
ret = [rowref[0]+1,]
else:
# We only support one level.
ret = None
return ret
def on_iter_children(self, rowref):
"""
Return the first child of the given rowref.
"""
if rowref:
# If the rowref is not none then we must be
# asking for the second level so the first
# child is always 0.
ret = [rowref[0],0]
else:
# If rowref is None the we are asking for the
# top level and that is always [0]
ret = [0,]
return ret
def on_iter_has_child(self, rowref):
return False
def on_iter_n_children(self, rowref):
return self._length
def on_iter_nth_child(self, parent, n):
if parent:
ret = [parent[0],n]
else:
ret = [n,]
return ret
def on_iter_parent(self, child):
if len(child) > 1:
return [child[0]]
return None

195
src/Models/_FastModel.py Normal file
View File

@@ -0,0 +1,195 @@
import gtk
import time
import logging
log = logging.getLogger(".")
class FastModel(gtk.GenericTreeModel):
"""A I{gtk.GenericTreeModel} that links to a BSDB cursor to provide
fast access to large tables. This is a pure virtual class, it must be
subclassed and the subclass must implement L{_get_table}, L{_get_cursor} and
L{_get_object_class}.
The primary trick is to use the path specification as the tree iter.
This means that when the TreeView asks for the iter for path=[1,2]
we just echo it straight back. The onlt hard part is making sure that
on_iter_next can do something sensible. It needs to know how many
non duplicate records are in the table and then it can just accept the
iter from the TreeView and increment it until it reaches the total
length.
The record itself is only fetched when its value is requested from
on_get_value() and when the number of childen need to calculated. The
cursor looks after the number of children calculation but it does require
walking the list of duplicate keys, usually this is quite short.
@ivar _db: handle of the Gramps DB
@ivar _table: main table to be displayed
@ivar _cursor: cursor for accessing the table.
@ivar _obj_class: the class of the object that is being pulled from
the database. This should probably be one of the primary RelLib
classes.
@ivar _num_children_cache: dictionary to hold the number of
children for each primary record so that we don't have to look
it up every time.
@ivar _length: the number of primary (non duplicate) records.
"""
column_types = (object,)
def __init__(self,db):
gtk.GenericTreeModel.__init__(self)
self._db = db
self._table = self._get_table(db)
self._cursor = self._get_cursor(db)
self._object_class = self._get_object_class(db)
self._length = self._get_length(db)
self._num_children_cache = {}
# Methods that must be implemented by subclasses.
def _get_table(self,db):
raise NotImplementedError("subclass of FastModel must implement _get_table")
def _get_cursor(self,db):
raise NotImplementedError("subclass of FastModel must implement _get_cursor")
def _get_object_class(self,db):
raise NotImplementedError("subclass of FastModel must implement _get_cursor")
def _get_length(self,db):
raise NotImplementedError("subclass of FastModel must implement _get_length")
# GenericTreeModel methods
def on_get_flags(self):
return gtk.TREE_MODEL_ITERS_PERSIST
def on_get_n_columns(self):
return len(self.__class__.column_types)
def on_get_column_type(self, index):
return self.column_types[index]
def on_get_iter(self, path):
return list(path)
def on_get_path(self, rowref):
return list(rowref)
def on_get_value(self, rowref, column):
"""
Fetch the real object from the database.
"""
# We only have one column
if column is 0:
obj = self._object_class()
# Use the rowref as the path, because the iter methods
# simple return the path as the iter it is safe to use
# it here.
record = self._cursor.lookup_path(rowref)
# This should never return none, but there is a subtle bug
# somewhere that I can't find and sometimes it does.
if record is not None:
obj.unserialize(record[1])
else:
log.warn("Failed to fetch a record from the cursor rowref = %s" % (str(rowref)))
return (obj,rowref)
def on_iter_next(self, rowref):
"""
Calculate the next iter at the same level in the tree.
"""
# The length of the rowref (i.e. the number of elements in the path)
# tells us the level in the tree.
if len(rowref) == 1:
# If we are at the top of the tree we just increment the
# first element in the iter until we reach the total length.
if rowref[0]+1 >= self._length:
ret = None
else:
ret = [rowref[0]+1,]
elif len(rowref) == 2:
# If we are at the second level we first check to see if we
# have the number of children of this row already in the cache
if not self._num_children_cache.has_key(rowref[0]):
# If not calculate the number of children and put it in the cache.
self._num_children_cache[rowref[0]] = self._cursor.get_n_children([rowref[0],])
num_children = self._num_children_cache[rowref[0]]
# Now increment the second element of the iter path until we
# reach the number of children.
if rowref[1]+1 < num_children:
ret = [rowref[0],rowref[1]+1]
else:
ret = None
else:
# We only support two levels.
ret = None
return ret
def on_iter_children(self, rowref):
"""
Return the first child of the given rowref.
"""
if rowref:
# If the rowref is not none then we must be
# asking for the second level so the first
# child is always 0.
ret = [rowref[0],0]
else:
# If rowref is None the we are asking for the
# top level and that is always [0]
ret = [0,]
return ret
def on_iter_has_child(self, rowref):
if rowref:
ret = self._cursor.has_children(rowref)
else:
ret = range(0,self._length)
return ret
def on_iter_n_children(self, rowref):
if rowref:
# If we are at the second level we first check to see if we
# have the number of children of this row already in the cache
if not self._num_children_cache.has_key(rowref[0]):
# If not calculate the number of children and put it in the cache.
self._num_children_cache[rowref[0]] = self._cursor.get_n_children([rowref[0],])
ret = self._num_children_cache[rowref[0]]
else:
ret = self._length
return ret
def on_iter_nth_child(self, parent, n):
if parent:
ret = [parent[0],n]
else:
ret = [n,]
return ret
def on_iter_parent(self, child):
if len(child) > 1:
return [child[0]]
return None

176
src/Models/_ListCursor.py Normal file
View File

@@ -0,0 +1,176 @@
import cPickle
import logging
log = logging.getLogger(".")
class ListCursor(object):
"""
Provides a wrapper around the cursor class that provides fast
traversal using treeview paths for models that are LISTONLY, i.e.
they have no tree structure.
It keeps track of the current index that the cursor is pointing
at.
@ivar _index: The current index pointed to by the cursor.
To speed up lookups the cursor is kept as close as possible to the
likely next lookup and is moved using next()/prev() were ever
possible.
@ivar _object_cache: A cache of previously fetched records. These are
indexed by the values of the L{_index}.
"""
def __init__(self,cursor):
"""
@param cursor: The cursor used to fetch the records.
@type cursor: An object supporting the cursor methods of a U{BSDB
cursor<http://pybsddb.sourceforge.net/bsddb3.html>}.
It must have a BTREE index type and DB_DUP to support duplicate
records. It should probably also have DB_DUPSORT if you want to
have sorted records.
"""
self._cursor = cursor
self._object_cache = {}
self.top()
def top(self):
self._cursor.first()
self._index = 0
def next(self):
"""
Move to the next record.
"""
data = self._cursor.next()
# If there was a next record that data will
# not be None
if data is not None:
# Up date the index pointers so that
# they point to the current record.
self._index+= 1
return data
def prev(self):
"""
Move to the previous record.
"""
data = self._cursor.prev()
# If there was a next record that data will
# not be None
if data is not None:
# Up date the index pointers so that
# they point to the current record.
self._index -= 1
return data
def has_children(self,path):
"""
This cursor is only for simple lists so no records have
children.
@param path: The path spec to check.
@type path: A TreeView path.
"""
return False
def get_n_children(self,path):
"""
Return the number of children that the record at I{path} has.
@param path: The path spec to check.
@type path: A TreeView path.
"""
return 0
def lookup(self,index,use_cache=True):
"""
Lookup a primary record.
@param index: The index of the primary record. This is its
possition in the sequence of non_duplicate keys.
@type index: int
@para use_case: If B{True} the record will be looked up in the
object cache and will be returned from there. This will not
update the possition of the cursor. If B{False} the record will
fetched from the cursor even if it is in the object cache and
cursor will be left possitioned on the record.
"""
# See if the record is in the cache.
if self._object_cache.has_key(index) and use_cache is True:
ret = self._object_cache[index]
# If the record is not in the cache or we are ignoring the
# cache.
else:
# If the cursor points to the record we want
# the first index will be equal to the
# index required
if index == self._index:
ret = self._cursor.current()
# If the current cursor is behind the
# requested index move it forward.
elif index < self._index:
while index < self._index:
ret = self.prev()
if ret is None:
log.warn("Failed to move back to index = %s" % (str(index)))
break
ret = self._cursor.current()
# If the current cursor is in front of
# requested index move it backward.
else:
while index > self._index:
ret = self.next()
if ret is None:
log.warn("Failed to move forward to index = %s" % (str(index)))
break
ret = self._cursor.current()
# when we have got the record save it in
# the cache
if ret is not None:
ret = self._unpickle(ret)
self._object_cache[index] = ret
return ret
def _unpickle(self,rec):
"""
It appears that reading an object from a cursor does not
automatically unpickle it. So this method provides
a convenient way to unpickle the object.
"""
if rec and type(rec[1]) == type(""):
tmp = [rec[0],None]
tmp[1] = cPickle.loads(rec[1])
rec = tmp
return rec
def lookup_path(self,path):
"""
Lookup a record from a patch specification.
@param path: The path spec to check.
@type path: A TreeView path.
"""
return self.lookup(path[0])

324
src/Models/_PathCursor.py Normal file
View File

@@ -0,0 +1,324 @@
import cPickle
import logging
log = logging.getLogger(".")
class PathCursor(object):
"""
Provides a wrapper around the cursor class that provides fast
traversal using treeview paths.
It keeps track of the current index that the cursor is pointing
at by using a two stage index. The first element of the index is
the sequence number of the record in the list of non_duplicate
keys and the second element is the sequence number of the record
within the duplicate keys to which it is a member.
For example, with the following table indexed on Surname::
Record Value Index
============ =====
Blogs, Jo [0,0]
Blogs, Jane [0,1]
Smith, Wilman [1,0]
Smith, John [1,1]
@ivar _index: The current index pointed to by the cursor.
To speed up lookups the cursor is kept as close as possible to the
likely next lookup and is moved using next_dup()/prev_dup() were ever
possible.
@ivar _object_cache: A cache of previously fetched records. These are
indexed by the values of the L{_index}.
"""
def __init__(self,cursor):
"""
@param cursor: The cursor used to fetch the records.
@type cursor: An object supporting the cursor methods of a U{BSDB
cursor<http://pybsddb.sourceforge.net/bsddb3.html>}.
It must have a BTREE index type and DB_DUP to support duplicate
records. It should probably also have DB_DUPSORT if you want to
have sorted records.
"""
self._cursor = cursor
self._object_cache = {}
self.top()
def top(self):
self._cursor.first()
self._index = [0,0]
def next_nodup(self):
"""
Move to the next non-duplcate record.
"""
data = self._cursor.next_nodup()
# If there was a next record that data will
# not be None
if data is not None:
# Up date the index pointers so that
# they point to the current record.
self._index[0] += 1
self._index[1] = 0
return data
def prev_nodup(self):
"""
Move to the previous non-duplicate record.
"""
data = self._cursor.prev_nodup()
# If there was a next record that data will
# not be None
if data is not None:
# Up date the index pointers so that
# they point to the current record.
self._index[0] -= 1
self._index[1] = 0
return data
def next_dup(self):
"""
Move to the next record with a duplicate key to the current record.
"""
data = self._cursor.next_dup()
# If there was a next record that data will
# not be None
if data is not None:
# Update the secondary index.
self._index[1] += 1
return data
def has_children(self,path):
"""
Check is the I{path} has any children.
At the moment this method lies. There is no fast way to check
if a given key has any duplicates and the TreeView insists on
checking for every row. So this methods returns True if the
path is 1 element long and False if it is more. This works
for us because we show the first record in a set of duplicates
as the first child. So all top level rows have at least one child.
@param path: The path spec to check.
@type path: A TreeView path.
"""
if len(path) == 1:
return True
else:
return False
def get_n_children(self,path):
"""
Return the number of children that the record at I{path} has.
@param path: The path spec to check.
@type path: A TreeView path.
"""
# Only top level records can have children.
if len(path) > 1:
return 0
# Fetch the primary record
ret = self.lookup(path[0],use_cache=False)
if ret is not None:
# Now count the duplicates. We start at 1 because
# we want to include the primary in the duplicates.
count = 1
ret = self.next_dup()
while ret:
ret = self.next_dup()
count += 1
self._index[1] += 1
ret = count
else:
# If we failed to find the primary something is
# wrong.
ret = 0
return ret
def lookup(self,index,use_cache=True):
"""
Lookup a primary record.
@param index: The index of the primary record. This is its
possition in the sequence of non_duplicate keys.
@type index: int
@para use_case: If B{True} the record will be looked up in the
object cache and will be returned from there. This will not
update the possition of the cursor. If B{False} the record will
fetched from the cursor even if it is in the object cache and
cursor will be left possitioned on the record.
"""
# See if the record is in the cache.
if self._object_cache.has_key(index) and use_cache is True:
ret = self._object_cache[index]['primary']
# If the record is not in the cache or we are ignoring the
# cache.
else:
# If the cursor points to a duplicate record
# it will have a second index value of 0.
if self._index[1] != 0:
# We need to move the cursor to the
# first of a set of duplicates so that
# we can then shift it to the required
# index.
self.next_nodup()
# If the cursor points to the record we want
# the first index will be equal to the
# index required
if index == self._index[0]:
ret = self._cursor.current()
# If the current cursor is behind the
# requested index move it forward.
elif index < self._index[0]:
while index < self._index[0]:
ret = self.prev_nodup()
if ret is None:
log.warn("Failed to move back to index = %s" % (str(index)))
break
# Because prev_nodup() leaves the cursor on
# the last of a set of duplicates we need
# to go up one further and then back down
# again.
ret = self.prev_nodup()
if ret is None:
# We are at the start
self.top()
ret = self._cursor.current()
else:
ret = self.next_nodup()
# If the current cursor is in front of
# requested index move it backward.
else:
while index > self._index[0]:
ret = self.next_nodup()
if ret is None:
log.warn("Failed to move forward to index = %s" % (str(index)))
break
ret = self._cursor.current()
# when we have got the record save it in
# the cache
if ret is not None:
ret = self._unpickle(ret)
self._object_cache[index] = {'primary':ret}
return ret
def _unpickle(self,rec):
"""
It appears that reading an object from a cursor does not
automatically unpickle it. So this method provides
a convenient way to unpickle the object.
"""
if rec and type(rec[1]) == type(""):
tmp = [rec[0],None]
tmp[1] = cPickle.loads(rec[1])
rec = tmp
return rec
def lookup_path(self,path):
"""
Lookup a record from a patch specification.
@param path: The path spec to check.
@type path: A TreeView path.
"""
# If the path is for a primary record it will only
# be 1 element long.
if len(path) == 1:
ret = self.lookup(path[0])
# If it is for a secondary object we need to
# traverse the duplicates.
else:
# First check to see if the record has already
# been fetched.
if self._object_cache.has_key(path[0]) and \
self._object_cache[path[0]].has_key(path[1]):
# return record from cache.
ret = self._object_cache[path[0]][path[1]]
else:
# If we already in the duplicates for this
# primary index then the first index will
# be the same as the first element of the
# path.
if self._index[0] == path[0]:
# If the second elements match we are
# already looking at the correct record.
if self._index[1] == path[1]:
ret = self._cursor.current()
# If the cursor is in front we can
# move it back. Unfortunately there is no
# prev_dup() method so we have to
# reposition of the cursor at the start
# of the duplicates and step forward
else:
self.prev_nodup()
self.next_nodup()
ret = self.lookup(path[0],use_cache=False)
# If the request if not for the first duplicate
# we step forward the number of duplicates
# that have been requested.
count = 0
while count < path[1]:
ret = self.next_dup()
count += 1
# If the primary elements do not match we
# must move the cursor to the start of the
# duplicates that are requested.
else:
self.prev_nodup()
self.next_nodup()
ret = self.lookup(path[0],use_cache=False)
# If the request if not for the first duplicate
# we step forward the number of duplicates
# that have been requested.
count = 0
while count < path[1]:
ret = self.next_dup()
count += 1
# Put the fetched record in the cache
if ret is not None:
ret = self._unpickle(ret)
self._object_cache[path[0]][path[1]] = ret
return ret

29
src/Models/_PersonFilterModel.py Normal file
View File

@@ -0,0 +1,29 @@
import gtk
import time
import bsddb
import cPickle
import logging
log = logging.getLogger(".")
from _PathCursor import PathCursor
from _ListCursor import ListCursor
from _FastFilterModel import FastFilterModel
import RelLib
class PersonFilterModel(FastFilterModel):
"""Provides a fast model interface to the Person table.
"""
def __init__(self,db,apply_filter):
FastFilterModel.__init__(self,db,apply_filter)
def _get_object_class(self,db):
return RelLib.Person
def _get_fetch_func(self,db):
return db.get_person_from_handle

36
src/Models/_PersonListModel.py Normal file
View File

@@ -0,0 +1,36 @@
import gtk
import time
import bsddb
import cPickle
import logging
log = logging.getLogger(".")
from _PathCursor import PathCursor
from _ListCursor import ListCursor
from _FastModel import FastModel
import RelLib
class PersonListModel(FastModel):
"""Provides a fast model interface to the Person table.
"""
def __init__(self,db):
FastModel.__init__(self,db)
def _get_table(self,db):
return db.surnames
def _get_cursor(self,db):
return ListCursor(db.surnames.cursor())
def _get_object_class(self,db):
return RelLib.Person
def _get_length(self,db):
return self._table.stat()['ndata']
def on_get_flags(self):
return gtk.TREE_MODEL_LIST_ONLY|gtk.TREE_MODEL_ITERS_PERSIST

33
src/Models/_PersonTreeModel.py Normal file
View File

@@ -0,0 +1,33 @@
import gtk
import time
import bsddb
import cPickle
import logging
log = logging.getLogger(".")
from _PathCursor import PathCursor
from _ListCursor import ListCursor
from _FastModel import FastModel
import RelLib
class PersonTreeModel(FastModel):
"""Provides a fast model interface to the Person table.
"""
def __init__(self,db):
FastModel.__init__(self,db)
def _get_table(self,db):
return db.surnames
def _get_cursor(self,db):
return PathCursor(db.surnames.cursor())
def _get_object_class(self,db):
return RelLib.Person
def _get_length(self,db):
return self._table.stat()['nkeys']

4
src/Models/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
from _PersonListModel import PersonListModel
from _PersonTreeModel import PersonTreeModel
from _PersonFilterModel import PersonFilterModel