Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Minor tweaks

Browse Source 
svn: r6723
This commit is contained in:
Alex Roitman 2006-05-19 18:54:21 +00:00
parent 07c9ca05c9
commit 96fa4ed00a
1 changed files with 107 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
src/GrampsDb/_GrampsBSDDB.py
View File

@ -29,7 +29,7 @@ Provides the Berkeley DB (BSDDB) database backend for GRAMPS
# Standard python modules # Standard python modules
# #
#------------------------------------------------------------------------- #-------------------------------------------------------------------------
import cPickle import cPickle as pickle
import os import os
import time import time
import locale import locale
@ -56,6 +56,7 @@ from _GrampsDbBase import *
from _DbUtils import db_copy from _DbUtils import db_copy
import const import const
import Errors import Errors
from BasicUtils import UpdateCallback
_MINVERSION = 5 _MINVERSION = 5
_DBVERSION = 9 _DBVERSION = 9
@ -80,8 +81,6 @@ def find_primary_handle(key,data):
def find_referenced_handle(key,data): def find_referenced_handle(key,data):
return str((data)[1][1]) return str((data)[1][1])
import cPickle as pickle
class GrampsBSDDBCursor(GrampsCursor): class GrampsBSDDBCursor(GrampsCursor):
def __init__(self,source,txn=None): def __init__(self,source,txn=None):
@ -143,7 +142,7 @@ class GrampsBSDDBDupCursor(GrampsBSDDBAssocCursor):
# GrampsBSDDB # GrampsBSDDB
# #
#------------------------------------------------------------------------- #-------------------------------------------------------------------------
class GrampsBSDDB(GrampsDbBase): class GrampsBSDDB(GrampsDbBase,UpdateCallback):
"""GRAMPS database object. This object is a base class for other """GRAMPS database object. This object is a base class for other
objects.""" objects."""
@ -560,7 +559,7 @@ class GrampsBSDDB(GrampsDbBase):
def find_backlink_handles(self, handle, include_classes=None): def find_backlink_handles(self, handle, include_classes=None):
""" """
Find all objects that hold a reference to the object handle. Find all objects that hold a reference to the object handle.
Returns an interator over alist of (class_name,handle) tuples. Returns an interator over a list of (class_name,handle) tuples.
@param handle: handle of the object to search for. @param handle: handle of the object to search for.
@type handle: database handle @type handle: database handle
@ -592,8 +591,9 @@ class GrampsBSDDB(GrampsDbBase):
# (referenced_object_class_name, referenced_object_handle)) # (referenced_object_class_name, referenced_object_handle))
# so we need the first tuple to give us the type to compare # so we need the first tuple to give us the type to compare
data = cPickle.loads(data) data = pickle.loads(data)
if include_classes == None or KEY_TO_CLASS_MAP[data[0][0]] in include_classes: if include_classes == None or \
KEY_TO_CLASS_MAP[data[0][0]] in include_classes:
yield (KEY_TO_CLASS_MAP[data[0][0]],data[0][1]) yield (KEY_TO_CLASS_MAP[data[0][0]],data[0][1])
ret = referenced_cur.next_dup() ret = referenced_cur.next_dup()
@ -603,7 +603,9 @@ class GrampsBSDDB(GrampsDbBase):
return return
def _delete_primary_from_reference_map(self,handle,transaction,txn=None): def _delete_primary_from_reference_map(self,handle,transaction,txn=None):
"""Remove all references to the primary object from the reference_map""" """
Remove all references to the primary object from the reference_map.
"""
primary_cur = self.get_reference_map_primary_cursor() primary_cur = self.get_reference_map_primary_cursor()
@ -622,7 +624,7 @@ class GrampsBSDDB(GrampsDbBase):
# so we need the second tuple give us a reference that we can # so we need the second tuple give us a reference that we can
# combine with the primary_handle to get the main key. # combine with the primary_handle to get the main key.
main_key = (handle, cPickle.loads(data)[1][1]) main_key = (handle, pickle.loads(data)[1][1])
self._remove_reference(main_key,transaction,txn) self._remove_reference(main_key,transaction,txn)
@ -642,11 +644,6 @@ class GrampsBSDDB(GrampsDbBase):
update = self.reference_map_primary_map.has_key(str(handle)) update = self.reference_map_primary_map.has_key(str(handle))
if update: if update:
# FIXME: this needs to be properly integrated into the transaction
# framework so that the reference_map changes are part of the
# transaction
# First thing to do is get hold of all rows in the reference_map # First thing to do is get hold of all rows in the reference_map
# table that hold a reference from this primary obj. This means # table that hold a reference from this primary obj. This means
# finding all the rows that have this handle somewhere in the # finding all the rows that have this handle somewhere in the
@ -674,7 +671,7 @@ class GrampsBSDDB(GrampsDbBase):
# secondary DBs are not DBShelf's, so we need to do pickling # secondary DBs are not DBShelf's, so we need to do pickling
# and unpicking ourselves here # and unpicking ourselves here
existing_reference = cPickle.loads(data)[1] existing_reference = pickle.loads(data)[1]
existing_references.add( existing_references.add(
(KEY_TO_CLASS_MAP[existing_reference[0]], (KEY_TO_CLASS_MAP[existing_reference[0]],
existing_reference[1])) existing_reference[1]))
@ -694,28 +691,24 @@ class GrampsBSDDB(GrampsDbBase):
new_references = current_references.difference(existing_references) new_references = current_references.difference(existing_references)
else: else:
# No existing refs are found:
# all we have is new, nothing to remove
no_longer_required_references = set()
new_references = set(obj.get_referenced_handles_recursively()) new_references = set(obj.get_referenced_handles_recursively())
# handle addition of new references # handle addition of new references
for (ref_class_name,ref_handle) in new_references:
data = ((CLASS_TO_KEY_MAP[obj.__class__.__name__],handle),
(CLASS_TO_KEY_MAP[ref_class_name],ref_handle),)
self._add_reference((handle,ref_handle),data,transaction,txn)
if len(new_references) > 0: # handle deletion of old references
for (ref_class_name,ref_handle) in new_references: for (ref_class_name,ref_handle) in no_longer_required_references:
data = ((CLASS_TO_KEY_MAP[obj.__class__.__name__],handle), try:
(CLASS_TO_KEY_MAP[ref_class_name],ref_handle),) self._remove_reference((handle,ref_handle),transaction,txn)
self._add_reference((handle,ref_handle),data,transaction,txn) except:
# ignore missing old reference
if update: pass
# handle deletion of old references
if len(no_longer_required_references) > 0:
for (ref_class_name,ref_handle) in \
no_longer_required_references:
try:
self._remove_reference(
(handle,ref_handle),transaction,txn)
#self.reference_map.delete(str((handle,ref_handle),),
# txn=self.txn)
except: # ignore missing old reference
pass
def _remove_reference(self,key,transaction,txn=None): def _remove_reference(self,key,transaction,txn=None):
""" """
@ -724,7 +717,7 @@ class GrampsBSDDB(GrampsDbBase):
""" """
if not self.readonly: if not self.readonly:
if transaction.batch: if transaction.batch:
self.reference_map.delete(str(key),txn=txn)#=the_txn) self.reference_map.delete(str(key),txn=txn)
if not self.UseTXN: if not self.UseTXN:
self.reference_map.sync() self.reference_map.sync()
else: else:
@ -742,56 +735,58 @@ class GrampsBSDDB(GrampsDbBase):
return return
if transaction.batch: if transaction.batch:
#the_txn = self.env.txn_begin() self.reference_map.put(str(key),data,txn=txn)
self.reference_map.put(str(key),data,txn=txn)#=the_txn)
if not self.UseTXN: if not self.UseTXN:
self.reference_map.sync() self.reference_map.sync()
#the_txn.commit()
else: else:
transaction.add(REFERENCE_KEY,str(key),None,data) transaction.add(REFERENCE_KEY,str(key),None,data)
transaction.reference_add.append((str(key),data)) transaction.reference_add.append((str(key),data))
def reindex_reference_map(self): def reindex_reference_map(self):
"""Reindex all primary records in the database. This will be a """
slow process for large databases. Reindex all primary records in the database.
This will be a slow process for large databases.
At present this method does not clear the reference_map before it At present this method does not clear the reference_map before it
reindexes. This is fine when if reindex is run to index new content or reindexes. This is fine when if reindex is run to index new content
when upgrading from a non-reference_map version of the database. But it or when upgrading from a non-reference_map version of the database.
might be a problem if reindex is used to repair a broken index because any But it might be a problem if reindex is used to repair a broken index
references to primary objects that are no longer in the database will because any references to primary objects that are no longer in the
remain in the reference_map index. So if you want to reindex for repair database will remain in the reference_map index.
purposes you need to clear the reference_map first.
So if you want to reindex for repair purposes you need to
clear the reference_map first.
""" """
# Make a dictionary of the functions and classes that we need for # Make a dictionary of the functions and classes that we need for
# each of the primary object tables. # each of the primary object tables.
primary_tables = {'Person': {'cursor_func': self.get_person_cursor, primary_tables = {
'class_func': Person}, 'Person': {'cursor_func': self.get_person_cursor,
'Family': {'cursor_func': self.get_family_cursor, 'class_func': Person},
'class_func': Family}, 'Family': {'cursor_func': self.get_family_cursor,
'Event': {'cursor_func': self.get_event_cursor, 'class_func': Family},
'class_func': Event}, 'Event': {'cursor_func': self.get_event_cursor,
'Place': {'cursor_func': self.get_place_cursor, 'class_func': Event},
'class_func': Place}, 'Place': {'cursor_func': self.get_place_cursor,
'Source': {'cursor_func': self.get_source_cursor, 'class_func': Place},
'class_func': Source}, 'Source': {'cursor_func': self.get_source_cursor,
'MediaObject': {'cursor_func': self.get_media_cursor, 'class_func': Source},
'class_func': MediaObject}, 'MediaObject': {'cursor_func': self.get_media_cursor,
'Repository': {'cursor_func': self.get_repository_cursor, 'class_func': MediaObject},
'class_func': Repository}, 'Repository': {'cursor_func': self.get_repository_cursor,
} 'class_func': Repository},
}
# Now we use the functions and classes defined above to loop through each of the # Now we use the functions and classes defined above
# primary object tables. # to loop through each of the primary object tables.
for primary_table_name in primary_tables.keys(): for primary_table_name in primary_tables.keys():
cursor = primary_tables[primary_table_name]['cursor_func']() cursor = primary_tables[primary_table_name]['cursor_func']()
data = cursor.first() data = cursor.first()
# Grap the real object class here so that the lookup does # Grab the real object class here so that the lookup does
# not happen inside the main loop. # not happen inside the cursor loop.
class_func = primary_tables[primary_table_name]['class_func'] class_func = primary_tables[primary_table_name]['class_func']
while data: while data:
@ -972,59 +967,68 @@ class GrampsBSDDB(GrampsDbBase):
vals.sort() vals.sort()
return [item[1] for item in vals] return [item[1] for item in vals]
## def get_repository_type_list(self):
## vals = list(set(self.repository_types.keys()))
## vals.sort(locale.strcoll)
## return vals
def _get_obj_from_gramps_id(self,val,tbl,class_init): def _get_obj_from_gramps_id(self,val,tbl,class_init):
if tbl.has_key(str(val)): if tbl.has_key(str(val)):
data = tbl.get(str(val),txn=self.txn) data = tbl.get(str(val),txn=self.txn)
obj = class_init() obj = class_init()
obj.unserialize(cPickle.loads(data)) obj.unserialize(pickle.loads(data))
return obj return obj
else: else:
return None return None
def get_person_from_gramps_id(self,val): def get_person_from_gramps_id(self,val):
"""finds a Person in the database from the passed gramps' ID. """
If no such Person exists, a new Person is added to the database.""" Finds a Person in the database from the passed gramps' ID.
If no such Person exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.id_trans,Person) return self._get_obj_from_gramps_id(val,self.id_trans,Person)
def get_family_from_gramps_id(self,val): def get_family_from_gramps_id(self,val):
"""finds a Family in the database from the passed gramps' ID. """
If no such Family exists, a new Person is added to the database.""" Finds a Family in the database from the passed gramps' ID.
If no such Family exists, None is return.
"""
return self._get_obj_from_gramps_id(val,self.fid_trans,Family) return self._get_obj_from_gramps_id(val,self.fid_trans,Family)
def get_event_from_gramps_id(self,val): def get_event_from_gramps_id(self,val):
"""finds a Family in the database from the passed gramps' ID. """
If no such Family exists, a new Person is added to the database.""" Finds an Event in the database from the passed gramps' ID.
If no such Family exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.eid_trans,Event) return self._get_obj_from_gramps_id(val,self.eid_trans,Event)
def get_place_from_gramps_id(self,val): def get_place_from_gramps_id(self,val):
"""finds a Place in the database from the passed gramps' ID. """
If no such Place exists, a new Person is added to the database.""" Finds a Place in the database from the passed gramps' ID.
If no such Place exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.pid_trans,Place) return self._get_obj_from_gramps_id(val,self.pid_trans,Place)
def get_source_from_gramps_id(self,val): def get_source_from_gramps_id(self,val):
"""finds a Source in the database from the passed gramps' ID. """
If no such Source exists, a new Person is added to the database.""" Finds a Source in the database from the passed gramps' ID.
If no such Source exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.sid_trans,Source) return self._get_obj_from_gramps_id(val,self.sid_trans,Source)
def get_object_from_gramps_id(self,val): def get_object_from_gramps_id(self,val):
"""finds a MediaObject in the database from the passed gramps' ID. """
If no such MediaObject exists, a new Person is added to the database.""" Finds a MediaObject in the database from the passed gramps' ID.
If no such MediaObject exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.oid_trans,MediaObject) return self._get_obj_from_gramps_id(val,self.oid_trans,MediaObject)
def get_repository_from_gramps_id(self,val): def get_repository_from_gramps_id(self,val):
"""finds a MediaObject in the database from the passed gramps' ID. """
If no such MediaObject exists, a new Person is added to the database.""" Finds a Repository in the database from the passed gramps' ID.
If no such MediaObject exists, None is returned.
"""
return self._get_obj_from_gramps_id(val,self.rid_trans,Repository) return self._get_obj_from_gramps_id(val,self.rid_trans,Repository)
def _commit_base(self, obj, data_map, key, update_list, add_list, def _commit_base(self, obj, data_map, key, update_list, add_list,
transaction, change_time): transaction, change_time):
""" """
Commits the specified Person to the database, storing the changes Commits the specified object to the database, storing the changes
as part of the transaction. as part of the transaction.
""" """
if self.readonly or not obj or not obj.handle: if self.readonly or not obj or not obj.handle:
@ -1220,22 +1224,9 @@ class GrampsBSDDB(GrampsDbBase):
db_map.put(handle,data,txn=self.txn) db_map.put(handle,data,txn=self.txn)
self.emit(signal,([handle],)) self.emit(signal,([handle],))
def update_empty(self,newval):
pass
def update_real(self,newval):
if newval != self.oldval:
self.callback(newval)
self.oldval = newval
def gramps_upgrade(self,callback=None): def gramps_upgrade(self,callback=None):
self.callback = callback UpdateCallback.__init__(self,callback)
if '__call__' in dir(callback): # callback is really callable
self.oldval = 0
self.update = self.update_real
else:
self.update = self.update_empty
child_rel_notrans = [ child_rel_notrans = [
"None", "Birth", "Adopted", "Stepchild", "None", "Birth", "Adopted", "Stepchild",
"Sponsored", "Foster", "Unknown", "Other", ] "Sponsored", "Foster", "Unknown", "Other", ]
@ -1313,7 +1304,10 @@ class GrampsBSDDB(GrampsDbBase):
print "Upgrading to DB version 9 -- this may take a while" print "Upgrading to DB version 9 -- this may take a while"
# The very very first thing is to check for duplicates in the # The very very first thing is to check for duplicates in the
# primary tables and remove them. # primary tables and remove them.
status,length = low_level_9(self) self.set_total(7)
status,length = low_level_9(self,self.update)
self.set_total(length)
# Remove column metadata, since columns have changed. # Remove column metadata, since columns have changed.
# This will reset all columns to defaults # This will reset all columns to defaults
@ -1353,13 +1347,12 @@ class GrampsBSDDB(GrampsDbBase):
# The rest of the upgrade deals with real data, not metadata # The rest of the upgrade deals with real data, not metadata
# so starting (batch) transaction here. # so starting (batch) transaction here.
trans = self.transaction_begin("",True) trans = self.transaction_begin("",True)
current = 0
# Numerous changes were made between dbversions 8 and 9. # Numerous changes were made between dbversions 8 and 9.
# If nothing else, we switched from storing pickled gramps classes # If nothing else, we switched from storing pickled gramps classes
# to storing builting objects, via running serialize() recursively # to storing builtin objects, via running serialize() recursively
# until the very bottom. Every stored objects needs to be # until the very bottom.
# re-committed here. # Every stored object needs to be re-committed here.
# Change every Source to have reporef_list # Change every Source to have reporef_list
for handle in self.source_map.keys(): for handle in self.source_map.keys():
@ -1372,8 +1365,7 @@ class GrampsBSDDB(GrampsDbBase):
source.pubinfo, source.note, source.media_list, source.pubinfo, source.note, source.media_list,
source.abbrev, source.change, source.datamap) = info source.abbrev, source.change, source.datamap) = info
self.commit_source(source,trans) self.commit_source(source,trans)
current += 1 self.update()
self.update(100*current/length)
# Family upgrade # Family upgrade
for handle in self.family_map.keys(): for handle in self.family_map.keys():
@ -1419,8 +1411,7 @@ class GrampsBSDDB(GrampsDbBase):
family.lds_ord_list = [lds_seal] family.lds_ord_list = [lds_seal]
self.commit_family(family,trans) self.commit_family(family,trans)
current += 1 self.update()
self.update(100*current/length)
# Person upgrade # Person upgrade
# Needs to be run after the family upgrade completed. # Needs to be run after the family upgrade completed.
@ -1507,8 +1498,7 @@ class GrampsBSDDB(GrampsDbBase):
in [lds_bapt,lds_endow,lds_seal] if item] in [lds_bapt,lds_endow,lds_seal] if item]
self.commit_person(person,trans) self.commit_person(person,trans)
current += 1 self.update()
self.update(100*current/length)
# Event upgrade # Event upgrade
# Turns out that a lof ot events have duplicate gramps IDs # Turns out that a lof ot events have duplicate gramps IDs
@ -1575,8 +1565,7 @@ class GrampsBSDDB(GrampsDbBase):
event.set_note(note_text) event.set_note(note_text)
self.commit_event(event,trans) self.commit_event(event,trans)
current += 1 self.update()
self.update(100*current/length)
self.eid_trans.close() self.eid_trans.close()
# Place upgrade # Place upgrade
@ -1597,8 +1586,7 @@ class GrampsBSDDB(GrampsDbBase):
convert_url_9(url) convert_url_9(url)
self.commit_place(place,trans) self.commit_place(place,trans)
current += 1 self.update()
self.update(100*current/length)
# Media upgrade # Media upgrade
for handle in self.media_map.keys(): for handle in self.media_map.keys():
@ -1615,8 +1603,7 @@ class GrampsBSDDB(GrampsDbBase):
convert_attribute_9(attribute) convert_attribute_9(attribute)
self.commit_media_object(media_object,trans) self.commit_media_object(media_object,trans)
current += 1 self.update()
self.update(100*current/length)
self.transaction_commit(trans,"Upgrade to DB version 9") self.transaction_commit(trans,"Upgrade to DB version 9")
# Close secodnary index # Close secodnary index
@ -1653,7 +1640,7 @@ def convert_url_9(url):
new_type = UrlType.CUSTOM new_type = UrlType.CUSTOM
url.type = UrlType(new_type) url.type = UrlType(new_type)
def low_level_9(the_db): def low_level_9(the_db,update):
""" """
This is a low-level repair routine. This is a low-level repair routine.
@ -1672,6 +1659,8 @@ def low_level_9(the_db):
print "Low-level repair: table: %s" % the_map[0] print "Low-level repair: table: %s" % the_map[0]
status,length = _table_low_level_9(the_db.env,the_map[1]) status,length = _table_low_level_9(the_db.env,the_map[1])
if update:
update()
if status: if status:
print "Done." print "Done."
the_length += length the_length += length