6194: Database corrupted - TypeError: unhashable type: 'list'
Fixed upgrade to deal with sourceref in media references in Sources. Also fixed Check and Repair to repair previously broken databases.
This commit is contained in:
kulath
parent
b971373182
commit
56c67576d5
@ -27,7 +27,7 @@ from gen.lib.markertype import MarkerType
|
||||
from gen.lib.tag import Tag
|
||||
import time
|
||||
import logging
|
||||
LOG = logging.getLogger(".citation")
|
||||
LOG = logging.getLogger(".upgrade")
|
||||
|
||||
from gen.ggettext import gettext as _
|
||||
|
||||
@ -43,7 +43,7 @@ from gen.db import BSDDBTxn
|
||||
from gen.lib.nameorigintype import NameOriginType
|
||||
from gen.db.write import _mkname, SURNAMES
|
||||
from gen.db.dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY,
|
||||
MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY)
|
||||
MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY)
|
||||
from QuestionDialog import (InfoDialog)
|
||||
|
||||
def gramps_upgrade_16(self):
|
||||
@ -58,19 +58,22 @@ def gramps_upgrade_16(self):
|
||||
(4) add backlinks for references from object to Citations
|
||||
(5) add backlinks for references from Citation to Source
|
||||
|
||||
the backlinks are all updated at the end by calling
|
||||
reindex_reference_map
|
||||
the backlinks are all updated on return to write.py gramps_upgrade by
|
||||
calling reindex_reference_map
|
||||
|
||||
"""
|
||||
length = (len(self.note_map) + len(self.person_map) +
|
||||
# Only People, Families, Events, Media Objects, Places, Sources and
|
||||
# Repositories need to be updated, because these are the only primary
|
||||
# objects that can have source citations.
|
||||
length = (len(self.person_map) +
|
||||
len(self.event_map) + len(self.family_map) +
|
||||
len(self.repository_map) + len(self.media_map) +
|
||||
len(self.place_map) + len(self.source_map)) + 10
|
||||
len(self.place_map) + len(self.source_map))
|
||||
self.set_total(length)
|
||||
|
||||
# Setup data for upgrade statistics information dialogue
|
||||
keyorder = [PERSON_KEY, FAMILY_KEY, EVENT_KEY, MEDIA_KEY,
|
||||
PLACE_KEY, REPOSITORY_KEY]
|
||||
PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY]
|
||||
key2data = {
|
||||
PERSON_KEY : 0,
|
||||
FAMILY_KEY : 1,
|
||||
@ -78,6 +81,7 @@ def gramps_upgrade_16(self):
|
||||
MEDIA_KEY: 3,
|
||||
PLACE_KEY: 4,
|
||||
REPOSITORY_KEY: 5,
|
||||
SOURCE_KEY : 6,
|
||||
}
|
||||
key2string = {
|
||||
PERSON_KEY : _('%6d People upgraded with %6d citations in %6d secs\n'),
|
||||
@ -86,8 +90,9 @@ def gramps_upgrade_16(self):
|
||||
MEDIA_KEY : _('%6d Media Objects upgraded with %6d citations in %6d secs\n'),
|
||||
PLACE_KEY : _('%6d Places upgraded with %6d citations in %6d secs\n'),
|
||||
REPOSITORY_KEY : _('%6d Repositories upgraded with %6d citations in %6d secs\n'),
|
||||
SOURCE_KEY : _('%6d Sources upgraded with %6d citations in %6d secs\n'),
|
||||
}
|
||||
data_upgradeobject = [0] * 6
|
||||
data_upgradeobject = [0] * 7
|
||||
|
||||
# Initialise the citation gramps ID number
|
||||
self.cmap_index = 0
|
||||
@ -99,6 +104,15 @@ def gramps_upgrade_16(self):
|
||||
start_time = time.time()
|
||||
for person_handle in self.person_map.keys():
|
||||
person = self.person_map[person_handle]
|
||||
try:
|
||||
# The parameters are evaluated before deciding whether logging is on
|
||||
# or not. Since the retrieval of names is so complex, I think it is
|
||||
# safer to protect this with a try except block, even though it
|
||||
# seems to work for names being present and not.
|
||||
LOG.debug("upgrade person %s %s" % (person[3][4],
|
||||
" ".join([name[0] for name in person[3][5]])))
|
||||
except:
|
||||
pass
|
||||
(handle, gramps_id, gender, primary_name, alternate_names,
|
||||
death_ref_index, birth_ref_index, event_ref_list, family_list,
|
||||
parent_family_list, media_list, address_list, attribute_list,
|
||||
@ -141,6 +155,7 @@ def gramps_upgrade_16(self):
|
||||
attribute_list, urls, lds_seal_list,
|
||||
new_citation_list, note_list, change, tag_list,
|
||||
private, person_ref_list)
|
||||
LOG.debug(" upgrade new_person %s" % [new_person])
|
||||
with BSDDBTxn(self.env, self.person_map) as txn:
|
||||
txn.put(str(handle), new_person)
|
||||
self.update()
|
||||
@ -160,7 +175,7 @@ def gramps_upgrade_16(self):
|
||||
start_time = time.time()
|
||||
for media_handle in self.media_map.keys():
|
||||
media = self.media_map[media_handle]
|
||||
LOG.debug("upgrade media %s" % media[4])
|
||||
LOG.debug("upgrade media object %s" % media[4])
|
||||
(handle, gramps_id, path, mime, desc,
|
||||
attribute_list, source_list, note_list, change,
|
||||
date, tag_list, private) = media
|
||||
@ -175,12 +190,11 @@ def gramps_upgrade_16(self):
|
||||
LOG.debug(" upgrade new_media %s" % [new_media])
|
||||
with BSDDBTxn(self.env, self.media_map) as txn:
|
||||
txn.put(str(handle), new_media)
|
||||
LOG.debug(" update ref map media %s" % [handle,
|
||||
self.get_object_from_handle(handle) ])
|
||||
self.update()
|
||||
|
||||
LOG.debug("Media upgrade %d citations upgraded in %d seconds" %
|
||||
(self.cmap_index - start_num_citations,
|
||||
LOG.debug("%d media objects upgraded with %d citations in %d seconds" %
|
||||
(len(self.media_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
int(time.time() - start_time)))
|
||||
data_upgradeobject[key2data[MEDIA_KEY]] = (len(self.media_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
@ -193,6 +207,7 @@ def gramps_upgrade_16(self):
|
||||
start_time = time.time()
|
||||
for place_handle in self.place_map.keys():
|
||||
place = self.place_map[place_handle]
|
||||
LOG.debug("upgrade place %s" % place[2])
|
||||
(handle, gramps_id, title, long, lat,
|
||||
main_loc, alt_loc, urls, media_list, source_list, note_list,
|
||||
change, private) = place
|
||||
@ -209,6 +224,7 @@ def gramps_upgrade_16(self):
|
||||
long, lat, main_loc, alt_loc, urls,
|
||||
media_list, new_citation_list, note_list,
|
||||
change, private)
|
||||
LOG.debug(" upgrade new_place %s" % [new_place])
|
||||
with BSDDBTxn(self.env, self.place_map) as txn:
|
||||
txn.put(str(handle), new_place)
|
||||
self.update()
|
||||
@ -228,6 +244,7 @@ def gramps_upgrade_16(self):
|
||||
start_time = time.time()
|
||||
for family_handle in self.family_map.keys():
|
||||
family = self.family_map[family_handle]
|
||||
LOG.debug("upgrade family (gramps_id) %s" % family[1])
|
||||
(handle, gramps_id, father_handle, mother_handle,
|
||||
child_ref_list, the_type, event_ref_list, media_list,
|
||||
attribute_list, lds_seal_list, source_list, note_list,
|
||||
@ -257,11 +274,12 @@ def gramps_upgrade_16(self):
|
||||
child_ref_list, the_type, event_ref_list, media_list,
|
||||
attribute_list, lds_seal_list, new_citation_list,
|
||||
note_list, change, tag_list, private)
|
||||
LOG.debug(" upgrade new_family %s" % [new_family])
|
||||
with BSDDBTxn(self.env, self.family_map) as txn:
|
||||
txn.put(str(handle), new_family)
|
||||
self.update()
|
||||
|
||||
LOG.debug("%d familys upgraded with %d citations in %d seconds. " %
|
||||
LOG.debug("%d families upgraded with %d citations in %d seconds. " %
|
||||
(len(self.family_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
time.time() - start_time))
|
||||
@ -271,13 +289,11 @@ def gramps_upgrade_16(self):
|
||||
# ---------------------------------
|
||||
# Modify Events
|
||||
# ---------------------------------
|
||||
upgrade_time = 0
|
||||
backlink_time = 0
|
||||
start_num_citations = self.cmap_index
|
||||
start_time = time.time()
|
||||
for event_handle in self.event_map.keys():
|
||||
t1 = time.time()
|
||||
event = self.event_map[event_handle]
|
||||
LOG.debug("upgrade event %s" % event[4])
|
||||
(handle, gramps_id, the_type, date, description, place,
|
||||
source_list, note_list, media_list, attribute_list,
|
||||
change, private) = event
|
||||
@ -297,19 +313,15 @@ def gramps_upgrade_16(self):
|
||||
new_citation_list, note_list, media_list,
|
||||
attribute_list,
|
||||
change, private)
|
||||
LOG.debug(" upgrade new_event %s" % [new_event])
|
||||
with BSDDBTxn(self.env, self.event_map) as txn:
|
||||
txn.put(str(handle), new_event)
|
||||
t2 = time.time()
|
||||
upgrade_time += t2 - t1
|
||||
t3 = time.time()
|
||||
backlink_time += t3 - t2
|
||||
self.update()
|
||||
|
||||
LOG.debug("%d events upgraded with %d citations in %d seconds. "
|
||||
"Backlinks took %d seconds" %
|
||||
LOG.debug("%d events upgraded with %d citations in %d seconds. " %
|
||||
(len(self.event_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
int(upgrade_time), int(backlink_time)))
|
||||
time.time() - start_time))
|
||||
data_upgradeobject[key2data[EVENT_KEY]] = (len(self.event_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
time.time() - start_time)
|
||||
@ -321,6 +333,7 @@ def gramps_upgrade_16(self):
|
||||
start_time = time.time()
|
||||
for repository_handle in self.repository_map.keys():
|
||||
repository = self.repository_map[repository_handle]
|
||||
LOG.debug("upgrade repository %s" % repository[3])
|
||||
(handle, gramps_id, the_type, name, note_list,
|
||||
address_list, urls, change, private) = repository
|
||||
if address_list:
|
||||
@ -329,17 +342,52 @@ def gramps_upgrade_16(self):
|
||||
if address_list:
|
||||
new_repository = (handle, gramps_id, the_type, name, note_list,
|
||||
address_list, urls, change, private)
|
||||
LOG.debug(" upgrade new_repository %s" % [new_repository])
|
||||
with BSDDBTxn(self.env, self.repository_map) as txn:
|
||||
txn.put(str(handle), new_repository)
|
||||
self.update()
|
||||
|
||||
LOG.debug("%d repositorys upgraded with %d citations in %d seconds. " %
|
||||
LOG.debug("%d repositories upgraded with %d citations in %d seconds. " %
|
||||
(len(self.repository_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
time.time() - start_time))
|
||||
data_upgradeobject[key2data[REPOSITORY_KEY]] = (len(self.repository_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
time.time() - start_time)
|
||||
|
||||
# ---------------------------------
|
||||
# Modify Source
|
||||
# ---------------------------------
|
||||
start_num_citations = self.cmap_index
|
||||
start_time = time.time()
|
||||
for source_handle in self.source_map.keys():
|
||||
source = self.source_map[source_handle]
|
||||
LOG.debug("upgrade source %s" % source[2])
|
||||
(handle, gramps_id, title, author,
|
||||
pubinfo, note_list, media_list,
|
||||
abbrev, change, datamap, reporef_list,
|
||||
private) = source
|
||||
if media_list:
|
||||
media_list = upgrade_media_list_16(
|
||||
self, media_list)
|
||||
|
||||
new_source = (handle, gramps_id, title, author,
|
||||
pubinfo, note_list, media_list,
|
||||
abbrev, change, datamap, reporef_list,
|
||||
private)
|
||||
LOG.debug(" upgrade new_source %s" % [new_source])
|
||||
with BSDDBTxn(self.env, self.source_map) as txn:
|
||||
txn.put(str(handle), new_source)
|
||||
self.update()
|
||||
|
||||
LOG.debug("%d sources upgraded with %d citations in %d seconds" %
|
||||
(len(self.source_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
int(time.time() - start_time)))
|
||||
data_upgradeobject[key2data[SOURCE_KEY]] = (len(self.source_map.keys()),
|
||||
self.cmap_index - start_num_citations,
|
||||
time.time() - start_time)
|
||||
|
||||
# ---------------------------------
|
||||
|
||||
|
||||
@ -535,12 +583,6 @@ def convert_source_list_to_citation_list_16(self, source_list):
|
||||
with BSDDBTxn(self.env, self.citation_map) as txn:
|
||||
txn.put(str(new_handle), new_citation)
|
||||
self.cmap_index += 1
|
||||
# # add backlinks for references from Citation to Source
|
||||
# with BSDDBTxn(self.env) as txn:
|
||||
# self.update_reference_map(
|
||||
# self.get_citation_from_handle(new_handle),
|
||||
# transaction,
|
||||
# txn.txn)
|
||||
citation_list.append((new_handle))
|
||||
return citation_list
|
||||
|
||||
|
||||
@ -183,6 +183,7 @@ class Check(tool.BatchTool):
|
||||
checker.check_repo_references()
|
||||
checker.check_note_references()
|
||||
checker.check_tag_references()
|
||||
checker.check_media_sourceref()
|
||||
self.db.enable_signals()
|
||||
self.db.request_rebuild()
|
||||
|
||||
@ -223,6 +224,7 @@ class CheckIntegrity(object):
|
||||
self.invalid_dates = []
|
||||
self.removed_name_format = []
|
||||
self.empty_objects = defaultdict(list)
|
||||
self.replaced_sourceref = []
|
||||
self.last_img_dir = config.get('behavior.addmedia-image-dir')
|
||||
self.progress = ProgressMeter(_('Checking Database'),'')
|
||||
self.explanation = gen.lib.Note(_('Objects referenced by this note '
|
||||
@ -1813,6 +1815,64 @@ class CheckIntegrity(object):
|
||||
if len(self.invalid_tag_references) == 0:
|
||||
logging.info(' OK: no tag reference problems found')
|
||||
|
||||
def check_media_sourceref(self):
|
||||
"""
|
||||
This repairs a problem with database upgrade from database schema
|
||||
version 15 to 16. Mediarefs on source primary objects can contain
|
||||
sourcerefs, and these were not converted to citations.
|
||||
"""
|
||||
total = (
|
||||
self.db.get_number_of_sources()
|
||||
)
|
||||
|
||||
self.progress.set_pass(_('Looking for media source reference problems'),
|
||||
total)
|
||||
logging.info('Looking for media source reference problems')
|
||||
|
||||
for handle in self.db.source_map.keys():
|
||||
self.progress.step()
|
||||
info = self.db.source_map[handle]
|
||||
source = gen.lib.Source()
|
||||
source.unserialize(info)
|
||||
new_media_ref_list = []
|
||||
for media_ref in source.get_media_list():
|
||||
citation_list = media_ref.get_citation_list()
|
||||
new_citation_list = []
|
||||
for citation_handle in citation_list:
|
||||
# Either citation_handle is a handle, in which case it has
|
||||
# been converted, or it is a 6-tuple, in which case it now
|
||||
# needs to be converted.
|
||||
if len(citation_handle) == 6:
|
||||
if len(citation_handle) == 6:
|
||||
sourceref = citation_handle
|
||||
else:
|
||||
sourceref = eval(citation_handle)
|
||||
new_citation = gen.lib.Citation()
|
||||
new_citation.set_date_object(sourceref[0])
|
||||
new_citation.set_privacy(sourceref[1])
|
||||
new_citation.set_note_list(sourceref[2])
|
||||
new_citation.set_confidence_level(sourceref[3])
|
||||
new_citation.set_reference_handle(sourceref[4])
|
||||
new_citation.set_page(sourceref[5])
|
||||
citation_handle = Utils.create_id()
|
||||
new_citation.set_handle(citation_handle)
|
||||
self.replaced_sourceref.append(handle)
|
||||
logging.warning(' FAIL: the source "%s" has a media '
|
||||
'reference with a source citation '
|
||||
'which is invalid' % (source.gramps_id))
|
||||
self.db.add_citation(new_citation, self.trans)
|
||||
|
||||
new_citation_list.append(citation_handle)
|
||||
|
||||
media_ref.set_citation_list(new_citation_list)
|
||||
new_media_ref_list.append(media_ref)
|
||||
|
||||
source.set_media_list(new_media_ref_list)
|
||||
self.db.commit_source(source, self.trans)
|
||||
|
||||
if len(self.replaced_sourceref) > 0:
|
||||
logging.info(' OK: no broken source citations on mediarefs found')
|
||||
|
||||
def class_person(self, handle):
|
||||
person = gen.lib.Person()
|
||||
person.set_handle(handle)
|
||||
@ -1919,6 +1979,7 @@ class CheckIntegrity(object):
|
||||
note_references = len(self.invalid_note_references)
|
||||
tag_references = len(self.invalid_tag_references)
|
||||
name_format = len(self.removed_name_format)
|
||||
replaced_sourcerefs = len(self.replaced_sourceref)
|
||||
empty_objs = sum(len(obj) for obj in self.empty_objects.itervalues())
|
||||
|
||||
errors = (photos + efam + blink + plink + slink + rel +
|
||||
@ -2153,6 +2214,13 @@ class CheckIntegrity(object):
|
||||
name_format) % {'quantity' : name_format}
|
||||
)
|
||||
|
||||
if replaced_sourcerefs:
|
||||
self.text.write(
|
||||
ngettext("%(quantity)d invalid source citation was fixed\n",
|
||||
"%(quantity)d invalid source citations were fixed\n",
|
||||
replaced_sourcerefs) % {'quantity' : replaced_sourcerefs}
|
||||
)
|
||||
|
||||
if empty_objs > 0 :
|
||||
self.text.write(_("%(empty_obj)d empty objects removed:\n"
|
||||
" %(person)d person objects\n"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user