* src/GrampsDb/_GrampsBSDDB.py (transaction_begin,

transaction_commit): only do secondary index magic if not
instructde otherwise; (BdbTransaction.__init__): accept additional
argument, pass it on to Transaction class.
* src/GrampsDb/_GrampsDbBase.py (Transaction.__init__): accept
additional argument for no_magic.
* src/GrampsDb/_ReadGedcom.py (NoteParser): count people during
first pass; (parse_gedcom_file): start transaction with or without
secondary index magic.
* src/GrampsDb/_ReadXML.py (LineParser):count people during
first pass; (parse): start transaction with or without secondary
index magic.


svn: r5865

gramps2/ChangeLog

@@ -1,3 +1,17 @@
+2006-02-02  Alex Roitman  <shura@gramps-project.org>
+	* src/GrampsDb/_GrampsBSDDB.py (transaction_begin,
+	transaction_commit): only do secondary index magic if not
+	instructde otherwise; (BdbTransaction.__init__): accept additional
+	argument, pass it on to Transaction class.
+	* src/GrampsDb/_GrampsDbBase.py (Transaction.__init__): accept
+	additional argument for no_magic.
+	* src/GrampsDb/_ReadGedcom.py (NoteParser): count people during
+	first pass; (parse_gedcom_file): start transaction with or without
+	secondary index magic.
+	* src/GrampsDb/_ReadXML.py (LineParser):count people during
+	first pass; (parse): start transaction with or without secondary
+	index magic.
+
 2006-02-01  Don Allingham  <don@gramps-project.org>
 	* src/EditPerson.py: unused code
 	* src/ImgManip.py: convienence code

gramps2/src/GrampsDb/_GrampsBSDDB.py

@@ -968,23 +968,19 @@ class GrampsBSDDB(GrampsDbBase):
             add_func(obj,transaction)
         return obj
 
-    def transaction_begin(self,msg="",batch=False):
+    def transaction_begin(self,msg="",batch=False,no_magic=False):
         """
         Creates a new Transaction tied to the current UNDO database. The
         transaction has no effect until it is committed using the
         transaction_commit function of the this database object.
         """
 
-        transaction = BdbTransaction(msg,self.undodb,batch)
+        transaction = BdbTransaction(msg,self.undodb,batch,no_magic)
         if transaction.batch:
             self.env.txn_checkpoint()
             self.env.set_flags(db.DB_TXN_NOSYNC,1)      # async txn
-            # FIXME: There appears to have a bug in bsddb
-            # preventing us from rebuilding the secondary indices.
-            # In particular, the rebuild_secondary will also fail,
-            # but we can't help it. Disabling the secondary index
-            # removal/rebuilding for batch transactions for now.
 
+            if not transaction.no_magic:
                 # Disconnect unneeded secondary indices
                 self.surnames.close()
                 junk = db.DB(self.env)
@@ -1015,20 +1011,17 @@ class GrampsBSDDB(GrampsDbBase):
             self.env.txn_checkpoint()
             self.env.set_flags(db.DB_TXN_NOSYNC,0)      # sync txn
 
-            # FIXME: There appears to have a bug in bsddb
+            if not transaction.no_magic:
-            # preventing us from rebuilding the secondary indices.
+                # create new secondary indices to replace the ones removed
-            # In particular, the rebuild_secondary will also fail,
-            # but we can't help it. Disabling the secondary index
-            # removal/rebuilding for batch transactions for now.
-
                 open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT
                 dupe_flags = db.DB_DUP|db.DB_DUPSORT
-            # create new secondary indices to replace the ones removed
+
                 self.surnames = db.DB(self.env)
                 self.surnames.set_flags(dupe_flags)
                 self.surnames.open(self.full_name,"surnames",
                                    db.DB_BTREE,flags=open_flags)
-            self.person_map.associate(self.surnames,find_surname,open_flags)
+                self.person_map.associate(self.surnames,find_surname,
+                                          open_flags)
             
                 self.reference_map_referenced_map = db.DB(self.env)
                 self.reference_map_referenced_map.set_flags(dupe_flags)
@@ -1499,8 +1492,8 @@ class GrampsBSDDB(GrampsDbBase):
 
 
 class BdbTransaction(Transaction):
-    def __init__(self,msg,db,batch=False):
+    def __init__(self,msg,db,batch=False,no_magic=False):
-        Transaction.__init__(self,msg,db,batch)
+        Transaction.__init__(self,msg,db,batch,no_magic)
         self.reference_del = []
         self.reference_add = []
 

gramps2/src/GrampsDb/_GrampsDbBase.py

@@ -1865,18 +1865,29 @@ class Transaction:
     Defines a group of database commits that define a single logical
     operation.
     """
-    def __init__(self,msg,db,batch=False):
+    def __init__(self,msg,db,batch=False,no_magic=False):
         """
-        Creates a new transaction. A Transaction instance should not be created
+        Creates a new transaction. A Transaction instance should not be
-        directly, but by the GrampsDbBase class or classes derived from
+        created directly, but by the GrampsDbBase class or classes derived
-        GrampsDbBase. The db parameter is a list-like interface that stores
+        from GrampsDbBase. The db parameter is a list-like interface that
-        the commit data. This could be a simple list, or a RECNO-style database
+        stores the commit data. This could be a simple list, or a RECNO-style
-        object.
+        database object.
+
+        The batch parameter is set to True for large transactions. For such
+        transactions, the list of changes is not maintained, and no undo
+        is possible.
+
+        The no_magic parameter is ignored for non-batch transactions, and
+        is also of no importance for DB backends other than BSD DB. For
+        the BSDDB, when this paramter is set to True, some secondary
+        indices will be removed at the beginning and then rebuilt at
+        the end of such transaction (only if it is batch).
         """
         self.db = db
         self.first = None
         self.last = None
         self.batch = batch
+        self.no_magic = no_magic
         self.length = 0
 
         self.person_add = []

gramps2/src/GrampsDb/_ReadGedcom.py

@@ -1,7 +1,7 @@
 #
 # Gramps - a GTK+/GNOME based genealogy program
 #
-# Copyright (C) 2000-2005  Donald N. Allingham
+# Copyright (C) 2000-2006  Donald N. Allingham
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -198,7 +198,7 @@ def import2(database, filename, callback, codeset, use_trans):
     try:
         np = NoteParser(filename, False)
         g = GedcomParser(database,filename, callback, codeset, np.get_map(),
-                         np.get_lines())
+                         np.get_lines(),np.get_persons())
     except IOError,msg:
         ErrorDialog(_("%s could not be opened\n") % filename,str(msg))
         return
@@ -275,6 +275,7 @@ class IdFinder:
 noteRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+NOTE(.*)$")
 contRE = re.compile(r"\s*\d+\s+CONT\s(.*)$")
 concRE = re.compile(r"\s*\d+\s+CONC\s(.*)$")
+personRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+INDI(.*)$")
 
 #-------------------------------------------------------------------------
 #
@@ -305,6 +306,7 @@ class NoteParser:
         self.name_map = {}
 
         self.count = 0
+        self.person_count = 0
         f = open(filename,"rU")
         innote = False
         
@@ -333,6 +335,9 @@ class NoteParser:
                     self.name_map["@%s@" % data] = noteobj
                     noteobj.append(match.groups()[1])
                     innote = True
+                elif personRE.match(line):
+                    self.person_count += 1
+                
         f.close()
 
     def get_map(self):
@@ -341,6 +346,9 @@ class NoteParser:
     def get_lines(self):
         return self.count
 
+    def get_persons(self):
+        return self.person_count
+
 #-------------------------------------------------------------------------
 #
 # Reader - serves as the lexical analysis engine
@@ -421,9 +429,10 @@ class GedcomParser:
     SyntaxError = "Syntax Error"
     BadFile = "Not a GEDCOM file"
 
-    def __init__(self, dbase, filename, callback, codeset, note_map, lines):
+    def __init__(self,dbase,filename,callback,codeset,note_map,lines,people):
 
         self.maxlines = lines
+        self.maxpeople = people
         self.interval = lines/100
         self.percent = 0
         self.callback = callback
@@ -637,9 +646,12 @@ class GedcomParser:
         self.backoff = True
 
     def parse_gedcom_file(self,use_trans=False):
+        if self.maxpeople < 1000:
+            no_magic = True
+        else:
+            no_magic = False
+        self.trans = self.db.transaction_begin("",not use_trans,no_magic)
 
-        self.trans = self.db.transaction_begin("",not use_trans)
-        #self.trans.set_batch(not use_trans)
         self.db.disable_signals()
         t = time.time()
         self.fam_count = 0

gramps2/src/GrampsDb/_ReadXML.py

@@ -30,6 +30,7 @@ import sets
 import shutil
 from xml.parsers.expat import ExpatError, ParserCreate
 from gettext import gettext as _
+import re
 
 #-------------------------------------------------------------------------
 #
@@ -67,6 +68,8 @@ try:
 except:
     gzip_ok = 0
 
+personRE = re.compile(r"\s*\<person\s(.*)$")
+
 #-------------------------------------------------------------------------
 #
 # Importing data into the currently open database. 
@@ -86,6 +89,7 @@ def importData(database, filename, callback=None,cl=0,use_trans=False):
 
     linecounter = LineParser(filename)
     lc = linecounter.get_count()
+    pc = linecounter.get_person_count()
     
     ro = database.readonly
     database.readonly = False
@@ -124,7 +128,7 @@ def importData(database, filename, callback=None,cl=0,use_trans=False):
             ErrorDialog(_("%s could not be opened") % filename)
             return
     try:
-        parser.parse(xml_file,use_trans,lc)
+        parser.parse(xml_file,use_trans,lc,pc)
     except IOError,msg:
         if cl:
             print "Error reading %s" % filename
@@ -208,6 +212,7 @@ class LineParser:
     def __init__(self, filename):
 
         self.count = 0
+        self.person_count = 0
 
         if gzip_ok:
             use_gzip = 1
@@ -230,14 +235,20 @@ class LineParser:
 
             for line in f:
                 self.count += 1
+                if personRE.match(line):
+                    self.person_count += 1
 
             f.close()
         except:
             self.count = 0
+            self.person_count = 0
 
     def get_count(self):
         return self.count
 
+    def get_person_count(self):
+        return self.person_count
+
 #-------------------------------------------------------------------------
 #
 # Gramps database parsing class.  Derived from SAX XML parser
@@ -570,9 +581,12 @@ class GrampsParser:
                 self.ridswap[gramps_id] = gramps_id
         return self.ridswap[gramps_id]
 
-    def parse(self,file,use_trans=False,linecount=0):
+    def parse(self,file,use_trans=False,linecount=0,personcount=0):
-
+        if personcount < 1000:
-        self.trans = self.db.transaction_begin("",batch=True)
+            no_magic = True
+        else:
+            no_magic = False
+        self.trans = self.db.transaction_begin("",batch=True,no_magic=no_magic)
         self.linecount = linecount
 
         self.db.disable_signals()