Improve Addon management performance

gramps/gen/dbstate.py

@@ -43,7 +43,7 @@ from .proxy.proxybase import ProxyDbBase
 from .utils.callback import Callback
 from .config import config
 from gramps.gen.db.dbconst import DBLOGNAME
-from gramps.gen.db.utils import make_database
+from gramps.gen.db.dummydb import DummyDb
 
 #-------------------------------------------------------------------------
 #
@@ -69,7 +69,7 @@ class DbState(Callback):
         place holder until a real DB is assigned.
         """
         Callback.__init__(self)
-        self.db = make_database("dummydb")
+        self.db = DummyDb()
         self.open = False  #  Deprecated - use DbState.is_open()
         self.stack = []
 
@@ -135,7 +135,7 @@ class DbState(Callback):
         self.emit('no-database', ())
         if self.is_open():
             self.db.close()
-        self.db = make_database("dummydb")
+        self.db = DummyDb()
         self.open = False
         self.emit('database-changed', (self.db, ))
 

gramps/gen/plug/_manager.py

@@ -39,7 +39,7 @@ import os
 import sys
 import re
 import logging
-LOG = logging.getLogger('.' + __name__)
+LOG = logging.getLogger('._manager')
 LOG.progagate = True
 from ..const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
@@ -100,8 +100,8 @@ class BasePluginManager:
         self.__modules           = {}
 
         self.__pgr = PluginRegister.get_instance()
-        self.__registereddir_set = set()
         self.__loaded_plugins = {}
+        self.__scanned_dirs = []
 
     def reg_plugins(self, direct, dbstate=None, uistate=None,
                     load_on_reg=False):
@@ -112,23 +112,24 @@ class BasePluginManager:
         If a relationship calculator for env var LANG is present, it is
         immediately loaded so it is available for all.
         """
-        # if the directory does not exist, do nothing
+        # if we've already scanned this directory or if the directory does not
-        if not os.path.isdir(direct):
+        # exist, we are done.  Should only happen in tests.
-            return False # return value is True for error
 
-        for (dirpath, dirnames, filenames) in os.walk(direct):
+        # LOG.warning("\nPlugin manager registration: %s, load_on_reg=%s,"
-            root, subdir = os.path.split(dirpath)
+        #             " been_here=%s, pahte exists:%s", direct, load_on_reg,
-            if subdir.startswith("."):
+        #             direct in self.__scanned_dirs, os.path.isdir(direct))
-                dirnames[:] = []
+
-                continue
+        if os.path.isdir(direct) and direct not in self.__scanned_dirs:
-            for dirname in dirnames:
+            self.__scanned_dirs.append(direct)
+            for (dirpath, dirnames, filenames) in os.walk(direct,
+                                                          topdown=True):
+                for dirname in dirnames[:]:
                     # Skip hidden and system directories:
-                if dirname.startswith(".") or dirname in ["po", "locale"]:
+                    if dirname.startswith(".") or dirname in ["po", "locale",
+                                                              "__pycache__"]:
                         dirnames.remove(dirname)
-            # if the path has not already been loaded, save it in the
+                # LOG.warning("Plugin dir scanned: %s", dirpath)
-            # registereddir_list list for use on reloading.
+                self.__pgr.scan_dir(dirpath, filenames, uistate=uistate)
-            self.__registereddir_set.add(dirpath)
-            self.__pgr.scan_dir(dirpath, uistate=uistate)
 
         if load_on_reg:
             # Run plugins that request to be loaded on startup and
@@ -136,6 +137,7 @@ class BasePluginManager:
             # first, remove hidden
             plugins_to_load = []
             for plugin in self.__pgr.filter_load_on_reg():
+                # LOG.warning("\nFound %s at registration", plugin.id)
                 if plugin.id in config.get("plugin.hiddenplugins"):
                     continue
                 plugins_to_load.append(plugin)
@@ -146,6 +148,8 @@ class BasePluginManager:
             max_count = len(plugins_to_load)
             while plugins_to_load:
                 for plugin in plugins_to_load[:]: # copy of list
+                    # LOG.warning("\nDependencies for %s at registration",
+                    #             plugin.id)
                     delay = False
                     for depend in plugin.depends_on:
                         if depend not in [p.id for p in plugins_sorted]:
@@ -167,8 +171,12 @@ class BasePluginManager:
                     break
             # now load them:
             for plugin in plugins_sorted:
+                # next line shouldn't be necessary, but this gets called a lot
+                # of times during Travis test; so avoid multiple copies
+                plugin.data = []
                 mod = self.load_plugin(plugin)
                 if hasattr(mod, "load_on_reg"):
+                    # LOG.warning("\nRun %s at registration", plugin.id)
                     try:
                         results = mod.load_on_reg(dbstate, uistate, plugin)
                     except:
@@ -496,6 +504,8 @@ class BasePluginManager:
                     retval.extend(data)
                 except:
                     retval.append(data)
+        # LOG.warning("Process plugin data=%s, %s, items=%s",
+        #             process is not None, category, len(retval))
         if process:
             return process(retval)
         return retval

gramps/gen/plug/_pluginreg.py

@@ -43,6 +43,8 @@ from gramps.version import VERSION as GRAMPSVERSION, VERSION_TUPLE
 from ..const import IMAGE_DIR
 from ..const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
+import logging
+LOG = logging.getLogger('._manager')
 
 #-------------------------------------------------------------------------
 #
@@ -1104,11 +1106,16 @@ class PluginRegister:
         if __debug__:
             self.stable_only = False
         self.__plugindata  = []
+        self.__id_to_pdata = {}
 
     def add_plugindata(self, plugindata):
+        """ This is used to add an entry to the registration list.  The way it
+        is used, this entry is not yet filled in, so we cannot use the id to
+        add to the __id_to_pdata dict at this time. """
         self.__plugindata.append(plugindata)
 
-    def scan_dir(self, dir, uistate=None):
+
+    def scan_dir(self, dir, filenames, uistate=None):
         """
         The dir name will be scanned for plugin registration code, which will
         be loaded in :class:`PluginData` objects if they satisfy some checks.
@@ -1123,9 +1130,8 @@ class PluginRegister:
         extlen = -len(ext)
         pymod = re.compile(r"^(.*)\.py$")
 
-        for filename in os.listdir(dir):
+        for filename in filenames:
-            name = os.path.split(filename)[1]
+            if not filename[extlen:] == ext:
-            if not name[extlen:] == ext:
                 continue
             lenpd = len(self.__plugindata)
             full_filename = os.path.join(dir, filename)
@@ -1150,9 +1156,14 @@ class PluginRegister:
             else:
                 local_gettext = glocale.translation.gettext
             try:
-                #execfile(full_filename,
                 exec (compile(stream, filename, 'exec'),
                       make_environment(_=local_gettext), {'uistate': uistate})
+                for pdata in self.__plugindata[lenpd:]:
+                    # should not be duplicate IDs in different plugins
+                    assert pdata.id not in self.__id_to_pdata
+                    # if pdata.id in self.__id_to_pdata:
+                    #     print("Error: %s is duplicated!" % pdata.id)
+                    self.__id_to_pdata[pdata.id] = pdata
             except ValueError as msg:
                 print(_('ERROR: Failed reading plugin registration %(filename)s') % \
                             {'filename' : filename})
@@ -1170,6 +1181,7 @@ class PluginRegister:
             rmlist = []
             ind = lenpd-1
             for plugin in self.__plugindata[lenpd:]:
+                #LOG.warning("\nPlugin scanned %s at registration", plugin.id)
                 ind += 1
                 plugin.directory = dir
                 if not valid_plugin_version(plugin.gramps_target_version):
@@ -1211,19 +1223,20 @@ class PluginRegister:
                 module = match.groups()[0]
                 plugin.mod_name = module
                 plugin.fpath = dir
+                #LOG.warning("\nPlugin added %s at registration", plugin.id)
             rmlist.reverse()
             for ind in rmlist:
+                del self.__id_to_pdata[self.__plugindata[ind].id]
                 del self.__plugindata[ind]
 
     def get_plugin(self, id):
         """
         Return the :class:`PluginData` for the plugin with id
         """
-        matches = [x for x in self.__plugindata if x.id == id]
+        assert(len(self.__id_to_pdata) == len(self.__plugindata))
-        matches.sort(key=lambda x: version(x.version))
+        # if len(self.__id_to_pdata) != len(self.__plugindata):
-        if len(matches) > 0:
+        #     print(len(self.__id_to_pdata), len(self.__plugindata))
-            return matches[-1]
+        return self.__id_to_pdata.get(id, None)
-        return None
 
     def type_plugins(self, ptype):
         """

gramps/plugins/db/dummydb.gpr.py

@@ -1,36 +0,0 @@
-#
-# Gramps - a GTK+/GNOME based genealogy program
-#
-# Copyright (C) 2016       Tim G L Lyons
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-from gramps.gen.plug._pluginreg import register, STABLE, DATABASE
-from gramps.gen.const import GRAMPS_LOCALE as glocale
-_ = glocale.translation.gettext
-
-register(DATABASE,
-         id    = 'dummydb',
-         name  = _("Dummy database"),
-         name_accell  = _("Dummy Database"),
-         description =  _("Dummy Database"),
-         version = '1.0.0',
-         gramps_target_version = "5.0",
-         status = STABLE,
-         fname = 'dummydb.py',
-         databaseclass = 'DummyDb',
-         authors=['Tim Lyons'],
-         authors_email=[""],
-)