Commit 322809d3 authored by Jean-Christophe Fillion-Robin's avatar Jean-Christophe Fillion-Robin
Browse files

PERF: Improve startup time by 8% with lazy loading of wrapped libraries

*** WORK IN PROGRESS: For now, you have to make sure SlicerApp-real and
Slicer launcher are built to ensure the successful generation of the json
files ***

Startup time reduced from 3.8s to 3.5s with a "cold cache"
and from 2.7s to 2.38s with a "warm cache".

For each logic/mrml/dm/widgets python modules, a json files listing
the associated attributes is generated. Then, when the application is
initialized, the "slicer" module is created as a "lazy" module with
the attributes associated with logic/mrml/dm/widgets set as "not loaded".

Finally, as soon as an attribute not yet loaded is accessed, the specialized
__getattribute__ loads the associated python module and update the module
dictionary.

The "lazy" module has been adapted from "itkLazy.py"

Results have been gathered on Ubuntu 15.10 on a workstation with the
following specs: 64GB / M.2 PCIe NVMe SSD / Quad Core 3.80GHz
parent ab4521d4
......@@ -7,6 +7,7 @@ set(Slicer_PYTHON_SCRIPTS
slicer/testing
slicer/util
freesurfer
lazy
mrml
saferef
teem
......
import imp
import json
import os
import sys
import types
not_loaded = 'not loaded'
def library_loader(module_name):
#print("Loading %s" % module_name)
fp, pathname, description = imp.find_module(module_name)
module = imp.load_module(module_name, fp, pathname, description)
return module
class LazyModule(types.ModuleType):
"""Subclass of ModuleType that implements a custom __getattribute__ method
to allow lazy-loading of attributes from slicer sub-modules."""
def __init__(self, name):
types.ModuleType.__init__(self, name)
self.__lazy_attributes = {}
#print("__lazy_attributes: %s" % len(self.__lazy_attributes))
def _update_lazy_attributes(self, lazy_attributes):
self.__lazy_attributes.update(lazy_attributes)
for k in lazy_attributes:
setattr(self, k, not_loaded)
def __getattribute__(self, attr):
value = types.ModuleType.__getattribute__(self, attr)
#print("__getattribute__ %s" % (attr))
if value is not_loaded:
module_name = self.__lazy_attributes[attr]
module = library_loader(module_name)
namespace = module.__dict__
# Load into 'namespace' first, then self.__dict__ (via setattr) to
# prevent the warnings about overwriting the 'NotLoaded' values
# already in self.__dict__ we would get if we just update
# self.__dict__.
for k, v in namespace.items():
if not k.startswith('_'):
setattr(self, k, v)
value = namespace[attr]
return value
def writeModuleAttributeFile(module_name, config_dir='.'):
try:
exec("import %s as module" % module_name)
except ImportError as details:
print("%s [skipped: failed to import: %s]" % (module_name, details))
return
attributes = []
for attr in dir(module):
if not attr.startswith('__'):
attributes.append(attr)
filename = os.path.join(config_dir, "%s.json" % module_name)
with open(filename, 'w') as output:
print("%s [done: %s]" % (module_name, filename))
output.write(json.dumps({"attributes":attributes}, indent=4))
def updateLazyModule(module, input_module_names=[], config_dir=None):
if isinstance(module, basestring):
if module not in sys.modules:
print("updateLazyModule failed: Couldn't find %s module" % module)
return
module = sys.modules[module]
if not isinstance(module, LazyModule):
print("updateLazyModule failed: module '%s' is not a LazyModule" % module)
return
if isinstance(input_module_names, basestring):
input_module_names = [input_module_names]
if config_dir is None:
config_dir = os.path.dirname(module.__path__[0])
for input_module_name in input_module_names:
filename = os.path.join(config_dir, "%s.json" % input_module_name)
with open(filename) as input:
module_attributes = json.load(input)['attributes']
#print("Updating %s with %d attributes" % (filename, len(module_attributes)))
module._update_lazy_attributes({attribute: input_module_name for attribute in module_attributes})
#print("Updated %s module with %d attributes from %s" % (module, len(module._LazyModule__lazy_attributes), input_module_name))
def createLazyModule(module_name, module_path, input_module_names=[], config_dir=None):
thisModule = sys.modules[module_name] if module_name in sys.modules else None
if isinstance(thisModule, LazyModule):
# Handle reload case where we've already done this once.
# If we made a new module every time, multiple reload()s would fail
# because the identity of sys.modules['itk'] would always be changing.
#print("slicer: Calling ctor of LazyModule")
thisModule.__init__(module_name)
else:
print("slicer: Creating new LazyModule")
thisModule = LazyModule(module_name)
# Set the __path__ attribute, which is required for this module to be used as a
# package
setattr(thisModule, '__path__', module_path)
sys.modules[module_name] = thisModule
updateLazyModule(thisModule, input_module_names, config_dir)
return thisModule
""" This module sets up root logging and loads the Slicer library modules into its namespace."""
import lazy
thisModule = lazy.createLazyModule(__name__, __path__)
del lazy
#-----------------------------------------------------------------------------
def _createModule(name, globals, docstring):
import imp
......@@ -14,14 +18,14 @@ def _createModule(name, globals, docstring):
#-----------------------------------------------------------------------------
# Create slicer.modules and slicer.moduleNames
_createModule('slicer.modules', globals(),
_createModule('slicer.modules', vars(thisModule),
"""This module provides an access to all instantiated Slicer modules.
The module attributes are the lower-cased Slicer module names, the
associated value is an instance of ``qSlicerAbstractCoreModule``.
""")
_createModule('slicer.moduleNames', globals(),
_createModule('slicer.moduleNames', vars(thisModule),
"""This module provides an access to all instantiated Slicer module names.
The module attributes are the Slicer modules names, the associated
......@@ -36,15 +40,19 @@ try:
except ImportError:
available_kits = []
from .util import importModuleObjects
for kit in available_kits:
try:
exec "from %s import *" % (kit)
importModuleObjects(kit, thisModule)
#exec "from %s import *" % (kit)
except ImportError as detail:
print detail
#-----------------------------------------------------------------------------
# Cleanup: Removing things the user shouldn't have to see.
del thisModule
del _createModule
del available_kits
del kit
......@@ -53,11 +53,11 @@ def sourceDir():
# Custom Import
#
def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*'):
importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch)
def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch, lazy)
def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch)
def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch, lazy)
# To avoid globbing multiple times the same directory, successful
# call to ``importClassesFromDirectory()`` will be indicated by
......@@ -66,7 +66,7 @@ def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
# Each entry is a tuple of form (directory, dest_module_name, type_name, filematch)
__import_classes_cache = set()
def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*'):
def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*', lazy=False):
# Create entry for __import_classes_cache
cache_key = ",".join([directory, dest_module_name, type_name, filematch])
......@@ -74,27 +74,31 @@ def importClassesFromDirectory(directory, dest_module_name, type_name, filematch
if cache_key in __import_classes_cache:
return
import glob, os, re, fnmatch
import glob, lazy, os, re, fnmatch
re_filematch = re.compile(fnmatch.translate(filematch))
for fname in glob.glob(os.path.join(directory, filematch)):
if not re_filematch.match(os.path.basename(fname)):
continue
try:
from_module_name = os.path.splitext(os.path.basename(fname))[0]
importModuleObjects(from_module_name, dest_module_name, type_name)
except ImportError as detail:
import sys
print(detail, file=sys.stderr)
from_module_name = os.path.splitext(os.path.basename(fname))[0]
if lazy:
lazy.updateLazyModule(dest_module_name, from_module_name, os.path.dirname(fname))
else:
try:
importModuleObjects(from_module_name, dest_module_name, type_name)
except ImportError as detail:
import sys
print(detail, file=sys.stderr)
__import_classes_cache.add(cache_key)
def importModuleObjects(from_module_name, dest_module_name, type_name):
def importModuleObjects(from_module_name, dest_module, type_name='*'):
"""Import object of type 'type_name' from module identified
by 'from_module_name' into the module identified by 'dest_module_name'."""
by 'from_module_name' into the module identified by 'dest_module'."""
# Obtain a reference to the module identifed by 'dest_module_name'
# Obtain a reference to the module identifed by 'dest_module'
import sys
dest_module = sys.modules[dest_module_name]
if isinstance(dest_module, basestring):
dest_module = sys.modules[dest_module]
# Skip if module has already been loaded
if from_module_name in sys.modules:
......@@ -112,7 +116,7 @@ def importModuleObjects(from_module_name, dest_module_name, type_name):
item = getattr(module, item_name)
# Add the object to dest_module_globals_dict if any
if type(item).__name__ == type_name:
if type(item).__name__ == type_name or (type_name == '*' and not item_name.startswith('_')):
setattr(dest_module, item_name, item)
#
......
......@@ -74,13 +74,13 @@ bool qSlicerLoadableModule::importModulePythonExtensions(
ctkScopedCurrentDir scopedCurrentDir(QFileInfo(modulePath).absolutePath());
pythonManager->executeString(QString(
"from slicer.util import importVTKClassesFromDirectory;"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*');"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*');"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*');"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*', lazy=True);"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*', lazy=True);"
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*', lazy=True);"
).arg(scopedCurrentDir.currentPath()));
pythonManager->executeString(QString(
"from slicer.util import importQtClassesFromDirectory;"
"importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*');"
"importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*', lazy=True);"
).arg(scopedCurrentDir.currentPath()));
return !pythonManager->pythonErrorOccured();
#else
......
......@@ -240,6 +240,22 @@ macro(SlicerMacroBuildModuleQtLibrary)
if(NOT "${MODULEQTLIBRARY_FOLDER}" STREQUAL "")
set_target_properties(${lib_name}PythonQt PROPERTIES FOLDER ${MODULEQTLIBRARY_FOLDER})
endif()
# XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
# XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
# Add target to generate module attributes file to allow lazy loading
set(module_name "${lib_name}PythonQt")
set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
set(code "${code}import lazy;")
set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
add_custom_command(TARGET ${module_name} POST_BUILD
COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --no-splash -c "${code}"
COMMENT "Generating ${module_name}.json"
VERBATIM
)
endif()
endmacro()
......@@ -81,4 +81,23 @@ macro(SlicerMacroPythonWrapModuleVTKLibrary)
KIT_PYTHON_LIBRARIES ${PYTHONWRAPMODULEVTKLIBRARY_Wrapped_LIBRARIES}
)
# XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
# XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
# Get path to real executable
get_filename_component(python_bin_dir ${PYTHON_EXECUTABLE} PATH)
set(real_python_executable ${python_bin_dir}/python${CMAKE_EXECUTABLE_SUFFIX})
# Add target to generate module attributes file to allow lazy loading
set(module_name "${PYTHONWRAPMODULEVTKLIBRARY_NAME}Python")
set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
set(code "${code}import lazy;")
set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
add_custom_command(TARGET ${module_name} POST_BUILD
COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --launch ${real_python_executable} -c "${code}"
COMMENT "Generating ${module_name}.json"
VERBATIM
)
endmacro()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment