python: defer import of dependencies after native lib init

Wrapping/Python/vtkmodules/__init__.py.in

@@ -3,7 +3,34 @@ Currently, this package is experimental and may change in the future.
 """
 from __future__ import absolute_import
 import sys
-import importlib
+from pathlib import Path
+import importlib.util
+from importlib.abc import Loader, MetaPathFinder
+from importlib.machinery import ExtensionFileLoader
+
+class VTKMetaHook(MetaPathFinder):
+    """Attach a custom  loaded for vtk native library loading to defer loading of pure python dependencies"""
+    def find_spec(self, fullname, path, target=None):
+        if fullname.startswith("vtkmodules.vtk"):
+            module_path = None
+            vtk_module_name = fullname.split(".")[1]
+            for base_path in path:
+                for f in Path(base_path).glob(f"{vtk_module_name}.*"):
+                    module_path = str(f.resolve())
+
+            return importlib.util.spec_from_file_location(fullname, module_path, loader=VTKLoader(fullname, module_path))
+        return None
+
+
+class VTKLoader(ExtensionFileLoader):
+    """Flush any pending dependency load once initialize() phase is done"""
+    def exec_module(self, module):
+        super().exec_module(module)
+        _flush_pending()
+
+
+# Register hook
+sys.meta_path.insert(0, VTKMetaHook())
 
 
 def _windows_dll_path():
@@ -59,8 +86,8 @@ def _load_vtkmodules_static():
 #------------------------------------------------------------------------------
 # list the contents
 __all__ = [
-@_vtkmodules_all@]
-
+    @_vtkmodules_all@
+]
 #------------------------------------------------------------------------------
 # get the version
 __version__ = "@VTK_MAJOR_VERSION@.@VTK_MINOR_VERSION@.@VTK_BUILD_VERSION@"
@@ -76,6 +103,7 @@ MODULE_MAPPER = {
     ],
 }
 LOADED_MODULES = set()
+PENDING_LOADED_MODULES = set()
 
 def register_vtk_module_dependencies(vtk_module_name, *import_names):
     """Method to call for registering external override on vtkmodule load"""
@@ -87,12 +115,18 @@ def register_vtk_module_dependencies(vtk_module_name, *import_names):
             importlib.import_module(import_name)
 
 
-def on_vtk_module_loaded(module_name):
+def on_vtk_module_initialize(module_name):
     """Automatically called by vtkmodule when they are loaded"""
     if module_name in LOADED_MODULES:
         return
 
-    LOADED_MODULES.add(module_name)
+    PENDING_LOADED_MODULES.add(module_name)
+
+
+def _flush_pending():
+    for module_name in PENDING_LOADED_MODULES:
+        LOADED_MODULES.add(module_name)
+        for import_name in MODULE_MAPPER.get(module_name, []):
+            importlib.import_module(import_name)
 
-    for import_name in MODULE_MAPPER.get(module_name, []):
-        importlib.import_module(import_name)
+    PENDING_LOADED_MODULES.clear()

Wrapping/PythonCore/vtkPythonUtil.cxx

@@ -1039,6 +1039,17 @@ bool vtkPythonUtil::ImportModule(const char* fullname, PyObject* globals)
 void vtkPythonUtil::AddModule(const char* name)
 {
   vtkPythonMap->ModuleList->push_back(name);
+
+  // Register module name into pending list for defered side module loading
+  PyObject* pModule = PyImport_ImportModule("vtkmodules");
+  PyObject* pFunc = PyObject_GetAttrString(pModule, "on_vtk_module_initialize");
+  PyObject* pArgs = PyTuple_New(1);
+  PyTuple_SetItem(pArgs, 0, PyUnicode_FromString(name));
+  PyObject* execVal = PyObject_CallObject(pFunc, pArgs);
+  Py_DECREF(execVal);
+  Py_DECREF(pArgs);
+  Py_DECREF(pFunc);
+  Py_DECREF(pModule);
 }
 
 //------------------------------------------------------------------------------

Wrapping/Tools/vtkWrapPythonInit.c

@@ -129,18 +129,6 @@ static void CreateImplFile(
   fprintf(fout, "\n");
   fprintf(fout, "  vtkPythonUtil::AddModule(\"%s\");\n\n", libName);
 
-  // For any loaded module check if we need to import pure python dependency
-  // by calling vtkmodules.on_vtk_module_loaded(libName)
-  fprintf(fout, "  PyObject *pModule = PyImport_ImportModule(\"vtkmodules\");\n");
-  fprintf(fout, "  PyObject *pFunc = PyObject_GetAttrString(pModule, \"on_vtk_module_loaded\");\n");
-  fprintf(fout, "  PyObject *pArgs = PyTuple_New(1);\n");
-  fprintf(fout, "  PyTuple_SetItem(pArgs, 0, PyUnicode_FromString(\"%s\"));\n", libName);
-  fprintf(fout, "  PyObject *execVal = PyObject_CallObject(pFunc, pArgs);\n");
-  fprintf(fout, "  Py_DECREF(execVal);\n");
-  fprintf(fout, "  Py_DECREF(pArgs);\n");
-  fprintf(fout, "  Py_DECREF(pFunc);\n");
-  fprintf(fout, "  Py_DECREF(pModule);\n");
-
   fprintf(fout, "  return m;\n");
   fprintf(fout, "}\n\n");
 }