Commit 78dcea63 authored by Utkarsh Ayachit's avatar Utkarsh Ayachit
Browse files

Python interpreter fixes.

This commit includes various cleanups to Python interpreter
initialization in VTK.

vtkpython/pvtkpython executables now simply use vtkPythonInterpreter.
Previously, the code (which was added before vtkPythonInterpreter was
created) made direct Python API calls. Now, we simply defer to
vtkPythonInterpreter. That makes it possible to have just one
location to setup Python initialization logic in VTK.

Of HPC installations, Python default logic to locate prefix/exec_prefix
relative to the executable name had the potential to fail when using
custom Python builds. The new code tries to setup executable name in the
directory with the Python libraries, if possible.

The code to locate VTK's python modules (both platform dependent and
independent components) has been updated to look for them relative to
the VTK libraries, rather than the executable.

vtkpython now respects `-v` and `-vv` command line arguments to print
debugging information about paths looked up when setting up module
search paths.
parent b94049f6
......@@ -16,3 +16,7 @@
#include "vtkObjectFactory.h"
vtkStandardNewMacro(vtkVersion);
const char* GetVTKVersion()
{
return vtkVersion::GetVTKVersion();
}
......@@ -59,6 +59,10 @@ private:
void operator=(const vtkVersion&) = delete;
};
extern "C" {
VTKCOMMONCORE_EXPORT const char* GetVTKVersion();
}
#endif
// VTK-HeaderTest-Exclude: vtkVersion.h
......@@ -24,4 +24,9 @@
/* Whether the real python debug library has been provided. */
#cmakedefine VTK_WINDOWS_PYTHON_DEBUGGABLE
/* build specific site-packages suffix. This is used to setup Python
* module paths during initialization.
*/
static const char* const VTK_PYTHON_SITE_PACKAGES_SUFFIX = "@VTK_PYTHON_SITE_PACKAGES_SUFFIX@";
#endif
......@@ -18,15 +18,26 @@
#include "vtkCommand.h"
#include "vtkObjectFactory.h"
#include "vtkPythonStdStreamCaptureHelper.h"
#include "vtkVersion.h"
#include "vtkWeakPointer.h"
#include <vtksys/SystemInformation.hxx>
#include <vtksys/SystemTools.hxx>
#include <algorithm>
#include <signal.h>
#include <sstream>
#include <string>
#include <vector>
#if defined(_WIN32) && !defined(__CYGWIN__)
// Implementation for Windows win32 code but not cygwin
#include <windows.h>
#else
#include <dlfcn.h>
#endif
#if PY_VERSION_HEX >= 0x03000000
#if defined(__APPLE__) && PY_VERSION_HEX < 0x03050000
extern "C" {
......@@ -35,21 +46,129 @@ extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char* s, Py_ssize_t size);
#endif
#endif
#if defined(_WIN32) && !defined(__CYGWIN__)
#define VTK_PATH_SEPARATOR "\\"
#else
#define VTK_PATH_SEPARATOR "/"
#endif
#define VTKPY_DEBUG_MESSAGE(x) \
if (vtkPythonInterpreter::GetPythonVerboseFlag() > 0) \
{ \
cout << "# vtk: " x << endl; \
}
#define VTKPY_DEBUG_MESSAGE_VV(x) \
if (vtkPythonInterpreter::GetPythonVerboseFlag() > 1) \
{ \
cout << "# vtk: " x << endl; \
}
namespace
{
class StringPool
template <class T>
void strFree(T* foo)
{
delete[] foo;
}
template <class T>
class PoolT
{
std::vector<T*> Strings;
public:
std::vector<char*> Strings;
~StringPool()
~PoolT()
{
for (size_t cc = 0; cc < this->Strings.size(); cc++)
for (T* astring : this->Strings)
{
delete[] this->Strings[cc];
strFree(astring);
}
}
T* push_back(T* val)
{
this->Strings.push_back(val);
return val;
}
};
using StringPool = PoolT<char>;
#if PY_VERSION_HEX >= 0x03000000
template <>
void strFree(wchar_t* foo)
{
#if PY_VERSION_HEX >= 0x03050000
PyMem_RawFree(foo);
#else
PyMem_Free(foo);
#endif
}
using WCharStringPool = PoolT<wchar_t>;
#endif
#if PY_VERSION_HEX >= 0x03000000
wchar_t* vtk_Py_DecodeLocale(const char* arg, size_t* size)
{
(void)size;
#if PY_VERSION_HEX >= 0x03050000
return Py_DecodeLocale(arg, size);
#elif defined(__APPLE__)
return _Py_DecodeUTF8_surrogateescape(arg, strlen(arg));
#else
return _Py_char2wchar(arg, size);
#endif
}
#endif
#if PY_VERSION_HEX >= 0x03000000
char* vtk_Py_EncodeLocale(const wchar_t* arg, size_t* size)
{
(void)size;
#if PY_VERSION_HEX >= 0x03050000
return Py_EncodeLocale(arg, size);
#else
return _Py_wchar2char(arg, size);
#endif
}
#endif
std::string GetLibraryForSymbol(const char* symbolname)
{
#if defined(_WIN32) && !defined(__CYGWIN__)
(void)symbolname;
// Use the path to the running exe as the start of a search prefix
HMODULE handle = GetModuleHandle(NULL);
if (!handle)
{ // Can't find ourselves????? this shouldn't happen
return std::string();
}
TCHAR path[MAX_PATH];
if (!GetModuleFileName(handle, path, MAX_PATH))
{
return std::string();
}
return std::string(path);
#else // *NIX and macOS
// Use the library location of VTK's python wrapping as a prefix to search
void* handle = dlsym(RTLD_NEXT, symbolname);
if (!handle)
{
return std::string();
}
Dl_info di;
int ret = dladdr(handle, &di);
if (ret == 0 || !di.dli_saddr || !di.dli_fname)
{
return std::string();
}
return std::string(di.dli_fname);
#endif
}
static std::vector<vtkWeakPointer<vtkPythonInterpreter> > GlobalInterpreters;
static std::vector<std::string> PythonPaths;
......@@ -67,6 +186,7 @@ void NotifyInterpreters(unsigned long eventid, void* calldata = nullptr)
inline void vtkPrependPythonPath(const char* pathtoadd)
{
VTKPY_DEBUG_MESSAGE("adding module search path " << pathtoadd);
vtkPythonScopeGilEnsurer gilEnsurer;
PyObject* path = PySys_GetObject(const_cast<char*>("path"));
#if PY_VERSION_HEX >= 0x03000000
......@@ -77,6 +197,15 @@ inline void vtkPrependPythonPath(const char* pathtoadd)
PyList_Insert(path, 0, newpath);
Py_DECREF(newpath);
}
inline void vtkSafePrependPythonPath(const std::string& pathtoadd)
{
VTKPY_DEBUG_MESSAGE_VV("trying " << pathtoadd);
if (!pathtoadd.empty() && vtksys::SystemTools::FileIsDirectory(pathtoadd))
{
vtkPrependPythonPath(pathtoadd.c_str());
}
}
}
bool vtkPythonInterpreter::InitializedOnce = false;
......@@ -84,6 +213,7 @@ bool vtkPythonInterpreter::CaptureStdin = false;
bool vtkPythonInterpreter::ConsoleBuffering = false;
std::string vtkPythonInterpreter::StdErrBuffer;
std::string vtkPythonInterpreter::StdOutBuffer;
int vtkPythonInterpreter::PythonVerboseFlag = 0;
vtkStandardNewMacro(vtkPythonInterpreter);
//----------------------------------------------------------------------------
......@@ -123,6 +253,9 @@ bool vtkPythonInterpreter::Initialize(int initsigs /*=0*/)
{
if (Py_IsInitialized() == 0)
{
// guide the mechanism to locate Python standard library, if possible.
vtkPythonInterpreter::SetupPythonPrefix();
Py_InitializeEx(initsigs);
#ifdef SIGINT
......@@ -150,12 +283,11 @@ bool vtkPythonInterpreter::Initialize(int initsigs /*=0*/)
// callbacks.
vtkPythonInterpreter::RunSimpleString("");
// Setup handlers for stdout/stdin/stderr.
vtkPythonStdStreamCaptureHelper* wrapperOut = NewPythonStdStreamCaptureHelper(false);
vtkPythonStdStreamCaptureHelper* wrapperErr = NewPythonStdStreamCaptureHelper(true);
// Redirect Python's stdout and stderr and stdin - GIL protected operation
{
// Setup handlers for stdout/stdin/stderr.
vtkPythonStdStreamCaptureHelper* wrapperOut = NewPythonStdStreamCaptureHelper(false);
vtkPythonStdStreamCaptureHelper* wrapperErr = NewPythonStdStreamCaptureHelper(true);
vtkPythonScopeGilEnsurer gilEnsurer;
PySys_SetObject(const_cast<char*>("stdout"), reinterpret_cast<PyObject*>(wrapperOut));
PySys_SetObject(const_cast<char*>("stderr"), reinterpret_cast<PyObject*>(wrapperErr));
......@@ -164,6 +296,8 @@ bool vtkPythonInterpreter::Initialize(int initsigs /*=0*/)
Py_DECREF(wrapperErr);
}
vtkPythonInterpreter::SetupVTKPythonPaths();
for (size_t cc = 0; cc < PythonPaths.size(); cc++)
{
vtkPrependPythonPath(PythonPaths[cc].c_str());
......@@ -191,40 +325,30 @@ void vtkPythonInterpreter::Finalize()
//----------------------------------------------------------------------------
void vtkPythonInterpreter::SetProgramName(const char* programname)
{
if (vtkPythonInterpreter::InitializedOnce || Py_IsInitialized() != 0)
{
return;
}
if (programname)
{
static StringPool pool;
pool.Strings.push_back(vtksys::SystemTools::DuplicateString(programname));
// From Python Docs: The argument should point to a zero-terminated character
// string in static storage whose contents will not change for the duration of
// the program's execution. No code in the Python interpreter will change the
// contents of this storage.
#if PY_VERSION_HEX >= 0x03000000
wchar_t* argv0;
const std::string& av0 = pool.Strings.back();
#if PY_VERSION_HEX >= 0x03050000
argv0 = Py_DecodeLocale(av0.c_str(), nullptr);
#elif defined(__APPLE__)
argv0 = _Py_DecodeUTF8_surrogateescape(av0.data(), av0.length());
#else
argv0 = _Py_char2wchar(av0.c_str(), nullptr);
#endif
wchar_t* argv0 = vtk_Py_DecodeLocale(programname, nullptr);
if (argv0 == 0)
{
fprintf(stderr, "Fatal vtkpython error: "
"unable to decode the program name\n");
static wchar_t empty[1] = { 0 };
argv0 = empty;
Py_SetProgramName(argv0);
}
else
{
static WCharStringPool wpool;
Py_SetProgramName(wpool.push_back(argv0));
}
Py_SetProgramName(argv0);
#else
Py_SetProgramName(pool.Strings.back());
static StringPool pool;
Py_SetProgramName(pool.push_back(vtksys::SystemTools::DuplicateString(programname)));
#endif
}
}
......@@ -259,58 +383,60 @@ void vtkPythonInterpreter::PrependPythonPath(const char* dir)
//----------------------------------------------------------------------------
int vtkPythonInterpreter::PyMain(int argc, char** argv)
{
if (!vtkPythonInterpreter::InitializedOnce && Py_IsInitialized() == 0 && argc > 0)
vtksys::SystemTools::EnableMSVCDebugHook();
vtkPythonInterpreter::PythonVerboseFlag = 0;
for (int cc = 0; cc < argc; ++cc)
{
vtkPythonInterpreter::SetProgramName(argv[0]);
if (argv[cc] && strcmp(argv[cc], "-v") == 0)
{
vtkPythonInterpreter::PythonVerboseFlag += 1;
}
if (argv[cc] && strcmp(argv[cc], "-vv") == 0)
{
vtkPythonInterpreter::PythonVerboseFlag = 2;
}
}
vtkPythonInterpreter::Initialize(1);
#if PY_VERSION_HEX >= 0x03000000
wchar_t* argv0;
#if PY_VERSION_HEX >= 0x03050000
argv0 = Py_DecodeLocale(argv[0], nullptr);
#elif defined(__APPLE__)
argv0 = _Py_DecodeUTF8_surrogateescape(argv[0], strlen(argv[0]));
#else
argv0 = _Py_char2wchar(argv[0], nullptr);
#endif
if (argv0 == 0)
{
static wchar_t empty[1] = { 0 };
argv0 = empty;
}
// Need two copies of args, because programs might modify the first
wchar_t** argvWide = new wchar_t*[argc];
wchar_t** argvWide2 = new wchar_t*[argc];
int argcWide = 0;
for (int i = 0; i < argc; i++)
{
#if PY_VERSION_HEX >= 0x03050000
argvWide[i] = Py_DecodeLocale(argv[i], nullptr);
#elif defined(__APPLE__)
argvWide[i] = _Py_DecodeUTF8_surrogateescape(argv[i], strlen(argv[i]));
#else
argvWide[i] = _Py_char2wchar(argv[i], nullptr);
#endif
argvWide2[i] = argvWide[i];
if (argvWide[i] == 0)
if (argv[i] && strcmp(argv[i], "--enable-bt") == 0)
{
vtksys::SystemInformation::SetStackTraceOnError(1);
continue;
}
if (argv[i] && strcmp(argv[i], "-V") == 0)
{
// print out VTK version and let argument pass to Py_Main(). At which point,
// Python will print its version and exit.
cout << vtkVersion::GetVTKSourceVersion() << endl;
}
argvWide[argcWide] = vtk_Py_DecodeLocale(argv[i], nullptr);
argvWide2[argcWide] = argvWide[argcWide];
if (argvWide[argcWide] == 0)
{
fprintf(stderr, "Fatal vtkpython error: "
"unable to decode the command line argument #%i\n",
i + 1);
for (int k = 0; k < i; k++)
for (int k = 0; k < argcWide; k++)
{
PyMem_Free(argvWide2[i]);
PyMem_Free(argvWide2[k]);
}
PyMem_Free(argv0);
delete[] argvWide;
delete[] argvWide2;
return 1;
}
argcWide++;
}
vtkPythonScopeGilEnsurer gilEnsurer;
int res = Py_Main(argc, argvWide);
PyMem_Free(argv0);
for (int i = 0; i < argc; i++)
int res = Py_Main(argcWide, argvWide);
for (int i = 0; i < argcWide; i++)
{
PyMem_Free(argvWide2[i]);
}
......@@ -319,8 +445,26 @@ int vtkPythonInterpreter::PyMain(int argc, char** argv)
return res;
#else
// process command line argments to remove unhandled args.
std::vector<char*> newargv;
for (int i=0; i < argc; ++i)
{
if (argv[i] && strcmp(argv[i], "--enable-bt") == 0)
{
vtksys::SystemInformation::SetStackTraceOnError(1);
continue;
}
if (argv[i] && strcmp(argv[i], "-V") == 0)
{
// print out VTK version and let argument pass to Py_Main(). At which point,
// Python will print its version and exit.
cout << vtkVersion::GetVTKSourceVersion() << endl;
}
newargv.push_back(argv[i]);
}
vtkPythonScopeGilEnsurer gilEnsurer(false, true);
return Py_Main(argc, argv);
return Py_Main(static_cast<int>(newargv.size()), &newargv[0]);
#endif
}
......@@ -424,3 +568,121 @@ vtkStdString vtkPythonInterpreter::ReadStdin()
NotifyInterpreters(vtkCommand::UpdateEvent, &string);
return string;
}
//----------------------------------------------------------------------------
void vtkPythonInterpreter::SetupPythonPrefix()
{
using systools = vtksys::SystemTools;
if (Py_GetPythonHome() != nullptr)
{
// if PYTHONHOME is set, we do nothing. Don't override an already
// overridden environment.
VTKPY_DEBUG_MESSAGE("`PYTHONHOME` already set. Leaving unchanged.");
return;
}
std::string pythonlib = GetLibraryForSymbol("Py_SetProgramName");
if (pythonlib.empty())
{
VTKPY_DEBUG_MESSAGE("static Python build or `Py_SetProgramName` library couldn't be found. "
"Set `PYTHONHOME` if Python standard library fails to load.");
return;
}
#if PY_VERSION_HEX >= 0x03000000
auto oldprogramname = vtk_Py_EncodeLocale(Py_GetProgramName(), nullptr);
#else
auto oldprogramname = Py_GetProgramName();
#endif
if (oldprogramname != nullptr && strcmp(oldprogramname, "python") != 0)
{
VTKPY_DEBUG_MESSAGE("program-name has been changed. Leaving unchanged.");
#if PY_VERSION_HEX >= 0x03000000
PyMem_Free(oldprogramname);
#endif
return;
}
#if PY_VERSION_HEX >= 0x03000000
PyMem_Free(oldprogramname);
#endif
const std::string newprogramname =
systools::GetFilenamePath(pythonlib) + VTK_PATH_SEPARATOR "vtkpython";
VTKPY_DEBUG_MESSAGE(
"calling Py_SetProgramName(" << newprogramname << ") to aid in setup of Python prefix.");
#if PY_VERSION_HEX >= 0x03000000
static WCharStringPool wpool;
Py_SetProgramName(wpool.push_back(vtk_Py_DecodeLocale(newprogramname.c_str(), nullptr)));
#else
static StringPool pool;
Py_SetProgramName(pool.push_back(systools::DuplicateString(newprogramname.c_str())));
#endif
}
//----------------------------------------------------------------------------
void vtkPythonInterpreter::SetupVTKPythonPaths()
{
using systools = vtksys::SystemTools;
#if defined(VTK_BUILD_SHARED_LIBS) // and !frozen_vtk_python
// add path for VTK shared libs.
VTKPY_DEBUG_MESSAGE("shared VTK build detected.");
const std::string vtklib = GetLibraryForSymbol("GetVTKVersion");
if (vtklib.empty())
{
VTKPY_DEBUG_MESSAGE(
"`GetVTKVersion` library couldn't be found. Will use `Py_GetProgramName` next.");
}
#else
// static build.
VTKPY_DEBUG_MESSAGE(
"static VTK build detected. Using `Py_GetProgramName` to locate python modules.");
const std::string vtklib;
#endif
std::vector<std::string> vtkprefix_components;
if (vtklib.empty())
{
std::string vtkprefix;
#if PY_VERSION_HEX >= 0x03000000
auto tmp = vtk_Py_EncodeLocale(Py_GetProgramName(), nullptr);
vtkprefix = tmp;
PyMem_Free(tmp);
#else
vtkprefix = Py_GetProgramName();
#endif
vtkprefix = systools::CollapseFullPath(vtkprefix);
systools::SplitPath(vtkprefix, vtkprefix_components);
}
else
{
systools::SplitPath(systools::GetFilenamePath(vtklib), vtkprefix_components);
}
const std::string sitepackages = VTK_PYTHON_SITE_PACKAGES_SUFFIX;
#if defined(_WIN32) && !defined(__CYGWIN__)
const std::string landmark = "vtk\\__init__.py";
#else
const std::string landmark = "vtk/__init__.py";
#endif
while (!vtkprefix_components.empty())
{
std::string curprefix = systools::JoinPath(vtkprefix_components);
const std::string pathtocheck = curprefix + VTK_PATH_SEPARATOR + sitepackages;
const std::string landmarktocheck = pathtocheck + VTK_PATH_SEPARATOR + landmark;
if (vtksys::SystemTools::FileExists(landmarktocheck))
{
VTKPY_DEBUG_MESSAGE_VV("trying VTK landmark file " << landmarktocheck << " -- success!");
vtkSafePrependPythonPath(pathtocheck);
break;
}
else
{
VTKPY_DEBUG_MESSAGE_VV("trying VTK landmark file " << landmarktocheck << " -- failed!");
}
vtkprefix_components.pop_back();
}
}
......@@ -91,14 +91,24 @@ public:
static bool IsInitialized();
/**
* Set the program name. This must be called before the first Initialize()
* call. If called afterwords, this will raise a warning.
* Set the program name. This internally calls `Py_SetProgramName`.
* Python uses the program name to determine values for prefix and exec_prefix
* paths that are used to locate Python standard libraries and hence call this
* if you if you know what you are doing.
*
* If not explicitly overridden, `Initialize` will try to guess a good default
* for the `Py_SetProgramName` to help find Python standard libraries based
* on Python libraries used to build VTK.
*/
static void SetProgramName(const char* programname);
/**
* Call this method to start the Python event loop (Py_Main()).
* This will initialize Python if not already initialized.
*
* @note This function handles `--enable-bt` command line argument before
* passing it on to the `Py_Main(..)` call. Thus, the `--enable-bt` flag is
* also removed from the arguments passed to `Py_Main`.
*/
static int PyMain(int argc, char** argv);
......@@ -131,6 +141,8 @@ public:
static bool GetCaptureStdin();
//@}
static int GetPythonVerboseFlag() { return vtkPythonInterpreter::PythonVerboseFlag; }
protected:
vtkPythonInterpreter();
~vtkPythonInterpreter() override;
......@@ -166,6 +178,22 @@ private:
static std::string StdErrBuffer;
static std::string StdOutBuffer;
//@}
/**
* Since vtkPythonInterpreter is often used outside CPython executable, e.g.
* vtkpython, the default logic to locate Python standard libraries used by
* Python (which depends on the executable path) may fail or pickup incorrect
* Python libs. This methods address the issue by setting program name to help
* guide Python's default prefix/exec_prefix searching logic.
*/
static void SetupPythonPrefix();
/**
* Add paths to VTK's Python modules.
*/
static void SetupVTKPythonPaths();
static int PythonVerboseFlag;
};
#endif