Commit 16bb32d4 authored by David Gobbi's avatar David Gobbi
Browse files

Add the new py3k buffer interface.

Python 3 introduced a multi-dimensional buffer interface, which was
backported to Python 2.6 and Python 2.7.
parent f2aebb86
......@@ -3,6 +3,7 @@ vtk_add_test_python(
PythonSmoke.py
TestArrays.py
TestArrayArguments.py
TestBuffer.py
TestEmptyInput.py
TestEnums.py
TestExecuteMethodFinalizeCrash.py
......
"""Test buffer protocol for VTK arrays
Python 2.6 introduced a new buffer protocol that can expose raw
memory as a multi-dimensional array. This is used, for example,
by numpy in order to automatically generate arrays from memory
exposed by other python extension modules.
Created on Aug 1, 2015 by David Gobbi
"""
import sys
import struct
import exceptions
import vtk
from vtk.test import Testing
# array types and the corresponding format
lsize = vtk.VTK_SIZEOF_LONG
idsize = vtk.VTK_SIZEOF_ID_TYPE
if idsize == 8:
idchar = 'q'
else:
idchar = 'i'
arrayType = {
'SignedChar':('b', 1), 'UnsignedChar':('B', 1),
'Short':('h', 2), 'UnsignedShort':('H', 2),
'Int':('i', 4), 'UnsignedInt':('I', 4),
'Long':('l', lsize), 'UnsignedLong':('L', lsize),
'IdType':(idchar, idsize),
'LongLong':('q', 8), 'UnsignedLongLong':('Q', 8)
}
class TestBuffer(Testing.vtkTest):
def testOneDimensionalDataArray(self):
"""Test one-dimensional data array."""
if sys.hexversion < 0x02070000:
return
for atype,ainfo in arrayType.items():
aclass = getattr(vtk, 'vtk' + atype + 'Array')
a = aclass()
a.InsertNextValue(10)
a.InsertNextValue(7)
a.InsertNextValue(85)
m = memoryview(a)
self.assertEqual(m.format, ainfo[0])
self.assertEqual(m.itemsize, ainfo[1])
self.assertEqual(m.strides, (ainfo[1],))
self.assertEqual(m.shape, (3,))
self.assertEqual(m.ndim, 1)
# test the contents of the memoryview
tp = struct.unpack(3*ainfo[0], m.tobytes())
self.assertEqual(tp, (10, 7, 85))
# now test re-creating the array from a buffer
b = aclass()
b.SetVoidArray(m, 3, True)
self.assertEqual(b.GetValue(0), 10)
self.assertEqual(b.GetValue(1), 7)
self.assertEqual(b.GetValue(2), 85)
def testTwoDimensionalDataArray(self):
"""Test data array with components."""
if sys.hexversion < 0x02070000:
return
for atype,ainfo in arrayType.items():
aclass = getattr(vtk, 'vtk' + atype + 'Array')
a = aclass()
a.SetNumberOfComponents(3)
a.InsertNextTuple((10, 7, 4))
a.InsertNextTuple((85, 8, 2))
m = memoryview(a)
self.assertEqual(m.format, ainfo[0])
self.assertEqual(m.itemsize, ainfo[1])
self.assertEqual(m.shape, (2, 3))
self.assertEqual(m.strides, (ainfo[1]*3, ainfo[1]))
self.assertEqual(m.ndim, 2)
# test the contents of the memoryview
tp = struct.unpack(6*ainfo[0], m.tobytes())
self.assertEqual(tp, (10, 7, 4, 85, 8, 2))
def testCharArray(self):
"""Test the special case of the char array."""
if sys.hexversion < 0x02070000:
return
# bit array is actually stored as a byte array
a = vtk.vtkCharArray()
a.SetNumberOfComponents(5)
a.InsertNextTupleValue("hello")
a.InsertNextTupleValue("world")
m = memoryview(a)
self.assertEqual(m.format, 'c')
self.assertEqual(m.itemsize, 1)
self.assertEqual(m.shape, (2, 5))
self.assertEqual(m.strides, (5, 1))
self.assertEqual(m.ndim, 2)
# test the contents of the memoryview
self.assertEqual(m.tobytes(), bytes("helloworld"))
def testBitArray(self):
"""Test the special case of the bit array."""
if sys.hexversion < 0x02070000:
return
# bit array is actually stored as a byte array
a = vtk.vtkBitArray()
a.InsertNextValue(0)
a.InsertNextValue(1)
a.InsertNextValue(1)
a.InsertNextValue(0)
a.InsertNextValue(1)
m = memoryview(a)
self.assertEqual(m.format, 'B')
self.assertEqual(m.itemsize, 1)
self.assertEqual(m.shape, (1,))
# test the contents of the memoryview
self.assertEqual(ord(m.tobytes()) & 0xF8, 0x68)
if __name__ == "__main__":
Testing.main([(TestBuffer, 'test')])
......@@ -641,14 +641,29 @@ static Py_ssize_t PyVTKMutableObject_GetCharBuf(
return -1;
}
#if PY_VERSION_HEX >= 0x02060000
static int PyVTKMutableObject_GetBuffer(
PyObject *self, Py_buffer *view, int flags)
{
PyObject *obj = ((PyVTKMutableObject *)self)->value;
return PyObject_GetBuffer(obj, view, flags);
}
static void PyVTKMutableObject_ReleaseBuffer(
PyObject *, Py_buffer *view)
{
PyBuffer_Release(view);
}
#endif
static PyBufferProcs PyVTKMutableObject_AsBuffer = {
PyVTKMutableObject_GetReadBuf, // bf_getreadbuffer
PyVTKMutableObject_GetWriteBuf, // bf_getwritebuffer
PyVTKMutableObject_GetSegCount, // bf_getsegcount
PyVTKMutableObject_GetCharBuf, // bf_getcharbuffer
#if PY_VERSION_HEX >= 0x02060000
0, // bf_getbuffer
0 // bf_releasebuffer
PyVTKMutableObject_GetBuffer, // bf_getbuffer
PyVTKMutableObject_ReleaseBuffer // bf_releasebuffer
#endif
};
......
......@@ -215,6 +215,7 @@ void PyVTKObject_Delete(PyObject *op)
Py_DECREF(self->vtk_dict);
delete [] self->vtk_observers;
delete [] self->vtk_buffer;
PyObject_GC_Del(op);
}
......@@ -263,6 +264,7 @@ PyGetSetDef PyVTKObject_GetSet[] = {
// for PyVTKObject, so that python can read from a vtkDataArray.
// This is particularly useful for NumPy.
#ifndef VTK_PY3K
//--------------------------------------------------------------------
static Py_ssize_t
PyVTKObject_AsBuffer_GetSegCount(PyObject *op, Py_ssize_t *lenp)
......@@ -309,6 +311,7 @@ PyVTKObject_AsBuffer_GetReadBuf(
da->GetNumberOfComponents()*
da->GetDataTypeSize();
}
return -1;
}
......@@ -320,15 +323,156 @@ PyVTKObject_AsBuffer_GetWriteBuf(
return PyVTKObject_AsBuffer_GetReadBuf(op, segment, ptrptr);
}
#endif
#if PY_VERSION_HEX >= 0x02060000
//--------------------------------------------------------------------
// Convert a VTK type to a python type char (struct module)
static const char *pythonTypeFormat(int t)
{
const char *b = 0;
switch (t)
{
case VTK_CHAR: b = "c"; break;
case VTK_SIGNED_CHAR: b = "b"; break;
case VTK_UNSIGNED_CHAR: b = "B"; break;
case VTK_SHORT: b = "h"; break;
case VTK_UNSIGNED_SHORT: b = "H"; break;
case VTK_INT: b = "i"; break;
case VTK_UNSIGNED_INT: b = "I"; break;
case VTK_LONG: b = "l"; break;
case VTK_UNSIGNED_LONG: b = "L"; break;
case VTK_LONG_LONG: b = "q"; break;
case VTK_UNSIGNED_LONG_LONG: b = "Q"; break;
case VTK___INT64: b = "q"; break;
case VTK_UNSIGNED___INT64: b = "Q"; break;
case VTK_FLOAT: b = "f"; break;
case VTK_DOUBLE: b = "d"; break;
#ifndef VTK_USE_64BIT_IDS
case VTK_ID_TYPE: b = "i"; break;
#elif defined(VTK_TYPE_USE_LONG_LONG) || (VTK_SIZEOF_LONG != 8)
case VTK_ID_TYPE: b = "q"; break;
#else
case VTK_ID_TYPE: b = "l"; break;
#endif
}
return b;
}
//--------------------------------------------------------------------
static int
PyVTKObject_AsBuffer_GetBuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyVTKObject *self = (PyVTKObject*)obj;
vtkDataArray *da = vtkDataArray::SafeDownCast(self->vtk_ptr);
if (da)
{
void *ptr = da->GetVoidPointer(0);
Py_ssize_t ntuples = da->GetNumberOfTuples();
int ncomp = da->GetNumberOfComponents();
int dsize = da->GetDataTypeSize();
const char *format = pythonTypeFormat(da->GetDataType());
Py_ssize_t size = ntuples*ncomp*dsize;
if (da->GetDataType() == VTK_BIT)
{
size = (ntuples*ncomp + 7)/8;
}
// start by building a basic "unsigned char" buffer
if (PyBuffer_FillInfo(view, obj, ptr, size, 0, flags) == -1)
{
return -1;
}
// check if a dimensioned array was requested
if (format != 0 && (flags & PyBUF_ND) != 0)
{
// first, build a simple 1D array
view->itemsize = dsize;
view->ndim = (ncomp > 1 ? 2 : 1);
view->format = (char *)format;
#if PY_VERSION_HEX >= 0x02070000
// use "smalltable" for 1D arrays, like memoryobject.c
view->shape = view->smalltable;
view->strides = &view->smalltable[1];
if (view->ndim > 1)
#endif
{
if (self->vtk_buffer && self->vtk_buffer[0] != view->ndim)
{
delete [] self->vtk_buffer;
self->vtk_buffer = 0;
}
if (self->vtk_buffer == 0)
{
self->vtk_buffer = new Py_ssize_t[2*view->ndim + 1];
self->vtk_buffer[0] = view->ndim;
}
view->shape = &self->vtk_buffer[1];
view->strides = &self->vtk_buffer[view->ndim + 1];
}
if (view->ndim == 1)
{
// simple one-dimensional array
view->shape[0] = ntuples*ncomp;
view->strides[0] = view->itemsize;
}
else
{
// use native C dimension ordering by default
char order = 'C';
if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)
{
// use fortran ordering only if explicitly requested
order = 'F';
}
// need to allocate space for the strides and shape
view->shape[0] = ntuples;
view->shape[1] = ncomp;
if (order == 'F')
{
view->shape[0] = ncomp;
view->shape[1] = ntuples;
}
PyBuffer_FillContiguousStrides(
view->ndim, view->shape, view->strides, dsize, order);
}
}
return 0;
}
PyErr_Format(PyExc_ValueError,
"Cannot get a buffer from %s.", Py_TYPE(obj)->tp_name);
return -1;
}
//--------------------------------------------------------------------
static void
PyVTKObject_AsBuffer_ReleaseBuffer(PyObject *obj, Py_buffer *view)
{
// nothing to do, the caller will decref the obj
(void)obj;
(void)view;
}
#endif
//--------------------------------------------------------------------
PyBufferProcs PyVTKObject_AsBuffer = {
#ifndef VTK_PY3K
PyVTKObject_AsBuffer_GetReadBuf, // bf_getreadbuffer
PyVTKObject_AsBuffer_GetWriteBuf, // bf_getwritebuffer
PyVTKObject_AsBuffer_GetSegCount, // bf_getsegcount
0, // bf_getcharbuffer
#endif
#if PY_VERSION_HEX >= 0x02060000
0, // bf_getbuffer
0 // bf_releasebuffer
PyVTKObject_AsBuffer_GetBuffer, // bf_getbuffer
PyVTKObject_AsBuffer_ReleaseBuffer // bf_releasebuffer
#endif
};
......@@ -438,6 +582,7 @@ PyObject *PyVTKObject_FromPointer(
self->vtk_flags = 0;
self->vtk_class = cls;
self->vtk_dict = pydict;
self->vtk_buffer = 0;
self->vtk_observers = 0;
self->vtk_weakreflist = NULL;
......
......@@ -53,12 +53,13 @@ public:
// plus a pointer to the associated vtkObjectBase and PyVTKClass.
struct PyVTKObject {
PyObject_HEAD
PyObject *vtk_dict;
PyObject *vtk_weakreflist;
PyVTKClass *vtk_class;
vtkObjectBase *vtk_ptr;
unsigned long *vtk_observers;
unsigned int vtk_flags;
PyObject *vtk_dict; // each object has its own dict
PyObject *vtk_weakreflist; // list of weak references via python
PyVTKClass *vtk_class; // information about the class
vtkObjectBase *vtk_ptr; // pointer to the C++ object
Py_ssize_t *vtk_buffer; // ndims, shape, strides for Py_buffer
unsigned long *vtk_observers; // used to find our observers
unsigned int vtk_flags; // flags (see list above)
};
extern VTKWRAPPINGPYTHONCORE_EXPORT PyGetSetDef PyVTKObject_GetSet[];
......
......@@ -161,59 +161,86 @@ inline bool vtkPythonGetStdStringValue(PyObject *o, std::string &a, const char *
//--------------------------------------------------------------------
// Overloaded methods, mostly based on the above templates
static bool vtkPythonGetValue(PyObject *o, const void *&a)
static bool vtkPythonGetValue(
PyObject *o, const void *&a, Py_buffer *view)
{
const char *format = 0;
void *p = 0;
Py_ssize_t sz = 0;
PyBufferProcs *b = Py_TYPE(o)->tp_as_buffer;
#if PY_VERSION_HEX >= 0x02060000
// use the new buffer interface
if (PyObject_CheckBuffer(o))
{
if (PyObject_GetBuffer(o, view, PyBUF_SIMPLE) == -1)
{
return false;
}
p = view->buf;
sz = view->len;
format = view->format;
}
else
#endif
// use the old buffer interface
if (b && b->bf_getreadbuffer && b->bf_getsegcount)
{
if (b->bf_getsegcount(o, NULL) == 1)
{
void *p;
Py_ssize_t sz = b->bf_getreadbuffer(o, 0, &p);
if (sz >= 0 && sz <= VTK_INT_MAX)
{
// check for pointer mangled as string
int s = static_cast<int>(sz);
a = vtkPythonUtil::UnmanglePointer(
reinterpret_cast<char *>(p), &s, "p_void");
if (s >= 0)
{
return true;
}
if (s == -1)
{
char buf[128];
sprintf(buf, "value is %.80s, required type is p_void",
reinterpret_cast<char *>(p));
PyErr_SetString(PyExc_TypeError, buf);
}
else
{
PyErr_SetString(PyExc_TypeError, "cannot get a void pointer");
}
}
else if (sz >= 0)
{
// directly use the pointer to the buffer contents
a = p;
return true;
}
sz = b->bf_getreadbuffer(o, 0, &p);
}
else
{
PyErr_SetString(PyExc_TypeError, "buffer must be single-segment");
return false;
}
}
if (p && sz >= 0 && sz <= VTK_INT_MAX &&
(format == 0 || format[0] == 'c' || format[0] == 'B'))
{
// check for pointer mangled as string
int s = static_cast<int>(sz);
a = vtkPythonUtil::UnmanglePointer(
reinterpret_cast<char *>(p), &s, "p_void");
if (s >= 0)
{
return true;
}
if (s == -1)
{
char buf[128];
sprintf(buf, "value is %.80s, required type is p_void",
reinterpret_cast<char *>(p));
PyErr_SetString(PyExc_TypeError, buf);
return false;
}
else
{
PyErr_SetString(PyExc_TypeError, "cannot get a void pointer");
return false;
}
PyErr_SetString(PyExc_TypeError, "buffer must be single-segment");
return false;
}
PyErr_SetString(PyExc_TypeError, "object does not have a readable buffer");
else if (p && sz >= 0)
{
// directly use the pointer to the buffer contents
a = p;
return true;
}
PyErr_SetString(PyExc_TypeError,
"object does not have a readable buffer");
return false;
}
inline
bool vtkPythonGetValue(PyObject *o, void *&a)
bool vtkPythonGetValue(PyObject *o, void *&a, Py_buffer *buf)
{
// should have an alternate form for non-const "void *" that uses
// writebuffer instead of readbuffer, but that would break existing code
const void *b = NULL;
bool r = vtkPythonGetValue(o, b);
bool r = vtkPythonGetValue(o, b, buf);
a = const_cast<void *>(b);
return r;
}
......@@ -976,8 +1003,6 @@ bool vtkPythonArgs::GetValue(PyObject *o, T &a) \
return vtkPythonGetValue(o, a); \
}
VTK_PYTHON_GET_ARG(void *)
VTK_PYTHON_GET_ARG(const void *)
VTK_PYTHON_GET_ARG(char *)
VTK_PYTHON_GET_ARG(const char *)
VTK_PYTHON_GET_ARG(std::string)
......@@ -1076,7 +1101,7 @@ VTK_PYTHON_GET_NARRAY_ARG(unsigned __int64)
#endif
//--------------------------------------------------------------------
// Define the special function pointer GetNextArg method
// Define the special function pointer GetValue method
bool vtkPythonArgs::GetFunction(PyObject *arg, PyObject *&o)
{
......@@ -1095,6 +1120,29 @@ bool vtkPythonArgs::GetFunction(PyObject *&o)
return vtkPythonArgs::GetFunction(arg, o);
}
//--------------------------------------------------------------------
// Define the void pointer GetValue method
#define VTK_PYTHON_GET_BUFFER(T) \
bool vtkPythonArgs::GetBuffer(T &a, Py_buffer *buf) \
{ \
PyObject *o = PyTuple_GET_ITEM(this->Args, this->I++); \
if (vtkPythonGetValue(o, a, buf)) \
{ \
return true; \
} \
this->RefineArgTypeError(this->I - this->M - 1); \
return false; \
} \
\
bool vtkPythonArgs::GetBuffer(PyObject *o, T &a, Py_buffer *buf) \
{ \
return vtkPythonGetValue(o, a, buf); \
}
VTK_PYTHON_GET_BUFFER(void *)
VTK_PYTHON_GET_BUFFER(const void *)
//--------------------------------------------------------------------
// Define all the SetArgValue methods for setting reference args
......
......@@ -211,10 +211,10 @@ public:
static bool GetFunction(PyObject *arg, PyObject *&o);
// Get the next arg as a void pointer (to a buffer object).
bool GetValue(void *&v);
static bool GetValue(PyObject *o, void *&v);
bool GetValue(const void *&v);
static bool GetValue(PyObject *o, const void *&v);
bool GetBuffer(void *&v, Py_buffer *buf);
static bool GetBuffer(PyObject *o, void *&v, Py_buffer *buf);
bool GetBuffer(const void *&v, Py_buffer *buf);
static bool GetBuffer(PyObject *o, const void *&v, Py_buffer *buf);
// Description:
// Get the next argument as a string.
......
......@@ -25,6 +25,15 @@
// ===== Macros needed for Python 3 ====
#ifdef VTK_PY3K
// Buffer compatibility
#if PY_VERSION_HEX < 0x03030000
#define VTK_PYBUFFER_INITIALIZER \
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, { 0, 0 }, 0 }
#else
#define VTK_PYBUFFER_INITIALIZER \
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
#endif
#endif
// ===== Macros needed for Python 2 ====
......@@ -41,4 +50,17 @@
#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
#endif
// Buffer struct initialization is different for every version
#if PY_VERSION_HEX < 0x02060000
typedef struct bufferinfo { PyObject *obj; } Py_buffer;
#define VTK_PYBUFFER_INITIALIZER \
{ 0 }
#elif PY_VERSION_HEX < 0x02070000
#define VTK_PYBUFFER_INITIALIZER \
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
#else
#define VTK_PYBUFFER_INITIALIZER \
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, { 0, 0 }, 0 }
#endif
#endif
......@@ -397,6 +397,12 @@ static void vtkWrapPython_GenerateObjectNew(
" return (PyObject *)pytype;\n"
" }\n\n");
/* add any flags specific to this type */
fprintf(fp,
"#if !defined(VTK_PY3K) && PY_VERSION_HEX >= 0x02060000\n"
" pytype->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER;\n"
"#endif\n\n");
/* find the first superclass that is a VTK class, create it first */
name = vtkWrapPython_GetSuperClass(data, hinfo);