Commit b0bb336e authored by Burlen Loring's avatar Burlen Loring
Browse files

Python based regression tests for the ADIOS capabilities.

Add Python based regression test exercising
ADIOSAnalysisAdaptor, ADIOSDataAdaptor, and VTKDataAdaptor.
These tests exercise FLEXPATH and MPI/BP methods on
uniform Cartesian(vtkImageData) and unstructured(vtkPolyData)
parent a47f0f9b
......@@ -56,4 +56,5 @@ if (ENABLE_SENSEI)
install(EXPORT sensei DESTINATION lib/cmake
EXPORT_LINK_INTERFACE_LIBRARIES)
add_subdirectory(testing)
endif()
if (BUILD_TESTING)
sensei_add_test(test_adios_flexpath
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_adios.sh 2
${CMAKE_CURRENT_SOURCE_DIR} test_adios_flexpath.bp
FLEXPATH FLEXPATH 2
FEATURES ${ENABLE_PYTHON} ${ENABLE_ADIOS})
sensei_add_test(test_adios_mpi_bp
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_adios.sh 2
${CMAKE_CURRENT_SOURCE_DIR} test_adios_mpi_bp.bp
MPI BP 2
FEATURES ${ENABLE_PYTHON} ${ENABLE_ADIOS})
endif()
#!/usr/bin/env bash
if [[ $# < 6 ]]
then
echo "test_adios.sh [nproc] [src dir] [file] [write method] [read method] [nits]"
exit 1
fi
nproc=$1
srcdir=$2
file=$3
write_method=$4
read_method=$5
nits=$6
delay=1s
trap 'echo $BASH_COMMAND' DEBUG
rm -f ${file}
mpiexec -np ${nproc} python ${srcdir}/test_adios_write.py ${file} ${write_method} ${nits} &
write_pid=$!
if [[ "${read_method}" == "BP" ]]
then
echo "waiting for writer(${write_pid}) to complete"
wait ${write_pid}
elif [[ "${read_method}" == "FLEXPATH" ]]
then
echo "waiting for writer to start ${delay}"
while [[ True ]]
do
if [[ -e "${file}_writer_info.txt" ]]
then
break
else
sleep ${delay}
fi
done
fi
mpiexec -np ${nproc} python ${srcdir}/test_adios_read.py ${file} ${read_method}
exit 0
from mpi4py import *
from multiprocessing import Process,Lock,Value
from sensei import VTKDataAdaptor,ADIOSDataAdaptor,ADIOSAnalysisAdaptor
import sys,os
import numpy as np
import vtk, vtk.util.numpy_support as vtknp
from time import sleep
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
n_ranks = comm.Get_size()
def error_message(msg):
sys.stderr.write('ERROR[%d] : %s\n'%(rank, msg))
def status_message(msg, io_rank=0):
if rank == io_rank:
sys.stderr.write('STATUS[%d] : %s\n'%(rank, msg))
def check_array(array):
# checks that array[i] == i
test_array = vtknp.vtk_to_numpy(array)
n_vals = len(test_array)
base_array = np.empty(n_vals, dtype=test_array.dtype)
i = 0
while i < n_vals:
base_array[i] = i
i += 1
ids = np.where(base_array != test_array)[0]
if len(ids):
error_message('wrong values at %s'%(str(ids)))
return -1
return 0
def read_data(file_name, method):
# initialize the data adaptor
status_message('initializing ADIOSDataAdaptor %s %s'%(file_name,method))
da = ADIOSDataAdaptor.New()
da.Open(MPI.COMM_WORLD, method, file_name)
# process all time steps
n_steps = 0
retval = 0
while True:
# get the time info
t = da.GetDataTime()
it = da.GetDataTimeStep()
status_message('received step %d time %0.1f'%(it, t))
# get a VTK dataset with all the arrays
ds = da.GetMesh(False)
# request each array
assocs = {vtk.VTK_POINT_DATA:'point', vtk.VTK_CELL_DATA:'cell'}
for assoc,assoc_name in assocs.iteritems():
n_arrays = da.GetNumberOfArrays(assoc)
i = 0
while i < n_arrays:
array_name = da.GetArrayName(assoc, i)
da.AddArray(ds, assoc, array_name)
i += 1
# this often will cause segv's if the dataset has been
# improperly constructed, thus serves as a good check
str_rep = str(ds)
# check the arrays have the expected data
it = ds.NewIterator()
while not it.IsDoneWithTraversal():
bds = it.GetCurrentDataObject()
idx = it.GetCurrentFlatIndex()
for assoc,assoc_name in assocs.iteritems():
n_arrays = da.GetNumberOfArrays(assoc)
status_message('checking %d %s data arrays ' \
'in block %d %s'%(n_arrays,assoc_name,idx,bds.GetClassName()), \
rank)
j = 0
while j < n_arrays:
array = bds.GetPointData().GetArray(j) \
if assoc == vtk.VTK_POINT_DATA else \
bds.GetCellData().GetArray(j)
if (check_array(array)):
error_message('Test failed on array %d "%s"'%( \
j, array.GetName()))
retval = -1
j += 1
it.GoToNextItem()
n_steps += 1
if (da.Advance()):
break
# close down the stream
da.Close()
status_message('closed stream after receiving %d steps'%(n_steps))
return retval
if __name__ == '__main__':
# process command line
file_name = sys.argv[1]
method = sys.argv[2]
# write data
ierr = read_data(file_name, method)
if ierr:
error_message('read failed')
# return the error code
sys.exit(ierr)
from mpi4py import *
from sensei import VTKDataAdaptor,ADIOSDataAdaptor,ADIOSAnalysisAdaptor
import sys,os
import numpy as np
import vtk, vtk.util.numpy_support as vtknp
from time import sleep
from random import seed,random
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
n_ranks = comm.Get_size()
def error_message(msg):
sys.stderr.write('ERROR[%d] : %s\n'%(rank, msg))
def status_message(msg, io_rank=0):
if rank == io_rank:
sys.stderr.write('STATUS[%d] : %s\n'%(rank, msg))
def get_data_array(name, size, dtype):
a = np.empty(size, dtype=dtype)
i = 0
while i < size:
a[i] = i
i += 1
va = vtknp.numpy_to_vtk(a, deep=1)
va.SetName(name)
return va
def get_data_arrays(size, dsa):
dsa.AddArray(get_data_array('char_array', size, np.int8))
dsa.AddArray(get_data_array('int_array', size, np.int32))
dsa.AddArray(get_data_array('long_array', size, np.int64))
dsa.AddArray(get_data_array('unsigned_char_array', size, np.uint8))
dsa.AddArray(get_data_array('unsigned_int_array', size, np.uint32))
dsa.AddArray(get_data_array('unsigned_long_array', size, np.uint64))
dsa.AddArray(get_data_array('float_array', size, np.float32))
dsa.AddArray(get_data_array('double_array', size, np.float64))
return dsa
def get_image(i0,i1,j0,j1,k0,k1):
im = vtk.vtkImageData()
im.SetExtent(i0,i1,j0,j1,k0,k1)
nx = i1 - i0 + 1
ny = j1 - j0 + 1
nz = k1 - k0 + 1
npts = (nx + 1)*(ny + 1)*(nz + 1)
ncells = nx*ny*nz
get_data_arrays(npts, im.GetPointData())
get_data_arrays(ncells, im.GetCellData())
return im
def points_to_polydata(x,y,z):
nx = len(x)
# points
xyz = np.zeros(3*nx, dtype=np.float32)
xyz[::3] = x[:]
xyz[1::3] = y[:]
xyz[2::3] = z[:]
vxyz = vtknp.numpy_to_vtk(xyz, deep=1)
vxyz.SetNumberOfComponents(3)
vxyz.SetNumberOfTuples(nx)
pts = vtk.vtkPoints()
pts.SetData(vxyz)
# cells
cids = np.empty(2*nx, dtype=np.int32)
cids[::2] = 1
cids[1::2] = np.arange(0,nx,dtype=np.int32)
cells = vtk.vtkCellArray()
cells.SetCells(nx, vtknp.numpy_to_vtk(cids, \
deep=1, array_type=vtk.VTK_ID_TYPE))
# package it all up in a poly data set
pd = vtk.vtkPolyData()
pd.SetPoints(pts)
pd.SetVerts(cells)
# add some scalar data
get_data_arrays(nx, pd.GetPointData())
get_data_arrays(nx, pd.GetCellData())
return pd
def get_polydata(nx):
seed(2)
x = np.empty(nx)
y = np.empty(nx)
z = np.empty(nx)
i = 0
while i < nx:
x[i] = random()
y[i] = random()
z[i] = random()
i += 1
pd = points_to_polydata(x,y,z)
get_data_arrays(nx, pd.GetPointData())
get_data_arrays(nx, pd.GetCellData())
return pd
def write_data(file_name, method, n_its):
# initialize the analysis adaptor
aw = ADIOSAnalysisAdaptor.New()
aw.SetFileName(file_name)
aw.SetMethod(method)
# create the dataset
# each rank contributes a x-z plane
mbim = vtk.vtkMultiBlockDataSet()
mbim.SetNumberOfBlocks(2*n_ranks)
mbim.SetBlock(2*rank, get_image(rank,rank,0,16,0,1))
mbim.SetBlock(2*rank+1, get_polydata(16))
i = 0
while i < n_its:
t = float(i)
it = i
# pass into the data adaptor
status_message('initializing the VTKDataAdaptor ' \
'step %d time %0.1f'%(it,t))
da = VTKDataAdaptor.New()
da.SetDataTime(t)
da.SetDataTimeStep(it)
da.SetDataObject(mbim)
# execute the analysis adaptor
status_message('executing ADIOSAnalysisAdaptor %s ' \
'step %d time %0.1f'%(method,it,t))
aw.Execute(da)
# free up data
da.ReleaseData()
da = None
i += 1
# force free up the adaptor
aw = None
status_message('finished writing %d steps'%(n_its))
# set the return value
return 0
if __name__ == '__main__':
# process command line
file_name = sys.argv[1]
method = sys.argv[2]
n_its = int(sys.argv[3])
# write data
ierr = write_data(file_name, method, n_its)
if ierr:
error_message('write failed')
# return the error code
sys.exit(ierr)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment