Merge topic 'revamp-testing-infra' into master

9bb5faff Use VTK_TEST_OUTPUT_DIR rather than rebuilding the path
e2d1a14e Support Python+MPI testing
3b70ea86 Support JUST_VALID for Python tests
063ddda4 Support flags to tests common at the module level
e7851823 Support custom ExternalData targets for tests
72cc9e3c Support VTKExternalData_OBJECT_STORES as well
d1548cd5 Store VTK's ExternalData in its source directory
5b742fd8 Support custom baselines in tests
e04656f3 Build the VTK test data directory properly
910a306b Allow overriding the baseline directory in tests
f79be069 Allow for overriding the data directory in tests
e05e78f0 Skip adding the executable if no tests are created
b7535cae Allow arguments to be passed directly to Python
b6e69cba ParallelMPI: Add missing return statement
3c35e552 Infovis: Add an include for newer Boost versions
a6faf425 Copy Statistics tests to StatisticsGnuR
...

Accelerators/Dax/Testing/Cxx/CMakeLists.txt

-vtk_add_test_cxx(
+vtk_add_test_cxx(${vtk-module}CxxTests tests
   TestDaxThreshold.cxx
   TestDaxThreshold2.cxx
   TestDaxMarchingCubes.cxx
   )
-
-
- vtk_test_cxx_executable(${vtk-module}CxxTests)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests)

Accelerators/Piston/Testing/Cxx/CMakeLists.txt

-vtk_add_test_cxx(
+vtk_add_test_cxx(${vtk-module}CxxTests tests
   TestUsePiston.cxx
 )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests)
 
 if(VTK_MPI_MAX_NUMPROCS GREATER 1)
   include(vtkMPI)
   find_package(MPI REQUIRED)
   include_directories(${MPI_INCLUDE_PATH})
 
-  set(PistonMPITests TestDMPFiltering TestCompositeRender)
-
-  foreach(test ${PistonMPITests})
-
-    vtk_module_test_executable(${vtk-module}Cxx-${test} ${test}.cxx)
-    ExternalData_add_test(VTKData
-      NAME ${vtk-module}Cxx-${test}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${vtk-module}Cxx-${test}>
-      -D ${VTK_TEST_DATA_DIR}
-      -T ${VTK_TEST_OUTPUT_DIR}
-      -V DATA{../Data/Baseline/${test}.png,:}
-      ${VTK_MPI_POSTFLAGS})
-    vtk_mpi_link(${vtk-module}Cxx-${test})
-  endforeach()
+  set(${vtk-module}Cxx-MPI_NUMPROCS 2)
+  vtk_add_test_mpi(${vtk-module}Cxx-MPI tests
+    TESTING_DATA
+    TestDMPFiltering.cxx
+    TestCompositeRender.cxx
+    )
+  vtk_test_mpi_executable(${vtk-module}Cxx-MPI tests)
 
 endif()

Accelerators/Piston/Testing/Cxx/TestCompositeRender.cxx

@@ -214,7 +214,7 @@ void MyProcess::Execute()
 }
 
 
-int main(int argc, char **argv)
+int TestCompositeRender(int argc, char *argv[])
 {
   // This is here to avoid false leak messages from vtkDebugLeaks when
   // using mpich. It appears that the root process which spawns all the

Accelerators/Piston/Testing/Cxx/TestDMPFiltering.cxx

@@ -225,7 +225,7 @@ void MyMain(vtkMultiProcessController *controller, void *arg)
   elev->Delete();
 }
 
-int main(int argc, char *argv[])
+int TestDMPFiltering(int argc, char *argv[])
 {
   // This is here to avoid false leak messages from vtkDebugLeaks when
   // using mpich. It appears that the root process which spawns all the

Accelerators/Piston/Testing/Python/CMakeLists.txt

-if (VTK_PYTHON_EXE)
-  #
-  # Test base functionality.
-  #
-  set(TestConversion_OPTS NO_VALID)
-  foreach ( tfile
-      TestConversion
-      TestContour
-      TestRendering
-      TestThreshold
-      TestSlice
-      TestSort
-      )
-    set(${tfile}_ARGS --normalize)
-    vtk_add_test_python(${tfile}.py NO_RT ${${tfile}_OPTS})
-  endforeach ( )
+#
+# Exercise origin and spacing aspect of vtkImageData
+#
+set(vtk_test_prefix Placed)
+vtk_add_test_python(
+  NO_OUTPUT NO_RT
+  TestContour.py
+  TestThreshold.py
+  TestSlice.py
+  TestSort.py
+  )
+unset(vtk_test_prefix)
 
-  #
-  # Exercise origin and spacing aspect of vtkImageData
-  #
-  foreach ( tfile
-      Contour
-      Threshold
-      Slice
-      Sort
-      )
-    ExternalData_add_test(VTKData
-      NAME ${vtk-module}Python-TestPlaced${tfile}
-      COMMAND ${VTK_PYTHON_EXE}
-      ${CMAKE_CURRENT_SOURCE_DIR}/Test${tfile}.py
-      -B "DATA{../Data/Baseline/,REGEX:Test${tfile}(_[0-9]+)?.png}"
-      -D ${VTK_TEST_DATA_DIR}/Data)
-  endforeach ( )
+#
+# Exercise thrust to gl direct on GPU rendering.
+#
+set(vtk_test_prefix Direct)
+set(Rendering_ARGS --gpu_render)
+vtk_add_test_python(
+  NO_RT NO_OUTPUT
+  TestRendering.py
+  )
+unset(vtk_test_prefix)
 
-  #
-  # Exercise thrust to gl direct on GPU rendering.
-  #
-  ExternalData_add_test(VTKData
-    NAME ${vtk-module}Python-TestDirectRendering
-    COMMAND ${VTK_PYTHON_EXE}
-    ${CMAKE_CURRENT_SOURCE_DIR}/TestRendering.py
-    -B "DATA{../Data/Baseline/,REGEX:TestRendering(_[0-9]+)?.png}"
-    -D ${VTK_TEST_DATA_DIR}/Data
-    --gpu_render)
-
-endif ()
+#
+# Test base functionality.
+#
+set(python_tests
+  TestConversion
+  TestContour
+  TestRendering
+  TestThreshold
+  TestSlice
+  TestSort
+  )
+set(test_files)
+foreach(test IN LISTS python_tests)
+  set(${test}_ARGS --normalize)
+  list(APPEND test_files ${test}.py)
+endforeach()
+vtk_add_test_python(
+  NO_RT NO_VALID
+  ${test_files}
+  )

CMake/vtkExternalData.cmake

@@ -4,9 +4,9 @@ include(${_VTKExternalData_DIR}/ExternalData.cmake)
 if(NOT VTK_DATA_STORE)
   # Select a default from the following.
   set(VTK_DATA_STORE_DEFAULT "")
-  if(EXISTS "${CMAKE_SOURCE_DIR}/.ExternalData/config/store")
+  if(EXISTS "${VTK_SOURCE_DIR}/.ExternalData/config/store")
     # Configuration left by developer setup script.
-    file(STRINGS "${CMAKE_SOURCE_DIR}/.ExternalData/config/store"
+    file(STRINGS "${VTK_SOURCE_DIR}/.ExternalData/config/store"
       VTK_DATA_STORE_DEFAULT LIMIT_COUNT 1 LIMIT_INPUT 1024)
   elseif(IS_DIRECTORY "${CMAKE_SOURCE_DIR}/../VTKExternalData")
     # Adjacent directory created by user.
@@ -16,6 +16,9 @@ if(NOT VTK_DATA_STORE)
     # Generic adjacent directory created by user.
     get_filename_component(VTK_DATA_STORE_DEFAULT
       "${CMAKE_SOURCE_DIR}/../ExternalData" ABSOLUTE)
+  elseif(DEFINED "ENV{VTKExternalData_OBJECT_STORES}")
+    # The VTKExternalData environment variable.
+    file(TO_CMAKE_PATH "$ENV{VTKExternalData_OBJECT_STORES}" VTK_DATA_STORE_DEFAULT)
   elseif(DEFINED "ENV{ExternalData_OBJECT_STORES}")
     # Generic ExternalData environment variable.
     file(TO_CMAKE_PATH "$ENV{ExternalData_OBJECT_STORES}" VTK_DATA_STORE_DEFAULT)

CMakeLists.txt

@@ -40,7 +40,8 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 # Test input data staging directory.
-set(VTK_TEST_DATA_DIR "${ExternalData_BINARY_ROOT}/Testing")
+file(RELATIVE_PATH vtk_reldir "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
+set(VTK_TEST_DATA_DIR "${ExternalData_BINARY_ROOT}/${vtk_reldir}/Testing")
 
 # Test input data directory.
 set(VTK_TEST_INPUT_DIR "${VTK_SOURCE_DIR}/Testing/Data")

Charts/Core/Testing/Cxx/CMakeLists.txt

-# add tests that do not require data
-vtk_add_test_cxx(NO_DATA NO_VALID
-  TestContextScene.cxx
-  TestControlPointsItem.cxx
-  TestControlPointsItemEvents.cxx
+set(TestChartUnicode_ARGS
+  DATA{../Data/Fonts/DejaVuSans.ttf}
+  -E 25
+  )
+set(TestChartsOn3D_ARGS
+  -E 16
+  )
+set(TestColorTransferFunction_ARGS
+  -E 80
+  )
+set(TestContextUnicode_ARGS
+  DATA{../Data/Fonts/DejaVuSans.ttf}
+  )
+set(TestControlPointsHandleItem_ARGS
+  -E 30
+  )
+set(TestLinePlot_ARGS
+  -E 25
+  )
+set(TestLinePlot2_ARGS
+  -E 25
+  )
+set(TestLinePlotInteraction_ARGS
+  -E 25
+  )
+set(TestMultipleRenderers_ARGS
+  -E 25
+  )
+set(TestMultipleScalarsToColors_ARGS
+  -E 25
+  )
+set(TestParallelCoordinates_ARGS
+  -E 15
+  )
+set(TestParallelCoordinatesDouble_ARGS
+  -E 15
   )
 
-set(TestChartUnicode_ARGS DATA{../Data/Fonts/DejaVuSans.ttf})
-set(TestContextUnicode_ARGS DATA{../Data/Fonts/DejaVuSans.ttf})
-
-# add tests that require data
-vtk_add_test_cxx(
+vtk_add_test_cxx(${vtk-module}CxxTests tests
+  TestContextScene.cxx,NO_DATA,NO_VALID
+  TestControlPointsItem.cxx,NO_DATA,NO_VALID
+  TestControlPointsItemEvents.cxx,NO_DATA,NO_VALID
   TestAxes.cxx
   TestBagPlot.cxx
   TestBarGraph.cxx
   TestBarGraphHorizontal.cxx
   TestBoxPlot.cxx
   TestCategoryLegend.cxx
-  TestColorTransferFunction.cxx,-E80
+  TestColorTransferFunction.cxx
   TestChartDouble.cxx
   TestChartMatrix.cxx
-  TestChartUnicode.cxx,-E25
-  TestChartsOn3D.cxx,-E16
+  TestChartUnicode.cxx
+  TestChartsOn3D.cxx
   TestChartXYZ.cxx
   TestContext.cxx
   TestContextImage.cxx
   TestContextUnicode.cxx
-  TestControlPointsHandleItem.cxx,-E30
+  TestControlPointsHandleItem.cxx
   TestDiagram.cxx
   TestFunctionalBagPlot.cxx
   TestHistogram2D.cxx
   TestInteractiveChartXYZ.cxx
   TestLegendHiddenPlots.cxx
-  TestLinePlot.cxx,-E25
+  TestLinePlot.cxx
   TestLinePlotDouble.cxx
   TestLinePlotDouble2.cxx
   TestLinePlot3D.cxx
   TestLinePlotAxisFonts.cxx
-  TestLinePlot2.cxx,-E25
-  TestLinePlotInteraction.cxx,-E25
+  TestLinePlot2.cxx
+  TestLinePlotInteraction.cxx
   TestLinePlotSelection.cxx
   TestLinePlotSelection2.cxx
   TestMultipleChartRenderers.cxx
-  TestMultipleRenderers.cxx,-E25
-  TestMultipleScalarsToColors.cxx,-E25
-  TestParallelCoordinates.cxx,-E15
-  TestParallelCoordinatesDouble.cxx,-E15
+  TestMultipleRenderers.cxx
+  TestMultipleScalarsToColors.cxx
+  TestParallelCoordinates.cxx
+  TestParallelCoordinatesDouble.cxx
   TestPieChart.cxx
   TestPlotMatrix.cxx
   TestScalarsToColors.cxx
@@ -58,5 +88,6 @@ vtk_add_test_cxx(
   TestSurfacePlot.cxx
   TestZoomAxis.cxx
   )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests RENDERING_FACTORY)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests
+  RENDERING_FACTORY
+  )

Charts/Core/Testing/Python/CMakeLists.txt

-if(VTK_PYTHON_EXE)
-  # These tests should use vtk.test.Testing.
-  set(tests
-    TestBarGraph
-    TestLinePlot
-    TestStackedPlot
-    TestLinePlotColors
-    TestParallelCoordinatesColors
-    TestScatterPlotColors
-    )
-  foreach(tfile ${tests})
-    vtk_add_test_python(${tfile}.py NO_RT NO_OUTPUT)
-  endforeach()
-else()
-  message(FATAL_ERROR "No Python tests added!")
-endif()
+# These tests should use vtk.test.Testing.
+vtk_add_test_python(
+  NO_RT NO_OUTPUT
+  TestBarGraph.py
+  TestLinePlot.py
+  TestStackedPlot.py
+  TestLinePlotColors.py
+  TestParallelCoordinatesColors.py
+  TestScatterPlotColors.py
+  )

Common/Color/Testing/Cxx/CMakeLists.txt

-vtk_add_test_cxx(NO_VALID
-  TestCategoricalColors.cxx
-  TestNamedColors.cxx
-  )
-
-vtk_add_test_cxx(
+vtk_add_test_cxx(${vtk-module}CxxTests tests
+  TestCategoricalColors.cxx,NO_VALID
   TestColorSeries.cxx
+  TestNamedColors.cxx,NO_VALID
   )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests)

Common/ComputationalGeometry/Testing/Cxx/CMakeLists.txt

-vtk_add_test_cxx(NO_DATA NO_VALID NO_OUTPUT
+vtk_add_test_cxx(${vtk-module}CxxTests tests
+  NO_DATA NO_VALID NO_OUTPUT
   UnitTestParametricSpline.cxx
   )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests)

Common/ComputationalGeometry/Testing/Python/CMakeLists.txt

-vtk_add_test_python(CSpline.py)
-vtk_add_test_python(KSpline.py)
-vtk_add_test_python(TestParametricFunctions.py NO_RT)
-vtk_add_test_python(closedSplines.py)
+vtk_add_test_python(
+  CSpline.py
+  KSpline.py
+  TestParametricFunctions.py,NO_RT
+  closedSplines.py
+  )

Common/ComputationalGeometry/Testing/Tcl/CMakeLists.txt

-vtk_add_test_tcl(TestParametricFunctions)
-vtk_add_test_tcl(CSpline)
-vtk_add_test_tcl(KSpline)
-vtk_add_test_tcl(closedSplines)
+vtk_add_test_tcl(
+  TestParametricFunctions.tcl
+  CSpline.tcl
+  KSpline.tcl
+  closedSplines.tcl
+  )

Common/Core/Testing/Cxx/CMakeLists.txt

@@ -4,7 +4,8 @@ set(TestSystemInformation_ARGS ${CMAKE_BINARY_DIR})
 # Tell TestXMLFileOutputWindow where to write test file
 set(TestXMLFileOutputWindow_ARGS ${CMAKE_BINARY_DIR}/Testing/Temporary/XMLFileOutputWindow.txt)
 
-vtk_add_test_cxx(NO_DATA NO_VALID NO_OUTPUT
+vtk_add_test_cxx(${vtk-module}CxxTests tests
+  NO_DATA NO_VALID NO_OUTPUT
   UnitTestMath.cxx
   TestArrayAPI.cxx
   TestArrayAPIConvenience.cxx
@@ -59,6 +60,6 @@ vtk_add_test_cxx(NO_DATA NO_VALID NO_OUTPUT
   #otherLookupTableWithEnabling.cxx
   otherStringArray.cxx
   )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests
-  vtkTestNewVar.cxx)
+vtk_test_cxx_executable(${vtk-module}CxxTests tests
+  vtkTestNewVar.cxx
+  )

Common/Core/Testing/Cxx/TestArrayCasting.cxx

@@ -172,7 +172,7 @@ struct Transpose
 //
 //
 
-int main(int vtkNotUsed(argc), char *vtkNotUsed(argv)[])
+int TestArrayCasting(int vtkNotUsed(argc), char *vtkNotUsed(argv)[])
 {
   try
     {

Common/Core/Testing/Python/CMakeLists.txt

-if(VTK_PYTHON_EXE)
-  foreach(tfile
-    PythonSmoke
-    TestArrayArguments
-    TestExecuteMethodFinalizeCrash
-    TestGhost
-    TestIgnoreBTX
-    TestMutable
-    TestNumpySupport
-    TestOperators
-    TestPointers
-    TestStrings
-    TestSubClass
-    TestTerminationCrash
-    TestVariant
-    TestWeakref
-    TestCommand
-    )
-    vtk_add_test_python(${tfile}.py NO_DATA NO_VALID NO_OUTPUT)
-  endforeach()
-endif()
+vtk_add_test_python(
+  NO_DATA NO_VALID NO_OUTPUT
+  PythonSmoke.py
+  TestArrayArguments.py
+  TestExecuteMethodFinalizeCrash.py
+  TestGhost.py
+  TestIgnoreBTX.py
+  TestMutable.py
+  TestNumpySupport.py
+  TestOperators.py
+  TestPointers.py
+  TestStrings.py
+  TestSubClass.py
+  TestTerminationCrash.py
+  TestVariant.py
+  TestWeakref.py
+  TestCommand.py
+  )

Common/Core/Testing/Tcl/CMakeLists.txt

-set(tests
-  TestArrays
-  TestEmptyInput
-  TestSetGet
-  TestString
-  TestIgnoreBTX
-  otherInterp
-  otherPrint
+vtk_add_test_tcl(
+  NO_DATA NO_RT
+  TestArrays.tcl
+  TestEmptyInput.tcl
+  TestSetGet.tcl
+  TestString.tcl
+  TestIgnoreBTX.tcl
+  otherInterp.tcl
+  otherPrint.tcl
   )
-
-foreach(test ${tests})
-  vtk_add_test_tcl(${test}.tcl NO_DATA NO_RT)
-endforeach()

Common/DataModel/Testing/Cxx/CMakeLists.txt

-vtk_add_test_cxx(NO_DATA NO_VALID NO_OUTPUT
+vtk_add_test_cxx(${vtk-module}CxxTests tests
+  NO_DATA NO_VALID NO_OUTPUT
   TestColor.cxx
   TestVector.cxx
   TestVectorOperators.cxx
@@ -44,13 +45,12 @@ vtk_add_test_cxx(NO_DATA NO_VALID NO_OUTPUT
   TestStructuredData.cxx
   TestDataObjectTypes.cxx
   )
-
-vtk_add_test_cxx(NO_VALID NO_OUTPUT
-  TestCellIterators.cxx
-  )
-
-vtk_add_test_cxx(
+vtk_add_test_cxx(${vtk-module}CxxTests data_tests
+  TestCellIterators.cxx,NO_VALID,NO_OUTPUT
   TestQuadraticPolygonFilters.cxx
   )
-
-vtk_test_cxx_executable(${vtk-module}CxxTests)
+set(all_tests
+  ${tests}
+  ${data_tests}
+  )
+vtk_test_cxx_executable(${vtk-module}CxxTests all_tests)

Common/DataModel/Testing/Python/CMakeLists.txt

-vtk_add_test_python(ImplicitSum.py)
-vtk_add_test_python(LineIntersectQuadraticCells.py)
-vtk_add_test_python(PerlinNoise.py)
-vtk_add_test_python(SelectionLoop.py)
-vtk_add_test_python(TestNumericArrayImageData.py NO_DATA NO_VALID)
-vtk_add_test_python(TestQuadricClustering.py)
-vtk_add_test_python(TestStructuredGrid.py)
-vtk_add_test_python(TestTemplates.py NO_DATA NO_VALID)
-vtk_add_test_python(headBone.py)
-vtk_add_test_python(scalarColors.py)
-vtk_add_test_python(scalarConn.py)
-vtk_add_test_python(TestConvexPointSet.py)
-vtk_add_test_python(otherDataSetAttributes.py NO_DATA NO_VALID NO_RT)
-vtk_add_test_python(quadricCut.py NO_RT)
-vtk_add_test_python(TestICPTransform.py NO_RT)
+vtk_add_test_python(
+  ImplicitSum.py
+  LineIntersectQuadraticCells.py
+  PerlinNoise.py
+  SelectionLoop.py
+  TestNumericArrayImageData.py,NO_DATA,NO_VALID
+  TestQuadricClustering.py
+  TestStructuredGrid.py
+  TestTemplates.py,NO_DATA,NO_VALID
+  headBone.py
+  scalarColors.py
+  scalarConn.py
+  TestConvexPointSet.py
+  otherDataSetAttributes.py,NO_DATA,NO_VALID,NO_RT
+  quadricCut.py,NO_RT
+  TestICPTransform.py,NO_RT
+  )