ParaViewTestingMacros.cmake 13.8 KB
Newer Older
1 2
include(vtkTestingMacros)

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
# Override vtk_add_test_* variables for use with ParaView.
macro (_paraview_override_vtk_dirs)
  set(VTK_TEST_DATA_DIR    ${PARAVIEW_TEST_OUTPUT_DATA_DIR})
  set(VTK_BASELINE_DIR     ${PARAVIEW_TEST_BASELINE_DIR})
  set(VTK_TEST_OUTPUT_DIR  ${PARAVIEW_TEST_OUTPUT_DIR})
  set(VTK_TEST_DATA_TARGET ParaViewData)
endmacro ()

function (paraview_add_test_mpi exe var)
  _paraview_override_vtk_dirs()
  vtk_add_test_mpi("${exe}" "${var}" ${ARGN})
  set("${var}" ${${var}}
    PARENT_SCOPE)
endfunction ()

function (paraview_add_test_cxx exe var)
  _paraview_override_vtk_dirs()
  vtk_add_test_cxx("${exe}" "${var}" ${ARGN})
  set("${var}" ${${var}}
    PARENT_SCOPE)
endfunction ()

function (paraview_add_test_python)
  set(VTK_PYTHON_EXE "$<TARGET_FILE:pvpython>")
27
  list(APPEND VTK_PYTHON_ARGS -dr
28 29 30 31 32 33 34
    ${PARAVIEW_PYTHON_ARGS})
  _paraview_override_vtk_dirs()
  vtk_add_test_python(${ARGN})
endfunction ()

function (paraview_add_test_python_mpi)
  set(VTK_PYTHON_EXE "$<TARGET_FILE:pvpython>")
35
  list(APPEND VTK_PYTHON_ARGS -dr
36 37 38 39 40 41 42
    ${PARAVIEW_PYTHON_ARGS})
  _paraview_override_vtk_dirs()
  vtk_add_test_python_mpi(${ARGN})
endfunction ()

function (paraview_add_test_pvbatch)
  set(VTK_PYTHON_EXE "$<TARGET_FILE:pvbatch>")
43
  list(APPEND VTK_PYTHON_ARGS -dr
44
    ${PARAVIEW_PVBATCH_ARGS})
45
  set(vtk_test_prefix "Batch-${vtk_test_prefix}")
46 47 48 49 50 51
  _paraview_override_vtk_dirs()
  vtk_add_test_python(${ARGN})
endfunction ()

function (paraview_add_test_pvbatch_mpi)
  set(VTK_PYTHON_EXE "$<TARGET_FILE:pvbatch>")
52
  list(APPEND VTK_PYTHON_ARGS -dr
53
    ${PARAVIEW_PVBATCH_ARGS})
54
  set(vtk_test_prefix "Batch-${vtk_test_prefix}")
55 56 57 58 59 60 61 62 63 64 65
  _paraview_override_vtk_dirs()
  vtk_add_test_python_mpi(${ARGN})
endfunction ()

function(paraview_add_test_driven)
  if (NOT (TARGET pvserver AND TARGET pvpython))
    return()
  endif ()
  set(VTK_PYTHON_EXE "$<TARGET_FILE:smTestDriver>")
  list(APPEND VTK_PYTHON_ARGS
    --server $<TARGET_FILE:pvserver>
66
    --client $<TARGET_FILE:pvpython> -dr)
67 68 69 70 71 72 73 74 75 76
  _paraview_override_vtk_dirs()
  vtk_add_test_python(${ARGN})
endfunction ()

function (paraview_test_load_baselines name)
  set(data)
  foreach (datafile IN LISTS ARGN)
    list(APPEND data
      "DATA{${PARAVIEW_TEST_BASELINE_DIR}/${datafile}}")
  endforeach ()
77
  _paraview_test_load_data(${data})
78 79 80 81 82 83 84 85
endfunction ()

function (paraview_test_load_baselines_dirs name)
  set(data)
  foreach (datafile IN LISTS ARGN)
    list(APPEND data
      "DATA{${PARAVIEW_TEST_BASELINE_DIR}/${datafile}/,REGEX:.*}")
  endforeach ()
86
  _paraview_test_load_data(${data})
87 88 89 90 91 92 93 94
endfunction ()

function (paraview_test_load_data name)
  set(data)
  foreach (datafile IN LISTS ARGN)
    list(APPEND data
      "DATA{${PARAVIEW_TEST_DATA_DIR}/${datafile}}")
  endforeach ()
95
  _paraview_test_load_data(${data})
96 97 98 99 100 101
endfunction ()

function (paraview_test_load_data_dirs name)
  set(data)
  foreach (datafile IN LISTS ARGN)
    list(APPEND data
102
      "DATA{${PARAVIEW_TEST_DATA_DIR}/${datafile}/,RECURSE:,REGEX:.*}")
103
  endforeach ()
104
  _paraview_test_load_data(${data})
105 106
endfunction ()

107 108
function (_paraview_test_load_data)
  ExternalData_Expand_Arguments("ParaViewData" files ${ARGN})
109 110
endfunction ()

111
# Set up some common testing environment.
112 113 114
SET (CLIENT_EXECUTABLE  "\$<TARGET_FILE:paraview>")
# FIXME: need to verify that the above points to the paraview executable within
# the app bundle on Mac.
115 116 117 118

MACRO (process_args out_extra_args)
  SET (temp_args)
  IF (ACT_BASELINE_DIR)
119 120 121 122 123
    IF (${test_name}_BASELINE)
      SET (temp_args "--test-baseline=DATA{${ACT_BASELINE_DIR}/${${test_name}_BASELINE}}")
    ELSE ()
      SET (temp_args "--test-baseline=DATA{${ACT_BASELINE_DIR}/${test_name}.png}")
    ENDIF ()
124
  ENDIF ()
125 126
  IF (${test_name}_THRESHOLD)
    SET (temp_args ${temp_args} "--test-threshold=${${test_name}_THRESHOLD}")
127
  ENDIF ()
128
  SET (${out_extra_args} ${${out_extra_args}} ${temp_args})
129
ENDMACRO ()
130 131 132


FUNCTION (add_pv_test prefix skip_test_flag_suffix)
133
  PV_PARSE_ARGUMENTS(ACT "TEST_SCRIPTS;BASELINE_DIR;COMMAND;LOAD_PLUGIN;PLUGIN_PATH;EXTRA_LABELS" "" ${ARGN})
134 135 136 137
  while (ACT_TEST_SCRIPTS)
    set (counter 0)
    set (extra_args)
    set (full_test_name)
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
138
    set (force_serial FALSE)
139 140 141 142
    # Leaving group size as 1.
    # We need to fix "resetting" of application after each test correctly,
    # before we re-enable to avoid flaky tests.
    set (TEST_GROUP_SIZE 1)
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
143

144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
    while (${counter} LESS ${TEST_GROUP_SIZE})
      list(LENGTH ACT_TEST_SCRIPTS num_tests)
      if (num_tests)
        # pop test name from the top.
        list(GET ACT_TEST_SCRIPTS 0 test)
        list(REMOVE_AT ACT_TEST_SCRIPTS 0)
        GET_FILENAME_COMPONENT(test_name ${test} NAME_WE)

        # If this is "break" test, make sure that no other tests were already
        # added.
        if (${counter} GREATER 0)
          if (${test_name}_BREAK)
            set (counter 100000) # stop the group;
            # push the test back into the list.
            list(INSERT ACT_TEST_SCRIPTS 0 ${test})
159 160
          endif ()
        endif ()
161

162 163 164 165 166
        if (${counter} LESS 100000)
          if (NOT ${test_name}${skip_test_flag_suffix})
            set (full_test_name "${full_test_name}.${test_name}")
            set (extra_args ${extra_args} "--test-script=${test}")
            process_args(extra_args)
167 168 169
          endif ()
        endif ()
      endif ()
170 171 172
      math(EXPR counter "${counter} + 1")
      if (DEFINED ${test_name}_BREAK)
        set (counter 100000) # stop the group.
173
      endif ()
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
174 175
      if (${test_name}_FORCE_SERIAL)
        set (force_serial TRUE)
176 177
      endif ()
    endwhile ()
178 179

    if (extra_args)
180 181
      ExternalData_add_test(ParaViewData
        NAME "${prefix}${full_test_name}"
182
        COMMAND smTestDriver
183 184 185 186
                --enable-bt
                ${ACT_COMMAND}
                ${extra_args}
                --exit)
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
187 188 189 190
      if (force_serial)
        set_tests_properties("${prefix}${full_test_name}" PROPERTIES RUN_SERIAL ON)
        message(STATUS "Running in serial \"${prefix}${full_test_name}\"")
      endif()
191

192
      # add the "PARAVIEW" label to the test properties. this allows for the user
193 194 195
      # to instruct cmake to run just the ParaView tests with the '-L' flag.
      # also add in any extra labels (e.g. CATALYST if they were passed in as arguments)
      set_tests_properties("${prefix}${full_test_name}" PROPERTIES LABELS "PARAVIEW;${ACT_EXTRA_LABELS}")
196 197
    endif ()
  endwhile ()
198

199
ENDFUNCTION ()
200

201 202 203
# Add macro to support addition of paraview web tests
FUNCTION(add_pvweb_tests prefix)
  PV_PARSE_ARGUMENTS(ACT
204
    "APP;TEST_SCRIPTS;BASELINE_DIR;COMMAND;ARGS;SERVER;BROWSER"
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
    ""
    ${ARGN})

  # If this batch of tests has baseline images that need to be
  # compared against, make sure to include the baseline image
  # directory.  Otherwise, we will just leave this variable blank.
  if(DEFINED ACT_BASELINE_DIR)
    set(BASELINE_IMG_DIR "--baseline-img-dir;${ACT_BASELINE_DIR}")
  else()
    set(BASELINE_IMG_DIR "")
  endif()

  # Check for any args that we got
  if(DEFINED ACT_ARGS)
    set(ARGS ${ACT_ARGS})
  else()
    set(ARGS "")
  endif()

  # If there are browsers specified, we will iterate over them
  # and run the test scripts on each one.  If there isn't a
  # browser specified, we'll make one called "no-browser" to
  # keep the code clean.
  if (NOT DEFINED ACT_BROWSER)
    set(ACT_BROWSER "nobrowser")
  endif()

232
  set(pvw_port_number 9743)
233

234 235 236 237 238
  while(ACT_BROWSER)
    # Pull another browser off the list
    list(GET ACT_BROWSER 0 browser)
    list(REMOVE_AT ACT_BROWSER 0)

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
    # Create a copy of the scripts list so we keep the original intact
    set(TEST_SCRIPTS_LIST ${ACT_TEST_SCRIPTS})

    while (TEST_SCRIPTS_LIST)
      # pop test script path from the top of the list
      list(GET TEST_SCRIPTS_LIST 0 test_path)
      list(REMOVE_AT TEST_SCRIPTS_LIST 0)
      GET_FILENAME_COMPONENT(script_name ${test_path} NAME_WE)

      set(short_script_name ${script_name})

      # Use a regular expression to remove the first 4 batches of
      # underscore-separated character strings
      if(${script_name} MATCHES "^[^_]+_[^_]+_[^_]+_[^_]+_(.+)")
        set(short_script_name ${CMAKE_MATCH_1})
254
      endif()
255 256 257 258 259 260 261 262 263 264

      set(test_name "${prefix}-${browser}.${ACT_APP}-${short_script_name}")
      set(test_image_file_name "${test_name}.png")

      ExternalData_add_test(ParaViewData
        NAME ${test_name}
        COMMAND ${ACT_COMMAND}
                ${ACT_SERVER}
                --content ${ParaView_BINARY_DIR}/www
                --data-dir ${PARAVIEW_TEST_OUTPUT_DATA_DIR}
265
                --port ${pvw_port_number}
266 267 268 269 270 271 272 273
                ${ARGS}
                ${BASELINE_IMG_DIR}
                --run-test-script ${test_path}
                --test-use-browser ${browser}
                --temporary-directory ${PARAVIEW_TEST_OUTPUT_DIR}
                --test-image-file-name ${test_image_file_name}
                )
      set_tests_properties(${test_name} PROPERTIES LABELS "PARAVIEW")
274 275 276 277
      if (${ACT_APP}-${short_script_name}_FORCE_SERIAL)
        set_tests_properties("${test_name}" PROPERTIES RUN_SERIAL ON)
        message(STATUS "Running in serial \"${test_name}\"")
      endif()
278
      MATH(EXPR pvw_port_number "${pvw_port_number}+1")
279
    endwhile()
280 281 282
  endwhile()
ENDFUNCTION()

283 284
# ----------------------------------------------------------------------------
# Test functions
285
FUNCTION (add_client_tests prefix)
286 287
  PV_EXTRACT_CLIENT_SERVER_ARGS(${ARGN})

288
  add_pv_test(${prefix} "_DISABLE_C"
289
    COMMAND --client ${CLIENT_EXECUTABLE}
290
            --enable-bt
291
            -dr
292
            ${CLIENT_SERVER_ARGS}
293 294
            --test-directory=${PARAVIEW_TEST_DIR}
    ${ARGN})
295
ENDFUNCTION ()
296 297

FUNCTION (add_client_server_tests prefix)
298
  PV_EXTRACT_CLIENT_SERVER_ARGS(${ARGN})
299 300
  add_pv_test(${prefix} "_DISABLE_CS"
    COMMAND
301
       --server $<TARGET_FILE:pvserver>
302
       --enable-bt
303
         ${CLIENT_SERVER_ARGS}
304
       --client ${CLIENT_EXECUTABLE}
305
       --enable-bt
306
         ${CLIENT_SERVER_ARGS}
307 308 309
       -dr
       --test-directory=${PARAVIEW_TEST_DIR}
    ${ARGN})
310
ENDFUNCTION ()
311 312

FUNCTION (add_client_render_server_tests prefix)
313
  PV_EXTRACT_CLIENT_SERVER_ARGS(${ARGN})
314 315
  add_pv_test(${prefix} "_DISABLE_CRS"
    COMMAND
316
       --data-server $<TARGET_FILE:pvdataserver>
317
       --enable-bt
318
            ${CLIENT_SERVER_ARGS}
319
       --render-server $<TARGET_FILE:pvrenderserver>
320
       --enable-bt
321
            ${CLIENT_SERVER_ARGS}
322
       --client ${CLIENT_EXECUTABLE}
323
       --enable-bt
324
            ${CLIENT_SERVER_ARGS}
325 326
       -dr
       --test-directory=${PARAVIEW_TEST_DIR}
327
    ${ARGN})
328
ENDFUNCTION ()
329 330

FUNCTION(add_multi_client_tests prefix)
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
331
  PV_PARSE_ARGUMENTS(ACT "TEST_SCRIPTS;BASELINE_DIR" "" ${ARGN})
332 333 334 335 336

  foreach (test_script ${ACT_TEST_SCRIPTS})
    get_filename_component(test_name ${test_script} NAME_WE)
    if (${test_name}_ENABLE_COLLAB)
      set (extra_args)
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
337
      set (use_old_panels)
338
      process_args(extra_args)
339

340 341
      ExternalData_add_test(ParaViewData
        NAME "${prefix}.${test_name}"
342
        COMMAND smTestDriver
343
        --test-multi-clients
344
        --server $<TARGET_FILE:pvserver>
345
        --enable-bt
346 347

        --client ${CLIENT_EXECUTABLE}
348
        --enable-bt
349 350 351 352
        -dr
        --test-directory=${PARAVIEW_TEST_DIR}
        --test-script=${test_script}
        --test-master
Utkarsh Ayachit's avatar
Utkarsh Ayachit committed
353
        ${use_old_panels}
354 355 356
        --exit

        --client ${CLIENT_EXECUTABLE}
357
        --enable-bt
358 359 360 361 362 363
        -dr
        --test-directory=${PARAVIEW_TEST_DIR}
        --test-slave
        ${extra_args}
        --exit
        )
364 365
      # all 'collab' tests are always run in serial i.e. one at a time.
      set_tests_properties("${prefix}.${test_name}" PROPERTIES RUN_SERIAL ON)
366
      set_tests_properties("${prefix}.${test_name}" PROPERTIES LABELS "PARAVIEW")
367
    endif()
368 369
  endforeach()
ENDFUNCTION()
370 371 372 373 374 375 376 377

FUNCTION(add_multi_server_tests prefix nbServers)
  PV_PARSE_ARGUMENTS(ACT "TEST_SCRIPTS;BASELINE_DIR" "" ${ARGN})

  foreach (test_script ${ACT_TEST_SCRIPTS})
    get_filename_component(test_name ${test_script} NAME_WE)
      set (extra_args)
      process_args(extra_args)
378 379
      ExternalData_add_test(ParaViewData
        NAME "${prefix}.${test_name}"
380
        COMMAND smTestDriver
381
        --test-multi-servers ${nbServers}
382
        --server $<TARGET_FILE:pvserver>
383
        --enable-bt
384 385

        --client ${CLIENT_EXECUTABLE}
386
        --enable-bt
387 388 389 390 391 392
        -dr
        --test-directory=${PARAVIEW_TEST_DIR}
        --test-script=${test_script}
        ${extra_args}
        --exit
        )
393
      set_tests_properties("${prefix}.${test_name}" PROPERTIES LABELS "PARAVIEW")
394 395
  endforeach()
ENDFUNCTION()
396

397
FUNCTION (add_tile_display_tests prefix tdx tdy )
398 399
  PV_PARSE_ARGUMENTS(ACT "TEST_SCRIPTS;BASELINE_DIR;LOAD_PLUGIN;PLUGIN_PATH" "" ${ARGN})
  PV_EXTRACT_CLIENT_SERVER_ARGS(${ARGN})
400 401

  MATH(EXPR REQUIRED_CPU '${tdx}*${tdy}-1') # -1 is for LESS
402
  if (${PARAVIEW_USE_MPI})
403 404 405
    if (NOT DEFINED VTK_MPI_MAX_NUMPROCS)
      set (VTK_MPI_MAX_NUMPROCS ${MPIEXEC_MAX_NUMPROCS})
    endif()
406 407 408 409 410 411
    if (${REQUIRED_CPU} LESS ${VTK_MPI_MAX_NUMPROCS})
      foreach (test_script ${ACT_TEST_SCRIPTS})

        get_filename_component(test_name ${test_script} NAME_WE)
        set (extra_args)
        process_args(extra_args)
412 413
        ExternalData_add_test(ParaViewData
            NAME "${prefix}-${tdx}x${tdy}.${test_name}"
414
            COMMAND smTestDriver
415
            --test-tiled ${tdx} ${tdy}
416
            --server $<TARGET_FILE:pvserver>
417
            --enable-bt
418
            ${CLIENT_SERVER_ARGS}
419
            --client ${CLIENT_EXECUTABLE}
420
            --enable-bt
421
            ${CLIENT_SERVER_ARGS}
422 423 424 425 426 427 428 429 430 431
            -dr
            --test-directory=${PARAVIEW_TEST_DIR}
            --test-script=${test_script}
            --tile-image-prefix=${PARAVIEW_TEST_DIR}/${test_name}

            ${extra_args}
            --exit
            )
        set_property(TEST "${prefix}-${tdx}x${tdy}.${test_name}"
                     PROPERTY ENVIRONMENT "PV_ICET_WINDOW_BORDERS=1")
432
        set_tests_properties("${prefix}-${tdx}x${tdy}.${test_name}" PROPERTIES RUN_SERIAL ON)
433
        set_tests_properties("${prefix}-${tdx}x${tdy}.${test_name}" PROPERTIES LABELS "PARAVIEW")
434 435
      endforeach()
    endif()
436
  endif()
437
ENDFUNCTION ()