Commit d4c0aab0 authored by Brad King's avatar Brad King Committed by Kitware Robot

Merge topic 'test-RunCMake'

55b2aa88 Use generalized RunCMake test infrastrucure for build_command test
eb33000d Use generalized RunCMake test infrastrucure for find_package test
42a81e71 Add stronger infrastructure for CMake-only tests
parents 6452aea2 55b2aa88
macro(add_CMakeCommands_test test)
add_test(CMakeCommands.${test} ${CMAKE_CMAKE_COMMAND}
-DCMake_SOURCE_DIR=${CMake_SOURCE_DIR} # TODO: Remove
-Ddir=${CMAKE_CURRENT_BINARY_DIR}/${test}
-Dgen=${CMAKE_TEST_GENERATOR}
-P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/test.cmake"
)
endmacro()
add_CMakeCommands_test(build_command)
add_CMakeCommands_test(find_package)
if(NOT DEFINED CMake_SOURCE_DIR)
message(FATAL_ERROR "CMake_SOURCE_DIR not defined")
endif()
if(NOT DEFINED dir)
message(FATAL_ERROR "dir not defined")
endif()
if(NOT DEFINED gen)
message(FATAL_ERROR "gen not defined")
endif()
message(STATUS "CTEST_FULL_OUTPUT (Avoid ctest truncation of output)")
# Run cmake:
#
function(run_cmake build_dir extra_args expected_result expected_output expected_error)
message(STATUS "run_cmake build_dir='${build_dir}' extra_args='${extra_args}'")
# Ensure build_dir exists:
#
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${build_dir})
# Run cmake:
#
execute_process(COMMAND ${CMAKE_COMMAND}
${extra_args}
-G ${gen} ${CMake_SOURCE_DIR}/Tests/CMakeCommands/build_command
RESULT_VARIABLE result
OUTPUT_VARIABLE stdout
ERROR_VARIABLE stderr
WORKING_DIRECTORY ${build_dir}
)
message(STATUS "result='${result}'")
message(STATUS "stdout='${stdout}'")
message(STATUS "stderr='${stderr}'")
message(STATUS "")
# Verify result and output match expectations:
#
if("0" STREQUAL "${expected_result}")
if(NOT "${result}" STREQUAL "0")
message(FATAL_ERROR
"error: result='${result}' is non-zero and different than expected_result='${expected_result}'")
endif()
else()
if("${result}" STREQUAL "0")
message(FATAL_ERROR
"error: result='${result}' is zero and different than expected_result='${expected_result}'")
endif()
endif()
foreach(e ${expected_output})
if(NOT stdout MATCHES "${e}")
message(FATAL_ERROR
"error: stdout does not match expected_output item e='${e}'")
else()
message(STATUS "info: stdout matches '${e}'")
endif()
endforeach()
foreach(e ${expected_error})
if(NOT stderr MATCHES "${e}")
message(FATAL_ERROR
"error: stderr does not match expected_error item e='${e}'")
else()
message(STATUS "info: stderr matches '${e}'")
endif()
endforeach()
message(STATUS "result, stdout and stderr match all expectations: test passes")
message(STATUS "")
endfunction()
# Expect this case to succeed:
run_cmake("${dir}/b1" "" 0
"Build files have been written to:"
"skipping cases 1, 2 and 3 because TEST_ERROR_CONDITIONS is OFF")
# Expect this one to fail:
run_cmake("${dir}/b2" "-DTEST_ERROR_CONDITIONS:BOOL=ON" 1
"Configuring incomplete, errors occurred!"
"build_command requires at least one argument naming a CMake variable;build_command unknown argument ")
......@@ -53,7 +53,7 @@ IF(BUILD_TESTING)
ADD_SUBDIRECTORY(CMakeLib)
ADD_SUBDIRECTORY(CMakeOnly)
ADD_SUBDIRECTORY(CMakeCommands)
ADD_SUBDIRECTORY(RunCMake)
ADD_SUBDIRECTORY(FindPackageModeMakefileTest)
......
......@@ -16,10 +16,15 @@ your test to the test runs.
This includes tests that will build something using try_compile() and friends,
but nothing that expects add_executable(), add_library(), or add_test() to run.
If this matches your test you should put it into the Tests/CMakeOnly/ directory.
Create a subdirectory named like your test and write the CMakeLists.txt you
need into that subdirectory. Use the add_CMakeOnly_test() macro from
Tests/CMakeOnly/CMakeLists.txt to add your test to the test runs.
If the test configures the project only once and it must succeed then put it
into the Tests/CMakeOnly/ directory. Create a subdirectory named like your
test and write the CMakeLists.txt you need into that subdirectory. Use the
add_CMakeOnly_test() macro from Tests/CMakeOnly/CMakeLists.txt to add your
test to the test runs.
If the test configures the project with multiple variations and verifies
success or failure each time then put it into the Tests/RunCMake/ directory.
Read the instructions in Tests/RunCMake/CMakeLists.txt to add a test.
3. If you are testing something from the Modules directory
......
# This directory contains tests that run CMake to configure a project
# but do not actually build anything. To add a test:
#
# 1.) Add a subdirectory named for the test.
#
# 2.) Call add_RunCMake_test and pass the test directory name.
#
# 3.) Create a RunCMakeTest.cmake script in the directory containing
# include(RunCMake)
# run_cmake(SubTest1)
# ...
# run_cmake(SubTestN)
# where SubTest1..SubTestN are sub-test names each corresponding to
# an independent CMake run and project configuration.
#
# 3.) Create a CMakeLists.txt file in the directory containing
# cmake_minimum_required(...)
# project(${RunCMake_TEST} NONE) # or languages needed
# include(${RunCMake_TEST}.cmake)
# where "${RunCMake_TEST}" is literal. A value for RunCMake_TEST
# will be passed to CMake by the run_cmake macro when running each
# sub-test.
#
# 4.) Create a <SubTest>.cmake file for each sub-test named above
# containing the actual test code. Optionally create files
# containing expected test results:
# <SubTest>-result.txt = Process result expected if not "0"
# <SubTest>-stdout.txt = Regex matching expected stdout content
# <SubTest>-stderr.txt = Regex matching expected stderr content
# Note that trailing newlines will be stripped from actual test
# output before matching against the stdout and stderr expressions.
macro(add_RunCMake_test test)
add_test(RunCMake.${test} ${CMAKE_CMAKE_COMMAND}
-DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
-DRunCMake_GENERATOR=${CMAKE_TEST_GENERATOR}
-DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
-DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${test}
-P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
)
endmacro()
add_RunCMake_test(build_command)
add_RunCMake_test(find_package)
if(NOT DEFINED dir)
message(FATAL_ERROR "dir not defined")
endif()
if(NOT DEFINED gen)
message(FATAL_ERROR "gen not defined")
endif()
foreach(arg
RunCMake_GENERATOR
RunCMake_SOURCE_DIR
RunCMake_BINARY_DIR
)
if(NOT DEFINED ${arg})
message(FATAL_ERROR "${arg} not given!")
endif()
endforeach()
# TODO: Generalize this for other tests.
function(run_test test)
set(top_src "${CMAKE_CURRENT_LIST_DIR}")
set(top_bin "${dir}")
function(run_cmake test)
set(top_src "${RunCMake_SOURCE_DIR}")
set(top_bin "${RunCMake_BINARY_DIR}")
if(EXISTS ${top_src}/${test}-result.txt)
file(READ ${top_src}/${test}-result.txt expect_result)
string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
......@@ -29,7 +30,8 @@ function(run_test test)
file(REMOVE_RECURSE "${binary_dir}")
file(MAKE_DIRECTORY "${binary_dir}")
execute_process(
COMMAND ${CMAKE_COMMAND} "${source_dir}" -G "${gen}" -DTEST=${test}
COMMAND ${CMAKE_COMMAND} "${source_dir}"
-G "${RunCMake_GENERATOR}" -DRunCMake_TEST=${test}
WORKING_DIRECTORY "${binary_dir}"
OUTPUT_VARIABLE actual_stdout
ERROR_VARIABLE actual_stderr
......@@ -65,16 +67,3 @@ function(run_test test)
message(STATUS "${test} - PASSED")
endif()
endfunction()
run_test(MissingNormal)
run_test(MissingNormalRequired)
run_test(MissingNormalVersion)
run_test(MissingNormalWarnNoModuleOld)
run_test(MissingNormalWarnNoModuleNew)
run_test(MissingModule)
run_test(MissingModuleRequired)
run_test(MissingConfig)
run_test(MissingConfigOneName)
run_test(MissingConfigRequired)
run_test(MissingConfigVersion)
run_test(MixedModeOptions)
cmake_minimum_required(VERSION 2.8)
project(${RunCMake_TEST} NONE)
include(${RunCMake_TEST}.cmake)
# This CMakeLists file is *sometimes expected* to result in a configure error.
#
# expect this to succeed:
......@@ -12,12 +16,9 @@
# ...even purposefully calling it with known-bad argument lists to cover
# error handling code.
#
cmake_minimum_required(VERSION 2.8)
project(test_build_command)
set(cmd "initial")
message("CTEST_FULL_OUTPUT")
message("0. begin")
if(TEST_ERROR_CONDITIONS)
......
skipping cases 1, 2 and 3 because TEST_ERROR_CONDITIONS is OFF
CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
build_command requires at least one argument naming a CMake variable
+
1. cmd='initial'
CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
build_command unknown argument "BOGUS"
+
2. cmd='initial'
CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
build_command unknown argument "STUFF"
Configuring incomplete, errors occurred!
include(RunCMake)
run_cmake(ErrorsOFF)
run_cmake(ErrorsON)
cmake_minimum_required(VERSION 2.8)
project(${TEST} NONE)
include(${TEST}.cmake)
project(${RunCMake_TEST} NONE)
include(${RunCMake_TEST}.cmake)
include(RunCMake)
run_cmake(MissingNormal)
run_cmake(MissingNormalRequired)
run_cmake(MissingNormalVersion)
run_cmake(MissingNormalWarnNoModuleOld)
run_cmake(MissingNormalWarnNoModuleNew)
run_cmake(MissingModule)
run_cmake(MissingModuleRequired)
run_cmake(MissingConfig)
run_cmake(MissingConfigOneName)
run_cmake(MissingConfigRequired)
run_cmake(MissingConfigVersion)
run_cmake(MixedModeOptions)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment