Commit b741ee82 authored by Kyle Edwards's avatar Kyle Edwards Committed by Brad King

Tests: Add test for CTest hardware allocation feature

parent 3c8a5aad
cmake_minimum_required(VERSION 3.15)
set(CASE_NAME "@CASE_NAME@")
if(CASE_NAME MATCHES "^(.*)-ctest-s")
set(projname "${CMAKE_MATCH_1}")
project(${projname} NONE)
include(CTest)
include("@RunCMake_SOURCE_DIR@/HardwareCommon.cmake")
include("@RunCMake_SOURCE_DIR@/${projname}.cmake")
endif()
function(setup_hardware_tests)
if(CTEST_HARDWARE_ALLOC_ENABLED)
add_test(NAME HardwareSetup COMMAND "${CMAKE_COMMAND}" -E remove -f "${CMAKE_BINARY_DIR}/cthwalloc.log")
endif()
endfunction()
function(add_hardware_test name sleep_time proc)
if(CTEST_HARDWARE_ALLOC_ENABLED)
add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}" "${proc}")
set_property(TEST "${name}" PROPERTY DEPENDS HardwareSetup)
else()
add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}")
endif()
set_property(TEST "${name}" PROPERTY PROCESSES "${proc}")
list(APPEND HARDWARE_TESTS "${name}")
set(HARDWARE_TESTS "${HARDWARE_TESTS}" PARENT_SCOPE)
endfunction()
function(cleanup_hardware_tests)
if(CTEST_HARDWARE_ALLOC_ENABLED)
file(WRITE "${CMAKE_BINARY_DIR}/hwtests.txt" "${HARDWARE_TESTS}")
endif()
endfunction()
include(RunCMake)
include(RunCTest)
###############################################################################
# Test cthwalloc itself - we want to make sure it's not just rubber-stamping
......@@ -47,7 +48,7 @@ alloc widgets 0 1
dealloc widgets 0 1
end test1
]])
run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets=2,transmogrifiers=1;2,widgets=1,widgets=2"
run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets:2,transmogrifiers:1;2,widgets:1,widgets:2"
CTEST_PROCESS_COUNT=3
CTEST_PROCESS_0=widgets,transmogrifiers
CTEST_PROCESS_0_WIDGETS=id:0,slots:2
......@@ -58,57 +59,57 @@ run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets=2,transmogr
"CTEST_PROCESS_2_WIDGETS=id:0,slots:1\\;id:2,slots:2"
)
set(RunCMake_TEST_NO_CLEAN 0)
run_cthwalloc_write_proc(cthwalloc-write-proc-good2 "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-good2 "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
CTEST_PROCESS_0_WIDGETS=id:3,slots:8
)
run_cthwalloc_write_proc(cthwalloc-write-proc-nocount "widgets=8")
run_cthwalloc_write_proc(cthwalloc-write-proc-badcount "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-nocount "widgets:8")
run_cthwalloc_write_proc(cthwalloc-write-proc-badcount "widgets:8"
CTEST_PROCESS_COUNT=2
)
run_cthwalloc_write_proc(cthwalloc-write-proc-nores "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-nores "widgets:8"
CTEST_PROCESS_COUNT=1
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badres "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-badres "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets,transmogrifiers
)
run_cthwalloc_write_proc(cthwalloc-write-proc-nowidgets "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-nowidgets "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets1 "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets1 "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
CTEST_PROCESS_0_WIDGETS=
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets2 "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets2 "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
"CTEST_PROCESS_0_WIDGETS=id:3,slots:8\\;id:0,slots:1"
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets3 "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets3 "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
CTEST_PROCESS_0_WIDGETS=id:3,slots:7
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets4 "widgets=8"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets4 "widgets:8"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
CTEST_PROCESS_0_WIDGETS=invalid
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets5 "widgets=2,widgets=2"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets5 "widgets:2,widgets:2"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
"CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets6 "widgets=2"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets6 "widgets:2"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
"CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
)
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets7 "widgets=2,widgets=2"
run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets7 "widgets:2,widgets:2"
CTEST_PROCESS_COUNT=1
CTEST_PROCESS_0=widgets
CTEST_PROCESS_0_WIDGETS=id:0,slots:2
......@@ -134,3 +135,33 @@ run_cthwalloc_verify(cthwalloc-verify-badtest4 "test1")
run_cthwalloc_verify(cthwalloc-verify-badtest5 "test1")
run_cthwalloc_verify(cthwalloc-verify-nobegin "test1")
run_cthwalloc_verify(cthwalloc-verify-noend "test1")
###############################################################################
# Now test the hardware allocation feature of CTest
###############################################################################
function(run_ctest_hardware name parallel random)
run_ctest("${name}-ctest-s-hw" "-DCTEST_HARDWARE_ALLOC_ENABLED=1" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
run_ctest("${name}-ctest-s-nohw" "-DCTEST_HARDWARE_ALLOC_ENABLED=0" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
endfunction()
function(verify_ctest_hardware)
file(READ "${RunCMake_TEST_BINARY_DIR}/hwtests.txt" hwtests)
execute_process(COMMAND "${CTHWALLOC_COMMAND}" verify "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" "${CMAKE_CURRENT_LIST_DIR}/hwspec.json" "${hwtests}"
OUTPUT_VARIABLE output ERROR_QUIET RESULT_VARIABLE result)
if(result)
string(APPEND RunCMake_TEST_FAILED "${output}")
set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
endif()
endfunction()
run_ctest_hardware(lotsoftests 10 1)
run_ctest_hardware(checkfree1 2 0)
run_ctest_hardware(checkfree2 1 0)
run_ctest_hardware(notenough1 1 0)
run_ctest_hardware(notenough2 1 0)
run_ctest_hardware(ensure_parallel 2 0)
set(ENV{CTEST_PROCESS_COUNT} 2)
run_ctest_hardware(process_count 1 0)
unset(ENV{CTEST_PROCESS_COUNT})
setup_hardware_tests()
add_hardware_test(Test1 1 "widgets:8")
add_hardware_test(Test2 1 "fluxcapacitors:50;fluxcapacitors:50,widgets:8")
add_hardware_test(Test3 1 "fluxcapacitors:121")
cleanup_hardware_tests()
setup_hardware_tests()
# This test is an attack on the hardware scheduling algorithm. It has been
# carefully crafted to fool the algorithm into thinking there isn't sufficient
# hardware for it.
add_hardware_test(Test1 1 "widgets:2;4,widgets:4")
cleanup_hardware_tests()
verify_ctest_hardware()
set(expected_contents [[
begin Test1
alloc transmogrifiers calvin 2
begin Test2
alloc transmogrifiers hobbes 2
dealloc transmogrifiers calvin 2
end Test1
dealloc transmogrifiers hobbes 2
end Test2
]])
file(READ "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" actual_contents)
if(NOT actual_contents STREQUAL expected_contents)
string(APPEND RunCMake_TEST_FAILED "cthwalloc.log contents did not match expected\n")
endif()
setup_hardware_tests()
add_hardware_test(Test1 4 "transmogrifiers:2")
# Mitigate possible race conditions to ensure that the events are logged in the
# exact order we want
add_test(NAME Test2Sleep COMMAND "${CMAKE_COMMAND}" -E sleep 2)
add_hardware_test(Test2 4 "transmogrifiers:2")
set_property(TEST Test2 APPEND PROPERTY DEPENDS Test2Sleep)
cleanup_hardware_tests()
......@@ -4,51 +4,50 @@
"widgets": [
{
"id": "0",
"capacity": 4
"slots": 4
},
{
"id": "1",
"capacity": 2
"slots": 2
},
{
"id": "2",
"capacity": 4
"slots": 4
},
{
"id": "3",
"capacity": 8
"slots": 8
},
{
"id": "4",
"capacity": 1
"slots": 1
},
{
"id": "5",
"capacity": 1
"slots": 1
},
{
"id": "6",
"capacity": 1
"slots": 1
},
{
"id": "7",
"capacity": 1
"id": "7"
}
],
"transmogrifiers": [
{
"id": "calvin",
"capacity": 2
"slots": 2
},
{
"id": "hobbes",
"capacity": 2
"slots": 2
}
],
"fluxcapacitors": [
{
"id": "outatime",
"capacity": 121
"slots": 121
}
]
}
......
setup_hardware_tests()
add_hardware_test(Test1 2 "widgets:8;2,widgets:2")
add_hardware_test(Test2 5 "fluxcapacitors:40")
add_hardware_test(Test3 1 "10,widgets:1,fluxcapacitors:2")
add_hardware_test(Test4 4 "fluxcapacitors:121")
foreach(i RANGE 5 50)
add_hardware_test(Test${i} 1 "2,widgets:1")
endforeach()
foreach(i RANGE 51 100)
add_hardware_test(Test${i} 1 "2,transmogrifiers:2")
endforeach()
cleanup_hardware_tests()
if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
endif()
^Insufficient hardware
CMake Error at [^
]*/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
Tests did not pass$
setup_hardware_tests()
add_hardware_test(Test1 1 "fluxcapacitors:200")
cleanup_hardware_tests()
if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
endif()
^Insufficient hardware
CMake Error at [^
]*/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
Tests did not pass$
setup_hardware_tests()
add_hardware_test(Test1 1 "terminators:2")
cleanup_hardware_tests()
setup_hardware_tests()
add_hardware_test(Test1 1 "widgets:1")
cleanup_hardware_tests()
set(CTEST_SITE "test-site")
set(CTEST_BUILD_NAME "test-build-name")
set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@")
set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@")
set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@")
set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
ctest_start(Experimental QUIET)
ctest_configure(OPTIONS
"-DCTEST_HARDWARE_ALLOC_ENABLED=${CTEST_HARDWARE_ALLOC_ENABLED};-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}"
)
ctest_build()
if(CTEST_HARDWARE_ALLOC_ENABLED)
set(hwspec HARDWARE_SPEC_FILE "@RunCMake_SOURCE_DIR@/hwspec.json")
endif()
ctest_test(${hwspec} RETURN_VALUE retval PARALLEL_LEVEL ${CTEST_PARALLEL} SCHEDULE_RANDOM ${CTEST_RANDOM})
if(retval)
message(FATAL_ERROR "Tests did not pass")
endif()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment