Feature Request: Allow install(TARGETS) to optionally also install(TARGETS) on the transitive closure of the provided targets dependencies
I have a project structure like this:
project/
- CMakeLists.txt
- subfolder/
- - subfolder/
- - - CMakeLists.txt
...
several hundred nested sets of subfolders, with each subfolder containing one or more targets, with arbitrary and unknown (and from the perspective of the install team, unknowable) dependencies on previously defined targets from previously seen subfolders. With dependency relationships potentially being different depending on which target host os, target cpu arch, and $<CONFIG> is being built for. I go into more detail about $<CONFIG> usage here: https://gitlab.kitware.com/cmake/cmake/-/issues/23272
...
install/
- CMakeLists.txt
The CMakeLists.txt in the install/ folder has the rules for installing the different targets.
The way this installation works for us is that we have some several dozen different installation targets that are all handled by the same build tree. Most of them are .zip files, some of them are file-copies to the host OS, and some of them are for creating deb/rpm packages.
We use a function like this for the zip targets
function(package_zip NAME)
set(options)
set(oneValueArgs)
set(multiValueArgs TARGETS)
cmake_parse_arguments(FUNC_ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
if(FUNC_ARGS_UNPARSED_ARGUMENTS)
message(FATAL_ERROR "Unknown arguments to package_zip ; ${FUNC_ARGS_UNPARSED_ARGUMENTS}")
endif()
# Get the entire list of targets, including the top-level ones.
get_recursive_dependency_list(FULL_TARGET_LIST ${FUNC_ARGS_TARGETS})
list(PREPEND FULL_TARGET_LIST ${FUNC_ARGS_TARGETS})
list(REMOVE_DUPLICATES FULL_TARGET_LIST)
filter_target_type(FULL_TARGET_LIST STATIC_LIBRARY)
filter_target_type(FULL_TARGET_LIST INTERFACE_LIBRARY)
split_out_imported_targets(FULL_TARGET_LIST IMPORTED_TARGET_LIST)
# Install each target into the folder created for cpack to pack into a zip.
install(TARGETS ${FULL_TARGET_LIST}
RUNTIME EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
LIBRARY EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
ARCHIVE EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
)
# Install any additional installation files to the destination.
# the actual destination directory is relative to the base-dir for each such file set
# and controlled by the target...
install(TARGETS ${FULL_TARGET_LIST}
FILE_SET additional_install_files EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
)
install(IMPORTED_RUNTIME_ARTIFACTS ${IMPORTED_TARGET_LIST}
RUNTIME EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
LIBRARY EXCLUDE_FROM_ALL COMPONENT ${NAME} DESTINATION .
)
set(CPACK_PACKAGE_NAME ${NAME})
set(CPACK_OUTPUT_CONFIG_FILE "${CMAKE_BINARY_DIR}/BuildCPackConfigs/${NAME}.cmake")
set(CPACK_GENERATOR "ZIP")
SET(CPACK_OUTPUT_FILE_PREFIX "${TIER_ROOT_CURRENT}/pub/gen/distrib")
set(CPACK_PACKAGE_FILE_NAME ${NAME})
set(CPACK_SOURCE_ZIP OFF)
set(CPACK_ARCHIVE_COMPONENT_INSTALL ON)
set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF)
set(CPACK_ARCHIVE_FILE_NAME "")
string(TOUPPER ${NAME} UPPER_COMP_NAME)
set(CPACK_ARCHIVE_${UPPER_COMP_NAME}_FILE_NAME ${NAME})
set(CPACK_COMPONENTS_ALL ${NAME})
set(CPACK_ARCHIVE_THREADS 0)
include(CPack)
# https://gitlab.kitware.com/cmake/cmake/-/issues/8438
if(NOT TARGET distrib)
add_custom_target(distrib)
endif()
add_custom_command(OUTPUT "${CPACK_OUTPUT_FILE_PREFIX}/${NAME}.zip"
MAIN_DEPENDENCY "${CPACK_OUTPUT_CONFIG_FILE}"
COMMAND ${CMAKE_CPACK_COMMAND} --config ${CPACK_OUTPUT_CONFIG_FILE})
add_custom_target("package_${NAME}.zip" DEPENDS "${CPACK_OUTPUT_FILE_PREFIX}/${NAME}.zip")
add_dependencies("package_${NAME}.zip" ${FUNC_ARGS_TARGETS})
add_dependencies(distrib "package_${NAME}.zip")
endfunction()
Where the whole point of this issue here on gitlab is this block of code below. I don't want to have this be part of my cmakelists.txt files. I want this to be something that cmake internally understands and handles.
# Get the entire list of targets, including the top-level ones.
get_recursive_dependency_list(FULL_TARGET_LIST ${FUNC_ARGS_TARGETS})
list(PREPEND FULL_TARGET_LIST ${FUNC_ARGS_TARGETS})
list(REMOVE_DUPLICATES FULL_TARGET_LIST)
filter_target_type(FULL_TARGET_LIST STATIC_LIBRARY)
filter_target_type(FULL_TARGET_LIST INTERFACE_LIBRARY)
split_out_imported_targets(FULL_TARGET_LIST IMPORTED_TARGET_LIST)
The way each target in my cmake codebase is configured is via one or more wrapper functions that add additional processing logic to each target. That wrapper function calls this:
function(initialize_dependency_tracking TARGETNAME)
set_target_properties(${TARGETNAME} PROPERTIES
PUBLIC_DEPENDENCIES ""
PRIVATE_DEPENDENCIES ""
INTERFACE_DEPENDENCIES ""
)
endfunction()
and then each target_libraries() function is wrapped by my own, which calls
function(establish_dependency TARGETNAME TYPE)
# For each dependency being established, add it to the dependent's list for the appropriate type.
foreach(DEP ${ARGN})
set_property(TARGET ${TARGETNAME} APPEND PROPERTY
ININBUILD_${TYPE}_DEPENDENCIES ${DEP}
)
endforeach()
endfunction()
To keep track of what targets depend on what, and with what relationship.
Then, when we go to actually configure the zip file to be built, we need to grab the recursive list of dependencies
function(get_recursive_dependency_list OUTPUTVARIABLE)
set(TARGET_DEPS) # Ensure we don't somehow pull this variable from the parent scope
foreach(TARGETNAME ${ARGN})
get_target_property(PUBLIC_DEPENDENCIES ${TARGETNAME} PUBLIC_DEPENDENCIES)
get_target_property(PRIVATE_DEPENDENCIES ${TARGETNAME} PRIVATE_DEPENDENCIES)
get_target_property(INTERFACE_DEPENDENCIES ${TARGETNAME} INTERFACE_DEPENDENCIES)
list(APPEND TARGET_DEPS ${PUBLIC_DEPENDENCIES} ${PRIVATE_DEPENDENCIES} ${INTERFACE_DEPENDENCIES})
endforeach()
list(REMOVE_DUPLICATES TARGET_DEPS)
if(TARGET_DEPS) # Avoid infinite recursion
set(RECURSIVE_DEPS)
get_recursive_dependency_list(RECURSIVE_DEPS ${TARGET_DEPS})
list(APPEND TARGET_DEPS ${RECURSIVE_DEPS})
list(REMOVE_DUPLICATES TARGET_DEPS)
endif()
set(${OUTPUTVARIABLE} ${TARGET_DEPS} PARENT_SCOPE)
endfunction()
and filter out the non-installable targets from the list
function(filter_target_type LISTVAR FILTERTYPE)
set(SCOPED_LIST_VAR ${${LISTVAR}})
foreach(TARGET ${${LISTVAR}})
get_target_property(TARGET_TYPE ${TARGET} TYPE)
if(TARGET_TYPE STREQUAL FILTERTYPE)
list(REMOVE_ITEM SCOPED_LIST_VAR ${TARGET})
endif()
endforeach()
set(${LISTVAR} ${SCOPED_LIST_VAR} PARENT_SCOPE)
endfunction()
And lastly, we need to filter but not throw away the list of targets which are IMPORTED, as they must be handled specially to avoid a configuration error.
E.g.
CMake Error at cmake/modules/common.cmake:206 (install):
install TARGETS given target "libaio" which does not exist.
Which is incorrect, as libaio is in fact a target that exists, but it's mad about install(TARGETS) being treated unnecessarily different from install(IMPORTED_RUNTIME_ARTIFACTS)
function(split_out_imported_targets LISTVAR IMPORTEDLISTVAR)
set(SCOPED_LIST_VAR ${${LISTVAR}})
set(SCOPED_IMPORTED_LIST_VAR ${${IMPORTEDLISTVAR}})
foreach(TARGET ${${LISTVAR}})
get_target_property(target_imported ${TARGET} IMPORTED)
if(target_imported)
list(REMOVE_ITEM SCOPED_LIST_VAR ${TARGET})
list(APPEND SCOPED_IMPORTED_LIST_VAR ${TARGET})
endif()
endforeach()
set(${LISTVAR} ${SCOPED_LIST_VAR} PARENT_SCOPE)
set(${IMPORTEDLISTVAR} ${SCOPED_IMPORTED_LIST_VAR} PARENT_SCOPE)
endfunction()
Tying all of this together, we eventually have install/CMakeLists.txt look something like:
package_zip(zip1 TARGET_1 TARGET_2 TARGET_3)
package_zip(zip2 TARGET_2 TARGET_3 TARGET_4 TARGET_5)
...
package_zip(zipN TARGET_N TARGET_N+1 TARGET_N+2...)
package_rpm(rpmN TARGET_N TARGET_N+1 TARGET_N+2...)
package_deb(debN TARGET_N TARGET_N+1 TARGET_N+2...)
install_to_disk(folderN TARGET_N TARGET_N+1 TARGET_N+2...)
By basically abusing the install()
function's COMPONENT parameter, we're able to have a single build tree be capable of producing multiple different packages created with CPack.
So what I'm looking for here is a replacement for the whole mess of having to, myself, recursively track the list of dependencies for my targets.
install()'s RUNTIME_DEPENDENCIES parameter is useless to me, as it
- Does not work with cross compilers, which I use.
- Doesn't operate on targets. It operates on files pulled in from the build host from arbitrary locations. (Or so I think is the case. Frankly the documentation for this parameter seems to assume that people who are reading it already know what it does).
If there were some mechanism that I could use to, any of:
- install(TARGET ${TARGET_NAME} RECURSIVE_DEPENDENCIES)
- install(TARGET $<RECURSIVE_DEPENDENCIES:${TARGET_NAME}>)
- get_target_property(recursive_dependencies ${TARGET_NAME} RECURSIVE_DEPENDENCIES) # only whatever is known at the time of the call, of course
then I could cut out almost all of the code i copied into this feature request from my cmake code, and additionally I would be able to combine two different build trees into a single build tree (see #23272)
While my ideal outcome is that I be able to simply pass some kind of RECURSIVE_DEPENDENCIES option to the install(TARGET) function, any of the 3 possibilities listed above work.
Furthermore, I would very much like to see 3. as well, as I have other places in my code where knowing the full transitive closure of dependencies is used for other purposes, like ensuring necessary dlls are in the PATH for executing unit tests for CTest.
##
# Build a CMake list containing the runtime library paths of all DLLs linked by the listed targets.
# This list of paths is intended to be the list of paths inside the build system, and not after an
# installation step is run. This function is intended to be used to determine the location of any DLLs
# needed to run a program built at build-time, as part of the build, e.g. a code generator, or unit test.
# This appears to only be needed on Windows hosts. On Unix hosts, CMakes built in handling of "RPATH"
# should be sufficient.
##
function(get_runtime_library_paths TARGETS OUTPUT_VAR)
get_recursive_dependency_list(FULL_TARGET_LIST ${TARGETS})
list(PREPEND FULL_TARGET_LIST ${TARGETS})
list(REMOVE_DUPLICATES FULL_TARGET_LIST)
filter_target_type(FULL_TARGET_LIST STATIC_LIBRARY)
filter_target_type(FULL_TARGET_LIST INTERFACE_LIBRARY)
# TODO: Instead of the function call and 2 for-loops below, we could potentially do:
# list(TRANSFORM FULL_TARGET_LIST REPLACE "^(.+)$" "$<$<BOOL:$<TARGET_PROPERTY:\\1,IMPORTED>>:$<TARGET_PROPERTY:\\1,IMPORTED_LOCATION>,$<TARGET_PROPERTY:\\1,BINARY_DIR>>")
# though this *does* have the consequence of not allowing duplicates to be filtered at cmake time.
# this is because each entry in the list will be unique, since they'll have a unique target name for each
# so what we would need to do is, where this function is called (for commented out unit test code only, at the moment)
# is skip doing the list-join in cmake, and instead use generator expressions for $<REMOVE_DUPLICATES> and then $<JOIN>
# so that 100% of the logic for how to transform the text happens inside the generator, instead of cmake.
#
# Frankly i'm not sure that using generator expressions for this is worth it, because the complexity of describing the
# transformation in the generator expression syntax is just so absolutely terrible and hard to debug.
# and really, i'm not sure it actually saves us any real CPU time anyway.
split_out_imported_targets(FULL_TARGET_LIST IMPORTED_TARGET_LIST)
set(LIBRARY_PATHS)
foreach(CUR_TARGET ${FULL_TARGET_LIST})
get_target_property(TARGET_LIBRARY_PATH ${CUR_TARGET} BINARY_DIR)
list(APPEND LIBRARY_PATHS "${TARGET_LIBRARY_PATH}")
endforeach()
foreach(CUR_TARGET ${IMPORTED_TARGET_LIST})
get_target_property(TARGET_LIBRARY_PATH ${CUR_TARGET} IMPORTED_LOCATION)
list(APPEND LIBRARY_PATHS "${TARGET_LIBRARY_PATH}")
endforeach()
list(REMOVE_DUPLICATES LIBRARY_PATHS)
set(${OUTPUT_VAR} ${LIBRARY_PATHS} PARENT_SCOPE)
endfunction()
# On windows, unit tests need the environment variable PATH set properly to be able to run.
# This is a major failing of windows, the lack of RPATH support, but its just what it is.
#
# See toplevel/CMakeLists.txt where we set CMAKE_LIBRARY_OUTPUT_DIRECTORY and CMAKE_RUNTIME_OUTPUT_DIRECTORY
# to work around this problem, as well as visual studio integration issues.
#
# There's also the property VS_DEBUGGER_ENVIRONMENT but that has lots of problems with generating valid strings...
if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
get_runtime_library_paths(${EXENAME} RUNTIME_LIB_PATHS)
list(TRANSFORM RUNTIME_LIB_PATHS REPLACE "^(.+)$" "PATH=path_list_prepend:\\1")
set_tests_properties(${EXENAME} PROPERTIES ENVIRONMENT_MODIFICATION "${RUNTIME_LIB_PATHS}")
endif()
As you can see above, my codebase has a lot of code written to deal with this limitation of the CMake handling of relationships between targets, and having the already known target relationships exposed directly to code in CMakeLists.txt would be very helpful.