From d1c7d808587995f9039d35ef5c1bfddd89e97d1c Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Thu, 1 Mar 2018 16:10:35 -0500 Subject: [PATCH 01/34] cmake: remove unused file --- cmake/CrossCompilationMacros.cmake | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cmake/CrossCompilationMacros.cmake diff --git a/cmake/CrossCompilationMacros.cmake b/cmake/CrossCompilationMacros.cmake deleted file mode 100644 index e69de29b..00000000 -- GitLab From 7850d41fe1152b96e57de6f92f76d5aa572a1200 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Fri, 2 Mar 2018 10:03:05 -0500 Subject: [PATCH 02/34] cmake: remove deprecated functions These functions aren't used anymore by the extant superbuilds. Since they've been replaced, we can just remove them. --- cmake/SuperbuildPackageMacros.cmake | 24 ----------------- cmake/SuperbuildRevisionMacros.cmake | 40 ---------------------------- 2 files changed, 64 deletions(-) diff --git a/cmake/SuperbuildPackageMacros.cmake b/cmake/SuperbuildPackageMacros.cmake index c0306e36..699b8601 100644 --- a/cmake/SuperbuildPackageMacros.cmake +++ b/cmake/SuperbuildPackageMacros.cmake @@ -112,27 +112,3 @@ function (superbuild_enable_install_target default) COMPONENT install) endif () endfunction () - -# DEPRECATED -# Adds a test to package the top-level superbuild. -# -# This is deprecated because CPack will rerun the build of the top-level -# project. Since superbuilds never have a "do-nothing" build in the presense of -# Git repositories, it is not recommended to use this. -function (superbuild_add_package_test generator) - message(AUTHOR_WARNING - "superbuild_add_package_test: This function is deprecated; " - "use the newer superbuild_add_extra_package_test mechanism instead.") - - add_test( - NAME "cpack-${generator}" - COMMAND "${CMAKE_CPACK_COMMAND}" - -V - -G "${generator}" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}") - - set_tests_properties("cpack-${generator}" - PROPERTIES - RESOURCE_LOCK cpack - ${ARGN}) -endfunction () diff --git a/cmake/SuperbuildRevisionMacros.cmake b/cmake/SuperbuildRevisionMacros.cmake index 508aa13d..d6c36cd1 100644 --- a/cmake/SuperbuildRevisionMacros.cmake +++ b/cmake/SuperbuildRevisionMacros.cmake @@ -66,46 +66,6 @@ function (superbuild_set_customizable_revision name) ${_args_UNPARSED_ARGUMENTS}) endfunction () -# Convenient way to declare a main project's source. -# -# Usage: -# -# superbuild_set_external_source(<name> -# <git-url> <git-ref> -# <tarball-url> <tarball-md5>) -# -# Adds options to build the project from a git repository, a tarball, or a -# source tree (linked from the source tree as -# ``${CMAKE_SOURCE_DIR}/source-${name}``). Usually relevant for the "primary" -# project(s) in a single superbuild. -function (superbuild_set_external_source name git_repo git_tag tarball_url tarball_md5) - option("${name}_FROM_GIT" "If enabled, fetch sources from GIT" ON) - cmake_dependent_option("${name}_FROM_SOURCE_DIR" "Use an existing source directory" OFF - "NOT ${name}_FROM_GIT" OFF) - - set(args) - if (${name}_FROM_GIT) - set(args - GIT_REPOSITORY "${git_repo}" - GIT_TAG "${git_tag}") - elseif (${name}_FROM_SOURCE_DIR) - set(args - SOURCE_DIR "${CMAKE_SOURCE_DIR}/source-${name}") - else () - set(args - URL "${tarball_url}" - URL_MD5 "${tarball_md5}") - endif () - - superbuild_set_customizable_revision("${name}" - ${args}) - - # Push the cmake_dependent_option to the parent scope. - set("${name}_FROM_SOURCE_DIR" - "${${name}_FROM_SOURCE_DIR}" - PARENT_SCOPE) -endfunction () - # A way to provide selections for a project's source. # # Usage: -- GitLab From 8a120cf130f29160924ba167b52ef74052f6f185 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:02:17 -0500 Subject: [PATCH 03/34] SuperbuildCrossMacros: use an uppercase variable for user settings --- cmake/SuperbuildCrossMacros.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/SuperbuildCrossMacros.cmake b/cmake/SuperbuildCrossMacros.cmake index 184a912b..de6ebc8b 100644 --- a/cmake/SuperbuildCrossMacros.cmake +++ b/cmake/SuperbuildCrossMacros.cmake @@ -21,9 +21,9 @@ endfunction () # right build hints and patches later on. # function (_superbuild_cross_target_machine) - set(cross_target "generic" + set(SUPERBUILD_CROSS_TARGET "generic" CACHE STRING "Platform to cross compile for, either generic|bgp_xlc|bgq_xlc|bgq_gnu|xk7_gnu") - set_property(CACHE cross_target PROPERTY STRINGS + set_property(CACHE SUPERBUILD_CROSS_TARGET PROPERTY STRINGS "generic" "bgp_xlc" "bgq_xlc" "bgq_gnu" "xk7_gnu") endfunction () @@ -31,7 +31,7 @@ endfunction () # Includes an optionally site-specific file from the cross-compiling directory. function (_superbuild_cross_include_file var name) set(site_file - "crosscompile/${cross_target}/${name}.cmake") + "crosscompile/${SUPERBUILD_CROSS_TARGET}/${name}.cmake") include("${site_file}" OPTIONAL RESULT_VARIABLE res) if (NOT res) @@ -49,7 +49,7 @@ endfunction () # function (_superbuild_cross_platform_settings) set(site_toolchain - "${CMAKE_CURRENT_LIST_DIR}/crosscompile/${cross_target}/toolchain.cmake.in") + "${CMAKE_CURRENT_LIST_DIR}/crosscompile/${SUPERBUILD_CROSS_TARGET}/toolchain.cmake.in") set(superbuild_cross_toolchain "${CMAKE_BINARY_DIR}/crosscompile/toolchain.cmake") -- GitLab From b685ded480ea7c48b9dc7144f96e7c1a9a7de1fc Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:02:38 -0500 Subject: [PATCH 04/34] SuperbuildCrossMacros: use crosscompile settings from the main project --- cmake/SuperbuildCrossMacros.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/SuperbuildCrossMacros.cmake b/cmake/SuperbuildCrossMacros.cmake index de6ebc8b..acabb32e 100644 --- a/cmake/SuperbuildCrossMacros.cmake +++ b/cmake/SuperbuildCrossMacros.cmake @@ -23,6 +23,7 @@ endfunction () function (_superbuild_cross_target_machine) set(SUPERBUILD_CROSS_TARGET "generic" CACHE STRING "Platform to cross compile for, either generic|bgp_xlc|bgq_xlc|bgq_gnu|xk7_gnu") + # TODO: This should be managed by the project, not hard-coded. set_property(CACHE SUPERBUILD_CROSS_TARGET PROPERTY STRINGS "generic" "bgp_xlc" "bgq_xlc" "bgq_gnu" "xk7_gnu") endfunction () @@ -49,7 +50,7 @@ endfunction () # function (_superbuild_cross_platform_settings) set(site_toolchain - "${CMAKE_CURRENT_LIST_DIR}/crosscompile/${SUPERBUILD_CROSS_TARGET}/toolchain.cmake.in") + "${CMAKE_SOURCE_DIR}/crosscompile/${SUPERBUILD_CROSS_TARGET}/toolchain.cmake.in") set(superbuild_cross_toolchain "${CMAKE_BINARY_DIR}/crosscompile/toolchain.cmake") -- GitLab From 078ae490161ca194d54649745b6eddf0cf135b1f Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:07:23 -0500 Subject: [PATCH 05/34] SuperbuildRevisionMacros: hide customizable revision This function should no longer be used outside of being an implementation detail. --- cmake/SuperbuildRevisionMacros.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/SuperbuildRevisionMacros.cmake b/cmake/SuperbuildRevisionMacros.cmake index d6c36cd1..8020bcf1 100644 --- a/cmake/SuperbuildRevisionMacros.cmake +++ b/cmake/SuperbuildRevisionMacros.cmake @@ -25,14 +25,14 @@ endfunction () # # Usage: # -# superbuild_set_customizable_revision(<name> <args>...) +# _superbuild_set_customizable_revision(<name> <args>...) # # Adds advanced variables for the following keys: # # GIT_REPOSITORY, GIT_TAG, URL, URL_HASH, URL_MD5, and SOURCE_DIR # # they are named ``${name_UPPER}_${key}``. -function (superbuild_set_customizable_revision name) +function (_superbuild_set_customizable_revision name) set(keys GIT_REPOSITORY GIT_TAG URL URL_HASH URL_MD5 @@ -315,7 +315,7 @@ function (superbuild_set_selectable_source name) superbuild_set_revision("${name}" ${selection_${selection}_args}) else () - superbuild_set_customizable_revision("${name}" + _superbuild_set_customizable_revision("${name}" ${selection_${selection}_args}) endif () endfunction () -- GitLab From 7e986454961b63ddc378c87db34ba9328aa11827 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:08:13 -0500 Subject: [PATCH 06/34] SuperbuildRevisionMacros: drop some deprecated names --- cmake/SuperbuildRevisionMacros.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/SuperbuildRevisionMacros.cmake b/cmake/SuperbuildRevisionMacros.cmake index 8020bcf1..c21f58e2 100644 --- a/cmake/SuperbuildRevisionMacros.cmake +++ b/cmake/SuperbuildRevisionMacros.cmake @@ -39,7 +39,6 @@ function (_superbuild_set_customizable_revision name) SOURCE_DIR) cmake_parse_arguments(_args "" "${keys}" "" ${ARGN}) set(customized_args) - string(TOUPPER "${name}" name_UPPER) foreach (key IN LISTS keys) if (_args_${key}) @@ -48,8 +47,7 @@ function (_superbuild_set_customizable_revision name) if (key STREQUAL "SOURCE_DIR") set(cache_type PATH) endif () - superbuild_deprecated_setting(option_default "${option_name}" "${name_UPPER}_${key}" "${_args_${key}}") - set("${option_name}" "${option_default}" + set("${option_name}" "${_args_${key}}" CACHE "${cache_type}" "${key} for project '${name}'") if (NOT key STREQUAL "SOURCE_DIR") mark_as_advanced(${option_name}) -- GitLab From 1f19cd1de9dce711dadb1f13a36147bd0a85547d Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:28:30 -0500 Subject: [PATCH 07/34] SuperbuildTestingMacros: bail if a test glob isn't known for a generator --- cmake/SuperbuildTestingMacros.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/SuperbuildTestingMacros.cmake b/cmake/SuperbuildTestingMacros.cmake index b3738e8c..b5dcf45f 100644 --- a/cmake/SuperbuildTestingMacros.cmake +++ b/cmake/SuperbuildTestingMacros.cmake @@ -7,6 +7,12 @@ function (superbuild_add_extract_test name glob_prefix generator output) set(_DragNDrop_test_glob "${glob_prefix}*.dmg") set(_TGZ_test_glob "${glob_prefix}*.tar.gz") + if (NOT DEFINED _${generator}_test_glob) + message(FATAL_ERROR + "No known glob to find packages created by the ${generator} CPack " + "generator.") + endif () + add_test( NAME "extract-${name}-${generator}" COMMAND "${CMAKE_COMMAND}" -- GitLab From d4297bdc4bfc031b178d510da6f9eb7be037bcc9 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Sat, 10 Mar 2018 16:58:57 -0500 Subject: [PATCH 08/34] SuperbuildMacros: use a `_` prefix for help string overrides --- cmake/SuperbuildMacros.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/SuperbuildMacros.cmake b/cmake/SuperbuildMacros.cmake index 4db049d1..33f9b9ca 100644 --- a/cmake/SuperbuildMacros.cmake +++ b/cmake/SuperbuildMacros.cmake @@ -108,8 +108,8 @@ function (superbuild_add_project name) endforeach () # Allow projects to override the help string specified in the project file. - if (DEFINED "superbuild_help_string_${name}") - set(help_string "${superbuild_help_string_${name}}") + if (DEFINED "_superbuild_help_string_${name}") + set(help_string "${_superbuild_help_string_${name}}") endif () if (NOT help_string) -- GitLab From 76d17a2d7fc7436af71a448a75a07de5db6e6e9b Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Sat, 10 Mar 2018 16:59:15 -0500 Subject: [PATCH 09/34] SuperbuildMacros: remove vestigal code This has been commented for years. Just remove it instead. --- cmake/SuperbuildMacros.cmake | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmake/SuperbuildMacros.cmake b/cmake/SuperbuildMacros.cmake index 33f9b9ca..9e16f57c 100644 --- a/cmake/SuperbuildMacros.cmake +++ b/cmake/SuperbuildMacros.cmake @@ -827,12 +827,6 @@ function (_superbuild_add_project_internal name) CFLAGS "${project_c_flags} ${project_c_flags_buildtype}") endif () - if (APPLE) - # disabling this since it fails when building numpy. - #list(APPEND build_env - # MACOSX_DEPLOYMENT_TARGET "${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif () - list(INSERT extra_paths 0 "${superbuild_install_location}/bin") list(REMOVE_DUPLICATES extra_paths) -- GitLab From a82f19c51a072076624431eb77e4b842f333cf47 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:31:42 -0500 Subject: [PATCH 10/34] fixup_bundle.apple: remove unnecessary `create_` from ctor names --- cmake/scripts/fixup_bundle.apple.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmake/scripts/fixup_bundle.apple.py b/cmake/scripts/fixup_bundle.apple.py index c3182bd7..079708d7 100755 --- a/cmake/scripts/fixup_bundle.apple.py +++ b/cmake/scripts/fixup_bundle.apple.py @@ -222,7 +222,7 @@ class Library(object): if self._dependencies is None: collection = {} for dep in self._get_dependencies(): - deplib = Library.create_from_reference(dep, self) + deplib = Library.from_reference(dep, self) if deplib is not None and \ not deplib.path == self.path: collection[dep] = deplib @@ -238,33 +238,33 @@ class Library(object): return ref @classmethod - def create_from_reference(cls, ref, loader): + def from_reference(cls, ref, loader): paths = [ref] - if ref.startswith('@executable_path'): + if ref.startswith('@executable_path/'): # If the loader does not have an executable path, it is a plugin or # a framework and we trust the executable which loads the plugin to # provide this library instead. if loader.executable_path is None: return None paths.append(ref.replace('@executable_path', loader.executable_path)) - elif ref.startswith('@loader_path'): + elif ref.startswith('@loader_path/'): paths.append(ref.replace('@loader_path', loader.loader_path)) - elif ref.startswith('@rpath'): + elif ref.startswith('@rpath/'): for rpath in loader.rpaths: paths.append(ref.replace('@rpath', rpath)) paths.append(os.path.join(os.path.dirname(loader.path), ref)) for path in paths: if os.path.exists(path): - return cls.create_from_path(path, parent=loader) + return cls.from_path(path, parent=loader) search_path = loader._find_library(ref) if os.path.exists(search_path): - return cls.create_from_path(search_path, parent=loader) + return cls.from_path(search_path, parent=loader) raise RuntimeError('Unable to find the %s library from %s' % (ref, loader.path)) __cache = {} @classmethod - def create_from_path(cls, path, parent=None): + def from_path(cls, path, parent=None): if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -280,7 +280,7 @@ class Library(object): return cls.__cache[path] @classmethod - def create_from_manifest(cls, path, installed_id): + def from_manifest(cls, path, installed_id): if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -441,7 +441,7 @@ def _os_makedirs(path): os.makedirs(path) -def _create_arg_parser(): +def _arg_parser(): import argparse parser = argparse.ArgumentParser(description='Install an OS X binary into a bundle') @@ -553,7 +553,7 @@ def _update_manifest(manifest, installed, path): def main(args): - parser = _create_arg_parser() + parser = _arg_parser() opts = parser.parse_args(args) if opts.type == 'executable': @@ -610,7 +610,7 @@ def main(args): # Seed the cache with manifest entries. for path, installed_id in manifest.items(): - Library.create_from_manifest(path, installed_id) + Library.from_manifest(path, installed_id) installed = {} _install_binary(main_exe, is_excluded, bundle_dest, installed, manifest, dry_run=opts.dry_run) -- GitLab From 2493f9e0910cbc911d80b890b596377b0883c1ce Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:33:10 -0500 Subject: [PATCH 11/34] fixup_bundle.unix: handle ${ORIGIN} as well Also update the TODO to mention that `${LIB}` and `${PLATFORM}` are not really necessary for the superbuild's use cases. --- cmake/scripts/fixup_bundle.unix.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/scripts/fixup_bundle.unix.py b/cmake/scripts/fixup_bundle.unix.py index d2edfd54..af885dcc 100755 --- a/cmake/scripts/fixup_bundle.unix.py +++ b/cmake/scripts/fixup_bundle.unix.py @@ -114,10 +114,14 @@ class Library(object): return loader_paths def _resolve_rpath(self, rpath): - if rpath.startswith('$ORIGIN'): + if rpath.startswith('$ORIGIN/'): rpath = rpath.replace('$ORIGIN', self.loader_path) + elif rpath.startswith('${ORIGIN}/'): + rpath = rpath.replace('${ORIGIN}', self.loader_path) - # TODO: Handle $LIB and $PLATFORM (also ${} forms). + # TODO: Handle $LIB and $PLATFORM. These really shouldn't be used in + # superbuilds however; libraries should all be under a plain ``lib`` + # directory. return rpath.strip() -- GitLab From d1dbebf0cdd1294035f4662c1d3e8bce40387f17 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:31:42 -0500 Subject: [PATCH 12/34] fixup_bundle.unix: remove unnecessary `create_` from ctor names --- cmake/scripts/fixup_bundle.unix.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/scripts/fixup_bundle.unix.py b/cmake/scripts/fixup_bundle.unix.py index af885dcc..d6a264b0 100755 --- a/cmake/scripts/fixup_bundle.unix.py +++ b/cmake/scripts/fixup_bundle.unix.py @@ -203,7 +203,7 @@ class Library(object): if self._dependencies is None: collection = {} for dep in self._get_dependencies(): - deplib = Library.create_from_reference(dep, self) + deplib = Library.from_reference(dep, self) if deplib is not None: collection[dep] = deplib self._dependencies = collection @@ -241,12 +241,12 @@ class Library(object): return cls.__search_cache @classmethod - def create_from_reference(cls, ref, loader): + def from_reference(cls, ref, loader): paths = [] if '/' in ref: if os.path.exists(ref): - return cls.create_from_path(ref, parent=loader) + return cls.from_path(ref, parent=loader) else: # TODO: Automatically search in the libdir we are placing dependent libraries. @@ -269,17 +269,17 @@ class Library(object): for path in paths: libpath = os.path.join(path, ref) if os.path.exists(libpath): - return cls.create_from_path(libpath, parent=loader) + return cls.from_path(libpath, parent=loader) search_path = loader._find_library(ref) if os.path.exists(search_path): - return cls.create_from_path(search_path, parent=loader) + return cls.from_path(search_path, parent=loader) raise RuntimeError('Unable to find the %s library from %s: %s' % (ref, loader.path, ', '.join(paths))) __cache = {} @classmethod - def create_from_path(cls, path, parent=None): + def from_path(cls, path, parent=None): if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -295,7 +295,7 @@ class Library(object): return cls.__cache[path] @classmethod - def create_from_manifest(cls, path): + def from_manifest(cls, path): if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -446,7 +446,7 @@ def _os_makedirs(path): os.makedirs(path) -def _create_arg_parser(): +def _arg_parser(): import argparse parser = argparse.ArgumentParser(description='Install an ELF binary into a bundle') @@ -543,7 +543,7 @@ def _update_manifest(manifest, installed, path, libdir): def main(args): - parser = _create_arg_parser() + parser = _arg_parser() opts = parser.parse_args(args) if opts.type == 'executable': @@ -591,7 +591,7 @@ def main(args): # Seed the cache with manifest entries. for path in manifest.get(opts.libdir, []): - Library.create_from_manifest(path) + Library.from_manifest(path) cur_manifest = manifest.setdefault(opts.libdir, []) -- GitLab From 4ad381f5d59df0c3649f6b2bc8a78378a439a5f9 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:31:42 -0500 Subject: [PATCH 13/34] fixup_bundle.windows: remove unnecessary `create_` from ctor names --- cmake/scripts/fixup_bundle.windows.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/scripts/fixup_bundle.windows.py b/cmake/scripts/fixup_bundle.windows.py index b6c6f6fc..d561fb91 100755 --- a/cmake/scripts/fixup_bundle.windows.py +++ b/cmake/scripts/fixup_bundle.windows.py @@ -103,7 +103,7 @@ class Library(object): for dep in self._get_dependencies(): if msvc_runtimes.match(dep): continue - deplib = Library.create_from_reference(dep, self) + deplib = Library.from_reference(dep, self) if deplib is not None: collection[dep] = deplib self._dependencies = collection @@ -119,7 +119,7 @@ class Library(object): __search_cache = None @classmethod - def create_from_reference(cls, ref, loader): + def from_reference(cls, ref, loader): paths = [] # Use the loader's location. @@ -137,17 +137,17 @@ class Library(object): for path in paths: libpath = os.path.join(path, ref) if os.path.exists(libpath): - return cls.create_from_path(libpath, parent=loader) + return cls.from_path(libpath, parent=loader) search_path = loader._find_library(ref) if os.path.exists(search_path): - return cls.create_from_path(search_path, parent=loader) + return cls.from_path(search_path, parent=loader) raise RuntimeError('Unable to find the %s library from %s: %s' % (ref, loader.path, ', '.join(paths))) __cache = {} @classmethod - def create_from_path(cls, path, parent=None): + def from_path(cls, path, parent=None): if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -163,7 +163,7 @@ class Library(object): return cls.__cache[path] @classmethod - def create_from_manifest(cls, path): + def from_manifest(cls, path): if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -216,7 +216,7 @@ def _os_makedirs(path): os.makedirs(path) -def _create_arg_parser(): +def _arg_parser(): import argparse parser = argparse.ArgumentParser(description='Install an ELF binary into a bundle') @@ -300,7 +300,7 @@ def _update_manifest(manifest, installed, path, location): def main(args): - parser = _create_arg_parser() + parser = _arg_parser() opts = parser.parse_args(args) if opts.type == 'executable': @@ -348,7 +348,7 @@ def main(args): manifest_dirs = set(opts.binary_libdir + [libdir,]) for mdir in manifest_dirs: for path in manifest.get(mdir, []): - Library.create_from_manifest(path) + Library.from_manifest(path) cur_manifest = manifest.setdefault(opts.libdir, []) -- GitLab From 50f4fcf3edc9fd517581792c3e7e8cb18ca79270 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 18:26:58 -0500 Subject: [PATCH 14/34] SuperbuildUtils: hide an internal function --- cmake/SuperbuildUtils.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/SuperbuildUtils.cmake b/cmake/SuperbuildUtils.cmake index d847d0e5..585003cc 100644 --- a/cmake/SuperbuildUtils.cmake +++ b/cmake/SuperbuildUtils.cmake @@ -74,25 +74,25 @@ function (superbuild_detect_64bit_target) endif () endfunction () -macro(superbuild_make_path_var var) +macro (_superbuild_make_path_var var) set(${var} ${ARGN}) list(REMOVE_ITEM ${var} "") if (UNIX) string(REPLACE ";" ":" ${var} "${${var}}") endif () -endmacro() +endmacro () function (superbuild_setup_flags) if (WIN32) return () endif () - superbuild_make_path_var(superbuild_ld_library_path + _superbuild_make_path_var(superbuild_ld_library_path "${superbuild_install_location}/lib" "$ENV{LD_LIBRARY_PATH}") set(superbuild_ld_library_path "${superbuild_ld_library_path}" PARENT_SCOPE) - superbuild_make_path_var(superbuild_pkg_config_path + _superbuild_make_path_var(superbuild_pkg_config_path "${superbuild_install_location}/lib/pkgconfig" "${superbuild_install_location}/share/pkgconfig" "$ENV{PKG_CONFIG_PATH}") -- GitLab From 51f6dec07458d88c69f02b0475ac241e4b699042 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 14 Mar 2018 13:35:51 -0400 Subject: [PATCH 15/34] SuperbuildUtils: use message(DEPRECATION) --- cmake/SuperbuildUtils.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/SuperbuildUtils.cmake b/cmake/SuperbuildUtils.cmake index 585003cc..c0c7f600 100644 --- a/cmake/SuperbuildUtils.cmake +++ b/cmake/SuperbuildUtils.cmake @@ -146,7 +146,8 @@ endmacro () function (superbuild_deprecated_setting output_default new old intended_default) set(default "${intended_default}") if (DEFINED "${old}") - message(WARNING "The '${old}' variable is deprecated for '${new}'.") + message(DEPRECATION + "The `${old}` variable is deprecated for `${new}`.") set(default "${${old}}") endif () -- GitLab From 0498464cbeeab75ba698f4b93074f0ab6fd306bf Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:37:53 -0500 Subject: [PATCH 16/34] docs: document user-facing variables and targets These are variables which are common to all superbuilds provided by the common-superbuild's infrastructure. --- cmake/SuperbuildUserVariables.md | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 cmake/SuperbuildUserVariables.md diff --git a/cmake/SuperbuildUserVariables.md b/cmake/SuperbuildUserVariables.md new file mode 100644 index 00000000..c1da02d3 --- /dev/null +++ b/cmake/SuperbuildUserVariables.md @@ -0,0 +1,85 @@ +# Superbuild user variables + +## Targets + + - `download-all` This target causes the download steps of all projects to be + run. This essentially creates a cache of source files so that a build can + be run without network access. See `SUPERBUILD_OFFLINE_BUILD` for handling + VCS repositories in offline mode. + +## Building + +### `CMAKE_BUILD_TYPE_<PROJECT>` + +Some projects may support having their build configuration chosen independently +of the overall superbuild. If so, this variable is exposed to change its value. +By default, the `<same>` value is used which uses the same build type as the +superbuild (`CMAKE_BUILD_TYPE`). + +### `DEVELOPER_MODE_<PROJECT>` + +If a project is enabled and supports developer mode, this variable controls +whether it uses the developer mode or not. When a project is in developer mode, +it is not built, but instead it is treated as "enabled" and an associated +`<PROJECT>-developer-config.cmake` file is written into the build directory of +the superbuild. This may be used with `-C` on a standalone build of the +associated project to have it use the superbuild dependencies. Note that +updating environment variables such as `PATH`, `LD_LIBRARY_PATH`, +`DYLD_LIBRARY_PATH`, or `PKG_CONFIG_PATH` to include the relevant paths in the +superbuild install prefix may be required in order to use the resulting build +tree. + +### `ENABLE_<PROJECT>` + +If set, the associated project will be built. All dependent required projects +are enabled as well. + +### `SUPERBUILD_DEFAULT_INSTALL` + +If the project creates an install target, it is installed as a specific package +layout. This variable allows for the selection of which package should be +installed. + +### `SUPERBUILD_OFFLINE_BUILD` + +If set, all implicit updates of direct VCS repositories will be disabled. This +means that if a project is watching the `master` branch of its source +repository, a reconfigure will not trigger a `git fetch` unless the source +directory does not exist at all. This is useful when performing builds on +machines with limited network access. + +### `SUPERBUILD_PROJECT_PARALLELISM` + +This defaults to the number of CPUs on the machine. It is used as the `-j` +argument for `Makefiles`-based CMake generators. If the number of CPUs cannot +be determined, a reasonably beefy machine is assumed (8 cores). + +### `SUPPRESS_<PROJECT>_OUTPUT` + +If set, output from the build and install steps of the given project will be +logged to a file instead of appearing in the main build output. This can be +used to hide project warnings and errors from CDash and interactive builds. + +### `USE_SYSTEM_<PROJECT>` + +Some projects may support finding an external ("system") copy of the project. +Those that do offer this variable which, if set, directs the project to find an +external copy of itself. + +## Advanced variables + +These variables are workarounds or control internal features. They should only +be used if needed. If these end up being necessary for your builds, please open +a report on the project's page so that we can better track down when these +workarounds are necessary. + +### `PASS_LD_LIBRARY_PATH_FOR_BUILDS` + +On Linux, there may be cases where errors of the form: + +``` +libz.so.1: no version information available +``` + +may appear. In this case, the `PASS_LD_LIBRARY_PATH_FOR_BUILDS` variable should +be turned off. -- GitLab From 2d6718dd113f04a50b3dd2deaa9893f243e448bf Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:39:04 -0500 Subject: [PATCH 17/34] docs: document variables internal to superbuilds These variables may be used by superbuilds to control internal details, but are not generally user-facing. --- cmake/SuperbuildVariables.md | 119 +++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 cmake/SuperbuildVariables.md diff --git a/cmake/SuperbuildVariables.md b/cmake/SuperbuildVariables.md new file mode 100644 index 00000000..c9f32b77 --- /dev/null +++ b/cmake/SuperbuildVariables.md @@ -0,0 +1,119 @@ +# Superbuild variables + +There are many variables that are used within the common superbuild +infrastructure to influence parts of the build that are project-controlled +rather than user-controlled. This documentation covers these variables. + +## Projects + +### `superbuild_build_phase` + +This variable indicates whether the configure is in the "scan" phase where +dependencies are gathered or the "build" phase where the actual commands to be +used are created. During the "build" phase, the following variables are available: + + - `<PROJECT>_enabled` If set, the project is enabled in the superbuild. + - `<PROJECT>_built_by_superbuild` If set, the project is also being built by + the superbuild (versus being externally provided). + +The values of these variables may only be trusted if `superbuild_build_phase` +is set. Generally, this means that they should not be used in conditionals to +determine dependencies of a project. Instead, triangle dependencies ("`A` +depends on `B` because `C` is enabled") should be handled by having an optional +dependency from `A` on `B` and `C` and if `superbuild_build_phase` is set, +erroring out if `C_enabled` is set and `B_enabled` is not. + +### `_superbuild_help_string_<PROJECT>` + +Instead of the default help string for the `ENABLE_<PROJECT>` option, a more +descriptive name may be provided. + +### `superbuild_install_location` + +This variable contains the path that will be given as the installation prefix +to all projects when building and installing. It is also where packaging steps +will look for files to install. If not specified, it defaults to +`${CMAKE_BINARY_DIR}/install`. + +### `_superbuild_<PROJECT>_selectable` + +If set, the project `PROJECT` is treated as if it has a `SELECTABLE` keyword. +See the documentation for [`superbuild_add_project`][superbuild_add_project]. + +[superbuild_add_project]: SuperbuildMacros.md#Adding-a-project-to-the-build + +### `_superbuild_list_separator` + +If a project needs to pass `;` as a flag to a build command, instead, this +variable should be used. + +## Building + +### `_superbuild_<PROJECT>_default_selection` + +For projects which use a source selection, the default selection may be +overriden from the top-level by specifying the selection with this variable. + +### `_superbuild_suppress_<PROJECT>_output` + +If defined, this is used as the default value for the +`SUPPRESS_<PROJECT>_OUTPUT` variable for the given project. + +### `superbuild_is_64bit` + +Whether the build is targeting a 32-bit build or 64-bit build. May be removed +in the future if 32-bit support is removed. + +Not available during cross-compiling. + +### Language flags + +The following variables contain (as a string, not a list), the flags to pass to +projects using `CFLAGS`, `CXXFLAGS`, and `LDFLAGS` as environment variables or +through CMake's cache variables. + + - `superbuild_c_flags` + - `superbuild_cpp_flags` + - `superbuild_cxx_flags` + - `superbuild_ld_flags` + +Superbuild projects may add flags to these by using: + + - `superbuild_extra_c_flags` + - `superbuild_extra_cpp_flags` + - `superbuild_extra_cxx_flags` + +On Windows, these variables are not supported. + +## Testing + +### `superbuild_ctest_custom_files` + +This is a list of paths to files which are included from the +`CTestCustom.cmake` file. It is read by CTest. Please see the +[CTest variable documentation][] for details (the `CTEST_CUSTOM_` variables). + +[CTest variable documentation]: https://cmake.org/cmake/help/git-master/manual/cmake-variables.7.html#variables-for-ctest + +## Packaging + +These variables are used in packaging tests (mainly `.bundle.cmake` files). + +### `superbuild_bundle_skip_system_libraries` + +If set in a `bundle.cmake` file, it can be used to skip the +`InstallRequiredSystemLibraries` bit implicit in a packaging test. + +### `superbuild_is_install_target` + +If set, the package is being installed to the host system, not into a package. +Use of this is mostly internal, but it is available if necessary. + +### Variables forwarded from the build + +These variables are provided by the superbuild during the packaging step. + + - `superbuild_install_location` + - `<PROJECT>_enabled` + - `<PROJECT>_built_by_superbuild` + - `USE_SYSTEM_<PROJECT>` -- GitLab From 777520f4d03776d4d551a7c2470ec7652fe32baa Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:01:18 -0500 Subject: [PATCH 18/34] docs: document CTestCustom.cmake --- cmake/CTestCustom.cmake.in | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/cmake/CTestCustom.cmake.in b/cmake/CTestCustom.cmake.in index 034a4beb..e697abe2 100644 --- a/cmake/CTestCustom.cmake.in +++ b/cmake/CTestCustom.cmake.in @@ -1,3 +1,10 @@ +#[==[.md +# `CTestCustom.cmake` + +This is a trampoline CTest custom file which includes any other +`CTestCustom.cmake` files provided by a superbuild. +#]==] + # INTERNAL # Sets up CTest's variables to ignore messages from a project. function (_project_ignore_regexes variable project) @@ -21,20 +28,27 @@ function (_project_ignore_regexes variable project) PARENT_SCOPE) endfunction () -# Ignore warnings from a project. -# -# Usage: -# -# ignore_project_warnings(<name>) +#[==[.md +## Ignoring project outputs + +Since superbuilds build many third party projects, it can be useful to ignore +all warnings and errors coming from a project. Note that these only suppress +from appearing on CDash. To completely silence a project even when building +interactively, it is better to use `SUPPRESS_<project>_OUTPUT` variables. + +Real errors will still fail the build, but output of configure-time failures +will not show up in CDash. + +``` +ignore_project_warnings(<project>) +ignore_project_errors(<project>) +``` +#]==] + macro (ignore_project_warnings project) _project_ignore_regexes(CTEST_CUSTOM_WARNING_EXCEPTION "${project}") endmacro () -# Ignore errors from a project. -# -# Usage: -# -# ignore_project_errors(<name>) macro (ignore_project_errors project) _project_ignore_regexes(CTEST_CUSTOM_ERROR_EXCEPTION "${project}") endmacro () -- GitLab From fe0684237577d02d720cc144592e0e6c96460b4a Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:01:58 -0500 Subject: [PATCH 19/34] docs: document SuperbuildCrossMacros --- cmake/SuperbuildCrossMacros.cmake | 41 ++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/cmake/SuperbuildCrossMacros.cmake b/cmake/SuperbuildCrossMacros.cmake index acabb32e..b1e6881f 100644 --- a/cmake/SuperbuildCrossMacros.cmake +++ b/cmake/SuperbuildCrossMacros.cmake @@ -1,3 +1,25 @@ +#[==[.md +# Cross compilation support + +This is untested, but exists from prior support in a previous implemenation of +the superbuild. Most of them are intended to be used by the common superbuild +itself, but there are some hooks projects can set. + +It seems that the module is largely ignored for now. +#]==] + +#[==[.md +## Hooks + +### `superbuild_cross_prepare` + +If defined, this command is called in order to to do any up-front configuration +settings which may be necessary. +#]==] + +# Sets up the build for a particular target platform. +# +# Currently unused. function (superbuild_cross_determine_target) if (NOT CMAKE_CROSSCOMPILING) return () @@ -7,8 +29,7 @@ function (superbuild_cross_determine_target) # right environment settings. _superbuild_cross_target_machine() - # Configure the platform dependent settings 64bit_build, static_only, mpi - # search path. + # Configure platform dependent settings 64bit_build, static_only. _superbuild_cross_platform_settings() if (COMMAND superbuild_cross_prepare) @@ -16,10 +37,7 @@ function (superbuild_cross_determine_target) endif () endfunction () -#============================================================================= -# Ask user what the target machine is, so that we can choose the right -# right build hints and patches later on. -# +# Provide a cache variable to indicate the targeted machine. function (_superbuild_cross_target_machine) set(SUPERBUILD_CROSS_TARGET "generic" CACHE STRING "Platform to cross compile for, either generic|bgp_xlc|bgq_xlc|bgq_gnu|xk7_gnu") @@ -28,8 +46,9 @@ function (_superbuild_cross_target_machine) "generic" "bgp_xlc" "bgq_xlc" "bgq_gnu" "xk7_gnu") endfunction () -#============================================================================= -# Includes an optionally site-specific file from the cross-compiling directory. +# Includes an optional site-specific file from the cross-compiling directory. +# +# Currently unused. function (_superbuild_cross_include_file var name) set(site_file "crosscompile/${SUPERBUILD_CROSS_TARGET}/${name}.cmake") @@ -44,10 +63,8 @@ function (_superbuild_cross_include_file var name) PARENT_SCOPE) endfunction () -#============================================================================= -# Configures the cmake files that describe how to cross compile paraview -# From the ${cross_target} directory into the build tree. -# +# Configures the cmake files that describe how to crosscompile for a given +# platform. From the ${SUPERBUILD_CROSS_TARGET} directory into the build tree. function (_superbuild_cross_platform_settings) set(site_toolchain "${CMAKE_SOURCE_DIR}/crosscompile/${SUPERBUILD_CROSS_TARGET}/toolchain.cmake.in") -- GitLab From 4da89ce2b02c048b628f4f756db146f2f191939e Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:05:43 -0500 Subject: [PATCH 20/34] docs: document SuperbuildExternalProject and related files --- cmake/SuperbuildExternalProject.cmake | 17 +++++++++++++---- cmake/superbuild_handle_environment.cmake.in | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmake/SuperbuildExternalProject.cmake b/cmake/SuperbuildExternalProject.cmake index acd46482..1ebfcf4d 100644 --- a/cmake/SuperbuildExternalProject.cmake +++ b/cmake/SuperbuildExternalProject.cmake @@ -1,5 +1,10 @@ -# This file implements the logic to inject environment variables into the build -# steps of projects. It is quite messy. +#[==[.md INTERNAL +# `SuperbuildExternalProject` + +This file is basically a wrapper around CMake's `ExternalProject` module with +additional support for managing environments, parallel build settings, download +management, and output suppression. +#]==] if (CMAKE_VERSION VERSION_LESS "3.9") # Needed for fixes. @@ -26,7 +31,7 @@ if (CMAKE_GENERATOR MATCHES "Makefiles") set(superbuild_make_program "${CMAKE_MAKE_PROGRAM}") endif () -# Add "PROCESS_ENVIRONMENT" to the list of keywords recognized. +# Add `PROCESS_ENVIRONMENT` to the list of keywords. string(REPLACE ")" "|PROCESS_ENVIRONMENT)" _ep_keywords__superbuild_ExternalProject_add "${_ep_keywords_ExternalProject_Add}") @@ -67,6 +72,7 @@ function (_superbuild_ep_strip_extra_arguments name) ExternalProject_add("${name}" "${arguments}") endfunction () +# Wraps a command in a CMake script which handles environment management. function (_superbuild_ep_wrap_command var target command_name) get_property(has_command TARGET "${target}" PROPERTY "_EP_${command_name}_COMMAND" SET) @@ -129,6 +135,7 @@ function (_superbuild_ExternalProject_add name) return () endif () + # Wrap the three main commands which require environment manipulation. _superbuild_ep_wrap_command(configure_command "sb-${name}" CONFIGURE) _superbuild_ep_wrap_command(build_command "sb-${name}" BUILD) _superbuild_ep_wrap_command(install_command "sb-${name}" INSTALL) @@ -145,7 +152,7 @@ function (_superbuild_ExternalProject_add name) list(APPEND args "${install_command}") - # Now strip PROCESS_ENVIRONMENT and commands from arguments. + # Now strip `PROCESS_ENVIRONMENT` and commands from arguments. set(skip FALSE) foreach (arg IN LISTS ARGN) if (arg MATCHES "${_ep_keywords__superbuild_ExternalProject_add}") @@ -182,6 +189,7 @@ function (_superbuild_ExternalProject_add name) # empty install, configure, build, etc. ExternalProject_add("${name}" "${args}") + # Hook up the download step with the `download-all` target. if (TARGET "${name}-download") add_dependencies(download-all "${name}-download") @@ -193,6 +201,7 @@ function (_superbuild_ExternalProject_add name) _EP_PROCESS_ENVIRONMENT) _ep_replace_location_tags("${name}" process_environment) + # Configure each CMake script used during the actual build. foreach (step IN ITEMS configure build install) if (req_${step}_command) set(step_command "${original_${step}_command}") diff --git a/cmake/superbuild_handle_environment.cmake.in b/cmake/superbuild_handle_environment.cmake.in index 05f5a5d7..f65a4bd0 100644 --- a/cmake/superbuild_handle_environment.cmake.in +++ b/cmake/superbuild_handle_environment.cmake.in @@ -1,8 +1,12 @@ +# Handle the environment when running a primary step (configure, build, +# install) of a project. + set(environment "@process_environment@") set(list_separator "@_superbuild_list_separator@") set(command "@step_command@") set(env_separator ":") +# These environment variables are prepended to using `env_separator`. set(prepend_envvars PATH CMAKE_PREFIX_PATH PKG_CONFIG_PATH) if (WIN32) -- GitLab From 163d0aa64ea4b8f6e6784877975ba589c6d67b09 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:08:36 -0500 Subject: [PATCH 21/34] docs: document SuperbuildRevisionMacros --- cmake/SuperbuildRevisionMacros.cmake | 194 ++++++++++++++++++--------- 1 file changed, 127 insertions(+), 67 deletions(-) diff --git a/cmake/SuperbuildRevisionMacros.cmake b/cmake/SuperbuildRevisionMacros.cmake index c21f58e2..c7733603 100644 --- a/cmake/SuperbuildRevisionMacros.cmake +++ b/cmake/SuperbuildRevisionMacros.cmake @@ -1,13 +1,39 @@ +#[==[.md +# Specifying revisions + +When building a package, its sources must come from somewhere. In order to keep +the "what to build" (sources) separate from the "how to build" (the project +files) and separately updatable, the superbuild gathers source information +separately. + +There are multiple ways to specify a revision with the functions in the +superbuild, but in general, a package will use the first-specified revision +that is encountered. This is so that a specific project can, for example, use a +different version of Boost than is provided in the common superbuild +infrastructure. +#]==] + include(CMakeParseArguments) -# Sets the arguments to ``ExternalProject_add`` to download the project. -# -# Usage: -# -# superbuild_set_revision(<name> <args>...) -# -# If this is called multiple times for a single project, only the first one -# takes effect. +#[==[.md +## Simple use case + +The simplest use case is when a project just has a given location for its +sources that isn't configurable at build time by a user. + +``` +superbuild_set_revision(<NAME> <ARG>...) +``` + +The `superbuild_set_revision` function stores the given `ARG` for use when then +`NAME` project is built. See the documentation for [ExternalProject][] for the +supported download location arguments. + +Note that validation of the arguments only happens when the project is being +built. + +[ExternalProject]: https://cmake.org/cmake/help/v3.9/module/ExternalProject.html +#]==] function (superbuild_set_revision name) get_property(have_revision GLOBAL PROPERTY @@ -20,18 +46,26 @@ function (superbuild_set_revision name) endif () endfunction () -# Sets arguments to ``ExternalProject_add`` to download the project, but adds -# cache variables so that they may be changed by the user. -# -# Usage: -# -# _superbuild_set_customizable_revision(<name> <args>...) -# -# Adds advanced variables for the following keys: -# -# GIT_REPOSITORY, GIT_TAG, URL, URL_HASH, URL_MD5, and SOURCE_DIR -# -# they are named ``${name_UPPER}_${key}``. +#[==[.md INTERNAL +## Customizable revisions + +``` +_superbuild_set_customizable_revision(<NAME> <ARG>...) +``` + +This is used for `CUSTOMIZABLE` revisions in the +`superbuild_set_selectable_source` function. The following keys are turned into +cache variables. Others are ignored. + + - `GIT_REPOSITORY` + - `GIT_TAG` + - `URL` + - `URL_HASH` + - `URL_MD5` + - `SOURCE_DIR` + +The cache variables are named `<NAME>_<KEY>`. +#]==] function (_superbuild_set_customizable_revision name) set(keys GIT_REPOSITORY GIT_TAG @@ -64,53 +98,79 @@ function (_superbuild_set_customizable_revision name) ${_args_UNPARSED_ARGUMENTS}) endfunction () -# A way to provide selections for a project's source. -# -# Usage: -# -# superbuild_set_selectable_source(<name> -# [SELECTS_WITH <parent_name>] -# <SELECT <selection_name> [DEFAULT] [CUSTOMIZABLE] [FALLBACK] -# <args...>>...) -# -# This may be used to provide multiple ways to build a project: -# -# superbuild_set_selectable_source(myproject -# SELECT v1.0 -# URL "https://hostname/path/to/myproject-1.0.tar.gz" -# URL_MD5 00000000000000000000000000000000 -# SELECT v2.0 DEFAULT -# URL "https://hostname/path/to/myproject-2.0.tar.gz" -# URL_MD5 00000000000000000000000000000000 -# SELECT git CUSTOMIZABLE -# GIT_REPOSITORY "https://path/to/myproject.git" -# GIT_TAG "origin/master" -# SELECT source CUSTOMIZABLE -# SOURCE_DIR "path/to/local/directory") -# -# superbuild_set_selectable_source(myprojectdocs -# SELECTS_WITH myproject -# SELECT v1.0 -# URL "https://hostname/path/to/myprojectdocs-1.0.tar.gz" -# URL_MD5 00000000000000000000000000000000 -# SELECT v2.0 -# URL "https://hostname/path/to/myprojectdocs-2.0.tar.gz" -# URL_MD5 00000000000000000000000000000000 -# SELECT git FALLBACK -# GIT_REPOSITORY "https://path/to/myprojectdocs.git" -# GIT_TAG "origin/master") -# -# Generally, the ``source`` selection should be used to set up an external -# directory as the source tree for the project. -# -# This will create a variable in the cache named ``${name}_SOURCE_SELECTION`` -# which may be used to select one of the sources. -# -# If ``SELECTS_WITH`` is given, the selection of the ``<parent_name>`` will be -# used as the selection for this project as well. The ``DEFAULT`` keyword may -# not be used in projects which use ``SELECTS_WITH``. Instead, the ``FALLBACK`` -# keyword may be used to indicate that that selection should be used if there -# is not a matching selection in this project. +#[==[.md +## User-selectable sources + +Some projects may have different locations for sources that a user might want +to choose between. To facilitate this, a project may have a "selectable" +source. This creates a user-facing cache variable `<NAME>_SOURCE_SELECTION` +which chooses a selection that then indicates where the sources should be +retrieved. It is an error to choose a selection that does not exist. + +The signature is: + +``` +superbuild_set_selectable_source(<NAME> + [SELECTS_WITH <PARENT>] + <SELECT <SELECTION> [DEFAULT] [CUSTOMIZABLE] [FALLBACK] + <ARG>...>...) +``` + +Each selection is followed by a set of arguments which is used as the source +arguments for the project if it is used. The default selection is either the +one marked by the `DEFAULT` argument or the first selection if none is +specified. + +A selection may be `CUSTOMIZABLE` which means that the values to the arguments +may be edited by the user. + +A project may also `SELECTS_WITH` another project. If given, the selection of +the `PARENT` project will be used as the selection for this project as well if +it exists. Rather than the `DEFAULT` keyword, a `SELECTS_WITH` project may use +`FALLBACK` which indicates the selection that should be used if the parent +project uses a selection that is not valid for the current project. + +Some conventions are used for certain selections, but are not enforced. +Usually, at least the `git` and `source` selections are available for "primary" +projects within a superbuild. Both of these should be marked as `CUSTOMIZABLE`. + +As an example: + +```cmake +superbuild_set_selectable_source(myproject + SELECT v1.0 + URL "https://hostname/path/to/myproject-1.0.tar.gz" + URL_MD5 00000000000000000000000000000000 + SELECT v2.0 DEFAULT + URL "https://hostname/path/to/myproject-2.0.tar.gz" + URL_MD5 00000000000000000000000000000000 + SELECT git CUSTOMIZABLE + GIT_REPOSITORY "https://path/to/myproject.git" + GIT_TAG "origin/master" + SELECT source CUSTOMIZABLE + SOURCE_DIR "path/to/local/directory") + +superbuild_set_selectable_source(myprojectdocs + SELECTS_WITH myproject + SELECT v1.0 + URL "https://hostname/path/to/myprojectdocs-1.0.tar.gz" + URL_MD5 00000000000000000000000000000000 + SELECT v2.0 + URL "https://hostname/path/to/myprojectdocs-2.0.tar.gz" + URL_MD5 00000000000000000000000000000000 + SELECT git FALLBACK + GIT_REPOSITORY "https://path/to/myprojectdocs.git" + GIT_TAG "origin/master") +``` + +In this example, the `myproject` project defaults to the `v2.0` selection. In +addition, a `v1.0` is available and the other two are `CUSTOMIZABLE` which +allows a user to set the values with the given arguments. The `myprojectdocs` +project indicates that it `SELECTS_WITH` `myproject`. This means that if the +selection for `myproject` exists for `myprojectdocs` as well, it will be used. +However, if it does not exist, the selection marked as the `FALLBACK` will be +used instead. +#]==] function (superbuild_set_selectable_source name) set(selections) set(customizable_selections) -- GitLab From aba5821265e5902a68bd7e0c9f868846de725d0a Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:27:38 -0500 Subject: [PATCH 22/34] docs: document SuperbuildPackageMacros and related files --- cmake/SuperbuildPackageMacros.cmake | 88 ++++++++++++++------ cmake/scripts/package_test.cmake | 4 + cmake/superbuild_package_cmakelists.cmake.in | 6 +- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/cmake/SuperbuildPackageMacros.cmake b/cmake/SuperbuildPackageMacros.cmake index 699b8601..9fd784a6 100644 --- a/cmake/SuperbuildPackageMacros.cmake +++ b/cmake/SuperbuildPackageMacros.cmake @@ -1,24 +1,52 @@ +#[==[.md +# Packaging support + +One use case of the superbuild is to build packages for projects in order to +distribute for use as release binaries. The superbuild creates these packages +as tests. The reason it isn't done as a CPack step of the build is because +CPack only supports a single package per build. Multiple formats are supported, +but not distinct packages. Another reason is that CPack runs the `install` step +of a project which redoes a build of all targets. The superbuild may have "use +master" sources and when making a package, we don't want to change what is +being packaged. + +The general strategy for the tests is to have a CMake project generated in the +build tree which includes the necessary `.bundle.cmake` files. These files +contain install code and can use CPack variables to change the generated +package. +#]==] + set(_superbuild_packaging_cmake_dir "${CMAKE_CURRENT_LIST_DIR}") -# Adds a test to package a "bundle" project. -# -# Usage: -# -# superbuild_add_extra_package_test(<name> <generator> -# [<property> <value>]...) -# -# This packages a bundle described by a ``${name}.bundle.cmake`` file in the -# project hierarchy. Variables may be put into the package's context by setting -# the ``superbuild_export_variables`` variable to a list of variables to put -# into the generated CMake project. -# -# Extra arguments are set as properties on the test. This should be used for -# things such as ``TIMEOUT`` and ``LABEL``. -# -# This creates a test named ``cpack-${name}-${generator}`` which generates the -# package requested. These tests are set up so that they cannot run in parallel -# with each other. This is because CPack uses the ``_CPack_Packages`` directory -# for itself in all tests. +# TODO: link to the right section in SuperbuildVariables.md +#[==[.md +## Adding a package test + +The `superbuild_add_extra_package_test` function handles the logic for adding a +test which builds the package. Due to the way CPack works, only one packaging +test may be run concurrently. + +``` +superbuild_add_extra_package_test(<NAME> <GENERATOR> + [<PROPERTY> <VALUE>]...) +``` + +Adds a test with the name `cpack-NAME-GENERATOR`. The packaging rules are +handled by including a `NAME.bundle.cmake` file. The same include paths used in +the build are available in the packaging steps. By default, only the variables +in the [SuperbuildVariables][] packaging section are available. Other variables +may be passed to the packaging step by adding the variable name to the +`superbuild_export_variables` list. The value of the variable available when +adding the test is used. + +All other arguments are set as properties on the tests. The only reserved +property is the `RESOURCE_LOCK` property. + +[SuperbuildVariables]: SuperbuildVariables.md +#]==] + +# TODO: use a PROPERTIES argument +# TODO: use a VARIABLES argument function (superbuild_add_extra_package_test name generator) set(superbuild_extra_variables) foreach (variable IN LISTS superbuild_export_variables) @@ -27,6 +55,7 @@ function (superbuild_add_extra_package_test name generator) endforeach () set(cpack_source_dir "${CMAKE_BINARY_DIR}/cpack/${name}/${generator}") + # Create a build directory so that the installation variant doesn't conflict. set(cpack_build_dir "${cpack_source_dir}/build") configure_file( "${_superbuild_packaging_cmake_dir}/superbuild_package_cmakelists.cmake.in" @@ -57,14 +86,19 @@ function (superbuild_add_extra_package_test name generator) ${ARGN}) endfunction () -# Add a "superbuild-install" target to install one of the packages. -# -# This function adds a target which acts like "make install" for a selected -# package. -# -# The ``default`` argument is used as the default package to build and the -# cache editors use the list of available packages as the selection choices for -# the ``SUPERBUILD_DEFAULT_INSTALL`` variable. +#[==[.md +In addition to packages, a package may be used as a template for the `install` +target of the superbuild. + +``` +superbuild_enable_install_target(<DEFAULT>) +``` + +This uses a user-selectable `.bundle.cmake` to control the `install` target of +the superbuild. The default should be in the form `<NAME>/<GENERATOR>`. An +error is produced if the test for the package does not exist. +#]==] + function (superbuild_enable_install_target default) get_property(all_packages GLOBAL PROPERTY _superbuild_packages) diff --git a/cmake/scripts/package_test.cmake b/cmake/scripts/package_test.cmake index d5fc6704..158364b8 100644 --- a/cmake/scripts/package_test.cmake +++ b/cmake/scripts/package_test.cmake @@ -1,3 +1,6 @@ +# This is the test script for running packaging commands. + +# First, configure the package test source directory. execute_process( COMMAND "${CMAKE_COMMAND}" -G "${cmake_generator}" @@ -9,6 +12,7 @@ if (res) message(FATAL_ERROR "CMake failed with exit code ${res}") endif () +# Then run CPack with the requested generator. execute_process( COMMAND "${CMAKE_CPACK_COMMAND}" -V diff --git a/cmake/superbuild_package_cmakelists.cmake.in b/cmake/superbuild_package_cmakelists.cmake.in index c872a655..d27db959 100644 --- a/cmake/superbuild_package_cmakelists.cmake.in +++ b/cmake/superbuild_package_cmakelists.cmake.in @@ -1,6 +1,10 @@ +# This file is the top-level CMakeLists.txt file for the packaging step. It +# provides variables to the `.bundle.cmake` code such as project variables and +# the like. + cmake_minimum_required(VERSION "@CMAKE_MINIMUM_REQUIRED_VERSION@") -# We need to explicitly state so form of language support. That is how CMake +# We need to explicitly state some form of language support. That is how CMake # detects what compilers are being used, and therefore what # InstallRequiredSystemLibraries should properly do. If we state NONE for # languages we will fail to install any of the system libraries. -- GitLab From fcb569cddfa02fa048385a9f3f53d52591b22fe1 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:28:18 -0500 Subject: [PATCH 23/34] docs: document SuperbuildTestingMacros and related files --- cmake/SuperbuildTestingMacros.cmake | 28 +++++++++++++++++++++++++++- cmake/scripts/extract_test.cmake | 7 +++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cmake/SuperbuildTestingMacros.cmake b/cmake/SuperbuildTestingMacros.cmake index b5dcf45f..213ef162 100644 --- a/cmake/SuperbuildTestingMacros.cmake +++ b/cmake/SuperbuildTestingMacros.cmake @@ -1,6 +1,32 @@ +#[==[.md +# Testing packages + +The packages created by the superbuild packaging support may also be tested. In +order to do so, the created package must first be extracted. +#]==] + set(_superbuild_testing_cmake_dir "${CMAKE_CURRENT_LIST_DIR}") -# Adds a test which extracts a CPack artifact. +#[==[.md +The `superbuild_add_extract_test` extracts a package generated by a packaging +test. + +``` +superbuild_add_extract_test(<NAME> <PREFIX> <GENERATOR> <OUTPUT> + [<PROPERTY> <VALUE>]...) +``` + +The `NAME` and `GENERATOR` arguments must match the corresponding package test +that should be extracted. There is also the `PREFIX` argument which should be a +prefix long enough to make the glob for the resulting package name unique. + +The `OUTPUT` is the path to the directory that the package should be extracted +into. One intermediate directory is removed from the extraction. For example, a +package containing `dirname/bin/program` and `dirname/include/header.h` will +have a `bin` and `include` directory under the given output directory. +#]==] + +# TODO: use a PROPERTIES argument function (superbuild_add_extract_test name glob_prefix generator output) set(_ZIP_test_glob "${glob_prefix}*.zip") set(_NSIS_test_glob "${glob_prefix}*.exe") diff --git a/cmake/scripts/extract_test.cmake b/cmake/scripts/extract_test.cmake index d67f4af1..c811a026 100644 --- a/cmake/scripts/extract_test.cmake +++ b/cmake/scripts/extract_test.cmake @@ -1,3 +1,4 @@ +# A wrapper around a process call which checks the command's result. function (_extract_process input) execute_process(${ARGN} RESULT_VARIABLE res @@ -8,6 +9,7 @@ function (_extract_process input) endif () endfunction () +# Extract a package with CMake. function (_extract_with_cmake output input) _extract_process("${input}" COMMAND "${CMAKE_COMMAND}" -E tar @@ -15,6 +17,7 @@ function (_extract_with_cmake output input) WORKING_DIRECTORY "${output}") endfunction () +# Detect a tarbomb package. function (_detect_tarbomb var dir) file(GLOB contents "${dir}/*") @@ -27,6 +30,7 @@ function (_detect_tarbomb var dir) PARENT_SCOPE) endfunction () +# Do the dance to mount and extract a .app from a .dmg file. function (_extract_dmg output mount input) _extract_process("${input}" COMMAND /bin/sh -c @@ -45,6 +49,9 @@ function (_extract_dmg output mount input) "${mount}") endfunction () +# Extract a binary from the given directory composed of the given glob into the +# output directory. For simple composition packages (tarballs, zip, etc.), +# non-tarbomb files have their intermediate directories removed. function (extract_binary dir glob output) file(REMOVE_RECURSE "${output}") file(MAKE_DIRECTORY "${output}") -- GitLab From 240a2e00e2ea74bedf97989cdaa8f8047cf1df75 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:28:51 -0500 Subject: [PATCH 24/34] docs: document SuperbuildUtils-unix --- cmake/SuperbuildUtils-unix.cmake | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cmake/SuperbuildUtils-unix.cmake b/cmake/SuperbuildUtils-unix.cmake index 0c3462f0..a5e49d9c 100644 --- a/cmake/SuperbuildUtils-unix.cmake +++ b/cmake/SuperbuildUtils-unix.cmake @@ -1,13 +1,23 @@ +#[==[.md INTERNAL +# Unix utility functions + +On Unix platforms (mainly Linux), some workarounds are sometimes necessary. +These functions implement these workarounds. They are used internally to the +superbuild and do not need to be called manually. +#]==] + include (CMakeDependentOption) +#[==[.md INTERNAL +Sometimes a build will conflict with a system library in use by an external +tool. Since the `LD_LIBRARY_PATH` environment variable is a global setting for +executables, this may cause interference. This function allows users to disable +the logic to continue a build. + +The errors that show up if this needs to be disabled are usually due to +mismatches in sonames or version information. +#]==] function (superbuild_unix_ld_library_path_hack var) - # XXX(Utkarsh): I am not exactly sure about the cause of these issues and how - # to cleanly overcome them, however, on some Linuxes when CMake is built - # shared, if we set LD_LIBRARY_PATH to the install/lib dir, we end up with - # errors due to conflicts with system libz. So for now, I am making this an - # option. By default we will let the superbuild scripts set LD_LIBRARY_PATH. - # However if users get errors like: libz.so.1: no version information - # available, then users should turn this flag off. cmake_dependent_option(PASS_LD_LIBRARY_PATH_FOR_BUILDS "Pass LD_LIBRARY_PATH to build scripts" ON "UNIX;NOT APPLE" OFF) mark_as_advanced(PASS_LD_LIBRARY_PATH_FOR_BUILDS) -- GitLab From e35b1dd841fdecc41fe67d90e36c22c91291fe47 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:29:01 -0500 Subject: [PATCH 25/34] docs: document SuperbuildUtils-apple --- cmake/SuperbuildUtils-apple.cmake | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/cmake/SuperbuildUtils-apple.cmake b/cmake/SuperbuildUtils-apple.cmake index c3667914..0171e5e4 100644 --- a/cmake/SuperbuildUtils-apple.cmake +++ b/cmake/SuperbuildUtils-apple.cmake @@ -1,3 +1,23 @@ +#[==[.md INTERNAL +# Apple utility functions + +On Apple platforms (mainly macOS), some wrangling with version variables and +the link is required. These functions handle that. They are used internally to +the superbuild and do not need to be called manually. +#]==] + +#[==[.md INTERNAL +## SDK support + +CMake uses various `CMAKE_OSX_` variables to control the SDK in use. The +`superbuild_osx_add_version_flags` function extracts information from these +flags and adds them to the `superbuild_c_flags` and `superbuild_cxx_flags` +variables in the calling scope. + +``` +superbuild_osx_add_version_flags() +``` +#]==] function (superbuild_osx_add_version_flags) if (NOT APPLE) return () @@ -13,7 +33,9 @@ function (superbuild_osx_add_version_flags) "-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") endif () if (CMAKE_OSX_SYSROOT) + # The "sysroot" may be an SDK name. Get the path to the sysroot from it. if (NOT IS_DIRECTORY "${CMAKE_OSX_SYSROOT}") + # Ask Xcode what the SDK path is. execute_process( COMMAND xcodebuild -version @@ -39,6 +61,17 @@ function (superbuild_osx_add_version_flags) endforeach () endfunction () +#[==[.md INTERNAL +## Version flags + +This `superbuild_osx_pass_version_flags` function sets the given variable to +contain a list of CMake `-D` command line flags to use the same Apple SDK, +architecture, and deployment target as the superbuild. + +``` +superbuild_osx_pass_version_flags(<VARIABLE>) +``` +#]==] function (superbuild_osx_pass_version_flags var) if (NOT APPLE) return () -- GitLab From c1747dac0bad5d509fd0830d44155b21207e6acd Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:29:41 -0500 Subject: [PATCH 26/34] docs: document SuperbuildVersionMacros --- cmake/SuperbuildVersionMacros.cmake | 71 ++++++++++++++++++----------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/cmake/SuperbuildVersionMacros.cmake b/cmake/SuperbuildVersionMacros.cmake index ae81f61b..2fe0d9c2 100644 --- a/cmake/SuperbuildVersionMacros.cmake +++ b/cmake/SuperbuildVersionMacros.cmake @@ -14,6 +14,21 @@ find_package(Git) +#[==[.md +# Detect the version of a source tree + +When creating packages, it can be useful to know the specific version of the +source that is being built. To that end, this module provides functions which +determine a version number for a source based on its source selection. It uses +the `<NAME>_SOURCE_DIR` (for a `source` selection) or the build tree location +(for a `git` selection) to query the version of the checked out code using `git +describe`. + +If it turns out that Git is either not available or the source directory is not +a Git checkout, it reads a file in the source tree if available and if all else +fails, falls back to a static string. +#]==] + function (_superbuild_detect_version_git var source_dir default version_file) set(major) set(minor) @@ -46,6 +61,7 @@ function (_superbuild_detect_version_git var source_dir default version_file) endif () endif () + # If `git describe` failed, check for a version file. if (result) set(version_path "${source_dir}/${version_file}") if (source_dir AND version_file AND EXISTS "${version_path}") @@ -57,6 +73,7 @@ function (_superbuild_detect_version_git var source_dir default version_file) endif () endif () + # Split the version number into fields. if (output MATCHES "([0-9]+)\\.([0-9]+)\\.([0-9]+)-?(.*)") message(STATUS "Determined source version for ${project}: ${CMAKE_MATCH_0}") set(full "${CMAKE_MATCH_0}") @@ -69,6 +86,7 @@ function (_superbuild_detect_version_git var source_dir default version_file) "Failed to determine the version for ${var}; got ${output}") endif () + # Set variables in the parent scope if they're available. if (full) set("${var}_VERSION" "${major}.${minor}" PARENT_SCOPE) set("${var}_VERSION_MAJOR" "${major}" PARENT_SCOPE) @@ -84,38 +102,39 @@ function (_superbuild_detect_version_git var source_dir default version_file) endif () endfunction () +# Set a variable in the parent scope properly while still making it available +# in the current scope.. macro (_superbuild_set_up variable value) set("${variable}" "${value}" PARENT_SCOPE) set("${variable}" "${value}") endmacro () -# Extracts the version for a project from its source information or falls back -# to a default. -# -# superbuild_set_version_variables(<project> <default> <include file> [version file]) -# -# This will write out a file to ``<include file>`` which may be included to set -# variables related to the versions of the given ``<project>``. If the version -# cannot be determined (e.g., because the project will be cloned during the -# build), the default will be used. -# -# If there is a source directory to be used for the project, the ``<version -# file>`` will be used to get the version number. If it is empty or not -# provided, the default will be used instead. -# -# The variables set are: -# -# <project>_version (as ``<major>.<minor>``) -# <project>_version_major -# <project>_version_minor -# <project>_version_patch -# <project>_version_patch_extra (e.g., ``rc1``) -# <project>_version_suffix (equivalent to ``-<patch_extra>`` if -# ``patch_extra`` is non-empty) -# <project>_version_full -# <project>_version_is_release (``TRUE`` if the suffix is empty, ``FALSE`` -# otherwise) +#[==[.md +``` +superbuild_set_version_variables(<PROJECT> <DEFAULT> <INCLUDE FILE> [<VERSION FILE>]) +``` + +This will write out a file to `<INCLUDE FILE>` which may be `include`'d after +this function to set variables related to the versions of the given +`<PROJECT>`. If the version cannot be determined (e.g., because the project +will be cloned during the build), the given `DEFAULT` version will be used. + +If there is a source directory to be used for the project, the `<VERSION FILE>` +will be used to get the version number. If it is empty or not provided, the +default will be used instead. + +The variables available after inclusion are: + + `<PROJECT>_version` (as `<major>.<minor>`) + `<PROJECT>_version_major` + `<PROJECT>_version_minor` + `<PROJECT>_version_patch` + `<PROJECT>_version_patch_extra` (e.g., `rc1`) + `<PROJECT>_version_suffix` (equivalent to `-<patch_extra>` if `patch_extra` is non-empty) + `<PROJECT>_version_full` + `<PROJECT>_version_is_release` (`TRUE` if the suffix is empty, `FALSE` otherwise) +#]==] function (superbuild_set_version_variables project default include_file) set(source_dir "") if ((NOT ${project}_FROM_GIT AND ${project}_FROM_SOURCE_DIR) OR ${project}_SOURCE_SELECTION STREQUAL "source") -- GitLab From 93950c8e00b178b2340f5903cd1ea595893bb877 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:31:28 -0500 Subject: [PATCH 27/34] docs: document fixup_bundle.apple --- cmake/scripts/fixup_bundle.apple.py | 188 +++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 4 deletions(-) diff --git a/cmake/scripts/fixup_bundle.apple.py b/cmake/scripts/fixup_bundle.apple.py index 079708d7..31e580d5 100755 --- a/cmake/scripts/fixup_bundle.apple.py +++ b/cmake/scripts/fixup_bundle.apple.py @@ -1,5 +1,13 @@ #!/usr/bin/env python2.7 +''' +A tool to install Mach-O binaries into an ``.app`` bundle. + +Other bundle types (particularly ``.framework`` and ``.plugin`` bundles) are +not supported yet. +''' + + import json import os import os.path @@ -9,6 +17,11 @@ import subprocess class Pipeline(object): + ''' + A simple class to handle a list of shell commands which need to pass input + to each other. + ''' + def __init__(self, *commands): if not commands: raise RuntimeError('Pipeline: at least one command must be given') @@ -16,7 +29,8 @@ class Pipeline(object): self._commands = commands def __call__(self): - last_input = open('/dev/null', 'r') + # Use /dev/null as the input for the first command. + last_input = open(os.devnull, 'r') for command_args in self._commands: command = subprocess.Popen(command_args, stdin=last_input, stdout=subprocess.PIPE) @@ -30,6 +44,31 @@ class Pipeline(object): class Library(object): + ''' + A representation of a library. + + This class includes information that a runtime loader needs in order to + perform its job. It tries to implement the behavior of ``dyld(1)`` as + closely as possible. + + Known Issues + ------------ + + ``@rpath/`` and ``DYLD_LIBRARY_PATH`` + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + When a library contains a reference to a library like + ``@rpath/libname.dylib``, if ``DYLD_LIBRARY_PATH`` is set to contain a path + which has a ``libname.dylib``, ``dyld(1)`` will find it even if no + ``LC_RPATH`` commands are present. This behavior is not documented and it + only seems to work if a library is directly underneath a + ``DYLD_LIBRARY_PATH`` path. If the library reference is + ``@rpath/dir/libname.dylib``, even if a ``dir/libname.dylib`` exists in a + ``DYLD_LIBRARY_PATH`` path, it will still not be found. It is unknown + whether this behavior is expected or not due to the lack of documentation. + The logic in this script includes neither of these behaviors. + ''' + def __init__(self, path, parent=None, search_paths=None): # This is the actual path to a physical file self._path = os.path.normpath(path) @@ -58,25 +97,44 @@ class Library(object): @property def path(self): + '''The absolute path to the library.''' return self._path @property def parent(self): + '''The binary which loaded the library.''' return self._parent @property def name(self): + '''The name of the library.''' return os.path.basename(self.path) @property def installed_id(self): + ''' + The ID of the library. + + This is the string by which the library will be referenced by other + binaries in the installation. + ''' return self._installed_id def set_installed_id(self, installed_id): + '''Set the ID of the library as it is installed as.''' self._installed_id = installed_id @property def dependent_reference(self): + ''' + The prefix to use for a library loaded by the library. + + This is used as the prefix for IDs for libraries loaded by the library. + It is based on the initial binary which loaded the library. For + example, executables use ``@executable_path`` and plugins use + ``@loader_path``. In a chain of loaded libraries, the loader (parent) + of a library determines the prefix. + ''' # Refer to libraries the same way that the library which is loading it # references it. if self.parent is None: @@ -85,6 +143,12 @@ class Library(object): @property def symlinks(self): + ''' + A list of symlinks to the library. + + Symlinks are looked for only beside the library and the names of these + files are returned, not their full paths. + ''' if self._symlinks is None: realpath = os.path.realpath(self.path) dirname = os.path.dirname(realpath) @@ -111,20 +175,34 @@ class Library(object): @property def executable_path(self): + '''The path to the loading executable (if available).''' if self._parent is not None: return self._parent.executable_path return self._executable_path @property def loader_path(self): + '''The path to use for ``@loader_path`` references from the library.''' return os.path.dirname(self.path) @property def is_framework(self): + '''Whether the library is a framework or not.''' return self.path.count('.framework') @property def framework_info(self): + ''' + Information for frameworks. + + The return value is a tuple of path (where the framework is located), + name (the ``NAME.framework`` part of its path), and associated library + (the path under the ``.framework`` directory which contains the actual + library binary). + + See the ``framework_path``, ``framework_name``, and + ``framework_library`` properties. + ''' if self._framework_info is None: if not self.is_framework: self._framework_info = (None, None, None) @@ -152,18 +230,41 @@ class Library(object): @property def framework_path(self): + ''' + The path which contains the ``.framework`` for the library. + + ``None`` if the library is not a framework. + ''' return self.framework_info[0] @property def framework_name(self): + ''' + The name of the framework containing the library. + + ``None`` if the library is not a framework. + ''' return self.framework_info[1] @property def framework_library(self): + ''' + The path to the library under the ``.framework`` directory. + + ``None`` if the library is not a framework. + ''' return self.framework_info[2] @property def rpaths(self): + ''' + The list of rpaths used when resolving ``@rpath/`` references in the + library. + + In addition to the ``LC_RPATH`` load commands contained within the + library, rpaths in the binaries which loaded the library are + referenced. These are included in the property. + ''' if self._rpaths is None: rpaths = [] if self._parent is not None: @@ -188,12 +289,14 @@ class Library(object): ]) rpaths.extend(get_rpaths().split('\n')) + # rpaths may contain magic ``@`` references. This property only + # contains full paths, so we resolve them now. resolved_rpaths = [] for rpath in rpaths: if rpath.startswith('@executable_path'): # If the loader does not have an executable path, it is a plugin or # a framework and we trust the executable which loads the plugin to - # provide this library instead. + # provide the library instead. if self.executable_path is None: continue resolved_rpaths.append(rpath.replace('@executable_path', self.executable_path)) @@ -206,6 +309,7 @@ class Library(object): return self._rpaths def _get_dependencies(self): + '''Get the dependent library IDs of the library.''' pipe = Pipeline([ 'otool', '-L', @@ -219,6 +323,7 @@ class Library(object): @property def dependencies(self): + '''Dependent libraries of the library.''' if self._dependencies is None: collection = {} for dep in self._get_dependencies(): @@ -230,6 +335,13 @@ class Library(object): return self._dependencies def _find_library(self, ref): + ''' + Find a library using search paths. + + Use of this method to find a dependent library indicates that the + library depdencies are not properly specified. As such, it warns when + it is used. + ''' print 'WARNING: dependency from %s to %s requires a search path' % (self.path, ref) for loc in self._search_paths: path = os.path.join(loc, ref) @@ -239,6 +351,7 @@ class Library(object): @classmethod def from_reference(cls, ref, loader): + '''Create a library representation given an ID and a loading binary.''' paths = [ref] if ref.startswith('@executable_path/'): # If the loader does not have an executable path, it is a plugin or @@ -265,6 +378,7 @@ class Library(object): @classmethod def from_path(cls, path, parent=None): + '''Create a library representation from a path.''' if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -281,6 +395,7 @@ class Library(object): @classmethod def from_manifest(cls, path, installed_id): + '''Create a library representation from a cached manifest entry.''' if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -292,6 +407,13 @@ class Library(object): class Executable(Library): + ''' + The main executable of an ``.app`` bundle. + + A specialization of a library which lives in ``Contents/MacOS`` inside of a + ``.app`` bundle. + ''' + def __init__(self, path, **kwargs): super(Executable, self).__init__(path, None, **kwargs) @@ -307,6 +429,15 @@ class Executable(Library): class Utility(Executable): + ''' + A utility executable in an ``.app`` bundle. + + A specialization of a library which lives in ``Contents/bin`` inside of a + ``.app`` bundle. These are not the executables which run when opening a + ``.app``, but they are available for use by the main executable or via the + command line. + ''' + def __init__(self, path, **kwargs): super(Utility, self).__init__(path, **kwargs) @@ -316,6 +447,19 @@ class Utility(Executable): class Plugin(Library): + ''' + A plugin library in an ``.app`` bundle. + + These libraries live under the ``Contents/Plugins`` directory in an + ``.app`` bundle. They are expected to be loaded by an executable and use + ``@executable_path/`` references where possible, but for any libraries + required by the plugin and not otherwise provided, ``@loader_path/`` is + used instead. + + Some plugins may require to be considered as their own ``@executable_path`` + reference. This may indicate errors in the building of the plugin. + ''' + def __init__(self, path, fake_exe_path=False, **kwargs): super(Plugin, self).__init__(path, None, **kwargs) @@ -332,6 +476,20 @@ class Plugin(Library): class Module(Library): + ''' + A library loaded programmatically at runtime. + + Modules are usually used by interpreted languages (as opposed to compiled + languages) and loaded at runtime. They may live at any depth in a ``.app`` + bundle. + + Currently it is assumed that the only executables which will load these + modules is a binary in the same ``.app`` bundle. It is unknown if this + assumption is actually valid and documentation is scarce. + + Some modules may require to be considered as their own ``@executable_path`` + reference. This may indicate errors in the building of the module. + ''' def __init__(self, path, bundle_location, fake_exe_path=False, **kwargs): super(Module, self).__init__(path, None, **kwargs) @@ -358,6 +516,14 @@ class Module(Library): class Framework(Library): + ''' + A ``.framework`` library. + + Currently unsupported. Documentation for frameworks needs to be found and + understood to find out what it really means to install a framework (as a + usable framework). + ''' + def __init__(self, path, **kwargs): super(Framework, self).__init__(path, None, **kwargs) @@ -373,7 +539,9 @@ class Framework(Library): def copy_library(destination, library, dry_run=False): + '''Copy a library into the ``.app`` bundle.''' if library.is_framework: + # Frameworks go into Contents/Frameworks. print 'Copying %s/%s ==> Contents/Frameworks' % (library.framework_path, library.framework_name) app_dest = os.path.join(destination, 'Contents', 'Frameworks') @@ -389,6 +557,7 @@ def copy_library(destination, library, dry_run=False): _os_makedirs(app_dest) shutil.copytree(os.path.join(library.framework_path, library.framework_name), destination, symlinks=True) else: + # Libraries go into Contents/Libraries. print 'Copying %s ==> Contents/Libraries' % library.path app_dest = os.path.join(destination, 'Contents', 'Libraries') @@ -402,6 +571,7 @@ def copy_library(destination, library, dry_run=False): _os_makedirs(app_dest) shutil.copy(library.path, destination) + # Create any symlinks we found for the library as well. for symlink in library.symlinks: print 'Creating symlink to Contents/Libraries/%s ==> %s' % (library.name, symlink) if not dry_run: @@ -417,12 +587,14 @@ def copy_library(destination, library, dry_run=False): ln() if not dry_run: + # We need to make the library writable first. chmod = Pipeline([ 'chmod', 'u+w', binary, ]) chmod() + # Set the ID on the binary. install_name_tool = Pipeline([ 'install_name_tool', '-id', library.installed_id, @@ -433,9 +605,11 @@ def copy_library(destination, library, dry_run=False): return binary -# A function to fix up the fact that os.makedirs chokes if the path already -# exists. def _os_makedirs(path): + ''' + A function to fix up the fact that os.makedirs chokes if the path already + exists. + ''' if os.path.exists(path): return os.makedirs(path) @@ -487,6 +661,7 @@ def _arg_parser(): def _install_binary(binary, is_excluded, bundle_dest, installed, manifest, dry_run=False): + '''Install the main binary into the package.''' # Start looking at our main executable's dependencies. deps = binary.dependencies.values() while deps: @@ -522,6 +697,10 @@ def _install_binary(binary, is_excluded, bundle_dest, installed, manifest, dry_r def _fix_installed_binaries(installed, dry_run=False): + ''' + This function updates all of the installed binaries to use consistent + library IDs when referring to each other. + ''' # Go through all of the binaries installed and fix up references to other things. for binary_info in installed.values(): binary, installed_path = binary_info @@ -544,6 +723,7 @@ def _fix_installed_binaries(installed, dry_run=False): def _update_manifest(manifest, installed, path): + '''Update the manifest file with a set of newly installed binaries.''' for input_path, binary_info in installed.items(): binary, _ = binary_info manifest[input_path] = binary.installed_id -- GitLab From a729d1485360f7a50b3dbebdedc48b30f14f388b Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:32:50 -0500 Subject: [PATCH 28/34] docs: document fixup_bundle.unix --- cmake/scripts/fixup_bundle.unix.py | 94 +++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/cmake/scripts/fixup_bundle.unix.py b/cmake/scripts/fixup_bundle.unix.py index d6a264b0..a394ed3b 100755 --- a/cmake/scripts/fixup_bundle.unix.py +++ b/cmake/scripts/fixup_bundle.unix.py @@ -1,5 +1,10 @@ #!/usr/bin/env python2.7 +''' +A tool to install ELF binaries into in installation prefix. +''' + + import json import os import os.path @@ -9,6 +14,11 @@ import subprocess class Pipeline(object): + ''' + A simple class to handle a list of shell commands which need to pass input + to each other. + ''' + def __init__(self, *commands): if not commands: raise RuntimeError('Pipeline: at least one command must be given') @@ -16,7 +26,8 @@ class Pipeline(object): self._commands = commands def __call__(self): - last_input = open('/dev/null', 'r') + # Use /dev/null as the input for the first command. + last_input = open(os.devnull, 'r') for command_args in self._commands: command = subprocess.Popen(command_args, stdin=last_input, stdout=subprocess.PIPE) @@ -30,6 +41,14 @@ class Pipeline(object): class Library(object): + ''' + A representation of a library. + + This class includes information that a runtime loader needs in order to + perform its job. It tries to implement the behavior of ``ld.so(8)`` as + closely as possible. + ''' + def __init__(self, path, parent=None, search_paths=None): # This is the actual path to a physical file self._path = os.path.normpath(path) @@ -57,22 +76,32 @@ class Library(object): @property def is_cached(self): + '''Whether the library has already been installed or not.''' return self._is_cached @property def path(self): + '''The absolute path to the library.''' return self._path @property def parent(self): + '''The binary which loaded the library.''' return self._parent @property def name(self): + '''The name of the library.''' return os.path.basename(self.path) @property def symlinks(self): + ''' + A list of symlinks to the library. + + Symlinks are looked for only beside the library and the names of these + files are returned, not their full paths. + ''' if self._symlinks is None: realpath = os.path.realpath(self.path) dirname = os.path.dirname(realpath) @@ -99,14 +128,23 @@ class Library(object): @property def loader_path(self): + '''The path to use for ``@loader_path`` references from the library.''' return os.path.dirname(self.path) @property def extra_loader_paths(self): + ''' + Extra paths which should be looked at because of a loader's search + path. + ''' return [] @property def loader_paths(self): + ''' + A list of paths to look for libraries due to where the loading + libraries look. + ''' loader_paths = [self.loader_path] loader_paths.extend(self.extra_loader_paths) if self.parent is not None: @@ -114,6 +152,7 @@ class Library(object): return loader_paths def _resolve_rpath(self, rpath): + '''Resolve an rpath which may contain a special token reference.''' if rpath.startswith('$ORIGIN/'): rpath = rpath.replace('$ORIGIN', self.loader_path) elif rpath.startswith('${ORIGIN}/'): @@ -127,6 +166,13 @@ class Library(object): @property def rpaths(self): + ''' + The list of rpaths used when resolving required library references. + + In addition to the ``DT_RPATH`` entries contained within the library, + rpaths in the binaries which loaded the library are referenced. These + are included in the property. + ''' if self._rpaths is None: rpaths = [] get_rpaths = Pipeline([ @@ -164,6 +210,12 @@ class Library(object): @property def runpaths(self): + ''' + The list of runpaths used when resolving required library references. + + These are specified by the library using ``DT_RUNPATH`` entries. Unlike + ``DT_RPATH``, these entries are not inherited by loaded libraries. + ''' if self._runpaths is None: runpaths = [] get_runpaths = Pipeline([ @@ -184,6 +236,12 @@ class Library(object): return self._runpaths def _get_dependencies(self): + ''' + Get the dependent libraries of the library. + + These are specified by ``DT_NEEDED`` entries and are specified as + soname values. + ''' pipe = Pipeline([ 'objdump', '-p', @@ -200,6 +258,7 @@ class Library(object): @property def dependencies(self): + '''Dependent libraries of the library.''' if self._dependencies is None: collection = {} for dep in self._get_dependencies(): @@ -210,6 +269,13 @@ class Library(object): return self._dependencies def _find_library(self, ref): + ''' + Find a library using search paths. + + Use of this method to find a dependent library indicates that the + library depdencies are not properly specified. As such, it warns when + it is used. + ''' print 'WARNING: dependency from %s to %s requires a search path' % (self.path, ref) for loc in self._search_paths: path = os.path.join(loc, ref) @@ -221,6 +287,7 @@ class Library(object): @classmethod def default_search_paths(cls): + '''A list of search paths implicit to the system.''' if cls.__search_cache is None: pipe = Pipeline([ '/sbin/ldconfig', @@ -242,6 +309,7 @@ class Library(object): @classmethod def from_reference(cls, ref, loader): + '''Create a library representation given a soname and a loading binary.''' paths = [] if '/' in ref: @@ -280,6 +348,7 @@ class Library(object): @classmethod def from_path(cls, path, parent=None): + '''Create a library representation from a path.''' if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -296,6 +365,7 @@ class Library(object): @classmethod def from_manifest(cls, path): + '''Create a library representation from a cached manifest entry.''' if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -309,6 +379,13 @@ class Library(object): class Module(Library): + ''' + A library loaded programmatically at runtime. + + Modules are loaded at runtime using ``dlopen``. They may contain extra + paths to search that are implicit to their loading executable. + ''' + def __init__(self, path, bundle_location, loader_paths=None, **kwargs): super(Module, self).__init__(path, None, **kwargs) @@ -328,11 +405,16 @@ class Module(Library): class Executable(Module): + ''' + An executable in the installation. + ''' + def __init__(self, path, **kwargs): super(Executable, self).__init__(path, 'bin', **kwargs) def copy_library(destination, libdir, library, sources, dry_run=False): + '''Copy a library into the ``.app`` bundle.''' if library._is_cached: return @@ -361,6 +443,7 @@ def copy_library(destination, libdir, library, sources, dry_run=False): ln() if not dry_run: + # We need to make the library writable first. chmod = Pipeline([ 'chmod', 'u+w', @@ -385,6 +468,7 @@ HAVE_CHRPATH = None def remove_prefix_rpaths(binary, sources): + '''Remove rpaths which reference the build machine.''' if not sources: return @@ -438,9 +522,11 @@ def remove_prefix_rpaths(binary, sources): chrpath() -# A function to fix up the fact that os.makedirs chokes if the path already -# exists. def _os_makedirs(path): + ''' + A function to fix up the fact that os.makedirs chokes if the path already + exists. + ''' if os.path.exists(path): return os.makedirs(path) @@ -489,6 +575,7 @@ def _arg_parser(): def _install_binary(binary, is_excluded, bundle_dest, dep_libdir, installed, manifest, sources, dry_run=False): + '''Install the main binary into the package.''' # Start looking at our main executable's dependencies. deps = binary.dependencies.values() while deps: @@ -532,6 +619,7 @@ def _install_binary(binary, is_excluded, bundle_dest, dep_libdir, installed, man def _update_manifest(manifest, installed, path, libdir): + '''Update the manifest file with a set of newly installed binaries.''' for input_path in installed.keys(): manifest.setdefault(libdir, []).append(input_path) -- GitLab From ad0bf04db06f1ce635fa9886ef980c2be7cde2b2 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:37:14 -0500 Subject: [PATCH 29/34] docs: document fixup_bundle.windows --- cmake/scripts/fixup_bundle.windows.py | 67 ++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/cmake/scripts/fixup_bundle.windows.py b/cmake/scripts/fixup_bundle.windows.py index d561fb91..90a74e12 100755 --- a/cmake/scripts/fixup_bundle.windows.py +++ b/cmake/scripts/fixup_bundle.windows.py @@ -1,5 +1,10 @@ #!/usr/bin/env python2.7 +''' +A tool to install PE-COFF binaries into in installation prefix. +''' + + import json import os import os.path @@ -9,6 +14,11 @@ import subprocess class Pipeline(object): + ''' + A simple class to handle a list of shell commands which need to pass input + to each other. + ''' + def __init__(self, *commands): if not commands: raise RuntimeError('Pipeline: at least one command must be given') @@ -16,6 +26,7 @@ class Pipeline(object): self._commands = commands def __call__(self): + # Use /dev/null as the input for the first command. last_input = open(os.devnull, 'r') for command_args in self._commands: @@ -30,6 +41,17 @@ class Pipeline(object): class Library(object): + ''' + A representation of a library. + + This class includes information that a runtime loader needs in order to + perform its job. It tries to implement the behavior of the Windows runtime + loader, but documentation is scarce. + + Basically, the approach is to rely on the location of loading libraries and + the magic of the ``PATH`` environment variable. + ''' + def __init__(self, path, parent=None, search_paths=None): # This is the actual path to a physical file self._path = os.path.normpath(path) @@ -54,25 +76,36 @@ class Library(object): @property def is_cached(self): + '''Whether the library has already been installed or not.''' return self._is_cached @property def path(self): + '''The absolute path to the library.''' return self._path @property def parent(self): + '''The binary which loaded the library.''' return self._parent @property def name(self): + '''The name of the library.''' return os.path.basename(self.path) @property def loader_path(self): + '''The path to use for ``@loader_path`` references from the library.''' return os.path.dirname(self.path) def _get_dependencies(self): + ''' + Get the dependent libraries of the library. + + This only extracts the required libraries. Libraries which are marked + as "delay load" libraries are ignored and will not be installed. + ''' pipe = Pipeline([ 'dumpbin', '/DEPENDENTS', @@ -97,6 +130,11 @@ class Library(object): @property def dependencies(self): + ''' + Dependent libraries of the library. + + Visual C runtime libraries are ignored. + ''' if self._dependencies is None: collection = {} msvc_runtimes = re.compile('MSVC[A-Z][0-9]*.dll') @@ -110,6 +148,13 @@ class Library(object): return self._dependencies def _find_library(self, ref): + ''' + Find a library using search paths. + + Use of this method to find a dependent library indicates that the + library depdencies are not properly specified. As such, it warns when + it is used. + ''' for loc in self._search_paths: path = os.path.join(loc, ref) if os.path.exists(path): @@ -120,6 +165,7 @@ class Library(object): @classmethod def from_reference(cls, ref, loader): + '''Create a library representation given a name and a loading binary.''' paths = [] # Use the loader's location. @@ -148,6 +194,7 @@ class Library(object): @classmethod def from_path(cls, path, parent=None): + '''Create a library representation from a path.''' if not os.path.exists(path): raise RuntimeError('%s does not exist' % path) @@ -164,6 +211,7 @@ class Library(object): @classmethod def from_manifest(cls, path): + '''Create a library representation from a cached manifest entry.''' if path in cls.__cache: raise RuntimeError('There is already a library for %s' % path) @@ -176,6 +224,12 @@ class Library(object): class Module(Library): + ''' + A library loaded programmatically at runtime. + + Modules are loaded at runtime using ``LoadLibrary``. + ''' + def __init__(self, path, bundle_location, **kwargs): super(Module, self).__init__(path, None, **kwargs) @@ -187,11 +241,16 @@ class Module(Library): class Executable(Module): + ''' + An executable in the installation. + ''' + def __init__(self, path, **kwargs): super(Executable, self).__init__(path, 'bin', **kwargs) def copy_library(destination, bundle_dest, library, dry_run=False): + '''Copy a library into the ``.app`` bundle.''' if library._is_cached: return @@ -208,9 +267,11 @@ def copy_library(destination, bundle_dest, library, dry_run=False): return binary -# A function to fix up the fact that os.makedirs chokes if the path already -# exists. def _os_makedirs(path): + ''' + A function to fix up the fact that os.makedirs chokes if the path already + exists. + ''' if os.path.exists(path): return os.makedirs(path) @@ -255,6 +316,7 @@ def _arg_parser(): def _install_binary(binary, is_excluded, bundle_dest, dep_libdir, installed, manifest, dry_run=False): + '''Install the main binary into the package.''' # Start looking at our main executable's dependencies. deps = binary.dependencies.values() while deps: @@ -289,6 +351,7 @@ def _install_binary(binary, is_excluded, bundle_dest, dep_libdir, installed, man def _update_manifest(manifest, installed, path, location): + '''Update the manifest file with a set of newly installed binaries.''' for input_path in installed.keys(): manifest.setdefault(location, []).append(input_path) -- GitLab From 385aa8007e60779a7b105a07645dcf497bca4189 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:52:20 -0500 Subject: [PATCH 30/34] docs: document SuperbuildInstallMacros for Unix --- cmake/SuperbuildInstallMacros.cmake | 294 ++++++++++++++++++---------- 1 file changed, 188 insertions(+), 106 deletions(-) diff --git a/cmake/SuperbuildInstallMacros.cmake b/cmake/SuperbuildInstallMacros.cmake index cb9333b1..73a1b5c2 100644 --- a/cmake/SuperbuildInstallMacros.cmake +++ b/cmake/SuperbuildInstallMacros.cmake @@ -1,9 +1,24 @@ +#[==[.md +# Installation support + +When creating packages, these functions may be used in order to ensure that the +resulting package includes the required executables as well as all runtime +dependencies. These are discovered using platform-specific utilties to find out +what libraries are required and then emulating the platform's library searching +logic in order to find dependent libraries. For more details, see each +platform's section. + +Due to platform differences, each platform has its own set of functions for +use. +#]==] + set(_superbuild_install_cmake_dir "${CMAKE_CURRENT_LIST_DIR}") set_property(GLOBAL PROPERTY superbuild_has_cleaned FALSE) include(CMakeParseArguments) +# Find a Python executable to run the `fixup_bundle` scripts. if (NOT superbuild_python_executable) find_package(PythonInterp 2.7) if (PYTHONINTERP_FOUND) @@ -16,28 +31,78 @@ if (NOT superbuild_python_executable) endif () endif () +# TODO: error on unrecognized arguments + # TODO: The functions in this file should be grouped and made OS-agnostic. # Keyword arguments should be used more and be uniform across all # platforms. -# ======================================================================= -# Linux -# ======================================================================= - -# INTERNAL -# Install a binary and its required libraries to a location. -# -# _superbuild_unix_install_binary( -# LIBDIR <libdir> -# BINARY <path> -# TYPE <module|executable> -# [CLEAN] -# [DESTINATION <destination>] -# [LOCATION <location>] -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [LOADER_PATHS <loader-paths>...] -# [SEARCH_DIRECTORIES <search-directory>...]) +#[==[.md +## ELF (Linux) + +The superbuild installs ELF binaries using a core function to construct a +command to run the `fixup_bundle.unix.py` script with the correct arguments. It +tries to emulate an ELF runtime loader to determine where to find dependent +files and it copies them to the installation directory. + +Calling this function directory should not be necessary. Instead, using the +more specific functions documented later is recommended. The core function is +used as a single place to document the various common arguments available to +the other functions. If an argument is specified by a function, it should not +be passed as the remaining arguments to the function. + +``` +_superbuild_unix_install_binary( + LIBDIR <libdir> + BINARY <path> + TYPE <module|executable> + [CLEAN] + [DESTINATION <destination>] + [LOCATION <location>] + [INCLUDE_REGEXES <include-regex>...] + [EXCLUDE_REGEXES <exclude-regex>...] + [LOADER_PATHS <loader-paths>...] + [SEARCH_DIRECTORIES <search-directory>...]) +``` + +A manifest file is kept in the binary directory of the packaging step. This +allows for a library to be installed just once for an entire package. It is +reset when the `CLEAN` argument is present. In addition, the install directory +is removed for non-install targets (based on `superbuild_is_install_target`) +when `CLEAN` is specified. Whether this is necessary or not is maintained +internally and it should almost never need to be provided. + +The `DESTINATION` is the absolute path to the installation prefix, including +`DESTDIR`. It defaults to `\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}`. It is +escaped because we want CMake to expand its value at install time, not +configure time. + +The `BINARY` argument is the path to the actual executable to install. It must +be an absolute path. + +The `TYPE` argument specifies whether an executable or module (e.g., plugin or +standalone library) is being installed. For a module, the `LOCATION` argument +must be given. This is the path under the installation prefix to place the +module. Executables are always installed into `bin`. The libraries that are +found to be required by the installed binary are placed in the subdirectory of +the install destination given by the `LIBDIR` argument. + +The `LOADER_PATHS` argument is a list of paths to use when installing a module +to search for libraries that are assumed to be available because of the loading +executable. This is intended to be where libraries assumed to be with an +executable live when installing a plugin for that executable. + +The `SEARCH_DIRECTORIES` argument is a list of paths to search for libraries if +the library cannot be found due to rpaths in the binary, `LOADER_PATHS`, or +other runtime-loader logic. If these paths are required to find a library, a +warning is printed at install time. + +By default, libraries from the "system" (basically standard `/lib` directories) +are ignored when installing. The `INCLUDE_REGEXES` and `EXCLUDE_REGEXES` +arguments are lists of Python regular expressions to either force-include or +force-exclude from installation. Inclusion overrides exclusion. The provided +regular expressions are also expected to match the full path of the library. +#]==] function (_superbuild_unix_install_binary) set(options CLEAN) @@ -143,6 +208,7 @@ function (_superbuild_unix_install_binary) COMPONENT superbuild) endfunction () +# A convenience function for installing an executable. function (_superbuild_unix_install_executable path libdir) _superbuild_unix_install_binary( BINARY "${path}" @@ -151,6 +217,7 @@ function (_superbuild_unix_install_executable path libdir) ${ARGN}) endfunction () +# A convenience function for installing a module. function (_superbuild_unix_install_module path subdir libdir) _superbuild_unix_install_binary( BINARY "${path}" @@ -160,29 +227,35 @@ function (_superbuild_unix_install_module path subdir libdir) ${ARGN}) endfunction () -# Install a "forward executable" from the installation tree to the package. -# -# Usage: -# -# superbuild_unix_install_program_fwd(<name> <library-paths> -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [SEARCH_DIRECTORIES <search-directory>...]) -# -# Note that this installs a program which was created using the KWSys forward -# executable mechanisms. For "regular" binaries, see -# ``superbuild_unix_install_program``. -# -# Installs a binary named ``<name>`` to the package. The ``<library-paths>`` is -# a list of directories to search for the real binary and its libraries. -# Relative paths are taken to be relative to the install directory. The -# libraries are installed to the subdirectory the actual executable is found -# in. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``SEARCH_DIRECTORIES`` is a list of directories -# to search for dependent libraries. +#[==[.md +### Forwarding executables + +In the ParaView world, "forwarding" executables are used to make packages +standalone. This functionality is provided by KWSys. This creates a small +binary in `bin/` which finds its companion "real" executable under the +corresponding `lib/` directory. It then sets the `LD_LIBRARY_PATH` environment +variable accordingly and executes the real binary. + +In order to install these kinds of executables, this function is provided: + +``` +superbuild_unix_install_program_fwd(<NAME> <LIBRARY PATHS> [<ARG>...]) +``` + +Installs a binary named `NAME` to the package. The `LIBRARY PATHS` argument is +a list of directories to search for the real binary and its libraries. These +paths are assumed to be relative to `superbuild_install_location`. The +libraries are installed to the subdirectory the actual executable is found in. + +Note that `LIBRARY PATHS` is a CMake list passed as a single argument. + +The following arguments are set by calling this function: + + - `BINARY` + - `LIBDIR` + - `LOCATION` + - `TYPE` +#]==] function (superbuild_unix_install_program_fwd name paths) set(found FALSE) foreach (path IN LISTS paths) @@ -200,46 +273,53 @@ function (superbuild_unix_install_program_fwd name paths) _superbuild_unix_install_executable("${superbuild_install_location}/bin/${name}" "lib") endfunction () -# Install a program from the installation tree to the package. -# -# Usage: -# -# superbuild_unix_install_program(<path> <libdir> -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [SEARCH_DIRECTORIES <search-directory>...]) -# -# Installs a program ``<name>`` into ``bin/`` and its dependent libraies into -# ``<libdir>``. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``SEARCH_DIRECTORIES`` is a list of directories -# to search for dependent libraries. +#[==[.md +### Executables + +Non-forwarding executables are binaries that may not work in the package +without the proper rpaths or `LD_LIBRARY_PATH` set when running the executable. + +``` +superbuild_unix_install_program(<PATH> <LIBRARY DIR> [<ARG>...]) +``` + +Installs a program at `PATH` into `bin/` and its dependent libraries into +`LIBRARY DIR` under the install destination. The program must be an absolute +path. + +The following arguments are set by calling this function: + + - `BINARY` + - `LIBDIR` + - `TYPE` (`executable`) +#]==] function (superbuild_unix_install_program name libdir) _superbuild_unix_install_executable("${name}" "${libdir}" ${ARGN}) endfunction () -# Install a plugin to the package. -# -# Usage: -# -# superbuild_unix_install_plugin(<filename> <libdir> <search-paths> -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [LOADER_PATHS <loader-paths>...] -# [SEARCH_DIRECTORIES <search-directory>...]) -# -# Install a plugin from ``<filename>`` to the package. The file is searched for -# in ``<search-paths>`` under the superbuild's install tree. Required libraries -# are installed to ``<libdir>``. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``LOADER_PATHS`` is a list of directories where -# the executable loading the plugin is looking for libraries. These are -# searched for dependencies first. ``SEARCH_DIRECTORIES`` is a list of -# directories to search for dependent libraries. +#[==[.md +### Plugins + +Plugins include libraries that are meant to be loaded at runtime. It also +includes libraries that are linked to, but need to be installed separately. + +``` +superbuild_unix_install_plugin(<PATH> <LIBRARY DIR> <SEARCH PATHS> [<ARG>...]) +``` + +Installs a library at `PATH` into `bin/` and its dependent libraries into +`LIBRARY DIR` under the install destination. If the path is not absolute, it is +searched for underneath `superbuild_install_location` with the given `PATH` +under each path in the `SEARCH PATHS` argument. + +Note that `SEARCH PATHS` is a CMake list passed as a single argument. + +The following arguments are set by calling this function: + + - `BINARY` + - `LIBDIR` + - `TYPE` (`module`) +#]==] function (superbuild_unix_install_plugin name libdir paths) if (IS_ABSOLUTE "${name}") _superbuild_unix_install_module("${name}" "${paths}" "${libdir}" ${ARGN}) @@ -261,35 +341,37 @@ function (superbuild_unix_install_plugin name libdir paths) endif () endfunction () -# Install Python modules to the package. -# -# Usage: -# -# superbuild_unix_install_python( -# MODULES <module>... -# LIBDIR <libdir> -# MODULE_DIRECTORIES <module-path>... -# [MODULE_DESTINATION <destination>] -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [LOADER_PATHS <loader-paths>...] -# [SEARCH_DIRECTORIES <library-path>...]) -# -# Installs Python modules or packages named ``<name>`` into the package. The -# ``LIBDIR`` argument is used to place dependent libraries into the correct -# subdirectory of the package. Libraries are searched for in -# ``<library-paths>`` and relative paths are taken to be relative to the -# superbuild's install directory. The modules are searched for in the -# ``<module-paths>``. Modules are installed into the proper location -# (``lib/python2.7/site-packages``) within the package, but may be placed -# inside of a different directory by using the ``MODULE_DESTINATION`` argument. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``LOADER_PATHS`` is a list of directories where -# the executable loading the Python modules is looking for libraries. These are -# searched for dependencies first. ``SEARCH_DIRECTORIES`` is a list of -# directories to search for dependent libraries. +#[==[.md +### Python packages + +The superbuild also provides functions to install Python modules and packages. + +``` +superbuild_unix_install_python( + MODULES <module>... + LIBDIR <libdir> + MODULE_DIRECTORIES <module-path>... + [MODULE_DESTINATION <destination>] + [INCLUDE_REGEXES <include-regex>...] + [EXCLUDE_REGEXES <exclude-regex>...] + [LOADER_PATHS <loader-paths>...] + [SEARCH_DIRECTORIES <library-path>...]) +``` + +The list of modules to installed is given to the `MODULES` argument. These +modules (or packages) are searched for at install time in the paths given to +the `MODULE_DIRECTORIES` argument. + +Modules are placed in the `MODULE_DESTINATION` under the expected Python module +paths in the package (`lib/python2.7`). By default, `/site-packages` is used. + +The `INCLUDE_REGEXES`, `EXCLUDE_REGEXES`, `LOADER_PATHS`, and +`SEARCH_DIRECTORIES` arguments used when installing compiled Python modules +through an internal `superbuild_unix_install_plugin` call. + +Note that modules in the list which cannot be found are ignored. This function +also assumes Python 2.7 for now. +#]==] function (superbuild_unix_install_python) set(values MODULE_DESTINATION -- GitLab From ba5ca797275786ed9289b1d118e94e0dab496fc4 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:52:20 -0500 Subject: [PATCH 31/34] docs: document SuperbuildInstallMacros for macOS --- cmake/SuperbuildInstallMacros.cmake | 188 +++++++++++++++------------- 1 file changed, 104 insertions(+), 84 deletions(-) diff --git a/cmake/SuperbuildInstallMacros.cmake b/cmake/SuperbuildInstallMacros.cmake index 73a1b5c2..25cfe6d4 100644 --- a/cmake/SuperbuildInstallMacros.cmake +++ b/cmake/SuperbuildInstallMacros.cmake @@ -441,41 +441,48 @@ function (superbuild_unix_install_python) COMPONENT superbuild) endfunction () -# ======================================================================= -# OS X -# ======================================================================= +#[==[.md +## Mach-O (macOS) -# Create an application bundle. -# -# Usage: -# -# superbuild_apple_create_app(<destination> <name> <binary> -# [INCLUDE_REGEXES <regex>...] -# [EXCLUDE_REGEXES <regex>...] -# [SEARCH_DIRECTORIES <library-path>...] -# [PLUGINS <plugin>...] -# [FAKE_PLUGIN_PATHS] [CLEAN]) -# -# Creates a ``<name>.app`` bundle. The bundle is placed in ``<destination>`` -# with ``<binary>`` (full path) as a main executable for the bundle (under the -# ``MacOS/`` directory). Libraries are searched for and placed into the bundle -# from the ``<library-paths>`` specified. Library IDs and link paths are -# rewritten to use ``@executable_path`` or ``@loader_path`` as necessary. -# -# To exclude libraries from the bundle, use Python regular expressions as -# arguments to the ``EXCLUDE_REGEXES`` keyword. To include any -# otherwise-excluded libraries, use ``INCLUDE_REGEXES``. System libraries and -# frameworks are excluded by default. -# -# The ``CLEAN`` argument starts a new bundle, otherwise the bundle is left -# as-is (and is expected to have been created by this call). -# -# Plugins may be listed under the ``PLUGINS`` keyword and will be installed to -# the ``Plugins/`` directory in the bundle. These are full paths to the plugin -# binaries. If ``FAKE_PLUGIN_PATHS`` is given, the plugin is treated as its -# own ``@executable_path`` which is useful when packaging plugins which may be -# used for multiple applications and may require additional libraries -# depending on the application. +The superbuild installs Mach-O binaries using a core function to construct an +`.app` bundle using the `fixup_bundle.apple.py` script with the correct +arguments. It tries to emulate an Mach-O runtime loader to determine where to +find dependent files and it copies them to the installation directory. It also +fixes up internal library references so that the resulting package is +self-contained and relocatable. + +### Create an application bundle. + +``` +superbuild_apple_create_app(<DESTINATION> <NAME> <BINARY> + [INCLUDE_REGEXES <regex>...] + [EXCLUDE_REGEXES <regex>...] + [SEARCH_DIRECTORIES <library-path>...] + [PLUGINS <plugin>...] + [FAKE_PLUGIN_PATHS] [CLEAN]) +``` + +Creates a `<NAME>.app` bundle. The bundle is placed in `<DESTINATION>` with +`<BINARY>` (an absolute path) as a main executable for the bundle (under the +`MacOS/` directory). Libraries are searched for and placed into the bundle from +the `SEARCH_DIRECTORIES` specified. Library IDs and link paths are rewritten to +use `@executable_path` or `@loader_path` as necessary. + +To exclude libraries from the bundle, use Python regular expressions as +arguments to the `EXCLUDE_REGEXES` keyword. To include any otherwise-excluded +libraries, use `INCLUDE_REGEXES`. System libraries and frameworks are excluded +by default. + +The `CLEAN` argument starts a new bundle, otherwise the bundle is left as-is +(and is expected to have been created by this call). + +Plugins may be listed under the `PLUGINS` keyword and will be installed to the +`Plugins/` directory in the bundle. These are full paths to the plugin +binaries. If `FAKE_PLUGIN_PATHS` is given, the plugin is treated as its own +`@executable_path` which is useful when packaging plugins which may be used for +multiple applications and may require additional libraries depending on the +application. +#]==] function (superbuild_apple_create_app destination name binary) set(options CLEAN @@ -542,23 +549,25 @@ function (superbuild_apple_create_app destination name binary) COMPONENT superbuild) endfunction () -# Add a utility executable to a bundle. -# -# Usage: -# -# superbuild_apple_install_utility(<destination> <name> <binary> -# [INCLUDE_REGEXES <regex>...] -# [EXCLUDE_REGEXES <regex>...] -# [SEARCH_DIRECTORIES <library-path>...]) -# -# Adds a binary to the ``bin/`` path of the bundle. Required libraries are -# installed and fixed up. -# -# Must match an existing call to ``superbuild_apple_create_app(<destination> -# <name>)``. -# -# See ``superbuild_apple_create_app`` for the documentation for -# ``INCLUDE_REGEXES``, ``EXCLUDE_REGEXES``, and ``SEARCH_DIRECTORIES``. +#[==[.md +### Utility executables + +``` +superbuild_apple_install_utility(<DESTINATION> <NAME> <BINARY> + [INCLUDE_REGEXES <regex>...] + [EXCLUDE_REGEXES <regex>...] + [SEARCH_DIRECTORIES <library-path>...]) +``` + +Adds a binary to the `bin/` path of the bundle. Required libraries are +installed and fixed up using `@executable_path`. + +A previous call must have been made with matching `DESTINATION` and `NAME` +arguments; this call will not create a new application bundle. + +The `INCLUDE_REGEXES`, `EXCLUDE_REGEXES`, and `SEARCH_DIRECTORIES` arguments +are the same as those for `superbuild_apple_create_app`. +#]==] function (superbuild_apple_install_utility destination name binary) set(multivalues INCLUDE_REGEXES @@ -601,24 +610,27 @@ function (superbuild_apple_install_utility destination name binary) COMPONENT superbuild) endfunction () -# Add a module library to a bundle. -# -# Usage: -# -# superbuild_apple_install_module(<destination> <name> <binary> <location> -# [INCLUDE_REGEXES <regex>...] -# [EXCLUDE_REGEXES <regex>...] -# [SEARCH_DIRECTORIES <library-path>...]) -# -# Adds a library to the ``<location>`` path of the bundle. Required libraries are -# installed and fixed up using ``@loader_path``. Use this to install things -# such as compiled language modules and the like. -# -# Must match an existing call to ``superbuild_apple_create_app(<destination> -# <name>)``. -# -# See ``superbuild_apple_create_app`` for the documentation for -# ``INCLUDE_REGEXES``, ``EXCLUDE_REGEXES``, and ``SEARCH_DIRECTORIES``. +#[==[.md +### Module libraries + +``` +superbuild_apple_install_module(<DESTINATION> <NAME> <BINARY> <LOCATION> + [INCLUDE_REGEXES <regex>...] + [EXCLUDE_REGEXES <regex>...] + [SEARCH_DIRECTORIES <library-path>...]) +``` + +Adds a library to the `<LOCATION>` path of the bundle. Required libraries which +have not been installed by previous executable installs are installed and fixed +up using `@loader_path`. Use this to install things such as compiled language +modules and the like. + +A previous call must have been made with matching `DESTINATION` and `NAME` +arguments; this call will not create a new application bundle. + +The `INCLUDE_REGEXES`, `EXCLUDE_REGEXES`, and `SEARCH_DIRECTORIES` arguments +are the same as those for `superbuild_apple_create_app`. +#]==] function (superbuild_apple_install_module destination name binary location) set(multivalues INCLUDE_REGEXES @@ -662,21 +674,29 @@ function (superbuild_apple_install_module destination name binary location) COMPONENT superbuild) endfunction () -# Add Python modules or packages to a bundle. -# -# Usage: -# -# superbuild_apple_install_python(<destination> <name> -# MODULES <module>... -# MODULE_DIRECTORIES <module-path>... -# [SEARCH_DIRECTORIES <library-path>...]) -# -# Adds Python modules or packages ``<modules>`` to the bundle. Required -# libraries are searched for in the ``<library-paths>`` and placed next to the -# module which requires it. -# -# See ``superbuild_unix_install_python`` for the documentation for -# ``MODULES`` and ``MODULE_DIRECTORIES``. +#[==[.md +### Python packages + +The superbuild also provides functions to install Python modules and packages. + +``` +superbuild_apple_install_python(<DESTINATION> <NAME> + MODULES <module>... + MODULE_DIRECTORIES <module-path>... + [SEARCH_DIRECTORIES <library-path>...]) +``` + +The list of modules to installed is given to the `MODULES` argument. These +modules (or packages) are searched for at install time in the paths given to +the `MODULE_DIRECTORIES` argument. + +Modules are placed in the `Python/` directory in the given application bundle. + +A previous call must have been made with matching `DESTINATION` and `NAME` +arguments; this call will not create a new application bundle. + +Note that modules in the list which cannot be found are ignored. +#]==] function (superbuild_apple_install_python destination name) set(multivalues SEARCH_DIRECTORIES -- GitLab From 47a37c5b1e4fa44de9a2e4f139106f2b9552454e Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Wed, 7 Mar 2018 17:52:20 -0500 Subject: [PATCH 32/34] docs: document SuperbuildInstallMacros for Windows --- cmake/SuperbuildInstallMacros.cmake | 211 ++++++++++++++++++---------- 1 file changed, 136 insertions(+), 75 deletions(-) diff --git a/cmake/SuperbuildInstallMacros.cmake b/cmake/SuperbuildInstallMacros.cmake index 25cfe6d4..3ae1c681 100644 --- a/cmake/SuperbuildInstallMacros.cmake +++ b/cmake/SuperbuildInstallMacros.cmake @@ -736,24 +736,70 @@ function (superbuild_apple_install_python destination name) COMPONENT superbuild) endfunction () -# ======================================================================= -# Windows -# ======================================================================= - -# INTERNAL -# Install a binary and its required libraries to a location. -# -# _superbuild_windows_install_binary( -# LIBDIR <libdir> -# BINARY <path> -# TYPE <module|executable> -# [CLEAN] -# [DESTINATION <destination>] -# [LOCATION <location>] -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [BINARY_LIBDIR <binary_libdirs>...] -# [SEARCH_DIRECTORIES <search-directory>...]) +#[==[.md +## PE-COFF (Windows) + +The superbuild installs PE-COFF binaries using a core function to construct a +command to run the `fixup_bundle.windows.py` script with the correct arguments. +It tries to emulate the runtime loader to determine where to find dependent +files and it copies them to the installation directory. + +Calling this function directory should not be necessary. Instead, using the +more specific functions documented later is recommended. The core function is +used as a single place to document the various common arguments available to +the other functions. If an argument is specified by a function, it should not +be passed as the remaining arguments to the function. + +``` +_superbuild_windows_install_binary( + LIBDIR <libdir> + BINARY <path> + TYPE <module|executable> + [CLEAN] + [DESTINATION <destination>] + [LOCATION <location>] + [INCLUDE_REGEXES <include-regex>...] + [EXCLUDE_REGEXES <exclude-regex>...] + [BINARY_LIBDIR <binary_libdirs>...] + [SEARCH_DIRECTORIES <search-directory>...]) +``` + +A manifest file is kept in the binary directory of the packaging step. This +allows for a library to be installed just once for an entire package. It is +reset when the `CLEAN` argument is present. In addition, the install directory +is removed for non-install targets (based on `superbuild_is_install_target`) +when `CLEAN` is specified. Whether this is necessary or not is maintained +internally and it should almost never need to be provided. + +The `DESTINATION` is the absolute path to the installation prefix, including +`DESTDIR`. It defaults to `\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}`. It is +escaped because we want CMake to expand its value at install time, not +configure time. + +The `BINARY` argument is the path to the actual executable to install. It must +be an absolute path. + +The `TYPE` argument specifies whether an executable or module (e.g., plugin or +standalone library) is being installed. For a module, the `LOCATION` argument +must be given. This is the path under the installation prefix to place the +module. Executables are always installed into `bin`. The libraries that are +found to be required by the installed binary are placed in the subdirectory of +the install destination given by the `LIBDIR` argument. + +The `BINARY_LIBDIR` argument is a list of paths which the binary is assumed to +already be searching when loading a module. + +The `SEARCH_DIRECTORIES` argument is a list of paths to search for libraries if +the library cannot be found due to rpaths in the binary, `LOADER_PATHS`, or +other runtime-loader logic. If these paths are required to find a library, a +warning is printed at install time. + +By default, Microsoft's C runtime libraries are ignored when installing. The +`INCLUDE_REGEXES` and `EXCLUDE_REGEXES` arguments are lists of Python regular +expressions to either force-include or force-exclude from installation. +Inclusion overrides exclusion. The provided regular expressions are also +expected to match the full path of the library. +#]==] function (_superbuild_windows_install_binary) set(options CLEAN) @@ -862,6 +908,7 @@ function (_superbuild_windows_install_binary) COMPONENT superbuild) endfunction () +# A convenience function for installing an executable. function (_superbuild_windows_install_executable path libdir) _superbuild_windows_install_binary( BINARY "${path}" @@ -870,6 +917,7 @@ function (_superbuild_windows_install_executable path libdir) ${ARGN}) endfunction () +# A convenience function for installing a module. function (_superbuild_windows_install_module path subdir libdir) _superbuild_windows_install_binary( BINARY "${path}" @@ -879,43 +927,53 @@ function (_superbuild_windows_install_module path subdir libdir) ${ARGN}) endfunction () -# Install a program from the installation tree to the package. -# -# Usage: -# -# superbuild_windows_install_program(<path> <libdir> -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [SEARCH_DIRECTORIES <search-directory>...]) -# -# Installs a program ``<name>`` into ``bin/`` and its dependent libraies into -# ``<libdir>``. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``SEARCH_DIRECTORIES`` is a list of directories -# to search for dependent libraries. +#[==[.md +### Executables + +Non-forwarding executables are binaries that may not work in the package +without the proper rpaths or `LD_LIBRARY_PATH` set when running the executable. + +``` +superbuild_windows_install_program(<NAME> <LIBDIR>) +``` + +Installs a program at `NAME` into `bin/` and its dependent libraries into +`LIBDIR` under the install destination. The program is assumed to be in the +installation prefix as `bin/<NAME>.exe`. + +The following arguments are set by calling this function: + + - `BINARY` + - `LIBDIR` + - `TYPE` (`executable`) +#]==] function (superbuild_windows_install_program name libdir) _superbuild_windows_install_executable("${superbuild_install_location}/bin/${name}.exe" "${libdir}" ${ARGN}) endfunction () -# Install a plugin to the package. -# -# Usage: -# -# superbuild_windows_install_plugin(<filename> <libdir> <search-paths> -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [SEARCH_DIRECTORIES <search-directory>...]) -# -# Install a plugin from ``<filename>`` to the package. The file is searched for -# in ``<search-paths>`` under the superbuild's install tree. Required libraries -# are installed to ``<libdir>``. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``SEARCH_DIRECTORIES`` is a list of directories -# to search for dependent libraries. +#[==[.md +### Plugins + +Plugins include libraries that are meant to be loaded at runtime. It also +includes libraries that are linked to, but need to be installed separately. + +``` +superbuild_windows_install_plugin(<FILENAME> <LIBDIR> <SEARCH PATHS> [<ARG>...]) +``` + +Installs a library named `FILENAME` into `bin/` and its dependent libraries +into `LIBDIR` under the install destination. The given filename is searched for +under each path in the `SEARCH PATHS` argument. + +Note that `SEARCH PATHS` is a CMake list passed as a single argument. + +The following arguments are set by calling this function: + + - `BINARY` + - `LIBDIR` + - `LOCATION` + - `TYPE` (`module`) +#]==] function (superbuild_windows_install_plugin name libdir paths) if (IS_ABSOLUTE "${name}") _superbuild_windows_install_module("${name}" "${paths}" "${libdir}" ${ARGN}) @@ -937,30 +995,33 @@ function (superbuild_windows_install_plugin name libdir paths) endif () endfunction () -# Install Python modules to the package. -# -# Usage: -# -# superbuild_windows_install_python( -# MODULES <module>... -# MODULE_DIRECTORIES <module-path>... -# [MODULE_DESTINATION <destination>] -# [INCLUDE_REGEXES <include-regex>...] -# [EXCLUDE_REGEXES <exclude-regex>...] -# [SEARCH_DIRECTORIES <library-path>...]) -# -# Installs Python modules or packages named ``<name>`` into the package. -# Libraries are searched for in ``<library-paths>`` and relative paths are -# taken to be relative to the superbuild's install directory. The modules are -# searched for in the ``<module-paths>``. Modules are installed into the proper -# location (``lib/python2.7/site-packages``) within the package, but may be -# placed inside of a different directory by using the ``MODULE_DESTINATION`` -# argument. -# -# The ``INCLUDE_REGEXES`` and ``EXCLUDE_REGEXES`` arguments may be used to -# include or exclude found paths from being installed to the package. They are -# Python regular expressions. ``SEARCH_DIRECTORIES`` is a list of directories -# to search for dependent libraries. +#[==[.md +### Python packages + +The superbuild also provides functions to install Python modules and packages. + +``` +superbuild_windows_install_python( + MODULES <module>... + MODULE_DIRECTORIES <module-path>... + [MODULE_DESTINATION <destination>] + [INCLUDE_REGEXES <include-regex>...] + [EXCLUDE_REGEXES <exclude-regex>...] + [SEARCH_DIRECTORIES <library-path>...]) +``` + +The list of modules to installed is given to the `MODULES` argument. These +modules (or packages) are searched for at install time in the paths given to +the `MODULE_DIRECTORIES` argument. + +Modules are placed in the `MODULE_DESTINATION` under the expected Python module +paths in the package (`bin/Lib`). By default, `/site-packages` is used. + +The `INCLUDE_REGEXES`, `EXCLUDE_REGEXES`, and `SEARCH_DIRECTORIES` used when +installing compiled Python modules through `superbuild_windows_install_plugin`. + +Note that modules in the list which cannot be found are ignored. +#]==] function (superbuild_windows_install_python) set(values MODULE_DESTINATION) -- GitLab From 216f4f8935f4930e098e1a2d81425661e15456fd Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Sat, 10 Mar 2018 16:02:40 -0500 Subject: [PATCH 33/34] docs: document SuperbuildUtils --- cmake/SuperbuildUtils.cmake | 88 ++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/cmake/SuperbuildUtils.cmake b/cmake/SuperbuildUtils.cmake index c0c7f600..80569c32 100644 --- a/cmake/SuperbuildUtils.cmake +++ b/cmake/SuperbuildUtils.cmake @@ -1,11 +1,20 @@ +#[==[.md INTERNAL +# Utility functions + +This module contains utility functions mostly used internally to the +superbuild. Some may be of use to corner-case project builds. +#]==] + include("SuperbuildUtils-apple") include("SuperbuildUtils-unix") +# TODO: In 3.9.0, use the GENERATOR_IS_MULTI_CONFIG global property. if (NOT CMAKE_CONFIGURATION_TYPES) set(_superbuild_build_type_force) if (NOT CMAKE_BUILD_TYPE) set(_superbuild_build_type_force FORCE) endif () + # Default to a Release build. set(CMAKE_BUILD_TYPE "Release" CACHE STRING "The build mode" ${_superbuild_build_type_force}) mark_as_advanced(CMAKE_BUILD_TYPE) @@ -13,6 +22,10 @@ if (NOT CMAKE_CONFIGURATION_TYPES) PROPERTY STRINGS "Release;RelWithDebInfo") if (NOT WIN32) + # Windows debug builds is not really supported at the moment. Getting all + # projects to agree on the C++ runtime library is not easy. Also, many + # projects hard-code library paths and on Windows, many projects change + # their names for debug builds. set_property(CACHE CMAKE_BUILD_TYPE APPEND PROPERTY STRINGS "Debug") @@ -22,6 +35,7 @@ if (NOT CMAKE_CONFIGURATION_TYPES) message(FATAL_ERROR "A build type (CMAKE_BUILD_TYPE) must be set.") endif () + # Ensure that the chosen build type is valid. get_property(build_type_options CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS) @@ -47,6 +61,14 @@ if (NOT CMAKE_CONFIGURATION_TYPES) endif () endif () +#[==[.md INTERNAL +# 64-bit support + +Some projects need to know if a build is 32-bit or 64-bit. This function sets +`superbuild_is_64bit`. + +In the future, 32-bit support may be removed completely. +#]==] function (superbuild_detect_64bit_target) if (CMAKE_CROSSCOMPILING) return () @@ -61,6 +83,7 @@ function (superbuild_detect_64bit_target) set(superbuild_is_64bit TRUE PARENT_SCOPE) elseif (void_ptr_size EQUAL 4) + # XXX: Error out here? Is there a reason to still support 32-bit? set(superbuild_is_64bit FALSE PARENT_SCOPE) else () @@ -74,6 +97,7 @@ function (superbuild_detect_64bit_target) endif () endfunction () +# A utility function to create a PATH-link environment variable value. macro (_superbuild_make_path_var var) set(${var} ${ARGN}) list(REMOVE_ITEM ${var} "") @@ -82,6 +106,16 @@ macro (_superbuild_make_path_var var) endif () endmacro () +#[==[.md INTERNAL +# Compiler flag setup + +Superbuild projects may set add additional language flags to be made available +to projects during their builds via the standard environment variables (more +detail is available in [their documentation](SuperbuildVariables.md#Language +flags)). The function handles the setup of the flags made available to projects +by integrating the environment at configure time, the superbuild's installation +directory for projects. +#]==] function (superbuild_setup_flags) if (WIN32) return () @@ -115,6 +149,12 @@ function (superbuild_setup_flags) endforeach () endfunction () +#[==[.md INTERNAL +# Preparing the build tree + +The build tree needs some preparation for some features. These steps are done +here. +#]==] macro (superbuild_prepare_build_tree) if (WIN32) # Windows doesn't like it if that directory does not exist even if it is @@ -128,21 +168,25 @@ macro (superbuild_prepare_build_tree) "${_superbuild_module_gen_dir}") endmacro () -# Bridge an old, deprecated, setting to a new replacement setting. -# -# Use this function when a user-visible flag is being renamed or otherwise -# replaced. If the old value is set, it will be given as the default value, -# otherwise the given default value will be used. This returned value should -# then be used in the ``set(CACHE)`` or ``option()`` call for the new value. -# -# If the old value is set, it will warn that it is deprecated for the new name. -# -# If replacing the setting ``OLD_SETTING`` with ``NEW_SETTING``, its usage -# would look like: -# -# superbuild_deprecated_setting(default_setting NEW_SETTING OLD_SETTING "default value") -# set(NEW_SETTING "${default_setting}" -# CACHE STRING "Documentation for the setting.") +#[==[.md +# Handling deprecated variables + +Use this function when a user-visible flag is being renamed or otherwise +replaced. If the old value is set, it will be given as the default value, +otherwise the given default value will be used. This returned value should then +be used in the `set(CACHE)` or `option()` call for the new value. + +If the old value is set, it will warn that it is deprecated for the new name. + +If replacing the setting `OLD_SETTING` with `NEW_SETTING`, its usage would look +like: + +```cmake +superbuild_deprecated_setting(default_setting NEW_SETTING OLD_SETTING "default value") +set(NEW_SETTING "${default_setting}" + CACHE STRING "Documentation for the setting.") +``` +#]==] function (superbuild_deprecated_setting output_default new old intended_default) set(default "${intended_default}") if (DEFINED "${old}") @@ -154,12 +198,14 @@ function (superbuild_deprecated_setting output_default new old intended_default) set("${output_default}" "${default}" PARENT_SCOPE) endfunction () -# Determine whether the common superbuild is up-to-date or not. -# -# The common superbuild is intended to be used as a submodule to projects. -# However, updating the submodule when updating the main superbuild is an easy -# step to forget. This function tries to determine whether the common -# superbuild is up-to-date or not and error out if it is not. +#[==[.md +# Catching a stale superbuild submodule + +The common superbuild is intended to be used as a submodule to projects. +However, updating the submodule when updating the main superbuild is an easy +step to forget. This function tries to determine whether the common superbuild +is up-to-date or not and error out if it is not. +#]==] function (_superbuild_check_up_to_date) file(RELATIVE_PATH common_superbuild_path "${CMAKE_SOURCE_DIR}" -- GitLab From ff148a4b9a65ad4c8012cc42d53b4d6a52aec802 Mon Sep 17 00:00:00 2001 From: Ben Boeckel <ben.boeckel@kitware.com> Date: Sat, 10 Mar 2018 16:58:46 -0500 Subject: [PATCH 34/34] docs: document SuperbuildMacros --- cmake/SuperbuildMacros.cmake | 605 +++++++++++++++++++++-------------- 1 file changed, 361 insertions(+), 244 deletions(-) diff --git a/cmake/SuperbuildMacros.cmake b/cmake/SuperbuildMacros.cmake index 9e16f57c..48d1a764 100644 --- a/cmake/SuperbuildMacros.cmake +++ b/cmake/SuperbuildMacros.cmake @@ -1,3 +1,44 @@ +#[==[.md +# Superbuild projects + +A superbuild is designed to build various projects into a single prefix as a +prepatory step for packaging. As a secondary goal, development of a project may +be possible using the superbuild to prepare a set of dependencies. + +## Multi-config generators + +The superbuild does not support the CMake generators which allow for build-time +switching of the build configuration, namely Visual Studio and Xcode. The build +and install trees for these generators introduces complexities that are not +supported. + +## Usability of project builds + +In general, there are three places a built project lives: + + - in its own build tree + - in the superbuild's install tree + - in a package generated by the superbuild + +Due to various platform limitations, getting all three to be usable at the same +time is not generally possible. The package has to work, so other use cases +defer to it. Binaries within packages are prepared by various installation +scripts which rewrite dependency information where necessary so that the +package is free of any superbuild references and the package is relocatable. +Since there exist projects which do not generate relocatable installs and they +tend to be non-relocatable for various reasons, it is easiest to just fix +binaries directly. + +The next most interesting binary location is the project's own build directory. +Keeping this usable can be important since it is here that most test suites +run. + +Lastly, there is the install tree. This is interesting in the development use +case, but support for can preclude packaging (e.g., on macOS ParaView's install +either includes an SDK for use by other projects or an application bundle). +Support is done on a best-effort basis in cases such as this. +#]==] + include(SuperbuildExternalProject) include(CMakeParseArguments) @@ -5,53 +46,56 @@ include(CMakeParseArguments) # anything else can use it that needs it. set(_superbuild_list_separator "-+-") -# Describe a project to be built as part of the superbuild. -# -# Usage: -# -# superbuild_add_project(<name> [ARGS...]) -# -# All ExternalProject keywords are valid here as well as the following -# extensions: -# -# ``CAN_USE_SYSTEM`` -# Marks that the project may be provided by the system. In this case, the -# ``${name}.system.cmake`` file will be used during the second phase if the -# system version is selected. -# ``MUST_USE_SYSTEM`` -# Where a project can be provided by the system, this flag may be specified -# to indicate that this platform *must* use the system's version rather -# than a custom built one. Usually only used in platform-specific files. -# ``DEFAULT_ON`` -# If present, the project will default to be built. -# ``DEVELOPER_MODE`` -# If present, the project will offer an option to build it in "developer" -# mode. Developer mode enables and builds all dependent projects, but skips -# the project itself. Instead, a file named -# ``${name}-developer-config.cmake`` is written to the build directory -# which may be passed to a standalone instance of the project using the -# ``-C`` option of CMake to initialize the cache to use the dependencies -# built as part of the superbuild. -# ``DEBUGGABLE`` -# If present, an option to change the build type for the project will be -# exposed. -# ``SELECTABLE`` -# If present, this project's ``ENABLE_`` option will be visible (and all -# non-selectable projects will be hidden). May be set externally with the -# ``_superbuild_${name}_selectable`` flag. -# ``HELP_STRING`` -# Set the description string for the option to enable the project. -# ``DEPENDS_OPTIONAL <project>...`` -# Projects which this one can use if it is enabled, but is not required for -# use. -# ``PROCESS_ENVIRONMENT <var> <value>...`` -# Sets environment variables for the configure, build, and install steps. -# Some are "magic" and are prepended to the current value (namely ``PATH``, -# ``LD_LIBRARY_PATH`` (Linux), and ``DYLD_LIBRARY_PATH`` (OS X). -# -# Projects which are depended on may declare that they have CMake variables and -# flags which must be set in dependent projects (e.g., a Python project would -# set ``PYTHON_EXECUTABLE`` to the location of its installed Python). +#[==[.md +# Adding a project to the superbuild + +Usage: + +``` +superbuild_add_project(<NAME> [ARGS...]) +``` + +All [`ExternalProject`][ExternalProject] keywords are valid here as well as the +following extensions: + +[ExternalProject]: https://cmake.org/cmake/help/v3.9/module/ExternalProject.html + + - `CAN_USE_SYSTEM` Marks that the project may be provided by the system. In + this case, the `${NAME}.system.cmake` file will be used during the second + phase if the system version is selected. + - `MUST_USE_SYSTEM` Where a project can be provided by the system, this flag + may be specified to indicate that this platform *must* use the system's + version rather than a custom built one. Usually only used in + platform-specific files. + - `DEFAULT_ON` If present, the project will default to be built. + - `DEVELOPER_MODE` If present, the project will offer an option to build it + in "developer" mode. Developer mode enables and builds all dependent + projects, but skips the project itself. Instead, a file named + `${NAME}-developer-config.cmake` is written to the build directory which + may be passed to a standalone instance of the project using the `-C` option + of CMake to initialize the cache to use the dependencies built as part of + the superbuild. + - `DEBUGGABLE` If present, an option to change the build type for the project + will be exposed. By default, the value of `<same>` is used to match the + superbuild's configuration. On Windows, mixing of `Debug` and non-`Debug` + build configurations will be caught and disallowed. + - `SELECTABLE` If present, this project's `ENABLE_` option will be visible + (and all non-selectable projects will be hidden). May be set externally + with the `_superbuild_${NAME}_selectable` flag. + - `HELP_STRING` + Set the description string for the option to enable the project. + - `DEPENDS_OPTIONAL <project>...` + Projects which this one can use if it is enabled, but is not required for + use. + - `PROCESS_ENVIRONMENT <var> <value>...` + Sets environment variables for the configure, build, and install steps. + Some are "magic" and are prepended to the current value (namely ``PATH``, + ``LD_LIBRARY_PATH`` (Linux), and ``DYLD_LIBRARY_PATH`` (OS X). + +Projects which are depended on may declare that they have CMake variables and +flags which must be set in dependent projects (e.g., a Python project would set +`PYTHON_EXECUTABLE` to the location of its installed Python). +#]==] function (superbuild_add_project name) _superbuild_project_check_name("${name}") @@ -117,6 +161,11 @@ function (superbuild_add_project name) endif () if (superbuild_build_phase) + # Build phase logic. This logic involves saving the final list of arguments + # that will be passed through to `ExternalProject_add`. It also inspects + # optional dependencies and adds them as real dependencies if they are + # enabled. + foreach (op_dep IN LISTS optional_depends) if (${op_dep}_enabled) list(APPEND ep_arguments @@ -143,10 +192,12 @@ function (superbuild_add_project name) endif () endforeach () + # Warn if optional dependencies are unknown. if (missing_deps_optional) string(REPLACE ";" ", " missing_deps_optional "${missing_deps_optional}") message(AUTHOR_WARNING "Optional dependencies for ${name} not found: ${missing_deps_optional}") endif () + # Error if required dependencies are unknown. if (missing_deps) string(REPLACE ";" ", " missing_deps "${missing_deps}") message(FATAL_ERROR "Dependencies for ${name} not found: ${missing_deps}") @@ -157,6 +208,9 @@ function (superbuild_add_project name) "${ep_arguments}" PARENT_SCOPE) else () + # Scanning phase logic. This involves setting up global properties to + # include the information required in later steps of the superbuild. + option("ENABLE_${name}" "${help_string}" "${default}") # Set the TYPE because it is overrided to INTERNAL if it is required by # dependencies later. @@ -213,16 +267,19 @@ function (superbuild_add_project name) endif () endfunction () -# Adds a project to the list, but with a no-op build step. Useful for "feature" -# projects to set flags. -# -# Usage: -# -# superbuild_add_dummy_project(<name> [ARGS...]) -# -# The only keyword arguments which do anything for dummy projects are the -# ``DEPENDS`` and ``DEPENDS_OPTIONAL`` keywords which are used to enforce build -# order. +#[==[.md +Some projects may not have any special logic and may just be there for users to +enable certain features in other projects. This function is provided for these +cases: + +``` +superbuild_add_dummy_project(<NAME> [ARGS...]) +``` + +The only keyword arguments which do anything for dummy projects are the +``DEPENDS`` and ``DEPENDS_OPTIONAL`` keywords which are used to enforce build +order. +#]==] function (superbuild_add_dummy_project _name) superbuild_add_project(${_name} "${ARGN}") @@ -231,27 +288,117 @@ function (superbuild_add_dummy_project _name) "${_name}_is_dummy" TRUE) endfunction () -# Apply a patch to a project. -# -# Usage: -# -# superbuild_apply_patch(<name> <patch-name> <description>) -# -# Applies a patch to the project during the build. The patch is assumed live at -# the following path: -# -# ${CMAKE_CURRENT_LIST_DIR}/patches/${name}-${patch-name}.patch -# -# Patches should not be applied to projects which are sourced from Git -# repositories due to bugs in ``git apply``. Use of this function on such -# projects will cause patches to, in all probability, be ignored or fail to -# apply. For those projects, create a fork, create commits, and point the -# repository to the fork instead. -# -# This function does check if the build tree lives under a git repository and -# errors out if so since then *all* patch applications will fail. -# -# Please send relevant patches upstream. +#[==[.md +## Python projects + +Since Python projects are common in the projects using the superbuild +mechanisms, there are two additional functions provided for building Python +libraries. +#]==] + +#[==[.md +### `setup.py` + +``` +superbuild_add_project_python(<NAME> <ARG>...) +``` + +Same as `superbuild_add_project`, but sets `PYTHONPATH` and build commands to +work properly out of the box. +#]==] +macro (superbuild_add_project_python _name) + if (WIN32) + set(_superbuild_python_path <INSTALL_DIR>/bin/Lib/site-packages) + set(_superbuild_python_args + "--prefix=bin") + else () + set(_superbuild_python_path <INSTALL_DIR>/lib/python2.7/site-packages) + set(_superbuild_python_args + "--single-version-externally-managed" + "--prefix=") + endif () + + superbuild_add_project("${_name}" + BUILD_IN_SOURCE 1 + DEPENDS python ${ARGN} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + "${superbuild_python_executable}" + setup.py + build + ${${_name}_python_build_args} + INSTALL_COMMAND + "${superbuild_python_executable}" + setup.py + install + --skip-build + --root=<INSTALL_DIR> + ${_superbuild_python_args} + ${${_name}_python_install_args} + PROCESS_ENVIRONMENT + PYTHONPATH ${_superbuild_python_path}) +endmacro () + +#[==[.md +### Wheels + +``` +superbuild_add_project_python_wheel(<NAME> <ARG>...) +``` + +Same as `superbuild_add_project`, but installs the project from a wheel. Note +that the source for such projects must be a wheel that may be extracted using +`pip`. +#]==] +macro (superbuild_add_project_python_wheel _name) + if (superbuild_build_phase AND NOT superbuild_python_pip) + message(FATAL_ERROR + "No `pip` available?") + endif () + + superbuild_add_project("${_name}" + BUILD_IN_SOURCE 1 + DOWNLOAD_NO_EXTRACT 1 + DEPENDS python ${ARGN} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + "" + INSTALL_COMMAND + ${superbuild_python_pip} + install + --no-index + --prefix=<INSTALL_DIR> + "<DOWNLOADED_FILE>") +endmacro () + +#[==[.md +# Applying patches to a project + +Some projects may require patches applied to the source tree in order to fix +errors in them. The `superbuild_apply_patch` function is provided to make it +easy to apply such patches. + +``` +superbuild_apply_patch(<NAME> <PATCH NAME> <DESCRIPTION>) +``` + +Applies a patch to the project during the build. The patch is assumed live at +`${CMAKE_CURRENT_LIST_DIR}/patches/${NAME}-${PATCH-NAME}.patch` from the call +site. + +Patches should not be applied to projects which are sourced from Git +repositories due to bugs in `git apply`. Use of this function on such projects +will cause patches to, in all probability, be ignored or fail to apply. For +those projects, create a fork, create commits, and point the repository to the +fork instead. + +This function does check if the build tree lives under a git repository and +errors out if so since then *all* patch applications will fail. + +Please forward relevant patches upstream. +#]==] function (superbuild_apply_patch _name _patch _comment) find_package(Git QUIET) if (NOT GIT_FOUND) @@ -306,34 +453,18 @@ function (superbuild_apply_patch _name _patch _comment) WORKING_DIRECTORY <SOURCE_DIR>) endfunction () -# Add CMake arguments to projects using this one. -# -# Usage: -# -# superbuild_add_extra_cmake_args([-DREQUIRED_VARIABLE:TYPE=VALUE]...) -# -# The ``-D`` and ``TYPE`` are required (due to the way ExternalProject does -# things internally). -function (superbuild_add_extra_cmake_args) - if (NOT superbuild_build_phase) - return () - endif () +#[==[.md +## Custom steps - _superbuild_check_current_project("superbuild_add_extra_cmake_args") +Add a custom step to the project. - set_property(GLOBAL APPEND - PROPERTY - "${current_project}_cmake_args" ${ARGN}) -endfunction () +``` +superbuild_project_add_step(<NAME> <ARG>...) +``` -# Add a custom step to the project. -# -# Usage: -# -# superbuild_project_add_step(myproject <step-arguments>...) -# -# See the documentation for ``ExternalProject_add_step`` for the arguments to -# this. +This function uses `ExternalProject_add_step` to create new steps during the +project's superbuild-level target. See its documentation for details. +#]==] function (superbuild_project_add_step name) if (NOT superbuild_build_phase) return () @@ -349,21 +480,57 @@ function (superbuild_project_add_step name) "${current_project}_step_${name}" ${ARGN}) endfunction () -# Add flags to projects using this one. -# -# Usage: -# -# superbuild_append_flags(<key> <value> [PROJECT_ONLY]) -# -# Adds flags to the build of this and, if ``PROJECT_ONLY`` is not specified, -# dependent projects. -# -# Valid values for ``<key>`` are: -# -# cxx_flags: add flags for C++ compilation. -# c_flags: add flags for C compilation. -# cpp_flags: add flags C and C++ preprocessors. -# ld_flags: add flags for linkers. +#[==[.md +# Usage requirements + +Projects may have "usage requirements" by passing CMake or compiler flags to +dependent projects. Note that these flags are only offered to projects which +directly depend on a project: they are not transitive. +#]==] + +#[==[.md +## CMake configuration usage requirements + +Usage: + +``` +superbuild_add_extra_cmake_args([-DREQUIRED_VARIABLE:TYPE=VALUE]...) +``` + +The `-D` and `TYPE` are required (due to some internal logic of +`ExternalProject`). +#]==] +function (superbuild_add_extra_cmake_args) + if (NOT superbuild_build_phase) + return () + endif () + + _superbuild_check_current_project("superbuild_add_extra_cmake_args") + + set_property(GLOBAL APPEND + PROPERTY + "${current_project}_cmake_args" ${ARGN}) +endfunction () + +#[==[.md +## Compiler flag usage requirements + +Add compiler flags to projects using this one. + +``` +superbuild_append_flags(<KEY> <VALUE> [PROJECT_ONLY]) +``` + +Adds flags to the build of this and, if `PROJECT_ONLY` is not specified, +dependent projects. + +Valid values for `KEY` are: + + - `cxx_flags`: add flags for C++ compilation. + - `c_flags`: add flags for C compilation. + - `cpp_flags`: add flags C and C++ preprocessors. + - `ld_flags`: add flags for linkers. +#]==] function (superbuild_append_flags key value) if (NOT superbuild_build_phase) return () @@ -399,14 +566,17 @@ function (superbuild_append_flags key value) "${property}" " ${value}") endfunction () -# Add directories to PATH for projects using this one. -# -# Usage: -# -# superbuild_add_path(<path>...) -# -# Adds the arguments to the ``PATH`` environment for projects which use this -# one. +#[==[.md +## `PATH` usage requirements + +Add directories to PATH for projects using this one. + +``` +superbuild_add_path(<PATH>...) +``` + +Adds the arguments to the `PATH` environment for projects which use this one. +#]==] function (superbuild_add_path) if (NOT superbuild_build_phase) return () @@ -419,15 +589,37 @@ function (superbuild_add_path) "${current_project}_path" ${ARGN}) endfunction () -# INTERNAL -# Get a list of the dependencies this project has. -# -# Usage: -# -# _superbuild_get_project_depends(<name> <prefix>) -# -# Returns a list of projects depended on by ``<name>`` in the -# ``${prefix}_depends`` variable. +#[==[.md +# Escaping `;` in lists + +Passing a semicolon (`;`) around in CMake is prone to error. The +`ExternalProject` module is especially prone to it. This function handles +escaping a string to be used in the project with semicolons. + +``` +superbuild_sanitize_lists_in_string(<PREFIX> <VAR>) +``` + +The value of `${VAR}` is sanitized and placed into the `${PREFIX}VAR` variable. +#]==] +function (superbuild_sanitize_lists_in_string out_var_prefix var) + string(REPLACE ";" "${_superbuild_list_separator}" command "${${var}}") + set("${out_var_prefix}${var}" "${command}" + PARENT_SCOPE) +endfunction () + +#[==[.md INTERNAL +# Internal utilities + +Get a list of the dependencies this project has. + +``` +_superbuild_get_project_depends(<NAME> <PREFIX>) +``` + +Returns a list of projects depended on by `<NAME>` in the `${PREFIX}_depends` +variable. +#]==] function (_superbuild_get_project_depends name prefix) if (NOT superbuild_build_phase) message(AUTHOR_WARNING "get_project_depends can only be used in build pass") @@ -464,29 +656,36 @@ function (_superbuild_get_project_depends name prefix) PARENT_SCOPE) endfunction () -# Include all projects. -# -# Usage: -# -# _superbuild_discover_projects(<projects...>) -# -# This runs the first pass which gathers the required dependency information -# from projects which may be enabled. +#[==[.md INTERNAL +## Scan phase + +``` +_superbuild_discover_projects(<PROJECT>...) +``` + +This runs the first pass which gathers the required dependency information from +projects which may be enabled. Essentially, each project in the list is +included. The `CMAKE_MODULE_PATH` is expected to have been prepared by this +time. +#]==] function (_superbuild_discover_projects) foreach (project IN LISTS ARGN) include("${project}") endforeach () endfunction () -# Entry point of the build logic. -# -# Usage: -# -# superbuild_process_dependencies() -# -# Parses all of the relevant variables created by the inclusion of all of the -# project files. It uses this information to create the build recipes for all -# of the projects with the flags propagated and dependencies sorted properly. +#[==[.md INTERNAL +## Build phase + +``` +superbuild_process_dependencies() +``` + +Parses all of the relevant properties created by the inclusion of all of the +project files. It uses this information to create the `ExternalProject` calls +for all of the projects with the flags propagated and dependencies sorted +properly. +#]==] function (superbuild_process_dependencies) set(enabled_projects) @@ -644,14 +843,18 @@ function (superbuild_process_dependencies) get_property(is_dummy GLOBAL PROPERTY "${project}_is_dummy") if (can_use_system AND USE_SYSTEM_${project}) + # Project from the system environment. + list(APPEND system_projects "${project}") _superbuild_add_dummy_project_internal("${project}") include("${project}.system") elseif (allow_developer_mode AND DEVELOPER_MODE_${project}) + # Project using developer mode. + set(requiring_packages) + # Verify all enabled dependents are in DEVELOPER_MODE. foreach (dep IN LISTS ${project}_needed_by) - # Verify all dependencies are in DEVELOPER_MODE. if (NOT DEVELOPER_MODE_${dep}) list(APPEND requiring_packages "${dep}") endif () @@ -659,7 +862,8 @@ function (superbuild_process_dependencies) if (requiring_packages) string(REPLACE ";" ", " requiring_packages "${requiring_packages}") - message(FATAL_ERROR "${project} is in developer mode, but is required by: ${requiring_packages}.") + message(FATAL_ERROR + "${project} is in developer mode, but is required by: ${requiring_packages}.") endif () include("${project}") @@ -675,11 +879,13 @@ function (superbuild_process_dependencies) set(is_buildable_project TRUE) endif () + # Write the developer config if necessary. if (allow_developer_mode AND is_buildable_project) _superbuild_write_developer_mode_cache("${project}" "${${project}_arguments}") endif () endforeach () + # Export enable project information. foreach (project IN LISTS all_projects) set("${project}_enabled" "${${project}_enabled}" @@ -693,7 +899,6 @@ function (superbuild_process_dependencies) PARENT_SCOPE) endfunction () -# INTERNAL # Sets properties properly when enabling a project. function (_superbuild_enable_project name needed_by) set("${name}_enabled" TRUE @@ -708,7 +913,6 @@ function (_superbuild_enable_project name needed_by) endif () endfunction () -# INTERNAL # Implementation of building a dummy project. function (_superbuild_add_dummy_project_internal name) _superbuild_get_project_depends("${name}" arg) @@ -724,7 +928,6 @@ function (_superbuild_add_dummy_project_internal name) INSTALL_COMMAND "") endfunction () -# INTERNAL # Implementation of building an actual project. function (_superbuild_add_project_internal name) set(cmake_params) @@ -914,7 +1117,6 @@ function (_superbuild_add_project_internal name) endif () endfunction () -# INTERNAL # Wrapper around ExternalProject's internal calls to gather the CMake flags # that would be passed to a project if it were enabled. function (_superbuild_write_developer_mode_cache name) @@ -963,7 +1165,6 @@ function (_superbuild_write_developer_mode_cache name) _ep_write_initial_cache("developer-${name}" "${cache_file}" "${cmake_args}") endfunction () -# INTERNAL # Queries dependencies for their CMake flags they declare. function (_superbuild_fetch_cmake_args name var) # Get extra cmake args from every dependent project, if any. @@ -981,17 +1182,7 @@ function (_superbuild_fetch_cmake_args name var) PARENT_SCOPE) endfunction () -# Readies an argument which may contain ';' for use in ExternalProject_add. -# -# Usually you shouldn't need this, but in case you do. -function (superbuild_sanitize_lists_in_string out_var_prefix var) - string(REPLACE ";" "${_superbuild_list_separator}" command "${${var}}") - set("${out_var_prefix}${var}" "${command}" - PARENT_SCOPE) -endfunction () - -# INTERNAL -# Checks that a project name is valid. +# Check that a project name is valid. # # Currently "valid" means alphanumeric with a non-numeric prefix. function (_superbuild_project_check_name name) @@ -1001,7 +1192,6 @@ function (_superbuild_project_check_name name) endif () endfunction () -# INTERNAL # Checkpoint function to ensure that the phases are well-separated. function (_superbuild_check_current_project func) if (NOT current_project) @@ -1009,76 +1199,3 @@ function (_superbuild_check_current_project func) return () endif () endfunction () - -# Add a project to be built via Python's setup.py routine. -# -# Usage: -# -# superbuild_add_project_python(<name> <args>...) -# -# Same as ``superbuild_add_project``, but sets the ``PYTHONPATH`` and build -# commands to work properly out of the box. See ``superbuild_add_project`` for -# its argument documentation. -macro (superbuild_add_project_python _name) - if (WIN32) - set(_superbuild_python_path <INSTALL_DIR>/bin/Lib/site-packages) - set(_superbuild_python_args - "--prefix=bin") - else () - set(_superbuild_python_path <INSTALL_DIR>/lib/python2.7/site-packages) - set(_superbuild_python_args - "--single-version-externally-managed" - "--prefix=") - endif () - - superbuild_add_project("${_name}" - BUILD_IN_SOURCE 1 - DEPENDS python ${ARGN} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - "${superbuild_python_executable}" - setup.py - build - ${${_name}_python_build_args} - INSTALL_COMMAND - "${superbuild_python_executable}" - setup.py - install - --skip-build - --root=<INSTALL_DIR> - ${_superbuild_python_args} - ${${_name}_python_install_args} - PROCESS_ENVIRONMENT - PYTHONPATH ${_superbuild_python_path}) -endmacro () - -# Add a project to be installed from a Python wheel. -# -# Usage: -# -# superbuild_add_project_python_wheel(<name> <args>...) -# -# Same as ``superbuild_add_project``, but installs using the source from a file -# as a wheel. -macro (superbuild_add_project_python_wheel _name) - if (superbuild_build_phase AND NOT superbuild_python_pip) - message(FATAL_ERROR - "No `pip` available?") - endif () - - superbuild_add_project("${_name}" - BUILD_IN_SOURCE 1 - DOWNLOAD_NO_EXTRACT 1 - DEPENDS python ${ARGN} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - "" - INSTALL_COMMAND - ${superbuild_python_pip} - install - --no-index - --prefix=<INSTALL_DIR> - "<DOWNLOADED_FILE>") -endmacro () -- GitLab