Merge topic 'python3-support'

2c93bd43 packaging: fix exporting variables
ba83d584 SuperbuildMacros: move PYTHONPATH computation
228718d1 fixup_bundle.unix: support Python3
d12dca62 python: make the superbuild_python_version available for packaging
1cddceb9 numpy: fix the system-numpy case
f6778aeb python: add support for python3 in packages
a35883cf python3: add a project
120c8dc9

 SuperbuildInstallMacros: use the python version in use
...
Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !283

CMakeLists.txt

@@ -155,11 +155,22 @@ endif ()
 if (COMMAND superbuild_add_packaging)
   include(SuperbuildPackageMacros)
 
+  set(superbuild_export_variables)
   if (superbuild_python_executable)
-    set(superbuild_export_variables
+    list(APPEND superbuild_export_variables
       superbuild_python_executable)
   endif ()
 
+  if (python_version)
+    list(APPEND superbuild_export_variables
+      python_version)
+  endif ()
+
+  if (superbuild_python_version)
+    list(APPEND superbuild_export_variables
+      superbuild_python_version)
+  endif ()
+
   superbuild_add_packaging()
 endif ()
 
@@ -212,3 +223,8 @@ if (BUILD_TESTING)
     PROPERTY
       TEST_INCLUDE_FILE "${CMAKE_CURRENT_BINARY_DIR}/superbuild_testing_trampoline.cmake")
 endif ()
+
+if (python3_enabled)
+  include(python3.functions)
+  superbuild_python_write_reqs("${CMAKE_BINARY_DIR}/requirements.txt")
+endif ()

cmake/SuperbuildInstallMacros-unix.cmake

@@ -335,14 +335,14 @@ modules (or packages) are searched for at install time in the paths given to
 the `MODULE_DIRECTORIES` argument.
 
 Modules are placed in the `MODULE_DESTINATION` under the expected Python module
-paths in the package (`lib/python2.7`). By default, `/site-packages` is used.
+paths in the package (`lib/python${superbuild_python_version}`). By default,
+`/site-packages` is used.
 
 The `INCLUDE_REGEXES`, `EXCLUDE_REGEXES`, `LOADER_PATHS`, and
 `SEARCH_DIRECTORIES` arguments used when installing compiled Python modules
 through an internal `superbuild_unix_install_plugin` call.
 
-Note that modules in the list which cannot be found are ignored. This function
-also assumes Python 2.7 for now.
+Note that modules in the list which cannot be found are ignored.
 #]==]
 function (superbuild_unix_install_python)
   set(values
@@ -415,7 +415,7 @@ function (superbuild_unix_install_python)
 
     foreach (python_module IN LISTS python_modules)
       superbuild_unix_install_python_module(\"\${CMAKE_INSTALL_PREFIX}\"
-        \"\${python_module}\" \"\${module_directories}\" \"lib/python2.7${_install_python_MODULE_DESTINATION}\")
+        \"\${python_module}\" \"\${module_directories}\" \"lib/python${superbuild_python_version}${_install_python_MODULE_DESTINATION}\")
     endforeach ()"
     COMPONENT superbuild)
 endfunction ()

cmake/SuperbuildInstallMacros.cmake

@@ -17,14 +17,14 @@ include(CMakeParseArguments)
 
 # Find a Python executable to run the `fixup_bundle` scripts.
 if (NOT superbuild_python_executable)
-  find_package(PythonInterp 2.7)
-  if (PYTHONINTERP_FOUND)
+  find_package(Python${python_version} COMPONENTS Interpreter)
+  if (Python${python_version}_EXECUTABLE)
     set(superbuild_python_executable
-      "${PYTHON_EXECUTABLE}")
+      "${Python${python_version}_EXECUTABLE}")
   else ()
     message(FATAL_ERROR
-      "Could not find a Python executable newer than 2.7; one is required "
-      "to create packages on Linux and Windows.")
+      "Could not find a Python executable newer than "
+      "${superbuild_python_version}; one is required to create packages.")
   endif ()
 endif ()
 

cmake/SuperbuildMacros.cmake

@@ -310,36 +310,62 @@ Same as `superbuild_add_project`, but sets build commands to
 work properly out of the box for setuputils.
 #]==]
 macro (superbuild_add_project_python _name)
-  if (WIN32)
-    set(_superbuild_python_args
-      "--prefix=bin")
+  cmake_parse_arguments(_superbuild_python_project
+    ""
+    "PACKAGE"
+    ""
+    ${ARGN})
+
+  if (ENABLE_python3 OR python3_enabled)
+    if (NOT DEFINED _superbuild_python_project_PACKAGE)
+      message(FATAL_ERROR
+        "Python3 requires that projects have a package specified")
+    endif ()
+
+    superbuild_require_python_package("${_name}" "${_superbuild_python_project_PACKAGE}")
   else ()
-    set(_superbuild_python_args
-      "--single-version-externally-managed"
-      "--install-lib=lib/python2.7/site-packages"
-      "--prefix=")
-  endif ()
+    if (WIN32)
+      set(_superbuild_python_args
+        "--prefix=bin")
+    else ()
+      set(_superbuild_python_args
+        "--single-version-externally-managed"
+        "--install-lib=lib/python${superbuild_python_version}/site-packages"
+        "--prefix=")
+    endif ()
 
-  superbuild_add_project("${_name}"
-    BUILD_IN_SOURCE 1
-    DEPENDS python ${ARGN}
-    CONFIGURE_COMMAND
-      ""
-    BUILD_COMMAND
-      "${superbuild_python_executable}"
-        setup.py
-        build
-        ${${_name}_python_build_args}
-    INSTALL_COMMAND
-      "${superbuild_python_executable}"
-        setup.py
-        install
-        --skip-build
-        --root=<INSTALL_DIR>
-        ${_superbuild_python_args}
-        ${${_name}_python_install_args})
+    superbuild_add_project("${_name}"
+      BUILD_IN_SOURCE 1
+      DEPENDS python python2 ${_superbuild_python_project_UNPARSED_ARGUMENTS}
+      CONFIGURE_COMMAND
+        ""
+      BUILD_COMMAND
+        "${superbuild_python_executable}"
+          setup.py
+          build
+          ${${_name}_python_build_args}
+      INSTALL_COMMAND
+        "${superbuild_python_executable}"
+          setup.py
+          install
+          --skip-build
+          --root=<INSTALL_DIR>
+          ${_superbuild_python_args}
+          ${${_name}_python_install_args})
+  endif ()
 endmacro ()
 
+function (superbuild_require_python_package _name package)
+  if (superbuild_build_phase)
+    set_property(GLOBAL APPEND
+      PROPERTY
+        _superbuild_python_packages "${package}")
+  endif ()
+
+  superbuild_add_dummy_project("${_name}"
+    "${ARGN}")
+endfunction ()
+
 #[==[.md
 ### Wheels
 
@@ -1058,6 +1084,15 @@ function (_superbuild_add_project_internal name)
     "${superbuild_install_location}/bin")
   list(REMOVE_DUPLICATES extra_paths)
 
+  if (WIN32)
+    set(superbuild_python_path <INSTALL_DIR>/bin/Lib/site-packages)
+  else ()
+    set(superbuild_python_path <INSTALL_DIR>/lib/python${superbuild_python_version}/site-packages)
+  endif ()
+  _superbuild_make_path_var(superbuild_python_path
+    "$ENV{PYTHONPATH}"
+    ${superbuild_python_path})
+
   if (WIN32)
     string(REPLACE ";" "${_superbuild_list_separator}" extra_paths "${extra_paths}")
     string(REPLACE ";" "${_superbuild_list_separator}" superbuild_python_path "${superbuild_python_path}")
@@ -1103,8 +1138,8 @@ function (_superbuild_add_project_internal name)
 
   # prepare any separators in supplied environment variable
   set(converted_cmake_prefix_path "")
-  foreach (_path ${CMAKE_PREFIX_PATH})
-     string(APPEND converted_cmake_prefix_path "${_superbuild_list_separator}${_path}")
+  foreach (_path IN LISTS CMAKE_PREFIX_PATH)
+    string(APPEND converted_cmake_prefix_path "${_superbuild_list_separator}${_path}")
   endforeach()
   # now ensure superbuild's special directory comes first
   set(prepended_cmake_prefix_path

cmake/SuperbuildUtils.cmake

@@ -132,16 +132,6 @@ function (superbuild_setup_flags)
     "$ENV{PKG_CONFIG_PATH}")
   set(superbuild_pkg_config_path "${superbuild_pkg_config_path}" PARENT_SCOPE)
 
-  if (WIN32)
-    set(superbuild_python_path <INSTALL_DIR>/bin/Lib/site-packages)
-  else ()
-    set(superbuild_python_path <INSTALL_DIR>/lib/python2.7/site-packages)
-  endif ()
-  _superbuild_make_path_var(superbuild_python_path
-    "$ENV{PYTHONPATH}"
-    ${superbuild_python_path})
-  set(superbuild_python_path "${superbuild_python_path}" PARENT_SCOPE)
-
   if (CMAKE_CROSSCOMPILING)
     return ()
   endif ()

cmake/scripts/fixup_bundle.unix.py

@@ -37,7 +37,7 @@ class Pipeline(object):
         stdout, stderr = command.communicate()
         if command.returncode:
             raise RuntimeError('failed to execute pipeline:\n%s' % stderr)
-        return stdout
+        return stdout.decode('utf-8')
 
 
 class Library(object):
@@ -586,7 +586,7 @@ def _arg_parser():
 def _install_binary(binary, is_excluded, bundle_dest, dep_libdir, installed, manifest, sources, dry_run=False):
     '''Install the main binary into the package.'''
     # Start looking at our main executable's dependencies.
-    deps = binary.dependencies.values()
+    deps = list(binary.dependencies.values())
     while deps:
         dep = deps.pop(0)
 

prebuilt-python/CMakeLists.txt

 cmake_minimum_required(VERSION 3.0)
 
-project(prebuilt-python)
+project(prebuilt-python2)
 
 if (NOT WIN32 OR NOT MSVC90)
   message(FATAL_ERROR "This project requires Visual Studio 2008 to work")
@@ -15,6 +15,7 @@ function (superbuild_find_projects var)
     ftjam
     png
     python
+    python2
     pythoncycler
     pythondateutil
     pythonpyparsing
@@ -32,8 +33,8 @@ function (superbuild_find_projects var)
 endfunction ()
 
 function (superbuild_sanity_check)
-  if (NOT python_enabled)
-    message(FATAL_ERROR "The python project must be enabled.")
+  if (NOT python2_enabled)
+    message(FATAL_ERROR "The python2 project must be enabled.")
   endif ()
 endfunction ()
 

prebuilt-python/projects/win32/matplotlib.bundle.cmake

 set(package_name matplotlib)
 set(version 1.1.1)
-include(python.bundle.common)
+include(python2.bundle.common)
 
 set(modules matplotlib)
-include(python.package.bundle)
+include(python2.package.bundle)
 
 install(
   DIRECTORY   "${superbuild_install_location}/lib/site-packages/matplotlib/mpl-data/"

prebuilt-python/projects/win32/numpy.bundle.cmake

 set(package_name numpy)
 set(version 1.15.1)
-include(python.bundle.common)
+include(python2.bundle.common)
 
 set(numpy_egg_name
   "numpy-${version}-py2.7-win-amd64.egg")
@@ -21,4 +21,4 @@ install(
 set(modules numpy)
 list(APPEND python_extra_modules_directories
   "${superbuild_install_location}/${numpy_egg_dir}")
-include(python.package.bundle)
+include(python2.package.bundle)

prebuilt-python/projects/win32/python.bundle.cmake → prebuilt-python/projects/win32/python2.bundle.cmake

 set(package_name python)
 set(version 2.7.15)
-include(python.bundle.common)
+include(python2.bundle.common)
 
 superbuild_windows_install_program(python
   "bin"
@@ -20,5 +20,5 @@ install(
   FILES       "${superbuild_install_location}/lib/python27.lib"
   DESTINATION "bin/libs")
 
-include(python.functions)
-superbuild_install_superbuild_python()
+include(python2.functions)
+superbuild_install_superbuild_python2()

prebuilt-python/projects/win32/python.cmake → prebuilt-python/projects/win32/python2.cmake

@@ -22,7 +22,7 @@ if (NOT DEVENV_EXE)
     "compile the actual Python project itself.")
 endif ()
 
-superbuild_add_project(python
+superbuild_add_project(python2
   DEFAULT_ON
   DEPENDS bzip2 zlib
   BUILD_IN_SOURCE 1
@@ -39,7 +39,7 @@ superbuild_add_project(python
       -Dsource_location:PATH=<SOURCE_DIR>
       -Doutput_location:PATH=${python_executable_dir}
       -Dinstall_location:PATH=<INSTALL_DIR>
-      -P "${CMAKE_CURRENT_LIST_DIR}/scripts/python.install.cmake")
+      -P "${CMAKE_CURRENT_LIST_DIR}/scripts/python2.install.cmake")
 
 set(python_projects_to_build
   _ctypes

projects/apple-unix/pythonmpi4py.cmake

-superbuild_add_project_python(pythonmpi4py
-  DEPENDS pythonsetuptools mpi)

projects/apple-unix/scipy.cmake

-set(scipy_process_environment)
-if (lapack_enabled)
-  list(APPEND scipy_process_environment
-    MKL     "None"
-    ATLAS   "None"
-    BLAS    "<INSTALL_DIR>/lib"
-    LAPACK  "<INSTALL_DIR>/lib")
-endif ()
+if (ENABLE_python3 OR python3_enabled)
+  superbuild_require_python_package(scipy "scipy")
+else ()
+  set(scipy_process_environment)
+  if (lapack_enabled)
+    list(APPEND scipy_process_environment
+      MKL     "None"
+      ATLAS   "None"
+      BLAS    "<INSTALL_DIR>/lib"
+      LAPACK  "<INSTALL_DIR>/lib")
+  endif ()
 
-if (fortran_enabled)
-  list(APPEND scipy_process_environment
-    FC ${CMAKE_Fortran_COMPILER})
-endif ()
+  if (fortran_enabled)
+    list(APPEND scipy_process_environment
+      FC ${CMAKE_Fortran_COMPILER})
+  endif ()
 
-superbuild_add_project(scipy
-  DEPENDS python fortran numpy lapack
-  BUILD_IN_SOURCE 1
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND
-    ${superbuild_python_executable}
-      setup.py
-      config_fc
-      "--f90exec=${CMAKE_Fortran_COMPILER}"
-      build
-  INSTALL_COMMAND
-    ${superbuild_python_executable}
-      setup.py
-      install
-      --prefix=<INSTALL_DIR>
-  PROCESS_ENVIRONMENT
-    PYTHONPATH "<INSTALL_DIR>/lib/python2.7/site-packages"
-    ${scipy_process_environment})
+  superbuild_add_project(scipy
+    DEPENDS python fortran numpy lapack
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND
+      ${superbuild_python_executable}
+        setup.py
+        config_fc
+        "--f90exec=${CMAKE_Fortran_COMPILER}"
+        build
+    INSTALL_COMMAND
+      ${superbuild_python_executable}
+        setup.py
+        install
+        --prefix=<INSTALL_DIR>
+    PROCESS_ENVIRONMENT
+      PYTHONPATH "<INSTALL_DIR>/lib/python${superbuild_python_version}/site-packages"
+      ${scipy_process_environment})
 
-# The superbuild setting LDFLAGS (even to empty) causes SciPy to not add its
-# flags, so add required flags manually.
-if (APPLE)
-  # Scipy does not link libpython directly, but instead looks for them at load
-  # time.
-  superbuild_append_flags(ld_flags "-undefined dynamic_lookup" PROJECT_ONLY)
-elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-  superbuild_append_flags(ld_flags "-shared" PROJECT_ONLY)
+  # The superbuild setting LDFLAGS (even to empty) causes SciPy to not add its
+  # flags, so add required flags manually.
+  if (APPLE)
+    # Scipy does not link libpython directly, but instead looks for them at load
+    # time.
+    superbuild_append_flags(ld_flags "-undefined dynamic_lookup" PROJECT_ONLY)
+  elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+    superbuild_append_flags(ld_flags "-shared" PROJECT_ONLY)
+  endif ()
 endif ()

projects/apple/matplotlib.cmake

 include("${CMAKE_CURRENT_LIST_DIR}/../matplotlib.cmake")
 
-if (cxx11_enabled)
+if (cxx11_enabled AND NOT (ENABLE_python3 OR python3_enabled))
   superbuild_apply_patch(matplotlib setupext
     "Use libc++ on OS X")
 endif ()

projects/apple/python.cmake → projects/apple/python2.cmake

-superbuild_add_project(python
+superbuild_add_project(python2
   MUST_USE_SYSTEM)
 
+set(superbuild_python_version "2.7")
+
 file(MAKE_DIRECTORY
   "${superbuild_install_location}/lib/python2.7/site-packages")

projects/matplotlib.cmake

-set(matplotlib_process_environment)
-if (NOT WIN32)
-  set(matplotlib_process_environment
-    PYTHONPATH "<INSTALL_DIR>/lib/python2.7/site-packages")
-endif ()
+if (ENABLE_python3 OR python3_enabled)
+  include(matplotlib.system)
+else ()
+  set(matplotlib_process_environment)
+  if (NOT WIN32)
+    set(matplotlib_process_environment
+      PYTHONPATH "<INSTALL_DIR>/lib/python${superbuild_python_version}/site-packages")
+  endif ()
 
-superbuild_add_project(matplotlib
-  CAN_USE_SYSTEM
-  DEPENDS python numpy png freetype zlib pythondateutil pytz pythonpyparsing pythoncycler pythonsetuptools
-  DEPENDS_OPTIONAL cxx11
-  BUILD_IN_SOURCE 1
-  CONFIGURE_COMMAND
-    "${CMAKE_COMMAND}"
-      "-Dpatches_location:PATH=${CMAKE_CURRENT_LIST_DIR}/patches"
-      "-Dsource_location:PATH=<SOURCE_DIR>"
-      "-Dinstall_location:PATH=<INSTALL_DIR>"
-      -P "${CMAKE_CURRENT_LIST_DIR}/scripts/matplotlib.patch.cmake"
-  BUILD_COMMAND
-    "${CMAKE_COMMAND}"
-      "-DPYTHON_EXECUTABLE:PATH=${superbuild_python_executable}"
-      "-Dsource_location:PATH=<SOURCE_DIR>"
-      "-Dinstall_location:PATH=<INSTALL_DIR>"
-      -P "${CMAKE_CURRENT_LIST_DIR}/scripts/matplotlib.build.cmake"
-  INSTALL_COMMAND ""
-  PROCESS_ENVIRONMENT
-    "${matplotlib_process_environment}")
+  superbuild_add_project(matplotlib
+    CAN_USE_SYSTEM
+    DEPENDS python numpy png freetype zlib pythondateutil pytz pythonpyparsing pythoncycler pythonsetuptools
+    DEPENDS_OPTIONAL cxx11
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND
+      "${CMAKE_COMMAND}"
+        "-Dpatches_location:PATH=${CMAKE_CURRENT_LIST_DIR}/patches"
+        "-Dsource_location:PATH=<SOURCE_DIR>"
+        "-Dinstall_location:PATH=<INSTALL_DIR>"
+        -P "${CMAKE_CURRENT_LIST_DIR}/scripts/matplotlib.patch.cmake"
+    BUILD_COMMAND
+      "${CMAKE_COMMAND}"
+        "-DPYTHON_EXECUTABLE:PATH=${superbuild_python_executable}"
+        "-Dsource_location:PATH=<SOURCE_DIR>"
+        "-Dinstall_location:PATH=<INSTALL_DIR>"
+        -P "${CMAKE_CURRENT_LIST_DIR}/scripts/matplotlib.build.cmake"
+    INSTALL_COMMAND ""
+    PROCESS_ENVIRONMENT
+      "${matplotlib_process_environment}")
+endif ()

projects/matplotlib.system.cmake

-# Nothing to do; assuming matplotlib will be available at runtime in the Python
-# environment.
+superbuild_require_python_package(matplotlib "matplotlib")