CMake issueshttps://gitlab.kitware.com/cmake/cmake/-/issues2023-06-02T04:37:07-04:00https://gitlab.kitware.com/cmake/cmake/-/issues/24962FindOpenMP fails to discover OpenMP_Fortran_LIB_NAMES for CCE 15.0.12023-06-02T04:37:07-04:00Balthasar ReuterFindOpenMP fails to discover OpenMP_Fortran_LIB_NAMES for CCE 15.0.1This is on LUMI, using CMake 3.25.2. I have built a small reproducer here:
https://github.com/reuterbal/mfe_snippets/tree/main/cmake_omp_cce15
## C++
Works as expected. Reproduce with the following steps:
```
bareuter@uan02:/.../mfe_...This is on LUMI, using CMake 3.25.2. I have built a small reproducer here:
https://github.com/reuterbal/mfe_snippets/tree/main/cmake_omp_cce15
## C++
Works as expected. Reproduce with the following steps:
```
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> source env.sh
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> mkdir build.cxx
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> cd build.cxx
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.cxx> CXX=CC cmake ../cxx/
-- The CXX compiler identification is Clang 15.0.6
-- Cray Programming Environment 2.7.20 CXX
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/cray/pe/craype/2.7.20/bin/CC - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found OpenMP_CXX: -fopenmp (found version "5.0")
-- Found OpenMP: TRUE (found version "5.0") found components: CXX
-- Configuring done
-- Generating done
-- Build files have been written to: /.../mfe_snippets/cmake_omp_cce15/build.cxx
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.cxx> make
[ 50%] Building CXX object CMakeFiles/main.dir/main.cc.o
[100%] Linking CXX executable main
[100%] Built target main
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.cxx> ./main
SUCCESS
```
## Fortran
Fails to link the binary. Reproduce with the following steps:
```
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> source env.sh
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> mkdir build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> cd build.fortran/
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> cmake ../fortran/
-- The Fortran compiler identification is Cray 15.0.1
-- Cray Programming Environment 2.7.20 Fortran
-- Detecting Fortran compiler ABI info
-- Detecting Fortran compiler ABI info - done
-- Check for working Fortran compiler: /opt/cray/pe/craype/2.7.20/bin/ftn - skipped
-- Found OpenMP_Fortran: -h omp (found version "4.5")
-- Found OpenMP: TRUE (found version "4.5") found components: Fortran
-- Configuring done
-- Generating done
-- Build files have been written to: /.../mfe_snippets/cmake_omp_cce15/build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> make
[ 50%] Building Fortran object CMakeFiles/myprog.dir/myprog.F90.o
[100%] Linking Fortran executable myprog
/opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: CMakeFiles/myprog.dir/myprog.F90.o: in function `main':
The Cpu Module:(.text+0x2c): undefined reference to `_cray$mt_execute_parallel'
/opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: CMakeFiles/myprog.dir/myprog.F90.o: in function `my_prog__cray$mt$p0001':
The Cpu Module:(.text+0x74): undefined reference to `omp_get_thread_num_'
/opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: The Cpu Module:(.text+0x88): undefined reference to `_cray$mt_taskwaitall'
make[2]: *** [CMakeFiles/myprog.dir/build.make:96: myprog] Error 1
make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/myprog.dir/all] Error 2
make: *** [Makefile:91: all] Error 2
```
## Fixing the Fortran manually by adding C++ and specifying LIB_NAMES
1. Edit `fortran/CMakeLists.txt` and change the `project` line to the following:
```cmake
project( find_libomp_fortran LANGUAGES Fortran CXX )
```
2. Manually specify `OpenMP_Fortran_LIB_NAMES`:
```
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> rm -rf build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> mkdir build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15> cd build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> grep OpenMP_CXX_LIB_NAMES ../build.cxx/CMakeCache.txt
OpenMP_CXX_LIB_NAMES:STRING=sci_cray_mpi_mp;sci_cray_mp;craymp
//ADVANCED property for variable: OpenMP_CXX_LIB_NAMES
OpenMP_CXX_LIB_NAMES-ADVANCED:INTERNAL=1
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> CXX=CC cmake ../fortran/ -DOpenMP_Fortran_LIB_NAMES:STRING="sci_cray_mpi_mp;sci_cray_mp;craymp"
-- The CXX compiler identification is Clang 15.0.6
-- Cray Programming Environment 2.7.20 Fortran
-- Cray Programming Environment 2.7.20 CXX
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/cray/pe/craype/2.7.20/bin/CC - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found OpenMP_Fortran: -h omp (found version "4.5")
-- Found OpenMP: TRUE (found version "4.5") found components: Fortran
-- Configuring done
-- Generating done
-- Build files have been written to: /.../mfe_snippets/cmake_omp_cce15/build.fortran
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> make
[ 50%] Building Fortran object CMakeFiles/myprog.dir/myprog.F90.o
[100%] Linking Fortran executable myprog
[100%] Built target myprog
bareuter@uan02:/.../mfe_snippets/cmake_omp_cce15/build.fortran> ./myprog
SUCCESS
```https://gitlab.kitware.com/cmake/cmake/-/issues/24402HPE-Cray environment and OpenMP linking issue2024-02-13T10:24:26-05:00etiennemlbHPE-Cray environment and OpenMP linking issueI was prompted to create an issue following this a post on the discourse.cmake.org:
https://discourse.cmake.org/t/hpe-cray-environment-and-openmp-linking-issue/7420
The TLDR is that under the HPE-Cray environment that I use (often found...I was prompted to create an issue following this a post on the discourse.cmake.org:
https://discourse.cmake.org/t/hpe-cray-environment-and-openmp-linking-issue/7420
The TLDR is that under the HPE-Cray environment that I use (often found on supercomputers) the OpenMP runtime is not linked when (and only when) I load a GPU target module (namely, `craype-accel-amd-gfx90a` for AMD's MI250 cards) before compiling and *initializing* CMake. This module prompt the CPE wrapper to add the flags like `--offload-arch=gfx90a`.
Below is a copy past of the post, presenting the issue and a reproducer.
------------------------------------------------------------------------
I use CMake 3.20.2, CPE/22/11 (cce/15.0.2) and ROCm 5.2.3.
Assuming the following CMake script:
```cmake
cmake_minimum_required(VERSION 3.20)
project("test" CXX)
find_package(OpenMP REQUIRED)
add_executable(test_executable test.cc)
target_link_libraries(test_executable
PUBLIC OpenMP::OpenMP_CXX)
```
Under a recent HPE-Cray supercomputer, using CMake to compile the following code on AMD's MI 250 GPUs (gfx90a):
```
int main() {
#pragma omp target
#pragma omp teams distribute parallel for
for(int i = 0; i < 10; ++i) {
}
}
```
would require to load some modules like so:
```
module load craype-accel-amd-gfx90a
module load craype-x86-trento
module load PrgEnv-cray
```
and would also require that you specify, in a toolchain file or directly as an environment variable, that you seek to use the Cray compiler (cce):
```
export CXX=CC
```
All that being done, after the CMake configuration ends and I start building, I get a linker error of the following kind:
```
d.lld: error: undefined symbol: __tgt_target_kernel
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main)
ld.lld: error: undefined symbol: __kmpc_fork_teams
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12)
ld.lld: error: undefined symbol: __kmpc_for_static_init_4
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0002)
ld.lld: error: undefined symbol: _cray$mt_kmpc_fork_call_with_flags
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
ld.lld: error: undefined symbol: __kmpc_for_static_fini
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0002)
ld.lld: error: undefined symbol: __tgt_register_requires
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(.omp_offloading.requires_reg)
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
```
If I change the test case like that (no offloading anymore):
```
int main() {
#pragma omp parallel for
for(int i = 0; i < 10; ++i) {
}
}
```
I get this linker error:
```
ld.lld: error: undefined symbol: _cray$mt_kmpc_fork_call_with_flags
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main)
ld.lld: error: undefined symbol: __kmpc_for_static_init_4
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main_cray$mt$p0001)
ld.lld: error: undefined symbol: __kmpc_for_static_fini
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main_cray$mt$p0001)
```
and if I unload the `craype-accel-amd-gfx90a` module **it works fine**, but of course, without offloading.
As I would like to use the OpenMP offloading capabilities this is quite problematic.
It seems that CMake does not produce a build script that links with the HPE-Cray OpenMP runtime if the `craype-accel-amd-gfx90a`:
Here is the output of ``ninja -v`` (I tried both ninja and make and it resulted in the same error).
In the case where `craype-accel-amd-gfx90a` **is not loaded**, note the presence of explicit linking of the OpenMP runtime libraries:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC CMakeFiles/test_executable.dir/test.cc.o -o test_executable /opt/cray/pe/libsci/22.11.1.2/CRAY/9.0/x86_64/lib/libsci_cray_mpi_mp.so /opt/cray/pe/libsci/22.11.1.2/CRAY/9.0/x86_64/lib/libsci_cray_mp.so /opt/cray/pe/cce/15.0.0/cce/x86_64/lib/libcraymp.so && :
```
In the case where `craype-accel-amd-gfx90a` **is loaded**, note the absence of `-fopenmp` or explicit linking of the OpenMP runtime libraries:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC CMakeFiles/test_executable.dir/test.cc.o -o test_executable && :
```
The compilation step is unchanged, the linking step is wrong when `craype-accel-amd-gfx90a` is loaded.
**I could workaround** the issue by adding the following to the CMake script:
```cmake
target_link_options(test_executable
PUBLIC -fopenmp)
```
Which gives:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp CMakeFiles/test_executable.dir/test.cc.o -o test_executable && :
```
All CMake configuration steps are similar (regardless of the module's presence):
```
-- The CXX compiler identification is Clang 15.0.2
-- Cray Programming Environment 2.7.19 CXX
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/cray/pe/craype/2.7.19/bin/CC - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
int main() {
#pragma omp target
#pragma omp teams distribute parallel for
for(int i = 0; i < 10; ++i) {
}
}
```
would require to load some modules like so:
```
module load craype-accel-amd-gfx90a
module load craype-x86-trento
module load PrgEnv-cray
```
and would also require that you specify, in a toolchain file or directly as an environment variable, that you seek to use the Cray compiler (cce):
```
export CXX=CC
```
All that being done, after the CMake configuration ends and I start building, I get a linker error of the following kind:
```
d.lld: error: undefined symbol: __tgt_target_kernel
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main)
ld.lld: error: undefined symbol: __kmpc_fork_teams
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12)
ld.lld: error: undefined symbol: __kmpc_for_static_init_4
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0002)
ld.lld: error: undefined symbol: _cray$mt_kmpc_fork_call_with_flags
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
ld.lld: error: undefined symbol: __kmpc_for_static_fini
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0001)
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(__omp_offloading_84f0b5a2_49018856_main_l12_cray$mt$p0002)
ld.lld: error: undefined symbol: __tgt_register_requires
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(.omp_offloading.requires_reg)
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
```
If I change the test case like that (no offloading anymore):
```
int main() {
#pragma omp parallel for
for(int i = 0; i < 10; ++i) {
}
}
```
I get this linker error:
```
ld.lld: error: undefined symbol: _cray$mt_kmpc_fork_call_with_flags
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main)
ld.lld: error: undefined symbol: __kmpc_for_static_init_4
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main_cray$mt$p0001)
ld.lld: error: undefined symbol: __kmpc_for_static_fini
>>> referenced by test.cc
>>> CMakeFiles/test_executable.dir/test.cc.o:(main_cray$mt$p0001)
```
and if I unload the `craype-accel-amd-gfx90a` module **it works fine**, but of course, without offloading.
As I would like to use the OpenMP offloading capabilities this is quite problematic.
It seems that CMake does not produce a build script that links with the HPE-Cray OpenMP runtime if the `craype-accel-amd-gfx90a`:
Here is the output of ``ninja -v`` (I tried both ninja and make and it resulted in the same error).
In the case where `craype-accel-amd-gfx90a` **is not loaded**, note the presence of explicit linking of the OpenMP runtime libraries:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC CMakeFiles/test_executable.dir/test.cc.o -o test_executable /opt/cray/pe/libsci/22.11.1.2/CRAY/9.0/x86_64/lib/libsci_cray_mpi_mp.so /opt/cray/pe/libsci/22.11.1.2/CRAY/9.0/x86_64/lib/libsci_cray_mp.so /opt/cray/pe/cce/15.0.0/cce/x86_64/lib/libcraymp.so && :
```
In the case where `craype-accel-amd-gfx90a` **is loaded**, note the absence of `-fopenmp` or explicit linking of the OpenMP runtime libraries:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC CMakeFiles/test_executable.dir/test.cc.o -o test_executable && :
```
The compilation step is unchanged, the linking step is wrong when `craype-accel-amd-gfx90a` is loaded.
**I could workaround** the issue by adding the following to the CMake script:
```cmake
target_link_options(test_executable
PUBLIC -fopenmp)
```
Which gives:
```
[1/2] /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp -MD -MT CMakeFiles/test_executable.dir/test.cc.o -MF CMakeFiles/test_executable.dir/test.cc.o.d -o CMakeFiles/test_executable.dir/test.cc.o -c ../test.cc
[2/2] : && /opt/cray/pe/craype/2.7.19/bin/CC -fopenmp CMakeFiles/test_executable.dir/test.cc.o -o test_executable && :
```
All CMake configuration steps are similar (regardless of the module's presence):
```
-- The CXX compiler identification is Clang 15.0.2
-- Cray Programming Environment 2.7.19 CXX
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/cray/pe/craype/2.7.19/bin/CC - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found OpenMP_CXX: -fopenmp (found version "5.0")
-- Found OpenMP: TRUE (found version "5.0")
-- Configuring done
-- Generating done
-- Build files have been written to: XXXX/build
```-- Found OpenMP_CXX: -fopenmp (found version "5.0")
-- Found OpenMP: TRUE (found version "5.0")
-- Configuring done
-- Generating done
-- Build files have been written to: XXXX/build
```https://gitlab.kitware.com/cmake/cmake/-/issues/18925Fortran module and submodule support for Cray Fortran2019-08-22T12:07:35-04:00Brad KingFortran module and submodule support for Cray FortranDiscussion in !2958 identified that Cray Fortran uses upper-case module names:
```
Submodules/PARENT.mod
Submodules/CHILD.mod
```
It may also be told to use lower-case names via the `-f` flag.
This will need additional work beyond tha...Discussion in !2958 identified that Cray Fortran uses upper-case module names:
```
Submodules/PARENT.mod
Submodules/CHILD.mod
```
It may also be told to use lower-case names via the `-f` flag.
This will need additional work beyond that MR to handle, even for plain modules.
The Ninja generator in particular needs to know the exact file name.
https://gitlab.kitware.com/cmake/cmake/-/issues/17413CrayPrgEnv.cmake mishandles CMAKE_<LANG>_IMPLICIT_INCLUDE_DIRECTORIES2019-02-08T16:59:47-05:00chuck cranorCrayPrgEnv.cmake mishandles CMAKE_<LANG>_IMPLICIT_INCLUDE_DIRECTORIESGCC 6 and newer changed their include files so that if you compile a C++ program with "-isystem /usr/include" it fails with "stdlib.h: No such file or directory"... see:
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70129
and
https://git...GCC 6 and newer changed their include files so that if you compile a C++ program with "-isystem /usr/include" it fails with "stdlib.h: No such file or directory"... see:
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70129
and
https://gitlab.kitware.com/cmake/cmake/issues/16291
One of our cmake-based projects is impacted by this on the Cray, so I stated looking at the CrayPrgEnv.cmake to see exactly what it is doing and I found several problems in the cmake code:
1. The function __cray_extract_implicit's first arg is ${src}. ${src} is the name of a source file to do a test compile with (e.g. "cc -v" or "c++ -v"). For CXX the CrayPrgEnv-CXX.cmake sets this to "${CMAKE_ROOT}/Modules/CMakeCXXCompilerABI.cpp"... Unfortunately, the __cray_extract_implicit function doesn't actually use the ${src} variable anywhere in the body of the function! This results in the output of execute_process() being incomplete (e.g. it does not contain any INCLUDE information). Thus, when you run "cmake" on a Cray with CrayPrgEnv the ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES} is incorrectly empty. The fix for this is to add "${src}" after "-o ${BIN}" in the COMMAND part of the call to execute_process().
2. if you manually fix the handling of ${src} in __cray_extract_implicit() as suggested above, you will get a partial (but incomplete!) list of includes in ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}. This is because __cray_extract_implicit()'s handling is incorrect. The code uses a regular expression in ${compiler_command} to look for a line like "/opt/gcc/.*/cc1plus" (or "/opt/gcc/.*/cc1" for C) to extract -I's from the line to generate the implicit include list. Unfortunately, this line does not include important directories like /usr/include in the list. But that information is included later on in the output between the lines "#include <...> search starts here:" and "End of search list." ... the output parsing in CrayPrgEnv.cmake needs to be adjusted to handle this. Here is some sample output from the GNU C compiler on a Cray:
```
/opt/gcc/7.1.0/snos/libexec/gcc/x86_64-suse-linux/7.1.0/cc1 -quiet -v -I /opt/cray/pe/libsci/17.09.1/GNU/6.1/x86_64/include -I /opt/cray/pe/mpt/7.6.2/gni/mpich-gnu/5.1/include -I /opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/include -I /opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/include -I /opt/cray/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari/include -I /opt/cray/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari/include -I /opt/cray/pe/pmi/5.0.12/include -I /opt/cray/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari/include -I /opt/cray/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari/include -I /opt/cray/wlm_detect/1.2.1-6.0.4.0_22.1__gd26a3dc.ari/include -I /opt/cray/krca/2.2.2-6.0.4.0_5.2__g4614cf3.ari/include -I /opt/cray-hss-devel/8.0.0/include -D __CRAYXC -D __CRAY_HASWELL -D __CRAYXT_COMPUTE_LINUX_TARGET -D __TARGET_LINUX__ /users/ccranor/tmp/cmake/share/cmake-3.10/Modules/CMakeCCompilerABI.c -quiet -dumpbase CMakeCCompilerABI.c -march=core-avx2 -auxbase CMakeCCompilerABI -version -o /tmp/ccsd57Wh.s
GNU C11 (GCC) version 7.1.0 20170502 (Cray Inc.) (x86_64-suse-linux)
compiled by GNU C version 7.1.0 20170502 (Cray Inc.), GMP version 6.0.0, MPFR version 3.1.3, MPC version 1.0.3, isl version isl-0.15-GMP
GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
ignoring nonexistent directory "/opt/gcc/7.1.0/snos/lib/gcc/x86_64-suse-linux/7.1.0/../../../../x86_64-suse-linux/include"
#include "..." search starts here:
#include <...> search starts here:
/opt/cray/pe/libsci/17.09.1/GNU/6.1/x86_64/include
/opt/cray/pe/mpt/7.6.2/gni/mpich-gnu/5.1/include
/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/include
/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/include
/opt/cray/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari/include
/opt/cray/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari/include
/opt/cray/pe/pmi/5.0.12/include
/opt/cray/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari/include
/opt/cray/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari/include
/opt/cray/wlm_detect/1.2.1-6.0.4.0_22.1__gd26a3dc.ari/include
/opt/cray/krca/2.2.2-6.0.4.0_5.2__g4614cf3.ari/include
/opt/cray-hss-devel/8.0.0/include
/opt/gcc/7.1.0/snos/lib/gcc/x86_64-suse-linux/7.1.0/include
/usr/local/include
/opt/gcc/7.1.0/snos/include
/opt/gcc/7.1.0/snos/lib/gcc/x86_64-suse-linux/7.1.0/include-fixed
/usr/include
End of search list.
GNU C11 (GCC) version 7.1.0 20170502 (Cray Inc.) (x86_64-suse-linux)
compiled by GNU C version 7.1.0 20170502 (Cray Inc.), GMP version 6.0.0, MPFR version 3.1.3, MPC version 1.0.3, isl version isl-0.15-GMP
```
Note that the last 5 includes (from "/opt/gcc/7.1.0/snos/lib/gcc/x86_64-suse-linux/7.1.0/include" to "/usr/include" are not in the "cc1" output line.
3. the CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES variable is not preserved between runs. It isn't a cache variable and it isn't saved in the output directory file:
"CMakeFiles/*/CmakeCXXCompiler.cmake" either. On CrayPrgEnv.cmake, this results in the include processing being done the first time you run cmake, but then not on later times (because the CRAY_<LANG>_EXTRACTED_IMPLICIT cache variable is set to 1). Because UnixPaths.cmake init's the value to "/usr/include" if you rerun cmake it gets set to "/usr/include" instead of the correct value. I believe that files like "Modules/CMakeCXXCompiler.cmake.in" should be modified to include CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES in the list of data that they preserve between runs. I tried this, and it works (although it has an extra copy of /usr/include in the path... I think one is from the CrayPrgEnv and the other is the one that UnixPaths appends).
Here is a demo of the problem using a simple CMakeLists.txt on a cray:
```
cmake_minimum_required (VERSION 3.5)
project (cray-test C CXX)
message ("STATUS: incdir: ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}")
message ("STATUS: lnkdir: ${CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES}")
message ("STATUS: lnklib: ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES}")
```
Here's the output from 3.6.2 on a Cray:
```
% cmake --version
cmake version 3.6.2
CMake suite maintained and supported by Kitware (kitware.com/cmake).
%
%
% env CC=cc CXX=CC cmake ..
-- The C compiler identification is Intel 17.0.4.20170411
-- The CXX compiler identification is Intel 17.0.4.20170411
-- Cray Programming Environment 2.5.12 C
-- Check for working C compiler: /opt/cray/pe/craype/2.5.12/bin/cc
-- Check for working C compiler: /opt/cray/pe/craype/2.5.12/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Cray Programming Environment 2.5.12 CXX
-- Check for working CXX compiler: /opt/cray/pe/craype/2.5.12/bin/CC
-- Check for working CXX compiler: /opt/cray/pe/craype/2.5.12/bin/CC -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
STATUS: incdir:
STATUS: lnkdir: /opt/cray/pe/libsci/17.09.1/INTEL/16.0/x86_64/lib;/opt/cray/dmapp/default/lib64;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/lib;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/lib64;/opt/cray/pe/atp/2.1.1/libApp;/opt/intel/2017.4.196/compilers_and_libraries_2017/linux/mkl/lib/intel64;/opt/intel/2017.4.196/compilers_and_libraries_2017.4.196/linux/compiler/lib/intel64_lin;/opt/gcc/6.3.0/snos/lib/gcc/x86_64-suse-linux/6.3.0;/opt/gcc/6.3.0/snos/lib64;/lib64;/usr/lib64;/opt/gcc/6.3.0/snos/lib;/lib;/usr/lib
STATUS: lnklib: intlc;AtpSigHandler;AtpSigHCommData;rca;sci_intel_mpi;sci_intel;mpich_intel;mpichcxx_intel;stdc++;imf;m;ifcore;ifport;pthread;imf;svml;irng;stdc++;m;ipgo;decimal;cilkrts;stdc++;irc;svml;c;irc_s;dl;c
-- Configuring done
-- Generating done
-- Build files have been written to: /users/ccranor/tmp/cmake4/b
%
```
notice that "incdir:" is empty the first time you run cmake. Now if you rerun cmake, "incdir:" becomes "/usr/include!
```
% env CC=cc CXX=CC cmake ..
-- Cray Programming Environment 2.5.12 C
-- Cray Programming Environment 2.5.12 CXX
STATUS: incdir: /usr/include
STATUS: lnkdir: /opt/cray/pe/libsci/17.09.1/INTEL/16.0/x86_64/lib;/opt/cray/dmapp/default/lib64;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/lib;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/lib64;/opt/cray/pe/atp/2.1.1/libApp;/opt/intel/2017.4.196/compilers_and_libraries_2017/linux/mkl/lib/intel64;/opt/intel/2017.4.196/compilers_and_libraries_2017.4.196/linux/compiler/lib/intel64_lin;/opt/gcc/6.3.0/snos/lib/gcc/x86_64-suse-linux/6.3.0;/opt/gcc/6.3.0/snos/lib64;/lib64;/usr/lib64;/opt/gcc/6.3.0/snos/lib;/lib;/usr/lib
STATUS: lnklib: intlc;AtpSigHandler;AtpSigHCommData;rca;sci_intel_mpi;sci_intel;mpich_intel;mpichcxx_intel;stdc++;imf;m;ifcore;ifport;pthread;imf;svml;irng;stdc++;m;ipgo;decimal;cilkrts;stdc++;irc;svml;c;irc_s;dl;c
-- Configuring done
-- Generating done
-- Build files have been written to: /users/ccranor/tmp/cmake4/b
%
```
I compiled a more recent version of cmake and fixed the ${src} bug in CrayPrgEnv. Here is output from that:
```
% rm -rf *
% ~/tmp/cmake/bin/cmake --version
cmake version 3.10.0-rc3
CMake suite maintained and supported by Kitware (kitware.com/cmake).
%
%
% env CC=cc CXX=CC ~/tmp/cmake/bin/cmake ..
-- The C compiler identification is Intel 17.0.4.20170411
-- The CXX compiler identification is Intel 17.0.4.20170411
-- Cray Programming Environment 2.5.12 C
-- Check for working C compiler: /opt/cray/pe/craype/2.5.12/bin/cc
-- Check for working C compiler: /opt/cray/pe/craype/2.5.12/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Cray Programming Environment 2.5.12 CXX
-- Check for working CXX compiler: /opt/cray/pe/craype/2.5.12/bin/CC
-- Check for working CXX compiler: /opt/cray/pe/craype/2.5.12/bin/CC -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
STATUS: incdir: /opt/cray/pe/libsci/17.09.1/INTEL/16.0/x86_64/include;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/include;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/include;/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/include;/opt/cray/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari/include;/opt/cray/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari/include;/opt/cray/pe/pmi/5.0.12/include;/opt/cray/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari/include;/opt/cray/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari/include;/opt/cray/wlm_detect/1.2.1-6.0.4.0_22.1__gd26a3dc.ari/include;/opt/cray/krca/2.2.2-6.0.4.0_5.2__g4614cf3.ari/include;/opt/cray-hss-devel/8.0.0/include;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/include;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/include;/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/include;/opt/cray/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari/include;/opt/cray/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari/include;/opt/cray/pe/pmi/5.0.12/include;/opt/cray/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari/include;/opt/cray/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari/include;/opt/cray/wlm_detect/1.2.1-6.0.4.0_22.1__gd26a3dc.ari/include;/opt/cray/krca/2.2.2-6.0.4.0_5.2__g4614cf3.ari/include;/opt/cray-hss-devel/8.0.0/include
STATUS: lnkdir: /opt/cray/pe/libsci/17.09.1/INTEL/16.0/x86_64/lib;/opt/cray/dmapp/default/lib64;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/lib;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/lib64;/opt/cray/pe/atp/2.1.1/libApp;/opt/intel/2017.4.196/compilers_and_libraries_2017/linux/mkl/lib/intel64;/opt/intel/2017.4.196/compilers_and_libraries_2017.4.196/linux/compiler/lib/intel64_lin;/opt/gcc/6.3.0/snos/lib/gcc/x86_64-suse-linux/6.3.0;/opt/gcc/6.3.0/snos/lib64;/lib64;/usr/lib64;/opt/gcc/6.3.0/snos/lib;/lib;/usr/lib
STATUS: lnklib: intlc;AtpSigHandler;AtpSigHCommData;rca;sci_intel_mpi;sci_intel;mpich_intel;mpichcxx_intel;stdc++;imf;m;ifcore;ifport;pthread;imf;svml;irng;stdc++;m;ipgo;decimal;cilkrts;stdc++;gcc;gcc_s;irc;svml;c;gcc;gcc_s;irc_s;dl;c
-- Configuring done
-- Generating done
-- Build files have been written to: /users/ccranor/tmp/cmake4/b
%
```
Now incdir: is partially correct (but doesn't include /usr/include, as I noted above).
But if I rerun "cmake" it gets reduced to "/usr/include" because the output isn't saved anywhere like CMakeCXXCompiler.cmake...
```
% env CC=cc CXX=CC ~/tmp/cmake/bin/cmake ..
-- Cray Programming Environment 2.5.12 C
-- Cray Programming Environment 2.5.12 CXX
STATUS: incdir: /usr/include
STATUS: lnkdir: /opt/cray/pe/libsci/17.09.1/INTEL/16.0/x86_64/lib;/opt/cray/dmapp/default/lib64;/opt/cray/pe/mpt/7.6.2/gni/mpich-intel/16.0/lib;/opt/cray/rca/2.2.11-6.0.4.0_13.2__g84de67a.ari/lib64;/opt/cray/pe/atp/2.1.1/libApp;/opt/intel/2017.4.196/compilers_and_libraries_2017/linux/mkl/lib/intel64;/opt/intel/2017.4.196/compilers_and_libraries_2017.4.196/linux/compiler/lib/intel64_lin;/opt/gcc/6.3.0/snos/lib/gcc/x86_64-suse-linux/6.3.0;/opt/gcc/6.3.0/snos/lib64;/lib64;/usr/lib64;/opt/gcc/6.3.0/snos/lib;/lib;/usr/lib
STATUS: lnklib: intlc;AtpSigHandler;AtpSigHCommData;rca;sci_intel_mpi;sci_intel;mpich_intel;mpichcxx_intel;stdc++;imf;m;ifcore;ifport;pthread;imf;svml;irng;stdc++;m;ipgo;decimal;cilkrts;stdc++;gcc;gcc_s;irc;svml;c;gcc;gcc_s;irc_s;dl;c
-- Configuring done
-- Generating done
-- Build files have been written to: /users/ccranor/tmp/cmake4/b
%
```Chuck AtkinsChuck Atkins