Enable GPU-to-GPU communication

Depends: !3024 (merged)

This MR adds the flag VTKm_ENABLE_GPU_MPI which when enabled, it will use GPU aware MPI.

• This will only work with GPUs and MPI implementation that supports GPU aware MPI calls.
• Enabling VTKm_ENABLE_GPU_MP without MPI/GPU support might results in errors when running VTK-m with DIY/MPI.
• Only the following tests can run with this feature if enabled:
  • UnitTestSerializationDataSet
  • UnitTestSerializationArrayHandle

Options used in crusher:

Followed OLCF docs about GPU aware MPI: https://docs.olcf.ornl.gov/systems/crusher_quick_start_guide.html#gpu-aware-mpi

CMake options:

set(CMAKE_BUILD_TYPE     "debug"                  CACHE STRING "")
set(CMAKE_PREFIX_PATH /gpfs/alpine/csc331/scratch/vbolea/gpu-to-gpu/kokkos_install CACHE FILEPATH "")

#set(CMAKE_C_COMPILER     /opt/rocm-4.5.0/llvm/bin/clang CACHE FILEPATH "")
#set(CMAKE_CXX_COMPILER   /opt/rocm-4.5.0/llvm/bin/clang++ CACHE FILEPATH "")
set(CMAKE_HIP_FLAGS      "-O0 -I/opt/cray/pe/mpich/8.1.14/ofi/crayclang/10.0/include" CACHE STRING "")
set(CMAKE_CXX_FLAGS      "-O0 -I/opt/cray/pe/mpich/8.1.14/ofi/crayclang/10.0/include" CACHE STRING "")

set(VTKm_ENABLE_KOKKOS ON CACHE STRING "")
set(VTKm_ENABLE_MPI ON CACHE STRING "")
set(VTKm_ENABLE_GPU_MPI ON CACHE STRING "")
set(CMAKE_HIP_FLAGS "-O0" CACHE STRING "")

set(Kokkos_CXX_COMPILER /opt/rocm-4.5.0/llvm/bin/clang++ CACHE FILEPATH "")
set(Kokkos_COMPILE_LAUNCHER /opt/rocm-4.5.0/bin/amdclang++ CACHE FILEPATH "")
set(CMAKE_POSITION_INDEPENDENT_CODE ON              CACHE BOOL "")
set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "")

set(CMAKE_EXE_LINKER_FLAGS "-L/opt/cray/pe/mpich/8.1.14/ofi/crayclang/10.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.14/gtl/lib -lmpi_gtl_hsa" CACHE STRING "")

set(MPIEXEC_EXECUTABLE /usr/bin/srun CACHE FILEPATH "")
set(MPIEXEC_PREFLAGS "-ACSC331;-pbatch;-t110;--nice=0;-N;1;--gpus-per-node;1;-c;1" CACHE STRING "")