# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Spock.
.spock_gcc_hip:
  variables:
    CCACHE_BASEDIR:          "/gpfs/alpine/world-shared/csc331/"
    CCACHE_DIR:              "/gpfs/alpine/world-shared/csc331/vtk-m/ci/ccache"
    CUSTOM_CI_BUILDS_DIR:    "/gpfs/alpine/world-shared/csc331/vtk-m/ci/runtime"

    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
    CCACHE_IGNOREOPTIONS:    "-isystem=*"
    CCACHE_NOHASHDIR:        "true"
    CCACHE_INSTALL_DIR:      "$CI_PROJECT_DIR/build"

    CMAKE_BUILD_TYPE:        "RelWithDebInfo"
    CMAKE_GENERATOR:         "Ninja"

    # This is needed for the smoke_test, while we use rocm 5 to build VTK-m the
    # smoke_test needs 4.5 since Kokkos is built agains rocm 4.5
    LD_LIBRARY_PATH:         "/opt/rocm-4.5.0/lib:${LD_LIBRARY_PATH}"
    LIBRARY_PATH:            "/opt/rocm-4.5.0/lib:${LIBRARY_PATH}"

    # While Kokkos and VTK-m uses ROCm 4.5.0 runtime/sdk, we need to build
    # VTK-m with HIPCC from ROCM 5
    CMAKE_HIP_COMPILER:      "/opt/rocm-default/llvm/bin/clang++"
    Kokkos_CXX_COMPILER:     "/opt/rocm-default/llvm/bin/clang++"
    CMAKE_HIP_ARCHITECTURES: "gfx908"

    CC: gcc
    CXX: g++
    # DefApps/default;craype;rocm;gcc should be loaded first
    JOB_MODULES: >-
      DefApps/default
      craype-accel-amd-gfx90a
      rocm/5
      gcc/10
      cmake/3.22
      git
      git-lfs
      kokkos/3.5.00
      lsf-tools
      ninja
      spectrum-mpi
      zstd
    VTKM_SETTINGS: kokkos+hip+spock+ccache+no_rendering
  interruptible: true

.setup_env_ecpci: &setup_env_ecpci |
  module purge
  module load ${JOB_MODULES}
  module list
  export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"

build:spock_gcc_hip:
  stage: build
  tags: [spock, shell]
  extends:
    - .spock_gcc_hip
    - .run_spock_ci
  before_script:
    - *setup_env_ecpci
    - mkdir -p "$CCACHE_INSTALL_DIR"
    - cmake --version
    - cmake -VV -P .gitlab/ci/config/ccache.cmake
    - ccache -z
    - ccache -s

    - git remote add lfs https://gitlab.kitware.com/vtk/vtk-m.git
    - git fetch lfs
    - git-lfs install
    - git-lfs pull lfs
  script:
    - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake
    - ctest -VV -S .gitlab/ci/ctest_configure.cmake
  artifacts:
    expire_in: 24 hours
    when: always
    paths:
      - build/
      - .gitlab/ccache/ccache

test:spock_gcc_hip:
  stage: test
  tags: [spock, slurm]
  extends:
    - .spock_gcc_hip
    - .run_spock_ci
  needs:
    - build:spock_gcc_hip
  dependencies:
    - build:spock_gcc_hip
  variables:
    SCHEDULER_PARAMETERS: "-ACSC331 -pecp -t120 --nice=0 -c32 --gpus=4 -N 1"
    # We need this to skip ctest_submit from being run inside a jsrun job
    GITLAB_CI_EMULATION: 1
    # Tests errors to address due to different env in Spock
    # Refer to issue: https://gitlab.kitware.com/vtk/vtk-m/-/issues/743
    CTEST_EXCLUSIONS: >-
      UnitTestMIRFilter
      UnitTestDistributedBranchDecompositionFilter
      UnitTestWorkletParticleAdvection
      UnitTestLagrangianFilter
  before_script:
    - *setup_env_ecpci
  script:
    - CTEST_MAX_PARALLELISM=32 ctest -VV -S .gitlab/ci/ctest_build.cmake
    - CTEST_MAX_PARALLELISM=4  ctest -VV -S .gitlab/ci/ctest_test.cmake
  after_script:
    - *setup_env_ecpci
    - ccache -s
    - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake
    - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake
  timeout: 150 minutes
