Merge topic 'ci_annotations'

75b3ec38 Install ninja and cmake for building adios2/vtkm deps
1a490d7d Download updated cmake and ninja in the script
eb93410d Update images to use a newer cmake
13255245 Add CDash link annotations to CI jobs

Acked-by: Kitware Robot <kwrobot@kitware.com>
Reviewed-by: Caitlin Ross <caitlin.ross@kitware.com>
Merge-request: !159

.gitlab-ci.yml

@@ -59,6 +59,7 @@ ubuntu18-gcc-vtkm19:test:
         - .ubuntu18_gcc_vtkm19
         - .cmake_test_linux
         - .linux_builder_tags
+        - .cmake_test_artifacts
         - .run_automatically
     dependencies:
         - ubuntu18-gcc-vtkm19:build
@@ -78,6 +79,7 @@ ubuntu18-clang:test:
         - .ubuntu18_clang
         - .cmake_test_linux
         - .linux_builder_tags
+        - .cmake_test_artifacts
         - .run_automatically
     dependencies:
         - ubuntu18-clang:build
@@ -97,6 +99,7 @@ ubuntu18-asan:test:
         - .ubuntu18_asan
         - .cmake_memcheck_linux
         - .linux_tester_priv_tags
+        - .cmake_test_artifacts
         - .run_automatically
     dependencies:
         - ubuntu18-asan:build
@@ -116,6 +119,7 @@ ubuntu18-ubsan:test:
         - .ubuntu18_ubsan
         - .cmake_memcheck_linux
         - .linux_tester_priv_tags
+        - .cmake_test_artifacts
         - .run_automatically
     dependencies:
         - ubuntu18-ubsan:build
@@ -127,6 +131,7 @@ ubuntu18-tidy:build:
         - .ubuntu18_tidy
         - .cmake_build_linux
         - .linux_builder_tags
+        - .cmake_build_artifacts
         - .run_automatically
 
 ubuntu18-coverage:build:
@@ -154,6 +159,7 @@ ubuntu18-coverage:analyze:
         - .ubuntu18_coverage
         - .cmake_coverage_linux
         - .linux_builder_tags
+        - .cmake_coverage_artifacts
         - .run_automatically
     dependencies:
         - ubuntu18-coverage:test

.gitlab/artifacts.yml

@@ -5,6 +5,9 @@
         expire_in: 1d
         # Because build logs are in here, we should always upload artifacts.
         when: always
+        reports:
+            annotations:
+                - build/annotations.json
         paths:
             # The artifacts of the build.
             # Generated headers.
@@ -26,12 +29,44 @@
 
             # CDash files.
             - build/DartConfiguration.tcl
+            - build/cdash-build-id
+
+.cmake_test_artifacts:
+    artifacts:
+        expire_in: 1d
+        # Because build logs are in here, we should always upload artifacts.
+        when: always
+        reports:
+            annotations:
+                - build/annotations.json
+        paths:
+            # The artifacts of the build.
+            # Generated headers.
+            - build/*/*.h
+
+            # CTest files.
+            - build/CTestCustom*.cmake
+            - build/CTestTestfile.cmake
+            - build/**/CTestTestfile.cmake
+            - build/Testing/
+
+            # CTest coverage files
+            - build/CMakeFiles/TargetDirectories.txt
+            - build/**/*.gcno
+            - build/**/*.gcda
+
+            # CDash files.
+            - build/DartConfiguration.tcl
+            - build/cdash-build-id
 
 .cmake_coverage_artifacts:
     artifacts:
         expire_in: 1d
         # Because build logs are in here, we should always upload artifacts.
         when: always
+        reports:
+            annotations:
+                - build/annotations.json
         paths:
             # The artifacts of the build.
             # Generated headers.
@@ -50,3 +85,4 @@
 
             # CDash files.
             - build/DartConfiguration.tcl
+            - build/cdash-build-id

.gitlab/ci/cmake.sh

+#!/bin/sh
+
+set -e
+
+case "$( uname -s )" in
+    Linux)
+        shatool="sha256sum"
+        version="3.26.3"
+        platform="linux-x86_64"
+        sha256sum="28d4d1d0db94b47d8dfd4f7dec969a3c747304f4a28ddd6fd340f553f2384dc2"
+        prefix="cmake"
+        ;;
+    *)
+        echo "Unrecognized platform $( uname -s )"
+        exit 1
+        ;;
+esac
+readonly shatool
+readonly version
+readonly platform
+readonly sha256sum
+readonly prefix
+
+filename="cmake-${version}-${platform}"
+tarball="${filename}.tar.gz"
+readonly filename
+readonly tarball
+
+#install cmake
+cd .gitlab
+
+curl -OL "https://github.com/Kitware/CMake/releases/download/v${version}/${tarball}"
+echo "${sha256sum}  ${tarball}" > cmake.sha256sum
+$shatool --check cmake.sha256sum
+tar xzf "${tarball}"
+mv "${filename}" "${prefix}"

.gitlab/ci/ctest_annotation.cmake

+function (ctest_annotation_report file)
+  set(label "")
+
+  if (EXISTS "${file}")
+    file(READ "${file}" json)
+  else ()
+    set(json "{\"CDash\": []}")
+  endif ()
+
+  foreach (arg IN LISTS ARGN)
+    if (NOT label)
+      set(label "${arg}")
+      continue ()
+    endif ()
+
+    set(item "{\"external_link\":{\"label\":\"${label}\",\"url\":\"${arg}\"}}")
+    set(label "")
+
+    string(JSON length LENGTH "${json}" "CDash")
+    string(JSON json SET "${json}" "CDash" "${length}" "${item}")
+  endforeach ()
+
+  file(WRITE "${file}" "${json}")
+endfunction ()
+
+if (NOT DEFINED build_id)
+  include("${CTEST_BINARY_DIRECTORY}/cdash-build-id" OPTIONAL)
+endif ()
+
+function (store_build_id build_id)
+  file(WRITE "${CTEST_BINARY_DIRECTORY}/cdash-build-id"
+    "set(build_id \"${build_id}\")\n")
+endfunction ()

.gitlab/ci/ctest_build.cmake

@@ -15,10 +15,19 @@ if (CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles")
 endif ()
 
 ctest_build(
+  NUMBER_ERRORS num_errors
   NUMBER_WARNINGS num_warnings
   RETURN_VALUE build_result)
 ctest_submit(PARTS Build)
 
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+if (build_id)
+  ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+    "Build Errors (${num_errors})" "https://open.cdash.org/viewBuildError.php?buildid=${build_id}"
+    "Build Warnings (${num_warnings})" "https://open.cdash.org/viewBuildError.php?type=1&buildid=${build_id}"
+  )
+endif ()
+
 if (build_result)
   message(FATAL_ERROR
     "Failed to build")

.gitlab/ci/ctest_configure.cmake

@@ -22,9 +22,20 @@ ctest_configure(
 # Read the files from the build directory.
 ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
 
-ctest_submit(PARTS Update)
+ctest_submit(PARTS Update
+  BUILD_ID build_id)
 ctest_submit(PARTS Configure)
 
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+if (build_id)
+  ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+    "Build Summary" "https://open.cdash.org/build/${build_id}"
+    "Update" "https://open.cdash.org/build/${build_id}/update"
+    "Configure" "https://open.cdash.org/build/${build_id}/configure"
+  )
+  store_build_id("${build_id}")
+endif ()
+
 if (configure_result)
   message(FATAL_ERROR
     "Failed to configure")

.gitlab/ci/ctest_coverage.cmake

@@ -18,6 +18,13 @@ ctest_coverage(
   RETURN_VALUE coverage_result)
 ctest_submit(PARTS Coverage)
 
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+if (build_id)
+  ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+    "Coverage Report" "https://open.cdash.org/viewCoverage.php?buildid=${build_id}"
+  )
+endif ()
+
 if (coverage_result)
   message(FATAL_ERROR
     "Failed to gather coverage")

.gitlab/ci/ctest_memcheck.cmake

@@ -22,6 +22,13 @@ ctest_memcheck(
   DEFECT_COUNT defects)
 ctest_submit(PARTS Memcheck)
 
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+if (build_id)
+  ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+    "Dynamic Analysis" "https://open.cdash.org/viewDynamicAnalysis.php?buildid=${build_id}"
+  )
+endif ()
+
 if (test_result)
   message(FATAL_ERROR
     "Failed to test")

.gitlab/ci/ctest_test.cmake

@@ -18,6 +18,17 @@ ctest_test(
   EXCLUDE "${test_exclusions}")
 ctest_submit(PARTS Test)
 
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+if (build_id)
+  ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+    "Build Summary" "https://open.cdash.org/build/${build_id}"
+    "All Tests"     "https://open.cdash.org/viewTest.php?buildid=${build_id}"
+    "Test Failures" "https://open.cdash.org/viewTest.php?onlyfailed&buildid=${build_id}"
+    "Tests Not Run" "https://open.cdash.org/viewTest.php?onlynotrun&buildid=${build_id}"
+    "Test Passes"   "https://open.cdash.org/viewTest.php?onlypassed&buildid=${build_id}"
+  )
+endif ()
+
 if (test_result)
   message(FATAL_ERROR
     "Failed to test")

.gitlab/ci/docker/install_cmake.sh

-#!/bin/sh
-
-cd /tmp
-mkdir -p Software
-
-#install cmake
-cd /tmp/Software
-curl -OL https://github.com/Kitware/CMake/releases/download/v3.16.3/cmake-3.16.3-Linux-x86_64.tar.gz
-tar xzf cmake-3.16.3-Linux-x86_64.tar.gz
-mkdir -p /opt/cmake
-mv cmake-3.16.3-Linux-x86_64/* /opt/cmake
-
-cd /tmp
-rm -rf Software/
-
-

.gitlab/ci/docker/ubuntu18-ompi/Dockerfile

@@ -4,10 +4,6 @@ MAINTAINER Caitlin Ross <caitlin.ross@kitware.com>
 COPY install_deps.sh /root/install_deps.sh
 RUN sh /root/install_deps.sh
 
-COPY install_cmake.sh /root/install_cmake.sh
-RUN sh /root/install_cmake.sh
-ENV PATH=/opt/cmake/bin:$PATH
-
 ARG ADIOS_HASH
 ENV ADIOS_HASH_ENV=$ADIOS_HASH
 COPY install_adios2.sh /root/install_adios2.sh

.gitlab/ci/docker/ubuntu18-ompi/install_deps.sh

 #!/bin/sh
 
+# System cmake/ninja are required for building dependencies
+
 # Install build requirements
 apt-get update -y
 apt-get install build-essential sudo curl \
                 openmpi-bin libopenmpi-dev \
                 python3-dev python3-mpi4py python3-numpy python3-pip \
                 libbz2-dev libpng-dev rapidjson-dev \
-                ninja-build \
+                cmake ninja-build \
                 git git-lfs vim \
                 libssl-dev \
                 clang-8 \

.gitlab/ci/docker/ubuntu18/Dockerfile

@@ -4,10 +4,6 @@ MAINTAINER Caitlin Ross <caitlin.ross@kitware.com>
 COPY install_deps.sh /root/install_deps.sh
 RUN sh /root/install_deps.sh
 
-COPY install_cmake.sh /root/install_cmake.sh
-RUN sh /root/install_cmake.sh
-ENV PATH=/opt/cmake/bin:$PATH
-
 ARG ADIOS_HASH
 ENV ADIOS_HASH_ENV=$ADIOS_HASH
 COPY install_adios2.sh /root/install_adios2.sh

.gitlab/ci/docker/ubuntu18/install_deps.sh

 #!/bin/sh
 
+# System cmake/ninja are required for building dependencies
+
 # Install build requirements
 apt-get update -y
 apt-get install build-essential sudo curl \
                 python3-dev python3-numpy python3-pip \
                 libbz2-dev libpng-dev rapidjson-dev \
-                ninja-build \
+                cmake ninja-build \
                 git git-lfs vim \
                 libssl-dev \
                 clang-8 \

.gitlab/ci/docker/update-images.sh

@@ -8,30 +8,28 @@ VTKM_SHORT_HASH=${VTKM_HASH:0:8}
 ADIOS_HASH=289fabbc1c19f18ba00f3075b0782857514a412f
 ADIOS_SHORT_HASH=${ADIOS_HASH:0:8}
 
-cp install_cmake.sh ubuntu18
+tag_suffix=${1:-latest}
+
 cp install_adios2.sh ubuntu18
 cp install_vtkm.sh ubuntu18
-IMAGE_NAME=kitware/vtk:ci-fides-vtkm${VTKM_SHORT_HASH}-adios${ADIOS_SHORT_HASH}-latest
+IMAGE_NAME=kitware/vtk:ci-fides-vtkm${VTKM_SHORT_HASH}-adios${ADIOS_SHORT_HASH}-${tag_suffix}
 docker image build --build-arg VTKM_HASH=$VTKM_HASH \
   --build-arg ADIOS_HASH=$ADIOS_HASH \
   -t $IMAGE_NAME \
   ubuntu18/
 rm ubuntu18/install_adios2.sh
 rm ubuntu18/install_vtkm.sh
-rm ubuntu18/install_cmake.sh
 docker push $IMAGE_NAME
 
-cp install_cmake.sh ubuntu18-ompi
 cp install_adios2.sh ubuntu18-ompi
 cp install_vtkm.sh ubuntu18-ompi
-IMAGE_NAME=kitware/vtk:ci-fides-vtkm${VTKM_SHORT_HASH}-adios${ADIOS_SHORT_HASH}-ompi-latest
+IMAGE_NAME=kitware/vtk:ci-fides-vtkm${VTKM_SHORT_HASH}-adios${ADIOS_SHORT_HASH}-ompi-${tag_suffix}
 docker image build --build-arg VTKM_HASH=$VTKM_HASH \
   --build-arg ADIOS_HASH=$ADIOS_HASH \
   -t $IMAGE_NAME \
   ubuntu18-ompi/
 rm ubuntu18-ompi/install_adios2.sh
 rm ubuntu18-ompi/install_vtkm.sh
-rm ubuntu18-ompi/install_cmake.sh
 docker push $IMAGE_NAME
 
 #edit names in .gitlab-ci.yml

.gitlab/ci/ninja.sh

+#!/bin/sh
+
+set -e
+
+readonly version="1.10.2"
+
+case "$( uname -s )" in
+    Linux)
+        shatool="sha256sum"
+        sha256sum="763464859c7ef2ea3a0a10f4df40d2025d3bb9438fcb1228404640410c0ec22d"
+        platform="linux"
+        ;;
+    Darwin)
+        shatool="shasum -a 256"
+        sha256sum="6fa359f491fac7e5185273c6421a000eea6a2f0febf0ac03ac900bd4d80ed2a5"
+        platform="mac"
+        ;;
+    *)
+        echo "Unrecognized platform $( uname -s )"
+        exit 1
+        ;;
+esac
+readonly shatool
+readonly sha256sum
+readonly platform
+
+readonly filename="ninja-$platform"
+readonly tarball="$filename.zip"
+
+cd .gitlab
+
+echo "$sha256sum  $tarball" > ninja.sha256sum
+curl -OL "https://github.com/ninja-build/ninja/releases/download/v$version/$tarball"
+$shatool --check ninja.sha256sum
+./cmake/bin/cmake -E tar xf "$tarball"

.gitlab/os-linux.yml

@@ -103,6 +103,9 @@
 ## Linux-specific scripts
 
 .before_script_linux: &before_script_linux
+    - .gitlab/ci/cmake.sh
+    - .gitlab/ci/ninja.sh
+    - export PATH=$PWD/.gitlab/:$PWD/.gitlab/cmake/bin/:$PATH
     - cmake --version
     - ninja --version