diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93d659fb2a1fe530dd830bf6ef1c680c9b3a8989..f321d2a20cc11988dae87169d47e03e2755be36d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ include: - # Metadata shared my many jobs + # Metadata shared by many jobs + - local: .gitlab/projects.yml - local: .gitlab/rules.yml - local: .gitlab/artifacts.yml @@ -27,7 +28,9 @@ stages: # - a build script # - tags for the jobs # - already provided for upload and CI update jobs -# - rules for when to run the job +# - the `.rules` block (see https://gitlab.com/gitlab-org/gitlab/-/issues/420603) +# - see `.gitlab/rules.yml` for variables jobs may set to control when the +# job is executed # # Additionally, jobs may also contain: # @@ -47,7 +50,7 @@ centos7-modelbuilder:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-modelbuilder:test: extends: @@ -55,16 +58,16 @@ centos7-modelbuilder:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-modelbuilder:build gdrive-linux-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .project_cmb + - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-modelbuilder:test @@ -77,14 +80,14 @@ centos7-vtk:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-vtk:test: extends: - .centos7_vtk - .cmake_test_linux - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-vtk:build @@ -98,7 +101,7 @@ macos-arm64-modelbuilder:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .run_manually + - .rules macos-arm64-modelbuilder:test: extends: @@ -106,16 +109,16 @@ macos-arm64-modelbuilder:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .run_automatically + - .rules needs: - macos-arm64-modelbuilder:build gdrive-macos-arm64-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .project_cmb + - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-modelbuilder:test @@ -126,7 +129,7 @@ macos-x86_64-modelbuilder:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-modelbuilder:test: extends: @@ -134,16 +137,16 @@ macos-x86_64-modelbuilder:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-modelbuilder:build gdrive-macos-x86_64-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .project_cmb + - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-modelbuilder:test @@ -156,14 +159,14 @@ macos-arm64-smtk:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .run_manually + - .rules macos-arm64-smtk:test: extends: - .macos_arm64_smtk - .cmake_test_macos - .macos_arm64_builder_tags - - .run_automatically + - .rules needs: - macos-arm64-smtk:build @@ -173,14 +176,14 @@ macos-x86_64-smtk:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-smtk:test: extends: - .macos_x86_64_smtk - .cmake_test_macos - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-smtk:build @@ -194,7 +197,7 @@ windows-vs2022-modelbuilder:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules # adding occt extends the build time timeout: 1h 30m @@ -204,16 +207,16 @@ windows-vs2022-modelbuilder:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-modelbuilder:build gdrive-windows-vs2022-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .project_cmb + - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-modelbuilder:test @@ -226,7 +229,7 @@ windows-vs2022-smtk:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules timeout: 1h 30m windows-vs2022-smtk:test: @@ -234,7 +237,7 @@ windows-vs2022-smtk:test: - .windows_vs2022_smtk - .cmake_test_windows - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-smtk:build @@ -249,7 +252,7 @@ centos7-docker-modelbuilder:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-docker-modelbuilder:test: extends: @@ -257,7 +260,7 @@ centos7-docker-modelbuilder:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-docker-modelbuilder:build @@ -273,7 +276,7 @@ centos7-cmb2d:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-cmb2d:test: extends: @@ -281,16 +284,16 @@ centos7-cmb2d:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-cmb2d:build gdrive-linux-cmb2d:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_cmb2d + - .rules variables: - GDRIVE_TARGET_PATH: packages/cmb2d GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-cmb2d:test @@ -307,7 +310,7 @@ centos7-truchas:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-truchas:test: extends: @@ -315,16 +318,16 @@ centos7-truchas:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-truchas:build -gdirve-linux-truchas:upload: +gdrive-linux-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_truchas + - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-truchas:test @@ -339,7 +342,7 @@ macos-arm64-truchas:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules macos-arm64-truchas:test: extends: @@ -347,16 +350,16 @@ macos-arm64-truchas:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules needs: - macos-arm64-truchas:build gdrive-macos-arm64-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_truchas + - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-truchas:test @@ -367,7 +370,7 @@ macos-x86_64-truchas:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules macos-x86_64-truchas:test: extends: @@ -375,16 +378,16 @@ macos-x86_64-truchas:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules needs: - macos-x86_64-truchas:build gdrive-macos-x86_64-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_truchas + - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-truchas:test @@ -399,7 +402,7 @@ windows-vs2022-truchas:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .weekly_only + - .rules timeout: 1h 30m windows-vs2022-truchas:test: @@ -408,16 +411,16 @@ windows-vs2022-truchas:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .weekly_only + - .rules needs: - windows-vs2022-truchas:build gdrive-windows-vs2022-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_truchas + - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-truchas:test @@ -435,7 +438,7 @@ centos7-aeva:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-aeva:test: extends: @@ -443,16 +446,16 @@ centos7-aeva:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-aeva:build gdrive-linux-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .project_aeva + - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-aeva:test @@ -467,7 +470,7 @@ macos-x86_64-aeva:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-aeva:test: extends: @@ -475,16 +478,16 @@ macos-x86_64-aeva:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-aeva:build gdrive-macos-x86_64-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .project_aeva + - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-aeva:test @@ -499,7 +502,7 @@ windows-vs2022-aeva:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules # adding occt extends the build time timeout: 1h 45m @@ -509,16 +512,16 @@ windows-vs2022-aeva:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-aeva:build gdrive-windows-vs2022-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .project_aeva + - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-aeva:test @@ -532,7 +535,8 @@ gdrive-windows-vs2022-aeva:upload: smtk-fedora:update-ci: extends: - .image_builder - - .upload_only + - .project_smtk + - .rules needs: - centos7-modelbuilder:test script: @@ -542,7 +546,8 @@ smtk-fedora:update-ci: # smtk-fedora-paraview59:update-ci: # extends: # - .image_builder -# - .upload_only +# - .project_smtk +# - .rules # needs: # - centos7-modelbuilder:test # script: @@ -552,7 +557,8 @@ smtk-fedora:update-ci: smtk-fedora-paraview:update-ci: extends: - .image_builder - - .upload_only + - .project_smtk + - .rules needs: - centos7-modelbuilder:test script: @@ -562,7 +568,8 @@ smtk-fedora-paraview:update-ci: smtk-fedora-vtk:update-ci: extends: - .image_builder - - .upload_only + - .project_smtk + - .rules needs: - centos7-vtk:test script: @@ -574,7 +581,8 @@ smtk-fedora-vtk:update-ci: smtk-macos-arm64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_smtk + - .rules needs: - macos-arm64-smtk:build # - job: macos-arm64-smtk:test @@ -583,13 +591,13 @@ smtk-macos-arm64:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: macos-arm64 smtk-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_smtk + - .rules needs: - macos-x86_64-smtk:build # - job: macos-x86_64-smtk:test @@ -598,7 +606,6 @@ smtk-macos-x86_64:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: macos-x86_64 ### Windows @@ -606,7 +613,8 @@ smtk-macos-x86_64:update-ci: smtk-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_smtk + - .rules needs: - windows-vs2022-smtk:build # - job: windows-vs2022-smtk:test @@ -615,7 +623,6 @@ smtk-windows-vs2022:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: windows-x86_64-vs2022 ## CMB @@ -625,7 +632,8 @@ smtk-windows-vs2022:update-ci: cmb-fedora:update-ci: extends: - .image_builder - - .upload_only + - .project_cmb + - .rules needs: - centos7-modelbuilder:test script: @@ -635,7 +643,8 @@ cmb-fedora:update-ci: # cmb-fedora-paraview59:update-ci: # extends: # - .image_builder -# - .upload_only +# - .project_cmb +# - .rules # needs: # - centos7-modelbuilder:test # script: @@ -647,7 +656,8 @@ cmb-fedora:update-ci: cmb-macos-arm64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_cmb + - .rules needs: - macos-arm64-modelbuilder:build - job: macos-arm64-modelbuilder:test @@ -656,13 +666,13 @@ cmb-macos-arm64:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: macos-arm64 cmb-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_cmb + - .rules needs: - macos-x86_64-modelbuilder:build # - job: macos-x86_64-modelbuilder:test @@ -671,7 +681,6 @@ cmb-macos-x86_64:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: macos-x86_64 ### Windows @@ -679,7 +688,8 @@ cmb-macos-x86_64:update-ci: cmb-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_cmb + - .rules needs: - windows-vs2022-modelbuilder:build # - job: windows-vs2022-modelbuilder:test @@ -688,7 +698,6 @@ cmb-windows-vs2022:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: windows-x86_64-vs2022 ## AEVA @@ -698,7 +707,8 @@ cmb-windows-vs2022:update-ci: aevasession-fedora:update-ci: extends: - .image_builder - - .upload_only + - .project_aeva + - .rules needs: - centos7-aeva:test script: @@ -708,7 +718,8 @@ aevasession-fedora:update-ci: aeva-fedora:update-ci: extends: - .image_builder - - .upload_only + - .project_aeva + - .rules needs: - centos7-aeva:test script: @@ -720,7 +731,8 @@ aeva-fedora:update-ci: aevasession-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_aeva + - .rules needs: - macos-x86_64-aeva:build - job: macos-x86_64-aeva:test @@ -729,13 +741,13 @@ aevasession-macos-x86_64:update-ci: TARGET_PROJECT: aevasession PROJECTS_TO_CLEAN: "aeva aevasession" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: macos-x86_64 aeva-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_aeva + - .rules needs: - macos-x86_64-aeva:build - job: macos-x86_64-aeva:test @@ -744,7 +756,6 @@ aeva-macos-x86_64:update-ci: TARGET_PROJECT: aeva PROJECTS_TO_CLEAN: "aeva" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: macos-x86_64 ### Windows @@ -752,7 +763,8 @@ aeva-macos-x86_64:update-ci: aevasession-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_aeva + - .rules needs: - windows-vs2022-aeva:build - job: windows-vs2022-aeva:test @@ -761,13 +773,13 @@ aevasession-windows-vs2022:update-ci: TARGET_PROJECT: aevasession PROJECTS_TO_CLEAN: "aeva aevasession" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: windows-x86_64-vs2022 aeva-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .project_aeva + - .rules needs: - windows-vs2022-aeva:build - job: windows-vs2022-aeva:test @@ -776,7 +788,6 @@ aeva-windows-vs2022:update-ci: TARGET_PROJECT: aeva PROJECTS_TO_CLEAN: "aeva" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: windows-x86_64-vs2022 # WindTunnel builds @@ -791,7 +802,7 @@ centos7-windtunnel:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-windtunnel:test: extends: @@ -799,16 +810,16 @@ centos7-windtunnel:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-windtunnel:build gdrive-linux-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_windtunnel + - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-windtunnel:test @@ -823,7 +834,7 @@ macos-x86_64-windtunnel:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules macos-x86_64-windtunnel:test: extends: @@ -831,16 +842,16 @@ macos-x86_64-windtunnel:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules needs: - macos-x86_64-windtunnel:build gdrive-macos-x86_64-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_windtunnel + - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-windtunnel:test @@ -851,7 +862,7 @@ macos-arm64-windtunnel:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules macos-arm64-windtunnel:test: extends: @@ -859,16 +870,16 @@ macos-arm64-windtunnel:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules needs: - macos-arm64-windtunnel:build gdrive-macos-arm64-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_windtunnel + - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-windtunnel:test @@ -883,7 +894,7 @@ windows-vs2022-windtunnel:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .weekly_only + - .rules # Machines are right on the edge 1 hour; avoid unnecessary rebuilds. timeout: 1h 30m @@ -893,16 +904,16 @@ windows-vs2022-windtunnel:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .weekly_only + - .rules needs: - windows-vs2022-windtunnel:build gdrive-windows-vs2022-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .project_windtunnel + - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-windtunnel:test diff --git a/.gitlab/ci/build-aeva-image.sh b/.gitlab/ci/build-aeva-image.sh index b5017ec5c4957b6d525df5338b53919cc321eec9..6ff0bfa0c64c17b62c64ede19f2551367a73fe64 100755 --- a/.gitlab/ci/build-aeva-image.sh +++ b/.gitlab/ci/build-aeva-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-aevasession-image.sh b/.gitlab/ci/build-aevasession-image.sh index 6fafb0f91171f96854326cc7bac45c2c2af3e8f2..d4a7bd79ed4d917a23efa503f4fd509dd34a1c48 100755 --- a/.gitlab/ci/build-aevasession-image.sh +++ b/.gitlab/ci/build-aevasession-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-cmb-image.sh b/.gitlab/ci/build-cmb-image.sh index f23c60bf14e965e868cab4fbace2da34aae4ec6f..1572728718a31f828837fe297ba05f496c431d8f 100755 --- a/.gitlab/ci/build-cmb-image.sh +++ b/.gitlab/ci/build-cmb-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-smtk-image.sh b/.gitlab/ci/build-smtk-image.sh index 92f8f48dfa3d0b0cbc42b4b058b9f411dc71a8ed..bf0ea9e2123c4efbb78f7b162bee2d60fe94279d 100755 --- a/.gitlab/ci/build-smtk-image.sh +++ b/.gitlab/ci/build-smtk-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index a92459c3f454baf891b759260877ae2c2f39de54..774fcb06371ef7d389ee69b6282f0b6ea7d8c56f 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -18,45 +18,61 @@ GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci .centos7_modelbuilder: - extends: .centos7 + extends: + - .centos7 + - .project_cmb variables: CMAKE_CONFIGURATION: centos7_modelbuilder .centos7_docker_modelbuilder: - extends: .centos7 + extends: + - .centos7 + - .project_cmb variables: CMAKE_CONFIGURATION: centos7_docker_modelbuilder .centos7_vtk: - extends: .centos7 + extends: + - .centos7 + - .project_smtk variables: CMAKE_CONFIGURATION: centos7_vtk .centos7_aeva: - extends: .centos7 + extends: + - .centos7 + - .project_aeva variables: CMAKE_CONFIGURATION: centos7_aeva .centos7_ace3p: - extends: .centos7 + extends: + - .centos7 + - .project_ace3p variables: CMAKE_CONFIGURATION: centos7_ace3p .centos7_truchas: - extends: .centos7 + extends: + - .centos7 + - .project_truchas variables: CMAKE_CONFIGURATION: centos7_truchas .centos7_cmb2d: - extends: .centos7 + extends: + - .centos7 + - .project_cmb2d variables: CMAKE_CONFIGURATION: centos7_cmb2d .centos7_windtunnel: - extends: .centos7 + extends: + - .centos7 + - .project_windtunnel variables: CMAKE_CONFIGURATION: centos7_windtunnel @@ -96,6 +112,8 @@ .cmake_build_linux: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_linux diff --git a/.gitlab/os-macos.yml b/.gitlab/os-macos.yml index bd797000f8782f7e4503755be98d805689fa6627..0253c16132faefc22ff3a8e7eb975c4228d71f21 100644 --- a/.gitlab/os-macos.yml +++ b/.gitlab/os-macos.yml @@ -25,68 +25,90 @@ GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmb-ci" .macos_x86_64_windtunnel: - extends: .macos_build + extends: + - .macos_build + - .project_windtunnel variables: CMAKE_CONFIGURATION: macos_x86_64_windtunnel .macos_arm64_windtunnel: - extends: .macos_build + extends: + - .macos_build + - .project_windtunnel variables: CMAKE_CONFIGURATION: macos_arm64_windtunnel .macos_arm64_truchas: - extends: .macos_build + extends: + - .macos_build + - .project_truchas variables: CMAKE_CONFIGURATION: macos_arm64_truchas .macos_x86_64_truchas: - extends: .macos_build + extends: + - .macos_build + - .project_truchas variables: CMAKE_CONFIGURATION: macos_x86_64_truchas .macos_x86_64_ace3p: - extends: .macos_build + extends: + - .macos_build + - .project_ace3p variables: CMAKE_CONFIGURATION: macos_x86_64_ace3p .macos_arm64_ace3p: - extends: .macos_build + extends: + - .macos_build + - .project_ace3p variables: CMAKE_CONFIGURATION: macos_arm64_ace3p .macos_x86_64_aeva: - extends: .macos_build + extends: + - .macos_build + - .project_aeva variables: CMAKE_CONFIGURATION: macos_x86_64_aeva DOWNLOAD_FORTRAN: "1" .macos_arm64_modelbuilder: - extends: .macos_build + extends: + - .macos_build + - .project_cmb variables: CMAKE_CONFIGURATION: macos_arm64_modelbuilder .macos_x86_64_modelbuilder: - extends: .macos_build + extends: + - .macos_build + - .project_cmb variables: CMAKE_CONFIGURATION: macos_x86_64_modelbuilder .macos_arm64_smtk: - extends: .macos_build + extends: + - .macos_build + - .project_smtk variables: CMAKE_CONFIGURATION: macos_arm64_smtk .macos_x86_64_smtk: - extends: .macos_build + extends: + - .macos_build + - .project_smtk variables: CMAKE_CONFIGURATION: macos_x86_64_smtk @@ -127,6 +149,8 @@ .cmake_build_macos: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_macos diff --git a/.gitlab/os-windows.yml b/.gitlab/os-windows.yml index 06cba193dd97e0ee6e67f1bc90909403a3421065..9f49e5be97faf32f097806cf56244baf924c9c65 100644 --- a/.gitlab/os-windows.yml +++ b/.gitlab/os-windows.yml @@ -36,37 +36,49 @@ VCVARSVERSION: "14.35.32215" .windows_vs2022_aeva: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_aeva variables: CMAKE_CONFIGURATION: windows_vs2022_aeva .windows_vs2022_windtunnel: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_windtunnel variables: CMAKE_CONFIGURATION: windows_vs2022_windtunnel .windows_vs2022_truchas: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_truchas variables: CMAKE_CONFIGURATION: windows_vs2022_truchas .windows_vs2022_ace3p: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_ace3p variables: CMAKE_CONFIGURATION: windows_vs2022_ace3p .windows_vs2022_modelbuilder: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_cmb variables: CMAKE_CONFIGURATION: windows_vs2022_modelbuilder .windows_vs2022_smtk: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_smtk variables: CMAKE_CONFIGURATION: windows_vs2022_smtk @@ -104,6 +116,8 @@ .cmake_build_windows: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_windows diff --git a/.gitlab/projects.yml b/.gitlab/projects.yml new file mode 100644 index 0000000000000000000000000000000000000000..0c797083d93c2172e055196e7eb449df27f58405 --- /dev/null +++ b/.gitlab/projects.yml @@ -0,0 +1,37 @@ +.project_cmb: + variables: + CMB_CI_FOR_PROJECT: /\bcmb\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/modelbuilder + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/cmb + +.project_smtk: + variables: + CMB_CI_FOR_PROJECT: /\bsmtk\b/ + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/smtk + +.project_aeva: + variables: + CMB_CI_FOR_PROJECT: /\baeva\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/aeva + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/aeva + +.project_ace3p: + variables: + CMB_CI_FOR_PROJECT: /\bace3p\b/ + +.project_truchas: + variables: + CMB_CI_FOR_PROJECT: /\btruchas\b/ + CMB_CI_WEEKLY_ONLY: "true" + GDRIVE_TARGET_PACKAGES_PATH: packages/truchas + +.project_cmb2d: + variables: + CMB_CI_FOR_PROJECT: /\bcmb2d\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/cmb2d + +.project_windtunnel: + variables: + CMB_CI_FOR_PROJECT: /\bwindtunnel\b/ + CMB_CI_WEEKLY_ONLY: "true" + GDRIVE_TARGET_PACKAGES_PATH: packages/windtunnel diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index ffffeabdd517bf208966acc8a91bc14acb3222a0..122ca2a850045a664a781d7c334879567654315f 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -1,48 +1,87 @@ # Rules for where jobs can run -.run_automatically: - rules: - - if: '$CI_MERGE_REQUEST_ID' - when: on_success - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: on_success - - when: never - -.run_manually: - rules: - - if: '$CI_MERGE_REQUEST_ID' - when: manual - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $CI_PIPELINE_SOURCE == "schedule"' - when: on_success - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: delayed - start_in: 30 minutes - - when: never +### +# Job variables: +# CMB_CI_RUN_MANUALLY +# Set to "true" when the job should be triggered manually. +# CMB_CI_IS_FOR_SCHEDULE +# Set to "true" if the job should only run for schedules. +# CMB_CI_UPLOAD_ONLY +# Set to "true" to only perform when an upload is requested. +# CMB_CI_WEEKLY_ONLY +# Set to "true" to make the job only run for weekly pipelines. +# CMB_CI_FOR_PROJECT +# The name of the project the job is "for". +# Pipeline variables: +# ENABLE_PACKAGE_UPLOAD +# Set to "true" when package upload jobs should be performed. +# ENABLE_WEEKLY_JOBS +# Set to "true" when weekly-only jobs should be performed. +# ENABLE_PROJECTS +# Set to a comma-separated list of projects that should be focused on. +# ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) +# Set to "true" when uploads should be performed. +### -.merged_only: +# When to even consider running a pipeline. +workflow: rules: + # Run for merge requests. - if: '$CI_MERGE_REQUEST_ID' + when: always + auto_cancel: + # Cancel all pipeline jobs if a new commit comes in on the branch. + on_new_commit: interruptible + # Do not run for other projects. + - if: '$CI_PROJECT_PATH != "cmb/cmb-superbuild"' when: never - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: on_success + # Run for protected branches. + - if: '$CI_COMMIT_REF_PROTECTED == "true"' + when: always + auto_cancel: + # Cancel all pipeline jobs if a new commit comes in on the branch. + on_new_commit: interruptible + # Run for tags. + - if: '$CI_COMMIT_TAG' + when: always + # Run for schedules. + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + auto_cancel: + # Never cancel scheduled pipelines because of new commits. + on_new_commit: none + # Skip pipelines in all other cases. - when: never -.upload_only: +.rules: rules: - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $GIRDER_UPLOAD_ENABLE == "true"' - when: on_success - - when: never - -.weekly_only: - rules: - - if: '$CI_MERGE_REQUEST_ID' + # Schedule-only jobs never run on MRs. + - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_IS_FOR_SCHEDULE' when: never - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $WEEKLY_SCHEDULE == "true"' + # MRs use manual triggers. + - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_RUN_MANUALLY == "true"' + when: manual + # Normal jobs run automatically. + - if: '$CI_MERGE_REQUEST_ID' when: on_success - - when: never - -.weekly_upload_only: - rules: - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $GIRDER_UPLOAD_ENABLE == "true" && $WEEKLY_SCHEDULE == "true"' + # If we have a project-specific pipeline, ignore jobs not matching that project. + - if: '$ENABLE_PROJECTS && $ENABLE_PROJECTS =~ $CMB_CI_FOR_PROJECT' + when: never + # Skip schedule jobs without a schedule. + - if: '$CI_PIPELINE_SOURCE != "schedule" && $CMB_CI_IS_FOR_SCHEDULE == "true"' + when: never + # Skip upload projects without an upload request. + - if: '$CMB_CI_UPLOAD_ONLY == "true" && $ENABLE_PACKAGE_UPLOAD != "true"' + when: never + # Skip weekly projects without a weekly pipeline. + - if: '$CMB_CI_WEEKLY_ONLY == "true" && $ENABLE_WEEKLY_JOBS != "true"' + when: never + # Jobs on a schedule run right away. + - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_success - - when: never + # Manual jobs run delayed (e.g., merges to branches). + - if: '$CMB_CI_RUN_MANUALLY == "true"' + when: delayed + start_in: 30 minutes + # Normal jobs on the project run automatically. + - when: on_success diff --git a/.gitlab/update-ci.yml b/.gitlab/update-ci.yml index 3a728b033713b380acd4e5b0c91aae29ce1f2f88..0d01066ba02f104257a9e1623c3d28a101d43174 100644 --- a/.gitlab/update-ci.yml +++ b/.gitlab/update-ci.yml @@ -16,10 +16,13 @@ - .gitlab/ci/prune-for-developer.sh - ls build/ - sha512sum build/* + # Bail before upload if we're not actually uploading bundles. + - '[ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ] || exit 0' - dnf install -y --setopt=install_weak_deps=False rclone - - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" + - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PACKAGE_PATH/" variables: GIT_SUBMODULE_STRATEGY: none + CMB_CI_IS_FOR_SCHEDULE: "true" .image_builder: image: "almalinux:8" @@ -32,6 +35,7 @@ variables: GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci + CMB_CI_IS_FOR_SCHEDULE: "true" dependencies: [] environment: diff --git a/.gitlab/upload.yml b/.gitlab/upload.yml index 76e3ae881e41651cdf6041ac2ab60cb3db6524ca..902bd089f6bb3524ac096e4d08d86b057f8fd2de 100644 --- a/.gitlab/upload.yml +++ b/.gitlab/upload.yml @@ -9,8 +9,11 @@ - build environment: name: gdrive-upload + variables: + CMB_CI_IS_FOR_SCHEDULE: "true" + CMB_CI_UPLOAD_ONLY: "true" script: - ls build/ - dnf install -y --setopt=install_weak_deps=False rclone - - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" + - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_CI_BUNDLE_PATH/" diff --git a/Scripts/ci-bundles/README.md b/Scripts/ci-bundles/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d56ea6bc54abc566b4233287d7743ea1d6892b7c --- /dev/null +++ b/Scripts/ci-bundles/README.md @@ -0,0 +1,57 @@ +# CI bundles from the superbuild + +CI bundles are created by the CMB superbuild's pipeline because it allows the +usage of an existing build to generate a tarball for usage in macOS and Windows +CI. Linux builds Docker images using recipes in each project but based on the +current superbuild's commit. This ensures that CI bundles are using the same +superbuild commit as it tends to move far faster than the CI Dockerfile +recipes. + +## Scripts + +The superbuild is home to a number of scripts to help build CI bundles. + +### Linux + +Docker images are built using scripts living in `.gitlab/ci/build-*-image.sh`. +These scripts clone the target project in question and build the pipeline's +superbuild commit inside of the container as specified in the target project's +relevant `.gitlab/ci/docker/*` directory. Each + +### macOS and Windows + +For macOS and Windows, a Linux container is handed the `install/` directory of +a superbuild build as well as some details from relevant build trees. The +`.gitlab/ci/prune-for-developer.sh` script is then run to strip out the target +project(s) from the `install/` tree so that it may be used as an SDK bundle for +the project in question. The tarball is then uploaded (if requested). + +## Uploading new bundles + +The bundles are always *created* so that when they break, there's at least a +signal that things are broken. Uploads only happen on-demand because the +bundles end up taking up a lot of cloud storage and are not actually used all +that often. + +To trigger the *upload* of new bundles to their respective cloud locations, +create [a new pipeline on the CMB superbuild][cmb-sb-new-pipeline] and set the +following variables: + + - `ENABLE_CI_BUNDLE_UPLOAD` to `true` + - `ENABLE_PROJECTS` to a comma-separated list of projects to create CI + bundles. If unset, all projects will be built. Supported projects are: + * `cmb` + * `smtk` + * `aeva` (uploads both `aeva/aeva` and `aeva/session` CI bundles) + +[cmb-sb-new-pipeline]: https://gitlab.kitware.com/cmb/cmb-superbuild/-/pipelines/new + +When usage of new bundles are *merged*, any containers must be marked as "safe" +in this repository's `Scripts/ci-bundles/dockerhub-keep-tags` file to avoid +deleting them. + +## Cleaning up old bundles + +The `Scripts/ci-bundles/rm-stale-*.bash` scripts may be used to prune older +bundles from the relevant cloud storage locations. The scripts keep any bundle +created within the last 2 weeks.