From c690e0ce0586b2879acfab88c5ac01ab6e824fe1 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 15:04:07 +0200 Subject: [PATCH 01/16] gitlab-ci: fix comment typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93d659fb..a337b7d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ include: - # Metadata shared my many jobs + # Metadata shared by many jobs - local: .gitlab/rules.yml - local: .gitlab/artifacts.yml -- GitLab From b0049c14939dfcd7e797d8412f0f2f8cbfaac5e7 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:30:29 +0200 Subject: [PATCH 02/16] gitlab-ci: fix `gdrive` job name typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a337b7d3..83f02460 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,7 +319,7 @@ centos7-truchas:test: needs: - centos7-truchas:build -gdirve-linux-truchas:upload: +gdrive-linux-truchas:upload: extends: - .gdrive_upload - .weekly_upload_only -- GitLab From 1145aa2978a324bc73a68ccf6ac77453ac3a2197 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 13:41:45 +0200 Subject: [PATCH 03/16] gitlab-ci: block uploading CI bundles without a variable set We've been uploading bundles every day "just in case" they've been useful. This has accumulated a lot of storage usage over time and needs to stop. Start migration to doing uploads on-demand only (the builds of the bundles still happens so that at least a "it can be built" notice is available. --- .gitlab/ci/build-aeva-image.sh | 6 ++++-- .gitlab/ci/build-aevasession-image.sh | 6 ++++-- .gitlab/ci/build-cmb-image.sh | 6 ++++-- .gitlab/ci/build-smtk-image.sh | 6 ++++-- .gitlab/rules.yml | 6 ++++++ .gitlab/update-ci.yml | 2 ++ 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.gitlab/ci/build-aeva-image.sh b/.gitlab/ci/build-aeva-image.sh index b5017ec5..6ff0bfa0 100755 --- a/.gitlab/ci/build-aeva-image.sh +++ b/.gitlab/ci/build-aeva-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-aevasession-image.sh b/.gitlab/ci/build-aevasession-image.sh index 6fafb0f9..d4a7bd79 100755 --- a/.gitlab/ci/build-aevasession-image.sh +++ b/.gitlab/ci/build-aevasession-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-cmb-image.sh b/.gitlab/ci/build-cmb-image.sh index f23c60bf..15727287 100755 --- a/.gitlab/ci/build-cmb-image.sh +++ b/.gitlab/ci/build-cmb-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/ci/build-smtk-image.sh b/.gitlab/ci/build-smtk-image.sh index 92f8f48d..bf0ea9e2 100755 --- a/.gitlab/ci/build-smtk-image.sh +++ b/.gitlab/ci/build-smtk-image.sh @@ -3,8 +3,10 @@ set -e set -x -# Do not push images for builds from not-cmb-superbuild. -if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ]; then +# Do not push images for builds from not-cmb-superbuild. Also require that +# bundle publishing is enabled. +if [ "$CI_PROJECT_PATH" = "cmb/cmb-superbuild" ] && + [ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ]; then push_image=true else push_image=false diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index ffffeabd..372be92f 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -1,5 +1,11 @@ # Rules for where jobs can run +### +# Pipeline variables: +# ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) +# Set to "true" when uploads should be performed. +### + .run_automatically: rules: - if: '$CI_MERGE_REQUEST_ID' diff --git a/.gitlab/update-ci.yml b/.gitlab/update-ci.yml index 3a728b03..497f29bc 100644 --- a/.gitlab/update-ci.yml +++ b/.gitlab/update-ci.yml @@ -16,6 +16,8 @@ - .gitlab/ci/prune-for-developer.sh - ls build/ - sha512sum build/* + # Bail before upload if we're not actually uploading bundles. + - '[ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ] || exit 0' - dnf install -y --setopt=install_weak_deps=False rclone - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" variables: -- GitLab From 2cf761a91ea169bb077235d07e7c31b56f2316c4 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 13:52:45 +0200 Subject: [PATCH 04/16] gitlab-ci: add workflow rules It allows some simplification of the rules as well, but not much. Further updates will use workflow rules to better control cancellation behaviors. --- .gitlab/rules.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index 372be92f..95cfb9a5 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -6,6 +6,27 @@ # Set to "true" when uploads should be performed. ### +# When to even consider running a pipeline. +workflow: + rules: + # Run for merge requests. + - if: '$CI_MERGE_REQUEST_ID' + when: always + # Do not run for other projects. + - if: '$CI_PROJECT_PATH != "cmb/cmb-superbuild"' + when: never + # Run for protected branches. + - if: '$CI_COMMIT_REF_PROTECTED == "true"' + when: always + # Run for tags. + - if: '$CI_COMMIT_TAG' + when: always + # Run for schedules. + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + # Skip pipelines in all other cases. + - when: never + .run_automatically: rules: - if: '$CI_MERGE_REQUEST_ID' -- GitLab From 88fb677dc5959cc2a11a24e6ed13656963bf3c6c Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 4 Apr 2025 23:12:10 +0200 Subject: [PATCH 05/16] gitlab-ci: cancel all jobs on MR and protected branch updates --- .gitlab/rules.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index 95cfb9a5..1ce4f69a 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -12,12 +12,18 @@ workflow: # Run for merge requests. - if: '$CI_MERGE_REQUEST_ID' when: always + auto_cancel: + # Cancel all pipeline jobs if a new commit comes in on the branch. + on_new_commit: interruptible # Do not run for other projects. - if: '$CI_PROJECT_PATH != "cmb/cmb-superbuild"' when: never # Run for protected branches. - if: '$CI_COMMIT_REF_PROTECTED == "true"' when: always + auto_cancel: + # Cancel all pipeline jobs if a new commit comes in on the branch. + on_new_commit: interruptible # Run for tags. - if: '$CI_COMMIT_TAG' when: always -- GitLab From 54af8cd2b507e1724bfd080314ddb8b40f9f8155 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 4 Apr 2025 23:12:30 +0200 Subject: [PATCH 06/16] gitlab-ci: don't auto-cancel scheduled pipelines If the built branch is updated, the schedule should still run. --- .gitlab/rules.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index 1ce4f69a..e28da339 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -30,6 +30,9 @@ workflow: # Run for schedules. - if: '$CI_PIPELINE_SOURCE == "schedule"' when: always + auto_cancel: + # Never cancel scheduled pipelines because of new commits. + on_new_commit: none # Skip pipelines in all other cases. - when: never -- GitLab From 8b202d9d4d1a1adce800ccb42fcaae44db31e3ea Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:09:29 +0200 Subject: [PATCH 07/16] gitlab-ci: remove unused `merged_only` rules block --- .gitlab/rules.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index e28da339..cb882ec5 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -55,14 +55,6 @@ workflow: start_in: 30 minutes - when: never -.merged_only: - rules: - - if: '$CI_MERGE_REQUEST_ID' - when: never - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: on_success - - when: never - .upload_only: rules: - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $GIRDER_UPLOAD_ENABLE == "true"' -- GitLab From 49d2d9736e02abe7e03d498acb89565697432bfc Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:10:21 +0200 Subject: [PATCH 08/16] gitlab-ci: port manual jobs to a single rules block --- .gitlab-ci.yml | 24 ++++++++++++------------ .gitlab/os-linux.yml | 2 ++ .gitlab/os-macos.yml | 2 ++ .gitlab/os-windows.yml | 2 ++ .gitlab/rules.yml | 25 +++++++++++++++---------- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83f02460..de6c7e94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ centos7-modelbuilder:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-modelbuilder:test: extends: @@ -77,7 +77,7 @@ centos7-vtk:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-vtk:test: extends: @@ -98,7 +98,7 @@ macos-arm64-modelbuilder:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .run_manually + - .rules macos-arm64-modelbuilder:test: extends: @@ -126,7 +126,7 @@ macos-x86_64-modelbuilder:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-modelbuilder:test: extends: @@ -156,7 +156,7 @@ macos-arm64-smtk:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .run_manually + - .rules macos-arm64-smtk:test: extends: @@ -173,7 +173,7 @@ macos-x86_64-smtk:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-smtk:test: extends: @@ -194,7 +194,7 @@ windows-vs2022-modelbuilder:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules # adding occt extends the build time timeout: 1h 30m @@ -226,7 +226,7 @@ windows-vs2022-smtk:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules timeout: 1h 30m windows-vs2022-smtk:test: @@ -249,7 +249,7 @@ centos7-docker-modelbuilder:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-docker-modelbuilder:test: extends: @@ -435,7 +435,7 @@ centos7-aeva:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .run_manually + - .rules centos7-aeva:test: extends: @@ -467,7 +467,7 @@ macos-x86_64-aeva:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .run_manually + - .rules macos-x86_64-aeva:test: extends: @@ -499,7 +499,7 @@ windows-vs2022-aeva:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .run_manually + - .rules # adding occt extends the build time timeout: 1h 45m diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index a92459c3..6515e47a 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -96,6 +96,8 @@ .cmake_build_linux: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_linux diff --git a/.gitlab/os-macos.yml b/.gitlab/os-macos.yml index bd797000..c27257c9 100644 --- a/.gitlab/os-macos.yml +++ b/.gitlab/os-macos.yml @@ -127,6 +127,8 @@ .cmake_build_macos: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_macos diff --git a/.gitlab/os-windows.yml b/.gitlab/os-windows.yml index 06cba193..07d6bb80 100644 --- a/.gitlab/os-windows.yml +++ b/.gitlab/os-windows.yml @@ -104,6 +104,8 @@ .cmake_build_windows: stage: build + variables: + CMB_CI_RUN_MANUALLY: "true" script: - *before_script_windows diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index cb882ec5..361baef3 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -1,6 +1,9 @@ # Rules for where jobs can run ### +# Job variables: +# CMB_CI_RUN_MANUALLY +# Set to "true" when the job should be triggered manually. # Pipeline variables: # ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) # Set to "true" when uploads should be performed. @@ -36,23 +39,25 @@ workflow: # Skip pipelines in all other cases. - when: never -.run_automatically: +.rules: rules: - - if: '$CI_MERGE_REQUEST_ID' - when: on_success - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' + # MRs use manual triggers. + - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_RUN_MANUALLY == "true"' + when: manual + # Jobs on a schedule run right away. + - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_success - - when: never + # Manual jobs run delayed (e.g., merges to branches). + - if: '$CMB_CI_RUN_MANUALLY == "true"' + when: delayed + start_in: 30 minutes -.run_manually: +.run_automatically: rules: - if: '$CI_MERGE_REQUEST_ID' - when: manual - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $CI_PIPELINE_SOURCE == "schedule"' when: on_success - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: delayed - start_in: 30 minutes + when: on_success - when: never .upload_only: -- GitLab From 1b4700636f61ace089ca2a26eeb39038e93142fc Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:11:16 +0200 Subject: [PATCH 09/16] gitlab-ci: port automatic jobs to a single rules block --- .gitlab-ci.yml | 24 ++++++++++++------------ .gitlab/rules.yml | 13 +++++-------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de6c7e94..c171c093 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ centos7-modelbuilder:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-modelbuilder:build @@ -84,7 +84,7 @@ centos7-vtk:test: - .centos7_vtk - .cmake_test_linux - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-vtk:build @@ -106,7 +106,7 @@ macos-arm64-modelbuilder:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .run_automatically + - .rules needs: - macos-arm64-modelbuilder:build @@ -134,7 +134,7 @@ macos-x86_64-modelbuilder:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-modelbuilder:build @@ -163,7 +163,7 @@ macos-arm64-smtk:test: - .macos_arm64_smtk - .cmake_test_macos - .macos_arm64_builder_tags - - .run_automatically + - .rules needs: - macos-arm64-smtk:build @@ -180,7 +180,7 @@ macos-x86_64-smtk:test: - .macos_x86_64_smtk - .cmake_test_macos - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-smtk:build @@ -204,7 +204,7 @@ windows-vs2022-modelbuilder:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-modelbuilder:build @@ -234,7 +234,7 @@ windows-vs2022-smtk:test: - .windows_vs2022_smtk - .cmake_test_windows - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-smtk:build @@ -257,7 +257,7 @@ centos7-docker-modelbuilder:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-docker-modelbuilder:build @@ -443,7 +443,7 @@ centos7-aeva:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .run_automatically + - .rules needs: - centos7-aeva:build @@ -475,7 +475,7 @@ macos-x86_64-aeva:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .run_automatically + - .rules needs: - macos-x86_64-aeva:build @@ -509,7 +509,7 @@ windows-vs2022-aeva:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .run_automatically + - .rules needs: - windows-vs2022-aeva:build diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index 361baef3..ec4da2ce 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -44,6 +44,9 @@ workflow: # MRs use manual triggers. - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_RUN_MANUALLY == "true"' when: manual + # Normal jobs run automatically. + - if: '$CI_MERGE_REQUEST_ID' + when: on_success # Jobs on a schedule run right away. - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_success @@ -51,14 +54,8 @@ workflow: - if: '$CMB_CI_RUN_MANUALLY == "true"' when: delayed start_in: 30 minutes - -.run_automatically: - rules: - - if: '$CI_MERGE_REQUEST_ID' - when: on_success - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild"' - when: on_success - - when: never + # Normal jobs on the project run automatically. + - when: on_success .upload_only: rules: -- GitLab From 20f8994b40eec595bcef62ed244e7f203971b166 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:14:51 +0200 Subject: [PATCH 10/16] gitlab-ci: port schedule-only jobs to a single rules block This also ports the "upload" flag but drops it from the `update-ci` jobs because their uploads are gated on the `ENABLE_CI_BUNDLE_UPLOAD` variable instead. --- .gitlab-ci.yml | 50 +++++++++++++++++++++---------------------- .gitlab/rules.yml | 21 ++++++++++++------ .gitlab/update-ci.yml | 2 ++ .gitlab/upload.yml | 3 +++ 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c171c093..7190452b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ centos7-modelbuilder:test: gdrive-linux-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -113,7 +113,7 @@ macos-arm64-modelbuilder:test: gdrive-macos-arm64-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -141,7 +141,7 @@ macos-x86_64-modelbuilder:test: gdrive-macos-x86_64-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -211,7 +211,7 @@ windows-vs2022-modelbuilder:test: gdrive-windows-vs2022-modelbuilder:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -450,7 +450,7 @@ centos7-aeva:test: gdrive-linux-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -482,7 +482,7 @@ macos-x86_64-aeva:test: gdrive-macos-x86_64-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -516,7 +516,7 @@ windows-vs2022-aeva:test: gdrive-windows-vs2022-aeva:upload: extends: - .gdrive_upload - - .upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -532,7 +532,7 @@ gdrive-windows-vs2022-aeva:upload: smtk-fedora:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-modelbuilder:test script: @@ -542,7 +542,7 @@ smtk-fedora:update-ci: # smtk-fedora-paraview59:update-ci: # extends: # - .image_builder -# - .upload_only +# - .rules # needs: # - centos7-modelbuilder:test # script: @@ -552,7 +552,7 @@ smtk-fedora:update-ci: smtk-fedora-paraview:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-modelbuilder:test script: @@ -562,7 +562,7 @@ smtk-fedora-paraview:update-ci: smtk-fedora-vtk:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-vtk:test script: @@ -574,7 +574,7 @@ smtk-fedora-vtk:update-ci: smtk-macos-arm64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-arm64-smtk:build # - job: macos-arm64-smtk:test @@ -589,7 +589,7 @@ smtk-macos-arm64:update-ci: smtk-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-x86_64-smtk:build # - job: macos-x86_64-smtk:test @@ -606,7 +606,7 @@ smtk-macos-x86_64:update-ci: smtk-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - windows-vs2022-smtk:build # - job: windows-vs2022-smtk:test @@ -625,7 +625,7 @@ smtk-windows-vs2022:update-ci: cmb-fedora:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-modelbuilder:test script: @@ -635,7 +635,7 @@ cmb-fedora:update-ci: # cmb-fedora-paraview59:update-ci: # extends: # - .image_builder -# - .upload_only +# - .rules # needs: # - centos7-modelbuilder:test # script: @@ -647,7 +647,7 @@ cmb-fedora:update-ci: cmb-macos-arm64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-arm64-modelbuilder:build - job: macos-arm64-modelbuilder:test @@ -662,7 +662,7 @@ cmb-macos-arm64:update-ci: cmb-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-x86_64-modelbuilder:build # - job: macos-x86_64-modelbuilder:test @@ -679,7 +679,7 @@ cmb-macos-x86_64:update-ci: cmb-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - windows-vs2022-modelbuilder:build # - job: windows-vs2022-modelbuilder:test @@ -698,7 +698,7 @@ cmb-windows-vs2022:update-ci: aevasession-fedora:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-aeva:test script: @@ -708,7 +708,7 @@ aevasession-fedora:update-ci: aeva-fedora:update-ci: extends: - .image_builder - - .upload_only + - .rules needs: - centos7-aeva:test script: @@ -720,7 +720,7 @@ aeva-fedora:update-ci: aevasession-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-x86_64-aeva:build - job: macos-x86_64-aeva:test @@ -735,7 +735,7 @@ aevasession-macos-x86_64:update-ci: aeva-macos-x86_64:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - macos-x86_64-aeva:build - job: macos-x86_64-aeva:test @@ -752,7 +752,7 @@ aeva-macos-x86_64:update-ci: aevasession-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - windows-vs2022-aeva:build - job: windows-vs2022-aeva:test @@ -767,7 +767,7 @@ aevasession-windows-vs2022:update-ci: aeva-windows-vs2022:update-ci: extends: - .upload_developer_install - - .upload_only + - .rules needs: - windows-vs2022-aeva:build - job: windows-vs2022-aeva:test diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index ec4da2ce..a6e0a70a 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -4,7 +4,13 @@ # Job variables: # CMB_CI_RUN_MANUALLY # Set to "true" when the job should be triggered manually. +# CMB_CI_IS_FOR_SCHEDULE +# Set to "true" if the job should only run for schedules. +# CMB_CI_UPLOAD_ONLY +# Set to "true" to only perform when an upload is requested. # Pipeline variables: +# ENABLE_PACKAGE_UPLOAD +# Set to "true" when package upload jobs should be performed. # ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) # Set to "true" when uploads should be performed. ### @@ -41,12 +47,21 @@ workflow: .rules: rules: + # Schedule-only jobs never run on MRs. + - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_IS_FOR_SCHEDULE' + when: never # MRs use manual triggers. - if: '$CI_MERGE_REQUEST_ID && $CMB_CI_RUN_MANUALLY == "true"' when: manual # Normal jobs run automatically. - if: '$CI_MERGE_REQUEST_ID' when: on_success + # Skip schedule jobs without a schedule. + - if: '$CI_PIPELINE_SOURCE != "schedule" && $CMB_CI_IS_FOR_SCHEDULE == "true"' + when: never + # Skip upload projects without an upload request. + - if: '$CMB_CI_UPLOAD_ONLY == "true" && $ENABLE_PACKAGE_UPLOAD != "true"' + when: never # Jobs on a schedule run right away. - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_success @@ -57,12 +72,6 @@ workflow: # Normal jobs on the project run automatically. - when: on_success -.upload_only: - rules: - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $GIRDER_UPLOAD_ENABLE == "true"' - when: on_success - - when: never - .weekly_only: rules: - if: '$CI_MERGE_REQUEST_ID' diff --git a/.gitlab/update-ci.yml b/.gitlab/update-ci.yml index 497f29bc..f93552da 100644 --- a/.gitlab/update-ci.yml +++ b/.gitlab/update-ci.yml @@ -22,6 +22,7 @@ - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" variables: GIT_SUBMODULE_STRATEGY: none + CMB_CI_IS_FOR_SCHEDULE: "true" .image_builder: image: "almalinux:8" @@ -34,6 +35,7 @@ variables: GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci + CMB_CI_IS_FOR_SCHEDULE: "true" dependencies: [] environment: diff --git a/.gitlab/upload.yml b/.gitlab/upload.yml index 76e3ae88..1b3f4456 100644 --- a/.gitlab/upload.yml +++ b/.gitlab/upload.yml @@ -9,6 +9,9 @@ - build environment: name: gdrive-upload + variables: + CMB_CI_IS_FOR_SCHEDULE: "true" + CMB_CI_UPLOAD_ONLY: "true" script: - ls build/ -- GitLab From 0b1dd4f32153465f9550dd741ece79b1466ec9e0 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:17:13 +0200 Subject: [PATCH 11/16] gitlab-ci: port weekly jobs to a single rules block Note that this actually removes these specific jobs from the weekly pool as the `CMB_CI_WEEKLY_ONLY` variable is not set anymore. That will be restored in a follow-up commit which moves it to a more logical location anyways. --- .gitlab-ci.yml | 54 +++++++++++++++++++++++------------------------ .gitlab/rules.yml | 21 ++++++------------ 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7190452b..5f2e33c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -273,7 +273,7 @@ centos7-cmb2d:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-cmb2d:test: extends: @@ -281,14 +281,14 @@ centos7-cmb2d:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-cmb2d:build gdrive-linux-cmb2d:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/cmb2d GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -307,7 +307,7 @@ centos7-truchas:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-truchas:test: extends: @@ -315,14 +315,14 @@ centos7-truchas:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-truchas:build gdrive-linux-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -339,7 +339,7 @@ macos-arm64-truchas:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules macos-arm64-truchas:test: extends: @@ -347,14 +347,14 @@ macos-arm64-truchas:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules needs: - macos-arm64-truchas:build gdrive-macos-arm64-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -367,7 +367,7 @@ macos-x86_64-truchas:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules macos-x86_64-truchas:test: extends: @@ -375,14 +375,14 @@ macos-x86_64-truchas:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules needs: - macos-x86_64-truchas:build gdrive-macos-x86_64-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -399,7 +399,7 @@ windows-vs2022-truchas:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .weekly_only + - .rules timeout: 1h 30m windows-vs2022-truchas:test: @@ -408,14 +408,14 @@ windows-vs2022-truchas:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .weekly_only + - .rules needs: - windows-vs2022-truchas:build gdrive-windows-vs2022-truchas:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -791,7 +791,7 @@ centos7-windtunnel:build: - .cmake_build_linux - .cmake_build_artifacts - .linux_builder_tags - - .weekly_only + - .rules centos7-windtunnel:test: extends: @@ -799,14 +799,14 @@ centos7-windtunnel:test: - .cmake_test_linux - .cmake_package_artifacts - .linux_test_tags - - .weekly_only + - .rules needs: - centos7-windtunnel:build gdrive-linux-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -823,7 +823,7 @@ macos-x86_64-windtunnel:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules macos-x86_64-windtunnel:test: extends: @@ -831,14 +831,14 @@ macos-x86_64-windtunnel:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_x86_64_builder_tags - - .weekly_only + - .rules needs: - macos-x86_64-windtunnel:build gdrive-macos-x86_64-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -851,7 +851,7 @@ macos-arm64-windtunnel:build: - .cmake_build_macos - .cmake_build_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules macos-arm64-windtunnel:test: extends: @@ -859,14 +859,14 @@ macos-arm64-windtunnel:test: - .cmake_test_macos - .cmake_package_artifacts - .macos_arm64_builder_tags - - .weekly_only + - .rules needs: - macos-arm64-windtunnel:build gdrive-macos-arm64-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci @@ -883,7 +883,7 @@ windows-vs2022-windtunnel:build: - .cmake_build_windows - .cmake_build_artifacts - .windows_builder_tags - - .weekly_only + - .rules # Machines are right on the edge 1 hour; avoid unnecessary rebuilds. timeout: 1h 30m @@ -893,14 +893,14 @@ windows-vs2022-windtunnel:test: - .cmake_test_windows - .cmake_package_artifacts - .windows_builder_tags - - .weekly_only + - .rules needs: - windows-vs2022-windtunnel:build gdrive-windows-vs2022-windtunnel:upload: extends: - .gdrive_upload - - .weekly_upload_only + - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index a6e0a70a..616ac903 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -8,9 +8,13 @@ # Set to "true" if the job should only run for schedules. # CMB_CI_UPLOAD_ONLY # Set to "true" to only perform when an upload is requested. +# CMB_CI_WEEKLY_ONLY +# Set to "true" to make the job only run for weekly pipelines. # Pipeline variables: # ENABLE_PACKAGE_UPLOAD # Set to "true" when package upload jobs should be performed. +# ENABLE_WEEKLY_JOBS +# Set to "true" when weekly-only jobs should be performed. # ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) # Set to "true" when uploads should be performed. ### @@ -62,6 +66,9 @@ workflow: # Skip upload projects without an upload request. - if: '$CMB_CI_UPLOAD_ONLY == "true" && $ENABLE_PACKAGE_UPLOAD != "true"' when: never + # Skip weekly projects without a weekly pipeline. + - if: '$CMB_CI_WEEKLY_ONLY == "true" && $ENABLE_WEEKLY_JOBS != "true"' + when: never # Jobs on a schedule run right away. - if: '$CI_PIPELINE_SOURCE == "schedule"' when: on_success @@ -71,17 +78,3 @@ workflow: start_in: 30 minutes # Normal jobs on the project run automatically. - when: on_success - -.weekly_only: - rules: - - if: '$CI_MERGE_REQUEST_ID' - when: never - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $WEEKLY_SCHEDULE == "true"' - when: on_success - - when: never - -.weekly_upload_only: - rules: - - if: '$CI_PROJECT_PATH == "cmb/cmb-superbuild" && $GIRDER_UPLOAD_ENABLE == "true" && $WEEKLY_SCHEDULE == "true"' - when: on_success - - when: never -- GitLab From 205870f77fb6b5c8726f7e4126ea211ca305e5d6 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:18:59 +0200 Subject: [PATCH 12/16] gitlab-ci: note the usage of the `.rules` block When https://gitlab.com/gitlab-org/gitlab/-/issues/420603 is fixed, `rules` can instead be provided by a `default` block. --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f2e33c2..5a331ab1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,9 @@ stages: # - a build script # - tags for the jobs # - already provided for upload and CI update jobs -# - rules for when to run the job +# - the `.rules` block (see https://gitlab.com/gitlab-org/gitlab/-/issues/420603) +# - see `.gitlab/rules.yml` for variables jobs may set to control when the +# job is executed # # Additionally, jobs may also contain: # -- GitLab From b5447bcbc6baaa9b267fb157f9313be9dd9ba580 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:20:53 +0200 Subject: [PATCH 13/16] gitlab-ci: classify jobs according to the "project" they belong to This will allow for running project-specific pipelines (e.g., to upload just SMTK's CI bundles). It also restores the "weekly" status to `truchas` and `windtunnel` jobs in a more logical place as we select weekly-ness based on the project. --- .gitlab-ci.yml | 35 +++++++++++++++++++++++++++++++++ .gitlab/os-linux.yml | 32 ++++++++++++++++++++++-------- .gitlab/os-macos.yml | 44 +++++++++++++++++++++++++++++++----------- .gitlab/os-windows.yml | 24 +++++++++++++++++------ .gitlab/projects.yml | 17 ++++++++++++++++ 5 files changed, 127 insertions(+), 25 deletions(-) create mode 100644 .gitlab/projects.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a331ab1..a1a82ddc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ include: # Metadata shared by many jobs + - local: .gitlab/projects.yml - local: .gitlab/rules.yml - local: .gitlab/artifacts.yml @@ -64,6 +65,7 @@ centos7-modelbuilder:test: gdrive-linux-modelbuilder:upload: extends: - .gdrive_upload + - .project_modelbuilder - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -115,6 +117,7 @@ macos-arm64-modelbuilder:test: gdrive-macos-arm64-modelbuilder:upload: extends: - .gdrive_upload + - .project_modelbuilder - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -143,6 +146,7 @@ macos-x86_64-modelbuilder:test: gdrive-macos-x86_64-modelbuilder:upload: extends: - .gdrive_upload + - .project_modelbuilder - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -213,6 +217,7 @@ windows-vs2022-modelbuilder:test: gdrive-windows-vs2022-modelbuilder:upload: extends: - .gdrive_upload + - .project_modelbuilder - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -290,6 +295,7 @@ centos7-cmb2d:test: gdrive-linux-cmb2d:upload: extends: - .gdrive_upload + - .project_cmb2d - .rules variables: GDRIVE_TARGET_PATH: packages/cmb2d @@ -324,6 +330,7 @@ centos7-truchas:test: gdrive-linux-truchas:upload: extends: - .gdrive_upload + - .project_truchas - .rules variables: GDRIVE_TARGET_PATH: packages/truchas @@ -356,6 +363,7 @@ macos-arm64-truchas:test: gdrive-macos-arm64-truchas:upload: extends: - .gdrive_upload + - .project_truchas - .rules variables: GDRIVE_TARGET_PATH: packages/truchas @@ -384,6 +392,7 @@ macos-x86_64-truchas:test: gdrive-macos-x86_64-truchas:upload: extends: - .gdrive_upload + - .project_truchas - .rules variables: GDRIVE_TARGET_PATH: packages/truchas @@ -417,6 +426,7 @@ windows-vs2022-truchas:test: gdrive-windows-vs2022-truchas:upload: extends: - .gdrive_upload + - .project_truchas - .rules variables: GDRIVE_TARGET_PATH: packages/truchas @@ -452,6 +462,7 @@ centos7-aeva:test: gdrive-linux-aeva:upload: extends: - .gdrive_upload + - .project_aeva - .rules variables: GDRIVE_TARGET_PATH: packages/aeva @@ -484,6 +495,7 @@ macos-x86_64-aeva:test: gdrive-macos-x86_64-aeva:upload: extends: - .gdrive_upload + - .project_aeva - .rules variables: GDRIVE_TARGET_PATH: packages/aeva @@ -518,6 +530,7 @@ windows-vs2022-aeva:test: gdrive-windows-vs2022-aeva:upload: extends: - .gdrive_upload + - .project_aeva - .rules variables: GDRIVE_TARGET_PATH: packages/aeva @@ -534,6 +547,7 @@ gdrive-windows-vs2022-aeva:upload: smtk-fedora:update-ci: extends: - .image_builder + - .project_smtk - .rules needs: - centos7-modelbuilder:test @@ -544,6 +558,7 @@ smtk-fedora:update-ci: # smtk-fedora-paraview59:update-ci: # extends: # - .image_builder +# - .project_smtk # - .rules # needs: # - centos7-modelbuilder:test @@ -554,6 +569,7 @@ smtk-fedora:update-ci: smtk-fedora-paraview:update-ci: extends: - .image_builder + - .project_smtk - .rules needs: - centos7-modelbuilder:test @@ -564,6 +580,7 @@ smtk-fedora-paraview:update-ci: smtk-fedora-vtk:update-ci: extends: - .image_builder + - .project_smtk - .rules needs: - centos7-vtk:test @@ -576,6 +593,7 @@ smtk-fedora-vtk:update-ci: smtk-macos-arm64:update-ci: extends: - .upload_developer_install + - .project_smtk - .rules needs: - macos-arm64-smtk:build @@ -591,6 +609,7 @@ smtk-macos-arm64:update-ci: smtk-macos-x86_64:update-ci: extends: - .upload_developer_install + - .project_smtk - .rules needs: - macos-x86_64-smtk:build @@ -608,6 +627,7 @@ smtk-macos-x86_64:update-ci: smtk-windows-vs2022:update-ci: extends: - .upload_developer_install + - .project_smtk - .rules needs: - windows-vs2022-smtk:build @@ -627,6 +647,7 @@ smtk-windows-vs2022:update-ci: cmb-fedora:update-ci: extends: - .image_builder + - .project_cmb - .rules needs: - centos7-modelbuilder:test @@ -637,6 +658,7 @@ cmb-fedora:update-ci: # cmb-fedora-paraview59:update-ci: # extends: # - .image_builder +# - .project_cmb # - .rules # needs: # - centos7-modelbuilder:test @@ -649,6 +671,7 @@ cmb-fedora:update-ci: cmb-macos-arm64:update-ci: extends: - .upload_developer_install + - .project_cmb - .rules needs: - macos-arm64-modelbuilder:build @@ -664,6 +687,7 @@ cmb-macos-arm64:update-ci: cmb-macos-x86_64:update-ci: extends: - .upload_developer_install + - .project_cmb - .rules needs: - macos-x86_64-modelbuilder:build @@ -681,6 +705,7 @@ cmb-macos-x86_64:update-ci: cmb-windows-vs2022:update-ci: extends: - .upload_developer_install + - .project_cmb - .rules needs: - windows-vs2022-modelbuilder:build @@ -700,6 +725,7 @@ cmb-windows-vs2022:update-ci: aevasession-fedora:update-ci: extends: - .image_builder + - .project_aeva - .rules needs: - centos7-aeva:test @@ -710,6 +736,7 @@ aevasession-fedora:update-ci: aeva-fedora:update-ci: extends: - .image_builder + - .project_aeva - .rules needs: - centos7-aeva:test @@ -722,6 +749,7 @@ aeva-fedora:update-ci: aevasession-macos-x86_64:update-ci: extends: - .upload_developer_install + - .project_aeva - .rules needs: - macos-x86_64-aeva:build @@ -737,6 +765,7 @@ aevasession-macos-x86_64:update-ci: aeva-macos-x86_64:update-ci: extends: - .upload_developer_install + - .project_aeva - .rules needs: - macos-x86_64-aeva:build @@ -754,6 +783,7 @@ aeva-macos-x86_64:update-ci: aevasession-windows-vs2022:update-ci: extends: - .upload_developer_install + - .project_aeva - .rules needs: - windows-vs2022-aeva:build @@ -769,6 +799,7 @@ aevasession-windows-vs2022:update-ci: aeva-windows-vs2022:update-ci: extends: - .upload_developer_install + - .project_aeva - .rules needs: - windows-vs2022-aeva:build @@ -808,6 +839,7 @@ centos7-windtunnel:test: gdrive-linux-windtunnel:upload: extends: - .gdrive_upload + - .project_windtunnel - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel @@ -840,6 +872,7 @@ macos-x86_64-windtunnel:test: gdrive-macos-x86_64-windtunnel:upload: extends: - .gdrive_upload + - .project_windtunnel - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel @@ -868,6 +901,7 @@ macos-arm64-windtunnel:test: gdrive-macos-arm64-windtunnel:upload: extends: - .gdrive_upload + - .project_windtunnel - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel @@ -902,6 +936,7 @@ windows-vs2022-windtunnel:test: gdrive-windows-vs2022-windtunnel:upload: extends: - .gdrive_upload + - .project_windtunnel - .rules variables: GDRIVE_TARGET_PATH: packages/windtunnel diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index 6515e47a..b0157fa6 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -18,45 +18,61 @@ GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci .centos7_modelbuilder: - extends: .centos7 + extends: + - .centos7 + - .project_modelbuilder variables: CMAKE_CONFIGURATION: centos7_modelbuilder .centos7_docker_modelbuilder: - extends: .centos7 + extends: + - .centos7 + - .project_modelbuilder variables: CMAKE_CONFIGURATION: centos7_docker_modelbuilder .centos7_vtk: - extends: .centos7 + extends: + - .centos7 + - .project_smtk variables: CMAKE_CONFIGURATION: centos7_vtk .centos7_aeva: - extends: .centos7 + extends: + - .centos7 + - .project_aeva variables: CMAKE_CONFIGURATION: centos7_aeva .centos7_ace3p: - extends: .centos7 + extends: + - .centos7 + - .project_ace3p variables: CMAKE_CONFIGURATION: centos7_ace3p .centos7_truchas: - extends: .centos7 + extends: + - .centos7 + - .project_truchas variables: CMAKE_CONFIGURATION: centos7_truchas .centos7_cmb2d: - extends: .centos7 + extends: + - .centos7 + - .project_cmb2d variables: CMAKE_CONFIGURATION: centos7_cmb2d .centos7_windtunnel: - extends: .centos7 + extends: + - .centos7 + - .project_windtunnel variables: CMAKE_CONFIGURATION: centos7_windtunnel diff --git a/.gitlab/os-macos.yml b/.gitlab/os-macos.yml index c27257c9..ca98146b 100644 --- a/.gitlab/os-macos.yml +++ b/.gitlab/os-macos.yml @@ -25,68 +25,90 @@ GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmb-ci" .macos_x86_64_windtunnel: - extends: .macos_build + extends: + - .macos_build + - .project_windtunnel variables: CMAKE_CONFIGURATION: macos_x86_64_windtunnel .macos_arm64_windtunnel: - extends: .macos_build + extends: + - .macos_build + - .project_windtunnel variables: CMAKE_CONFIGURATION: macos_arm64_windtunnel .macos_arm64_truchas: - extends: .macos_build + extends: + - .macos_build + - .project_truchas variables: CMAKE_CONFIGURATION: macos_arm64_truchas .macos_x86_64_truchas: - extends: .macos_build + extends: + - .macos_build + - .project_truchas variables: CMAKE_CONFIGURATION: macos_x86_64_truchas .macos_x86_64_ace3p: - extends: .macos_build + extends: + - .macos_build + - .project_ace3p variables: CMAKE_CONFIGURATION: macos_x86_64_ace3p .macos_arm64_ace3p: - extends: .macos_build + extends: + - .macos_build + - .project_ace3p variables: CMAKE_CONFIGURATION: macos_arm64_ace3p .macos_x86_64_aeva: - extends: .macos_build + extends: + - .macos_build + - .project_aeva variables: CMAKE_CONFIGURATION: macos_x86_64_aeva DOWNLOAD_FORTRAN: "1" .macos_arm64_modelbuilder: - extends: .macos_build + extends: + - .macos_build + - .project_modelbuilder variables: CMAKE_CONFIGURATION: macos_arm64_modelbuilder .macos_x86_64_modelbuilder: - extends: .macos_build + extends: + - .macos_build + - .project_modelbuilder variables: CMAKE_CONFIGURATION: macos_x86_64_modelbuilder .macos_arm64_smtk: - extends: .macos_build + extends: + - .macos_build + - .project_smtk variables: CMAKE_CONFIGURATION: macos_arm64_smtk .macos_x86_64_smtk: - extends: .macos_build + extends: + - .macos_build + - .project_smtk variables: CMAKE_CONFIGURATION: macos_x86_64_smtk diff --git a/.gitlab/os-windows.yml b/.gitlab/os-windows.yml index 07d6bb80..b7f7560f 100644 --- a/.gitlab/os-windows.yml +++ b/.gitlab/os-windows.yml @@ -36,37 +36,49 @@ VCVARSVERSION: "14.35.32215" .windows_vs2022_aeva: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_aeva variables: CMAKE_CONFIGURATION: windows_vs2022_aeva .windows_vs2022_windtunnel: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_windtunnel variables: CMAKE_CONFIGURATION: windows_vs2022_windtunnel .windows_vs2022_truchas: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_truchas variables: CMAKE_CONFIGURATION: windows_vs2022_truchas .windows_vs2022_ace3p: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_ace3p variables: CMAKE_CONFIGURATION: windows_vs2022_ace3p .windows_vs2022_modelbuilder: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_modelbuilder variables: CMAKE_CONFIGURATION: windows_vs2022_modelbuilder .windows_vs2022_smtk: - extends: .windows_vs2022 + extends: + - .windows_vs2022 + - .project_smtk variables: CMAKE_CONFIGURATION: windows_vs2022_smtk diff --git a/.gitlab/projects.yml b/.gitlab/projects.yml new file mode 100644 index 00000000..2384cebc --- /dev/null +++ b/.gitlab/projects.yml @@ -0,0 +1,17 @@ +.project_modelbuilder: {} + +.project_smtk: {} + +.project_aeva: {} + +.project_ace3p: {} + +.project_truchas: + variables: + CMB_CI_WEEKLY_ONLY: "true" + +.project_cmb2d: {} + +.project_windtunnel: + variables: + CMB_CI_WEEKLY_ONLY: "true" -- GitLab From e5c20ca7c996091e66bd94901db8e8fd6d6c5ec1 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:22:05 +0200 Subject: [PATCH 14/16] gitlab-ci: support running project-specific pipelines A schedule pipeline may set the `ENABLE_PROJECTS` variable to a comma-separate list of project names to run jobs for just those projects. --- .gitlab-ci.yml | 8 ++++---- .gitlab/os-linux.yml | 4 ++-- .gitlab/os-macos.yml | 4 ++-- .gitlab/os-windows.yml | 2 +- .gitlab/projects.yml | 22 +++++++++++++++++----- .gitlab/rules.yml | 7 +++++++ 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1a82ddc..92da504c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,7 +65,7 @@ centos7-modelbuilder:test: gdrive-linux-modelbuilder:upload: extends: - .gdrive_upload - - .project_modelbuilder + - .project_cmb - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -117,7 +117,7 @@ macos-arm64-modelbuilder:test: gdrive-macos-arm64-modelbuilder:upload: extends: - .gdrive_upload - - .project_modelbuilder + - .project_cmb - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -146,7 +146,7 @@ macos-x86_64-modelbuilder:test: gdrive-macos-x86_64-modelbuilder:upload: extends: - .gdrive_upload - - .project_modelbuilder + - .project_cmb - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder @@ -217,7 +217,7 @@ windows-vs2022-modelbuilder:test: gdrive-windows-vs2022-modelbuilder:upload: extends: - .gdrive_upload - - .project_modelbuilder + - .project_cmb - .rules variables: GDRIVE_TARGET_PATH: packages/modelbuilder diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index b0157fa6..774fcb06 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -20,7 +20,7 @@ .centos7_modelbuilder: extends: - .centos7 - - .project_modelbuilder + - .project_cmb variables: CMAKE_CONFIGURATION: centos7_modelbuilder @@ -28,7 +28,7 @@ .centos7_docker_modelbuilder: extends: - .centos7 - - .project_modelbuilder + - .project_cmb variables: CMAKE_CONFIGURATION: centos7_docker_modelbuilder diff --git a/.gitlab/os-macos.yml b/.gitlab/os-macos.yml index ca98146b..0253c161 100644 --- a/.gitlab/os-macos.yml +++ b/.gitlab/os-macos.yml @@ -84,7 +84,7 @@ .macos_arm64_modelbuilder: extends: - .macos_build - - .project_modelbuilder + - .project_cmb variables: CMAKE_CONFIGURATION: macos_arm64_modelbuilder @@ -92,7 +92,7 @@ .macos_x86_64_modelbuilder: extends: - .macos_build - - .project_modelbuilder + - .project_cmb variables: CMAKE_CONFIGURATION: macos_x86_64_modelbuilder diff --git a/.gitlab/os-windows.yml b/.gitlab/os-windows.yml index b7f7560f..9f49e5be 100644 --- a/.gitlab/os-windows.yml +++ b/.gitlab/os-windows.yml @@ -70,7 +70,7 @@ .windows_vs2022_modelbuilder: extends: - .windows_vs2022 - - .project_modelbuilder + - .project_cmb variables: CMAKE_CONFIGURATION: windows_vs2022_modelbuilder diff --git a/.gitlab/projects.yml b/.gitlab/projects.yml index 2384cebc..1160141a 100644 --- a/.gitlab/projects.yml +++ b/.gitlab/projects.yml @@ -1,17 +1,29 @@ -.project_modelbuilder: {} +.project_cmb: + variables: + CMB_CI_FOR_PROJECT: /\bcmb\b/ -.project_smtk: {} +.project_smtk: + variables: + CMB_CI_FOR_PROJECT: /\bsmtk\b/ -.project_aeva: {} +.project_aeva: + variables: + CMB_CI_FOR_PROJECT: /\baeva\b/ -.project_ace3p: {} +.project_ace3p: + variables: + CMB_CI_FOR_PROJECT: /\bace3p\b/ .project_truchas: variables: + CMB_CI_FOR_PROJECT: /\btruchas\b/ CMB_CI_WEEKLY_ONLY: "true" -.project_cmb2d: {} +.project_cmb2d: + variables: + CMB_CI_FOR_PROJECT: /\bcmb2d\b/ .project_windtunnel: variables: + CMB_CI_FOR_PROJECT: /\bwindtunnel\b/ CMB_CI_WEEKLY_ONLY: "true" diff --git a/.gitlab/rules.yml b/.gitlab/rules.yml index 616ac903..122ca2a8 100644 --- a/.gitlab/rules.yml +++ b/.gitlab/rules.yml @@ -10,11 +10,15 @@ # Set to "true" to only perform when an upload is requested. # CMB_CI_WEEKLY_ONLY # Set to "true" to make the job only run for weekly pipelines. +# CMB_CI_FOR_PROJECT +# The name of the project the job is "for". # Pipeline variables: # ENABLE_PACKAGE_UPLOAD # Set to "true" when package upload jobs should be performed. # ENABLE_WEEKLY_JOBS # Set to "true" when weekly-only jobs should be performed. +# ENABLE_PROJECTS +# Set to a comma-separated list of projects that should be focused on. # ENABLE_CI_BUNDLE_UPLOAD (used in job scripts) # Set to "true" when uploads should be performed. ### @@ -60,6 +64,9 @@ workflow: # Normal jobs run automatically. - if: '$CI_MERGE_REQUEST_ID' when: on_success + # If we have a project-specific pipeline, ignore jobs not matching that project. + - if: '$ENABLE_PROJECTS && $ENABLE_PROJECTS =~ $CMB_CI_FOR_PROJECT' + when: never # Skip schedule jobs without a schedule. - if: '$CI_PIPELINE_SOURCE != "schedule" && $CMB_CI_IS_FOR_SCHEDULE == "true"' when: never -- GitLab From 5c1bc4be34e5b481c3d27289aa0ffada299c3783 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 14:51:44 +0200 Subject: [PATCH 15/16] gitlab-ci: factor out upload destinations to the projects --- .gitlab-ci.yml | 26 -------------------------- .gitlab/projects.yml | 8 ++++++++ .gitlab/update-ci.yml | 2 +- .gitlab/upload.yml | 2 +- 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92da504c..f321d2a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,6 @@ gdrive-linux-modelbuilder:upload: - .project_cmb - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-modelbuilder:test @@ -120,7 +119,6 @@ gdrive-macos-arm64-modelbuilder:upload: - .project_cmb - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-modelbuilder:test @@ -149,7 +147,6 @@ gdrive-macos-x86_64-modelbuilder:upload: - .project_cmb - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-modelbuilder:test @@ -220,7 +217,6 @@ gdrive-windows-vs2022-modelbuilder:upload: - .project_cmb - .rules variables: - GDRIVE_TARGET_PATH: packages/modelbuilder GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-modelbuilder:test @@ -298,7 +294,6 @@ gdrive-linux-cmb2d:upload: - .project_cmb2d - .rules variables: - GDRIVE_TARGET_PATH: packages/cmb2d GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-cmb2d:test @@ -333,7 +328,6 @@ gdrive-linux-truchas:upload: - .project_truchas - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-truchas:test @@ -366,7 +360,6 @@ gdrive-macos-arm64-truchas:upload: - .project_truchas - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-truchas:test @@ -395,7 +388,6 @@ gdrive-macos-x86_64-truchas:upload: - .project_truchas - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-truchas:test @@ -429,7 +421,6 @@ gdrive-windows-vs2022-truchas:upload: - .project_truchas - .rules variables: - GDRIVE_TARGET_PATH: packages/truchas GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-truchas:test @@ -465,7 +456,6 @@ gdrive-linux-aeva:upload: - .project_aeva - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-aeva:test @@ -498,7 +488,6 @@ gdrive-macos-x86_64-aeva:upload: - .project_aeva - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-aeva:test @@ -533,7 +522,6 @@ gdrive-windows-vs2022-aeva:upload: - .project_aeva - .rules variables: - GDRIVE_TARGET_PATH: packages/aeva GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-aeva:test @@ -603,7 +591,6 @@ smtk-macos-arm64:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: macos-arm64 smtk-macos-x86_64:update-ci: @@ -619,7 +606,6 @@ smtk-macos-x86_64:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: macos-x86_64 ### Windows @@ -637,7 +623,6 @@ smtk-windows-vs2022:update-ci: TARGET_PROJECT: smtk PROJECTS_TO_CLEAN: "smtk" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/smtk PLATFORM: windows-x86_64-vs2022 ## CMB @@ -681,7 +666,6 @@ cmb-macos-arm64:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: macos-arm64 cmb-macos-x86_64:update-ci: @@ -697,7 +681,6 @@ cmb-macos-x86_64:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: macos-x86_64 ### Windows @@ -715,7 +698,6 @@ cmb-windows-vs2022:update-ci: TARGET_PROJECT: cmb PROJECTS_TO_CLEAN: "cmb" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/cmb PLATFORM: windows-x86_64-vs2022 ## AEVA @@ -759,7 +741,6 @@ aevasession-macos-x86_64:update-ci: TARGET_PROJECT: aevasession PROJECTS_TO_CLEAN: "aeva aevasession" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: macos-x86_64 aeva-macos-x86_64:update-ci: @@ -775,7 +756,6 @@ aeva-macos-x86_64:update-ci: TARGET_PROJECT: aeva PROJECTS_TO_CLEAN: "aeva" PREFIX: "/opt/glr/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: macos-x86_64 ### Windows @@ -793,7 +773,6 @@ aevasession-windows-vs2022:update-ci: TARGET_PROJECT: aevasession PROJECTS_TO_CLEAN: "aeva aevasession" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: windows-x86_64-vs2022 aeva-windows-vs2022:update-ci: @@ -809,7 +788,6 @@ aeva-windows-vs2022:update-ci: TARGET_PROJECT: aeva PROJECTS_TO_CLEAN: "aeva" PREFIX: "C:/glr/builds/cmb/cmb-ci" - GDRIVE_TARGET_PATH: ci/aeva PLATFORM: windows-x86_64-vs2022 # WindTunnel builds @@ -842,7 +820,6 @@ gdrive-linux-windtunnel:upload: - .project_windtunnel - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - centos7-windtunnel:test @@ -875,7 +852,6 @@ gdrive-macos-x86_64-windtunnel:upload: - .project_windtunnel - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-x86_64-windtunnel:test @@ -904,7 +880,6 @@ gdrive-macos-arm64-windtunnel:upload: - .project_windtunnel - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - macos-arm64-windtunnel:test @@ -939,7 +914,6 @@ gdrive-windows-vs2022-windtunnel:upload: - .project_windtunnel - .rules variables: - GDRIVE_TARGET_PATH: packages/windtunnel GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci needs: - windows-vs2022-windtunnel:test diff --git a/.gitlab/projects.yml b/.gitlab/projects.yml index 1160141a..0c797083 100644 --- a/.gitlab/projects.yml +++ b/.gitlab/projects.yml @@ -1,14 +1,19 @@ .project_cmb: variables: CMB_CI_FOR_PROJECT: /\bcmb\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/modelbuilder + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/cmb .project_smtk: variables: CMB_CI_FOR_PROJECT: /\bsmtk\b/ + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/smtk .project_aeva: variables: CMB_CI_FOR_PROJECT: /\baeva\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/aeva + GDRIVE_TARGET_CI_BUNDLE_PATH: ci/aeva .project_ace3p: variables: @@ -18,12 +23,15 @@ variables: CMB_CI_FOR_PROJECT: /\btruchas\b/ CMB_CI_WEEKLY_ONLY: "true" + GDRIVE_TARGET_PACKAGES_PATH: packages/truchas .project_cmb2d: variables: CMB_CI_FOR_PROJECT: /\bcmb2d\b/ + GDRIVE_TARGET_PACKAGES_PATH: packages/cmb2d .project_windtunnel: variables: CMB_CI_FOR_PROJECT: /\bwindtunnel\b/ CMB_CI_WEEKLY_ONLY: "true" + GDRIVE_TARGET_PACKAGES_PATH: packages/windtunnel diff --git a/.gitlab/update-ci.yml b/.gitlab/update-ci.yml index f93552da..0d01066b 100644 --- a/.gitlab/update-ci.yml +++ b/.gitlab/update-ci.yml @@ -19,7 +19,7 @@ # Bail before upload if we're not actually uploading bundles. - '[ "$ENABLE_CI_BUNDLE_UPLOAD" = "true" ] || exit 0' - dnf install -y --setopt=install_weak_deps=False rclone - - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" + - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PACKAGE_PATH/" variables: GIT_SUBMODULE_STRATEGY: none CMB_CI_IS_FOR_SCHEDULE: "true" diff --git a/.gitlab/upload.yml b/.gitlab/upload.yml index 1b3f4456..902bd089 100644 --- a/.gitlab/upload.yml +++ b/.gitlab/upload.yml @@ -16,4 +16,4 @@ script: - ls build/ - dnf install -y --setopt=install_weak_deps=False rclone - - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_PATH/" + - rclone --config $GDRIVE_RCLONE_CONFIG copy build/ "$GDRIVE_TARGET_HOST:$GDRIVE_TARGET_CI_BUNDLE_PATH/" -- GitLab From 9a1532e98354f1bc9867e01d7ba0bbf2f8a19b6a Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 3 Apr 2025 15:28:41 +0200 Subject: [PATCH 16/16] Scripts/ci-bundles: add docs for building new bundles --- Scripts/ci-bundles/README.md | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 Scripts/ci-bundles/README.md diff --git a/Scripts/ci-bundles/README.md b/Scripts/ci-bundles/README.md new file mode 100644 index 00000000..d56ea6bc --- /dev/null +++ b/Scripts/ci-bundles/README.md @@ -0,0 +1,57 @@ +# CI bundles from the superbuild + +CI bundles are created by the CMB superbuild's pipeline because it allows the +usage of an existing build to generate a tarball for usage in macOS and Windows +CI. Linux builds Docker images using recipes in each project but based on the +current superbuild's commit. This ensures that CI bundles are using the same +superbuild commit as it tends to move far faster than the CI Dockerfile +recipes. + +## Scripts + +The superbuild is home to a number of scripts to help build CI bundles. + +### Linux + +Docker images are built using scripts living in `.gitlab/ci/build-*-image.sh`. +These scripts clone the target project in question and build the pipeline's +superbuild commit inside of the container as specified in the target project's +relevant `.gitlab/ci/docker/*` directory. Each + +### macOS and Windows + +For macOS and Windows, a Linux container is handed the `install/` directory of +a superbuild build as well as some details from relevant build trees. The +`.gitlab/ci/prune-for-developer.sh` script is then run to strip out the target +project(s) from the `install/` tree so that it may be used as an SDK bundle for +the project in question. The tarball is then uploaded (if requested). + +## Uploading new bundles + +The bundles are always *created* so that when they break, there's at least a +signal that things are broken. Uploads only happen on-demand because the +bundles end up taking up a lot of cloud storage and are not actually used all +that often. + +To trigger the *upload* of new bundles to their respective cloud locations, +create [a new pipeline on the CMB superbuild][cmb-sb-new-pipeline] and set the +following variables: + + - `ENABLE_CI_BUNDLE_UPLOAD` to `true` + - `ENABLE_PROJECTS` to a comma-separated list of projects to create CI + bundles. If unset, all projects will be built. Supported projects are: + * `cmb` + * `smtk` + * `aeva` (uploads both `aeva/aeva` and `aeva/session` CI bundles) + +[cmb-sb-new-pipeline]: https://gitlab.kitware.com/cmb/cmb-superbuild/-/pipelines/new + +When usage of new bundles are *merged*, any containers must be marked as "safe" +in this repository's `Scripts/ci-bundles/dockerhub-keep-tags` file to avoid +deleting them. + +## Cleaning up old bundles + +The `Scripts/ci-bundles/rm-stale-*.bash` scripts may be used to prune older +bundles from the relevant cloud storage locations. The scripts keep any bundle +created within the last 2 weeks. -- GitLab