diff --git a/Scripts/ci-bundles/.gitignore b/Scripts/ci-bundles/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0f2e1a8de7d6869aa9119e56202bdfe97fac261e --- /dev/null +++ b/Scripts/ci-bundles/.gitignore @@ -0,0 +1,7 @@ +# Secrets for Google Drive pruning. +rclone-cmb-ci +# Mount point for `rclone`. +rclone-mount/ + +# Intermediate file for DockerHub pruning. +all-tags diff --git a/Scripts/ci-bundles/dockerhub-keep-tags b/Scripts/ci-bundles/dockerhub-keep-tags new file mode 100644 index 0000000000000000000000000000000000000000..f300f5a6d876e95b768430299c124a435c5595b0 --- /dev/null +++ b/Scripts/ci-bundles/dockerhub-keep-tags @@ -0,0 +1,243 @@ +# aeva +ci-aeva-fedora32-20200702 +ci-aeva-fedora32-20200820 +ci-aeva-fedora32-20201124 +ci-aeva-fedora32-20201128 +ci-aeva-fedora32-20201209 +ci-aeva-fedora32-20201223 +ci-aeva-fedora32-20210304 +ci-aeva-fedora33-20210322 +ci-aeva-fedora33-20210902 +ci-aeva-fedora33-20211110 +ci-aeva-fedora33-20211117 +ci-aeva-fedora33-20220303 +ci-aeva-fedora33-20220311 +ci-aeva-fedora33-20220509 +ci-aeva-fedora33-20220621 +ci-aeva-fedora33-20220818 +ci-aeva-fedora33-20220907 +ci-aeva-fedora33-20240127 +ci-aeva-fedora33-20250105 + +# aeva-session +ci-aevasession-fedora32-20200710 +ci-aevasession-fedora32-20200719 +ci-aevasession-fedora32-20200825 +ci-aevasession-fedora32-20200913 +ci-aevasession-fedora32-20201009 +ci-aevasession-fedora32-20201128 +ci-aevasession-fedora32-20201209 +ci-aevasession-fedora32-20201223 +ci-aevasession-fedora32-20210304 +ci-aevasession-fedora33-20210322 +ci-aevasession-fedora33-20210902 +ci-aevasession-fedora33-20211028 +ci-aevasession-fedora33-20211110 +ci-aevasession-fedora33-20211117 +ci-aevasession-fedora33-20220107 +ci-aevasession-fedora33-20220226 +ci-aevasession-fedora33-20220301 +ci-aevasession-fedora33-20220311 +ci-aevasession-fedora33-20220324 +ci-aevasession-fedora33-20220509 +ci-aevasession-fedora33-20220621 +ci-aevasession-fedora33-20220818 +ci-aevasession-fedora33-20220905 +ci-aevasession-fedora33-20220907 +ci-aevasession-fedora33-20221001 +ci-aevasession-fedora33-20221010 +ci-aevasession-fedora33-20221205 +ci-aevasession-fedora33-20221221 +ci-aevasession-fedora33-20240127 +ci-aevasession-fedora33-20250105 + +# cmb +ci-cmb-fedora31-20191120 +ci-cmb-fedora31-20191209 +ci-cmb-fedora31-20200117 +ci-cmb-fedora31-20200228 +ci-cmb-fedora31-20200601 +ci-cmb-fedora31-20200706 +ci-cmb-fedora31-20200910 +ci-cmb-fedora31-20201201 +ci-cmb-fedora31-20201209 +ci-cmb-fedora31-20210304 +ci-cmb-fedora33-20210420 +ci-cmb-fedora33-20210714 +ci-cmb-fedora33-20210829 +ci-cmb-fedora33-20211102 +ci-cmb-fedora33-20211111 +ci-cmb-fedora33-20220107 +ci-cmb-fedora33-20220110 +ci-cmb-fedora33-20220409 +ci-cmb-fedora33-20220427 +ci-cmb-fedora33-20220522 +ci-cmb-fedora33-20220527 +ci-cmb-fedora33-20220603 +ci-cmb-fedora33-20220622 +ci-cmb-fedora33-20220818 +ci-cmb-fedora33-20220905 +ci-cmb-fedora33-20230707 +ci-cmb-fedora33-20231026 +ci-cmb-fedora33-20240129 +ci-cmb-fedora33-20240724 +ci-cmb-fedora33-20240905 +ci-cmb-fedora33-20241101 +ci-cmb-fedora33-20250105 +ci-cmb-fedora33-paraview59-20220204 +ci-cmb-fedora33-paraview59-20220427 +ci-cmb-fedora33-paraview59-20220522 +ci-cmb-fedora33-paraview59-20220527 +ci-cmb-fedora33-paraview59-20220603 +ci-cmb-fedora33-paraview59-20220622 +ci-cmb-fedora33-paraview59-20220818 +ci-cmb-fedora33-paraview59-20220905 +ci-cmb-fedora33-paraview59-20230707 +ci-cmb-fedora33-paraview59-20231026 +ci-cmb-fedora33-paraview59-20240129 +ci-cmb-fedora33-paraview59-20240724 +ci-cmb-fedora35-paraview59-20220126 +ci-cmb-fedora41-20250226 + +# smtk +ci-smtk-fedora31-20191112 +ci-smtk-fedora31-20191119 +ci-smtk-fedora31-20191125 +ci-smtk-fedora31-20191207 +ci-smtk-fedora31-20200102 +ci-smtk-fedora31-20200117 +ci-smtk-fedora31-20200130 +ci-smtk-fedora31-20200204 +ci-smtk-fedora31-20200219 +ci-smtk-fedora31-20200810 +ci-smtk-fedora31-20201125 +ci-smtk-fedora31-20210219 +ci-smtk-fedora31-latest +ci-smtk-fedora31-paraview-20191119 +ci-smtk-fedora31-paraview-20191125 +ci-smtk-fedora31-paraview-20191204 +ci-smtk-fedora31-paraview-20191207 +ci-smtk-fedora31-paraview-20200102 +ci-smtk-fedora31-paraview-20200117 +ci-smtk-fedora31-paraview-20200130 +ci-smtk-fedora31-paraview-20200204 +ci-smtk-fedora31-paraview-20200219 +ci-smtk-fedora31-paraview-20200810 +ci-smtk-fedora31-paraview-20201125 +ci-smtk-fedora31-vtk-20191119 +ci-smtk-fedora31-vtk-20191125 +ci-smtk-fedora31-vtk-20191207 +ci-smtk-fedora31-vtk-20200102 +ci-smtk-fedora31-vtk-20200117 +ci-smtk-fedora31-vtk-20200130 +ci-smtk-fedora31-vtk-20200204 +ci-smtk-fedora31-vtk-20200219 +ci-smtk-fedora31-vtk-20200602 +ci-smtk-fedora31-vtk-20200810 +ci-smtk-fedora31-vtk-20201125 +ci-smtk-fedora31-vtk-python2-20191125 +ci-smtk-fedora31-vtk-python2-20191207 +ci-smtk-fedora31-vtk-python2-20200102 +ci-smtk-fedora31-vtk-python2-20200117 +ci-smtk-fedora31-vtk-python2-20200130 +ci-smtk-fedora31-vtk-python2-20200204 +ci-smtk-fedora31-vtk-python2-20200219 +ci-smtk-fedora31-vtk-python2-20200602 +ci-smtk-fedora31-vtk-python2-20200810 +ci-smtk-fedora31-vtk-python2-20201130 +ci-smtk-fedora33-20210420 +ci-smtk-fedora33-20210714 +ci-smtk-fedora33-20211024 +ci-smtk-fedora33-20211111 +ci-smtk-fedora33-20220103 +ci-smtk-fedora33-20220307 +ci-smtk-fedora33-20220321 +ci-smtk-fedora33-20220409 +ci-smtk-fedora33-20220425 +ci-smtk-fedora33-20220818 +ci-smtk-fedora33-20220902 +ci-smtk-fedora33-20230622 +ci-smtk-fedora33-20230628 +ci-smtk-fedora33-20230707 +ci-smtk-fedora33-20230717 +ci-smtk-fedora33-20231026 +ci-smtk-fedora33-20231127 +ci-smtk-fedora33-20240724 +ci-smtk-fedora33-20240905 +ci-smtk-fedora33-20241101 +ci-smtk-fedora33-20250105 +ci-smtk-fedora33-paraview-20210420 +ci-smtk-fedora33-paraview-20210714 +ci-smtk-fedora33-paraview-20210829 +ci-smtk-fedora33-paraview-20211024 +ci-smtk-fedora33-paraview-20211111 +ci-smtk-fedora33-paraview-20220103 +ci-smtk-fedora33-paraview-20220307 +ci-smtk-fedora33-paraview-20220409 +ci-smtk-fedora33-paraview-20220425 +ci-smtk-fedora33-paraview-20220818 +ci-smtk-fedora33-paraview-20220902 +ci-smtk-fedora33-paraview-20230622 +ci-smtk-fedora33-paraview-20230628 +ci-smtk-fedora33-paraview-20230707 +ci-smtk-fedora33-paraview-20230717 +ci-smtk-fedora33-paraview-20231026 +ci-smtk-fedora33-paraview-20231127 +ci-smtk-fedora33-paraview-20240724 +ci-smtk-fedora33-paraview-20240905 +ci-smtk-fedora33-paraview-20241101 +ci-smtk-fedora33-paraview-20250105 +ci-smtk-fedora33-paraview59-20220204 +ci-smtk-fedora33-paraview59-20220307 +ci-smtk-fedora33-paraview59-20220409 +ci-smtk-fedora33-paraview59-20220425 +ci-smtk-fedora33-paraview59-20220427 +ci-smtk-fedora33-paraview59-20220818 +ci-smtk-fedora33-paraview59-20220902 +ci-smtk-fedora33-paraview59-20230622 +ci-smtk-fedora33-paraview59-20230628 +ci-smtk-fedora33-paraview59-20230707 +ci-smtk-fedora33-paraview59-20230717 +ci-smtk-fedora33-paraview59-20231026 +ci-smtk-fedora33-paraview59-20231127 +ci-smtk-fedora33-paraview59-20240724 +ci-smtk-fedora33-vtk-20210420 +ci-smtk-fedora33-vtk-20210714 +ci-smtk-fedora33-vtk-20211024 +ci-smtk-fedora33-vtk-20211111 +ci-smtk-fedora33-vtk-20220103 +ci-smtk-fedora33-vtk-20220304 +ci-smtk-fedora33-vtk-20220425 +ci-smtk-fedora33-vtk-20220818 +ci-smtk-fedora33-vtk-20220902 +ci-smtk-fedora33-vtk-20230622 +ci-smtk-fedora33-vtk-20230628 +ci-smtk-fedora33-vtk-20230707 +ci-smtk-fedora33-vtk-20230717 +ci-smtk-fedora33-vtk-20231026 +ci-smtk-fedora33-vtk-20231127 +ci-smtk-fedora33-vtk-20240724 +ci-smtk-fedora33-vtk-20250105 +ci-smtk-fedora33-vtk-python2-20210420 +ci-smtk-fedora41-20250226 +ci-smtk-fedora41-20250305 +ci-smtk-fedora41-paraview-20250226 +ci-smtk-fedora41-paraview-20250305 +ci-smtk-fedora41-vtk-20250226 +ci-smtk-fedora41-vtk-20250305 +ci-smtk-latest + +# cmb-superbuild +ci-superbuild-centos7-20191111 +ci-superbuild-centos7-20191120 +ci-superbuild-centos7-20191127 +ci-superbuild-centos7-20200203 +ci-superbuild-centos7-20200205 +ci-superbuild-centos7-20200803 +ci-superbuild-centos7-20200831 +ci-superbuild-centos7-20210223 +ci-superbuild-centos7-20220201 +ci-superbuild-centos7-20220519 +ci-superbuild-centos7-20240312 +ci-superbuild-centos7-latest +ci-superbuild-fedora32-girder diff --git a/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash new file mode 100755 index 0000000000000000000000000000000000000000..b61f88ccfaee27d300f641f41b65e431199632ff --- /dev/null +++ b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +readonly repo="kitware/cmb" + +delete=false +size=false +while [ "$#" -gt 0 ]; do + case "$1" in + --delete) + delete=true + ;; + --size) + size=true + ;; + *) + echo >&2 "Unrecognized argument: '$1'" + exit 1 + ;; + esac + + shift +done +readonly delete +readonly size + +skopeo --override-os linux list-tags "docker://docker.io/$repo" | \ + jq -r .Tags.[] > "all-tags" + +# Find unused tags. This is done by listing all tags once and "keep" tags +# twice. We filter out the comments from the "keep" list, sort everything, +# and then list all lines that only show up once in the output. +unused_tags="$( cat "all-tags" "dockerhub-keep-tags" "dockerhub-keep-tags" | + grep -v -e '^$' -e '^#' | + sort | + uniq -u )" +readonly unused_tags + +two_weeks_ago="$( date --date='2 weeks ago' +%Y%m%d )" +readonly two_weeks_ago +echo "Keeping images with dates newer than '$two_weeks_ago'" + +undeleted_size=0 +deleted_size=0 +for unused_tag in $unused_tags; do + size_text= + if $size; then + image_size="$( skopeo inspect "docker://docker.io/$repo:$unused_tag" | + jq .LayersData.[].Size | + paste -sd+ | + bc )" + + size_text=" ($image_size bytes; $( echo "$image_size" | numfmt --to=iec ))" + fi + + date_tag="$( echo "$unused_tag" | rev | cut -d- -f1 | rev )" + if [[ "$date_tag" =~ [^0-9]+ ]]; then + echo "Non-date tag appears unused: '$unused_tag'$size_text" + continue + fi + if [ "$date_tag" -lt "$two_weeks_ago" ]; then + if $delete; then + echo "Deleting unused tag '$unused_tag'$size_text" + skopeo --override-os linux delete "docker://docker.io/$repo:$unused_tag" + else + echo "Would delete unused tag '$unused_tag'$size_text" + fi + + deleted_size="$(( deleted_size + image_size ))" + else + echo "Keeping recent unused tag '$unused_tag'$size_text" + + undeleted_size="$(( undeleted_size + image_size ))" + fi +done + +if $size; then + deleted_text="To-delete" + if $delete; then + deleted_text="Deleted" + fi + echo "$deleted_text images size: $deleted_size bytes ($( echo "$deleted_size" | numfmt --to=iec ))" + echo "Preserved images size: $undeleted_size bytes ($( echo "$undeleted_size" | numfmt --to=iec ))" +fi diff --git a/Scripts/ci-bundles/rm-stale-gdrive-bundles.bash b/Scripts/ci-bundles/rm-stale-gdrive-bundles.bash new file mode 100755 index 0000000000000000000000000000000000000000..8aa016ef64cac62673a88d22feb1662111c51933 --- /dev/null +++ b/Scripts/ci-bundles/rm-stale-gdrive-bundles.bash @@ -0,0 +1,84 @@ +#!/bin/sh + +set -e + +delete=false +size=false +while [ "$#" -gt 0 ]; do + case "$1" in + --delete) + delete=true + ;; + --size) + size=true + ;; + *) + echo >&2 "Unrecognized argument: '$1'" + exit 1 + ;; + esac + + shift +done +readonly delete +readonly size + +if ! [ -f "rclone-cmb-ci" ]; then + echo >&2 "Failed to find configuration for 'rclone'" + exit 1 +fi + +if ! rclone --config rclone-cmb-ci listremotes | grep -q -e 'gdrive-cmbsb:'; then + echo >&2 "Failed to find the 'gdrive-cmbsb' remote in the configuration file" + exit 1 +fi + +readonly rclone_mount_target="rclone-mount" +mkdir -p "$rclone_mount_target" +rclone --config rclone-cmb-ci mount --daemon gdrive-cmbsb:ci "$rclone_mount_target" +trap "fusermount -u '$rclone_mount_target'" EXIT + +two_weeks_ago="$( date --date='2 weeks ago' +%Y%m%d )" +readonly two_weeks_ago +echo "Keeping bundles with dates newer than '$two_weeks_ago'" + +undeleted_size=0 +deleted_size=0 +for project in aeva cmb smtk; do + for bundle in "$rclone_mount_target/$project/"*.tar.gz; do + bundle_name="$( basename "$bundle" )" + + size_text= + if $size; then + bundle_size="$( stat --format '%s' "$bundle" )" + + size_text=" ($bundle_size bytes; $( echo "$bundle_size" | numfmt --to=iec ))" + fi + + date_tag="$( echo "$bundle" | sed -e 's/.*-ci-developer-\([0-9]\+\)-.*/\1/' )" + + if [ "$date_tag" -lt "$two_weeks_ago" ]; then + if $delete; then + echo "Deleting stale bundle '$project/$bundle_name'$size_text" + rm "$bundle" + else + echo "Would delete stale bundle '$project/$bundle_name'$size_text" + fi + + deleted_size="$(( deleted_size + bundle_size ))" + else + echo "Keeping recent unused bundle '$project/$bundle_name'$size_text" + + undeleted_size="$(( undeleted_size + bundle_size ))" + fi + done +done + +if $size; then + deleted_text="To-delete" + if $delete; then + deleted_text="Deleted" + fi + echo "$deleted_text bundles size: $deleted_size bytes ($( echo "$deleted_size" | numfmt --to=iec ))" + echo "Preserved bundles size: $undeleted_size bytes ($( echo "$undeleted_size" | numfmt --to=iec ))" +fi