From 3f332108f52032a68d7cb05b911e74b1fca6a9ef Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 11 Apr 2025 15:50:08 +0200 Subject: [PATCH 1/3] ci-bundles: preserve the fedora36 girder upload image --- Scripts/ci-bundles/dockerhub-keep-tags | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/ci-bundles/dockerhub-keep-tags b/Scripts/ci-bundles/dockerhub-keep-tags index f300f5a6..3e978854 100644 --- a/Scripts/ci-bundles/dockerhub-keep-tags +++ b/Scripts/ci-bundles/dockerhub-keep-tags @@ -241,3 +241,4 @@ ci-superbuild-centos7-20220519 ci-superbuild-centos7-20240312 ci-superbuild-centos7-latest ci-superbuild-fedora32-girder +ci-superbuild-fedora36-girder -- GitLab From bbcc8654e001c22964083dc433be61e4c5e452aa Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 11 Apr 2025 15:55:51 +0200 Subject: [PATCH 2/3] ci-bundles: add instructions for performing batch deletion DockerHub lacks an official API to perform image deletion, but we can act like the browser that does it by reusing the request it uses with its authentication tokens and cookies and submitting a different image to delete. The token expires every hour, so it is still fairly hands-on, but is far faster than manual clicking. --- Scripts/ci-bundles/.gitignore | 3 +++ Scripts/ci-bundles/curl-delete.in | 19 +++++++++++++++++++ .../ci-bundles/rm-stale-dockerhub-tags.bash | 14 ++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100755 Scripts/ci-bundles/curl-delete.in diff --git a/Scripts/ci-bundles/.gitignore b/Scripts/ci-bundles/.gitignore index 0f2e1a8d..8028a3a2 100644 --- a/Scripts/ci-bundles/.gitignore +++ b/Scripts/ci-bundles/.gitignore @@ -3,5 +3,8 @@ rclone-cmb-ci # Mount point for `rclone`. rclone-mount/ +# Script to actually delete images. See `curl-delete.in` +curl-delete + # Intermediate file for DockerHub pruning. all-tags diff --git a/Scripts/ci-bundles/curl-delete.in b/Scripts/ci-bundles/curl-delete.in new file mode 100755 index 00000000..ac9d76a2 --- /dev/null +++ b/Scripts/ci-bundles/curl-delete.in @@ -0,0 +1,19 @@ +#!/bin/sh + +set -e + +# To use this script: +# +# 1. Go to the DockerHub repository management interface +# 2. Enable the network tab of your browser's Developer Tools. +# 3. Perform the deletion of an eligible tag (run the +# `./rm-stale-dockerhub-tags.bash` script without arguments to discover +# one). +# 4. After submitting the form, right click on the `DELETE` query performed by +# the form, right click and select "Copy as cURL" (or similar). +# 5. Save this file as `curl-delete` and paste the command at the end. +# 6. Remove the `-H 'Content-Length: …' argument. +# 7. Append with `"$@"` to forward arguments to the `curl` command (to pass +# data). +# 8. There is an embedded JWT in the query that expires after an hour (at most) +# that will need refreshing and restarting these steps. diff --git a/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash index b61f88cc..6f20e033 100755 --- a/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash +++ b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash @@ -44,9 +44,14 @@ echo "Keeping images with dates newer than '$two_weeks_ago'" undeleted_size=0 deleted_size=0 for unused_tag in $unused_tags; do + if $size || $delete; then + inspect="$( skopeo inspect "docker://docker.io/$repo:$unused_tag" )" + else + inspect='{}' + fi size_text= if $size; then - image_size="$( skopeo inspect "docker://docker.io/$repo:$unused_tag" | + image_size="$( echo "$inspect" | jq .LayersData.[].Size | paste -sd+ | bc )" @@ -59,10 +64,15 @@ for unused_tag in $unused_tags; do echo "Non-date tag appears unused: '$unused_tag'$size_text" continue fi + if [ "$date_tag" -lt "$two_weeks_ago" ]; then if $delete; then echo "Deleting unused tag '$unused_tag'$size_text" - skopeo --override-os linux delete "docker://docker.io/$repo:$unused_tag" + digest="$( echo "$inspect" | jq -r .Digest )" + data='{"digests":["'"$digest"'"],"delete_references":true}' + length="$( echo -n "$data" | wc -c )" + ./curl-delete --data "$data" -H "Content-Length: $length" + # skopeo --override-os linux delete "docker://docker.io/$repo:$unused_tag" else echo "Would delete unused tag '$unused_tag'$size_text" fi -- GitLab From 6ce66055fba8d43c5751d6c1ddd3521e3cede715 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 11 Apr 2025 17:17:01 +0200 Subject: [PATCH 3/3] ci-bundles: check dates before inspecting dockerhub --- Scripts/ci-bundles/rm-stale-dockerhub-tags.bash | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash index 6f20e033..b20aa304 100755 --- a/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash +++ b/Scripts/ci-bundles/rm-stale-dockerhub-tags.bash @@ -44,6 +44,12 @@ echo "Keeping images with dates newer than '$two_weeks_ago'" undeleted_size=0 deleted_size=0 for unused_tag in $unused_tags; do + date_tag="$( echo "$unused_tag" | rev | cut -d- -f1 | rev )" + if [[ "$date_tag" =~ [^0-9]+ ]]; then + echo "Non-date tag appears unused: '$unused_tag'$size_text" + continue + fi + if $size || $delete; then inspect="$( skopeo inspect "docker://docker.io/$repo:$unused_tag" )" else @@ -59,12 +65,6 @@ for unused_tag in $unused_tags; do size_text=" ($image_size bytes; $( echo "$image_size" | numfmt --to=iec ))" fi - date_tag="$( echo "$unused_tag" | rev | cut -d- -f1 | rev )" - if [[ "$date_tag" =~ [^0-9]+ ]]; then - echo "Non-date tag appears unused: '$unused_tag'$size_text" - continue - fi - if [ "$date_tag" -lt "$two_weeks_ago" ]; then if $delete; then echo "Deleting unused tag '$unused_tag'$size_text" -- GitLab