Chore: [AEA-0000] - delete old images fix

.github/scripts/delete_unused_images.sh

@@ -1,5 +1,39 @@
 #!/usr/bin/env bash
 
+DRY_RUN=false
+DELETE_PR=false
+DELETE_CI=false
+
+while [[ $# -gt 0 ]]; do
+	case "$1" in
+		--dry-run|-n)
+			DRY_RUN=true
+			shift
+			;;
+		--delete-pr)
+			DELETE_PR=true
+			shift
+			;;
+		--delete-ci)
+			DELETE_CI=true
+			shift
+			;;
+		--help|-h)
+			echo "Usage: $0 [--dry-run] [--delete-pr] [--delete-ci]"
+			exit 0
+			;;
+		*)
+			echo "Unknown option: $1" >&2
+			echo "Usage: $0 [--dry-run] [--delete-pr] [--delete-ci]" >&2
+			exit 1
+			;;
+	esac
+done
+
+if [[ "${DELETE_PR}" == "false" && "${DELETE_CI}" == "false" ]]; then
+	DELETE_PR=true
+fi
+
 get_container_package_name() {
 	local container_name=$1
 
@@ -40,16 +74,18 @@ delete_pr_images() {
 	tags=$(jq -r '[.[].metadata.container.tags[]?] | unique | .[]' <<<"${versions_json}")
 
 	if [[ -z "${tags}" ]]; then
+		echo "No tags found for container ${container_name}, skipping."
 		return 0
 	fi
 
 	while IFS= read -r tag; do
 		local pull_request
-		if [[ "${tag}" =~ ^pr-([0-9]+)- ]]; then
+		if [[ "${tag}" =~ ^pr-([0-9]+)(-.+)?$ ]]; then
 			pull_request=${BASH_REMATCH[1]}
-		elif [[ "${tag}" =~ ^githubactions-pr-([0-9]+)$ ]]; then
+		elif [[ "${tag}" =~ ^githubactions-pr-([0-9]+)(-.+)?$ ]]; then
 			pull_request=${BASH_REMATCH[1]}
 		else
+			echo "Tag ${tag} does not match expected PR tag format for container ${container_name}, skipping."
 			continue
 		fi
 
@@ -72,26 +108,89 @@ delete_pr_images() {
 				<<<"${versions_json}" \
 				| while IFS= read -r version_id; do
 					if [[ -n "${version_id}" ]]; then
-                        echo "Deleting image with tag ${tag} (version ID: ${version_id}) from container ${container_name}..."
-						gh api \
-						 	-H "Accept: application/vnd.github+json" \
-						 	-X DELETE \
-						 	"/orgs/nhsdigital/packages/container/${package_name}/versions/${version_id}"
+						if [[ "${DRY_RUN}" == "true" ]]; then
+							echo "[DRY RUN] Would delete image with tag ${tag} (version ID: ${version_id}) from container ${container_name}."
+						else
+							echo "Deleting image with tag ${tag} (version ID: ${version_id}) from container ${container_name}..."
+							gh api \
+						 		-H "Accept: application/vnd.github+json" \
+						 		-X DELETE \
+						 		"/orgs/nhsdigital/packages/container/${package_name}/versions/${version_id}"
+						fi
 					fi
 				done
 	done <<<"${tags}"
 }
 
+delete_ci_images() {
+	local container_name=$1
+	local package_name
+	local versions_json
+	local tags
+
+	if [[ -z "${container_name}" ]]; then
+		echo "Container name is required" >&2
+		return 1
+	fi
+
+	package_name=$(get_container_package_name "${container_name}")
+	versions_json=$(get_container_versions_json "${container_name}")
+	tags=$(jq -r '[.[].metadata.container.tags[]?] | unique | .[]' <<<"${versions_json}")
+
+	if [[ -z "${tags}" ]]; then
+		echo "No tags found for container ${container_name}, skipping."
+		return 0
+	fi
+
+	while IFS= read -r tag; do
+		if [[ ! "${tag}" =~ ^ci-[0-9a-fA-F]{8}.*$ ]] && [[ ! "${tag}" =~ ^githubactions-ci-[0-9a-fA-F]{8}.*$ ]]; then
+			echo "Tag ${tag} does not match expected CI tag format for container ${container_name}, skipping."
+			continue
+		fi
+
+		jq -r --arg tag "${tag}" '.[] | select(.metadata.container.tags[]? == $tag) | .id' \
+			<<<"${versions_json}" \
+			| while IFS= read -r version_id; do
+				if [[ -n "${version_id}" ]]; then
+					if [[ "${DRY_RUN}" == "true" ]]; then
+						echo "[DRY RUN] Would delete CI image with tag ${tag} (version ID: ${version_id}) from container ${container_name}."
+					else
+						echo "Deleting CI image with tag ${tag} (version ID: ${version_id}) from container ${container_name}..."
+						gh api \
+					 		-H "Accept: application/vnd.github+json" \
+					 		-X DELETE \
+					 		"/orgs/nhsdigital/packages/container/${package_name}/versions/${version_id}"
+					fi
+				fi
+			done
+	done <<<"${tags}"
+}
+
 
 language_folders=$(find src/languages -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | jq -R -s -c 'split("\n")[:-1]')
 project_folders=$(find src/projects -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | jq -R -s -c 'split("\n")[:-1]')
 
 for container_name in $(jq -r '.[]' <<<"${project_folders}"); do
-	delete_pr_images "${container_name}"
+	if [[ "${DELETE_PR}" == "true" ]]; then
+		delete_pr_images "${container_name}"
+	fi
+	if [[ "${DELETE_CI}" == "true" ]]; then
+		delete_ci_images "${container_name}"
+	fi
 done
 
 for container_name in $(jq -r '.[]' <<<"${language_folders}"); do
-	delete_pr_images "${container_name}"
+	if [[ "${DELETE_PR}" == "true" ]]; then
+		delete_pr_images "${container_name}"
+	fi
+	if [[ "${DELETE_CI}" == "true" ]]; then
+		delete_ci_images "${container_name}"
+	fi
 done
 
-delete_pr_images "base"
+if [[ "${DELETE_PR}" == "true" ]]; then
+	delete_pr_images "base"
+fi
+if [[ "${DELETE_CI}" == "true" ]]; then
+	delete_ci_images "base"
+fi

.github/workflows/ci.yml

@@ -43,6 +43,6 @@ jobs:
     needs: tag_release
     uses: ./.github/workflows/build_all_images.yml 
     with: 
-      docker_tag: '${{ needs.tag_release.outputs.version_tag }}'
+      docker_tag: 'ci-${{ needs.tag_release.outputs.version_tag }}'
       tag_latest: false
       NO_CACHE: false

.github/workflows/delete_old_images.yml

@@ -26,6 +26,13 @@ jobs:
 
       - name: delete unused images
         shell: bash
-        run: .github/scripts/delete_unused_images.sh
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            .github/scripts/delete_unused_images.sh --delete-pr
+          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
+            .github/scripts/delete_unused_images.sh --delete-ci
+          else
+            .github/scripts/delete_unused_images.sh
+          fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -130,8 +130,8 @@ The base image is built first, and then language images, and finally project ima
 Docker images are scanned for vulnerabilities using trivy as part of a build step, and the build fails if vulnerabilities are found not in .trivyignore file.
 
 For pull requests, images are tagged with the pr-{pull request id}-{short commit sha}.   
-For merges to main, images are tagged with the {short commit sha}.   
-Github actions images are tagged with githubactions-{tag}
+For merges to main, images are tagged with the ci-{short commit sha}.   
+Github actions images are tagged with githubactions-{full tag}
 Amd64 images are tagged with {tag}-amd64
 Arm64 images are tagged with {tag}-arm64
 Combined image manifest image is just tagged with {tag} so can be included in devcontainer.json and the correct image is pulled based on the host architecture.   
@@ -274,6 +274,25 @@ poetry run python \
   --output src/projects/fhir_facade_api/.trivyignore.new.yaml 
 ```
 
+## Cleaning up unused container images
+
+There is a script to delete unused container images. This runs on every merge to main, and deletes pull request images, and on a weekly schedule which deletes images created by ci.   
+You can run it manually using the following. Using the `dry-run` flag just shows what would be deleted
+
+```
+make github-login
+bash .github/scripts/delete_unused_images.sh --delete-pr --dry-run
+bash .github/scripts/delete_unused_images.sh --delete-ci --dry-run
+bash .github/scripts/delete_unused_images.sh --delete-pr --delete-ci
+```
+
+Flags:
+- `--dry-run` (`-n`) shows what would be deleted without deleting anything.
+- `--delete-pr` deletes images tagged with `pr-...` or `githubactions-pr-...` only when the PR is closed.
+- `--delete-ci` deletes images tagged with `ci-<8 hex sha>...` or `githubactions-ci-<8 hex sha>...`.
+
+If neither `--delete-pr` nor `--delete-ci` is set, the script defaults to `--delete-pr`.
+
 ## Common makefile targets
 There are a set of common Makefiles that are defined in `src/base/.devcontainer/Mk` and are included from `common.mk`. These are installed to /usr/local/share/eps/Mk on the base image so are available for all containers.
 

src/base/.devcontainer/Mk/trivy.mk

@@ -79,7 +79,7 @@ trivy-scan-java:
 		--output .trivy_out/dependency_results_java.txt \
 		--format table
 
-trivy-scan-docker:
+trivy-scan-docker: guard-DOCKER_IMAGE
 	mkdir -p .trivy_out/
 	trivy image $${DOCKER_IMAGE} \
 		--scanners vuln \