diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 16015ae..b08221a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu { - "name": "Ubuntu", + "name": "eps-devcontainers", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "build": { "dockerfile": "Dockerfile", diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..740676e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +node_modules +.venv +.out diff --git a/.github/scripts/delete_unused_images.sh b/.github/scripts/delete_unused_images.sh index 9ac66ed..68edc9f 100755 --- a/.github/scripts/delete_unused_images.sh +++ b/.github/scripts/delete_unused_images.sh @@ -44,14 +44,18 @@ delete_pr_images() { fi while IFS= read -r tag; do - if [[ "${tag}" =~ ^pr-[0-9]+- ]]; then - local pull_request + local pull_request + if [[ "${tag}" =~ ^pr-([0-9]+)- ]]; then + pull_request=${BASH_REMATCH[1]} + elif [[ "${tag}" =~ ^githubactions-pr-([0-9]+)$ ]]; then + pull_request=${BASH_REMATCH[1]} + else + continue + fi + local pr_json local pr_state - pull_request=${tag#pr-} - pull_request=${pull_request%%-*} - if ! pr_json=$(gh api \ -H "Accept: application/vnd.github+json" \ "/repos/NHSDigital/eps-devcontainers/pulls/${pull_request}"); then @@ -75,7 +79,6 @@ delete_pr_images() { "/orgs/nhsdigital/packages/container/${package_name}/versions/${version_id}" fi done - fi done <<<"${tags}" } diff --git a/.github/workflows/build_multi_arch_image.yml b/.github/workflows/build_multi_arch_image.yml index 0d08f4a..effc2b7 100644 --- a/.github/workflows/build_multi_arch_image.yml +++ b/.github/workflows/build_multi_arch_image.yml @@ -131,11 +131,17 @@ jobs: env: ARCHITECTURE: '${{ matrix.arch }}' DOCKER_TAG: '${{ inputs.docker_tag }}' - - name: Push tagged image + - name: Push tagged image and rebuild for github actions run: | echo "Pushing image..." docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}" echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY" + + echo "Rebuilding image for github actions with tag githubactions-${DOCKER_TAG}-${ARCHITECTURE}" + make build-githubactions-image BASE_IMAGE_NAME="${CONTAINER_NAME}" BASE_IMAGE_TAG="${DOCKER_TAG}-${ARCHITECTURE}" IMAGE_TAG="${DOCKER_TAG}-${ARCHITECTURE}" NO_CACHE="${{ inputs.NO_CACHE }}" + echo "Pushing github actions image..." + docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}" + echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY" env: DOCKER_TAG: ${{ inputs.docker_tag }} CONTAINER_NAME: '${{ inputs.container_name }}' @@ -144,9 +150,14 @@ jobs: if: ${{ inputs.tag_latest }} run: | docker tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}" "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}" - echo "Pushing image..." + echo "Pushing latest image..." docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}" echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY" + + docker tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}" "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}" + echo "Pushing github actions latest image..." + docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}" + echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY" env: DOCKER_TAG: ${{ inputs.docker_tag }} CONTAINER_NAME: '${{ inputs.container_name }}' @@ -172,6 +183,7 @@ jobs: run: | BUILD_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") export BUILD_TIMESTAMP + echo "Creating combined image for tag ${DOCKER_TAG}" docker buildx imagetools create \ --annotation "index:org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers" \ --annotation "index:org.opencontainers.image.description=EPS devcontainer ${CONTAINER_NAME}:${DOCKER_TAG}" \ @@ -184,6 +196,20 @@ jobs: "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-amd64" \ "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-arm64" echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}" >> "$GITHUB_STEP_SUMMARY" + + echo "Creating combined image for tag githubactions-${DOCKER_TAG}" + docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers" \ + --annotation "index:org.opencontainers.image.description=EPS devcontainer ${CONTAINER_NAME}:${DOCKER_TAG}" \ + --annotation "index:org.opencontainers.image.licenses=MIT" \ + --annotation "index:org.opencontainers.image.version=${DOCKER_TAG}" \ + --annotation "index:org.opencontainers.image.containerName=${CONTAINER_NAME}" \ + --annotation "index:org.opencontainers.image.created=${BUILD_TIMESTAMP}" \ + --annotation "index:org.opencontainers.image.authors=NHS England EPS Team" \ + --tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}" \ + "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-amd64" \ + "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-arm64" + echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}" >> "$GITHUB_STEP_SUMMARY" env: DOCKER_TAG: ${{ inputs.docker_tag }} CONTAINER_NAME: '${{ inputs.container_name }}' @@ -191,10 +217,18 @@ jobs: - name: Push multi-arch latest image if: ${{ inputs.tag_latest }} run: | + echo "Creating combined image for tag latest" docker buildx imagetools create -t "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest" \ "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-amd64" \ "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-arm64" echo "## PUSHED COMBINED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest" >> "$GITHUB_STEP_SUMMARY" + + echo "Creating combined image for tag githubactions-latest" + docker buildx imagetools create -t "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest" \ + "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-amd64" \ + "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-arm64" + echo "## PUSHED COMBINED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest" >> "$GITHUB_STEP_SUMMARY" + env: DOCKER_TAG: ${{ inputs.docker_tag }} CONTAINER_NAME: '${{ inputs.container_name }}' diff --git a/Makefile b/Makefile index fa8623f..8ea9469 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,5 @@ CONTAINER_PREFIX=ghcr.io/nhsdigital/eps-devcontainers/ -ifneq ($(strip $(PLATFORM)),) -PLATFORM_FLAG=--platform $(PLATFORM) -endif - ifeq ($(strip $(NO_CACHE)),true) NO_CACHE_FLAG=--no-cache endif @@ -30,9 +26,20 @@ build-image: guard-CONTAINER_NAME guard-BASE_VERSION_TAG guard-BASE_FOLDER guard --workspace-folder ./src/$${BASE_FOLDER}/$${CONTAINER_NAME} \ $(NO_CACHE_FLAG) \ --push false \ + --output type=image,name="${CONTAINER_PREFIX}$${CONTAINER_NAME}:$${IMAGE_TAG}",push=false,compression=zstd \ --cache-from "${CONTAINER_PREFIX}$${CONTAINER_NAME}:latest" \ --image-name "${CONTAINER_PREFIX}$${CONTAINER_NAME}:$${IMAGE_TAG}" +build-githubactions-image: guard-BASE_IMAGE_NAME guard-BASE_IMAGE_TAG guard-IMAGE_TAG + docker buildx build \ + -f src/githubactions/Dockerfile \ + $(NO_CACHE_FLAG) \ + --build-arg BASE_IMAGE_NAME="$${BASE_IMAGE_NAME}" \ + --build-arg BASE_IMAGE_TAG="$${BASE_IMAGE_TAG}" \ + --load \ + -t "${CONTAINER_PREFIX}$${BASE_IMAGE_NAME}:githubactions-$${IMAGE_TAG}" \ + . + scan-image: guard-CONTAINER_NAME guard-BASE_FOLDER @combined="src/$${BASE_FOLDER}/$${CONTAINER_NAME}/.trivyignore_combined.yaml"; \ common="src/common/.trivyignore.yaml"; \ diff --git a/README.md b/README.md index 6300a93..1377923 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ Images are built using using https://github.com/devcontainers/cli. We build a base image based on mcr.microsoft.com/devcontainers/base:ubuntu-22.04 that other images are then based on -The images have vsocde user setup as user 1001 so that they can be used in github actions - The base image contains - latest os packages - asdf @@ -109,11 +107,12 @@ This job should be used in github actions wherever you need to get the dev conta echo "DEVCONTAINER_IMAGE_VERSION=$DEVCONTAINER_VERSION" >> "$GITHUB_OUTPUT" ``` # Project structure -We have 3 types of dev container. These are defined under src +We have 4 types of dev container. These are defined under src `base` - this is the base image that all others are based on. `languages` - this installs specific versions of node and python. -`projects` - this is used for projects where more customization is needed than just a base language image +`projects` - this is used for projects where more customization is needed than just a base language image. +`githubactions` - this just takes an existing image and remaps vscode user to be 1001 so it can be used by github actions. Each image to be built contains a .devcontainer folder that defines how the devcontainer should be built. At a minimum, this should contain a devcontainer.json file. See https://containers.dev/implementors/json_reference/ for options for this @@ -122,16 +121,20 @@ Images under languages should point to a dockerfile under src/common that is bas We use trivy to scan for vulnerabilities in the built docker images. Known vulnerabilities in the base image are in `src/common/.trivyignore.yaml`. Vulnerabilities in specific images are in `.trivyignore.yaml` file in each images folder. These are combined before running a scan to exclude all known vulnerabilities # Pull requests and merge to main process -For each pull request, and merge to main, images are built and scanned using trivy, but the images are not pushed to github container registry -Docker images are built for each pull request, and on merges to main. -Docker images are built for amd64 and arm64 architecture, and a combined manifest is created and pushed as part of the build. +For each pull request, and merge to main, images are built and scanned using trivy, and pushed to github docker registry. +Docker images are built for amd64 and arm64 architecture, and a combined manifest is created and pushed as part of the build. +The main images have a vscode user with id 1000. A separately tagged image is also created with user vscode mapped to user id 1001 so they can be used by github actions. The base image is built first, and then language images, and finally project images. Docker images are scanned for vulnerabilities using trivy as part of a build step, and the build fails if vulnerabilities are found not in .trivyignore file. -For pull requests, images are tagged with the pr--. -For merges to main, images are tagged with the . +For pull requests, images are tagged with the pr-{pull request id}-{short commit sha}. +For merges to main, images are tagged with the {short commit sha}. +Github actions images are tagged with githubactions-{tag} +Amd64 images are tagged with {tag}-amd64 +Arm64 images are tagged with {tag}-arm64 +Combined image manifest image is just tagged with {tag} so can be included in devcontainer.json and the correct image is pulled based on the host architecture. When a pull request is merged to main or closed, all associated images are deleted from the registry using the github workflow delete_old_images @@ -154,7 +157,7 @@ CONTAINER_NAME=base \ ``` Language images ``` -CONTAINER_NAME=node_24_python_3_13 \ +CONTAINER_NAME=node_24_python_3_14 \ BASE_VERSION_TAG=local-build \ BASE_FOLDER=languages \ IMAGE_TAG=local-build \ @@ -168,7 +171,13 @@ CONTAINER_NAME=fhir_facade_api \ IMAGE_TAG=local-build \ make build-image ``` - +Github actions image +``` +BASE_IMAGE_NAME=base \ + BASE_IMAGE_TAG=local-build \ + IMAGE_TAG=local-build \ + make build-githubactions-image +``` ## Scanning images You can use these commands to scan images Base image @@ -213,13 +222,39 @@ CONTAINER_NAME=fhir_facade_api \ IMAGE_TAG=local-build \ make shell-image ``` +github actions image +``` +CONTAINER_NAME=base \ + IMAGE_TAG=githubactions-local-build \ + make shell-image +``` -## Using local or pull request images +## Using local or pull request images in visual studio code You can use local or pull request images by changing IMAGE_VERSION in devcontainer.json. For an image built locally, you should put the IMAGE_VERSION=local-build. For an image built from a pull request, you should put the IMAGE_VERSION=. You can only use images built from a pull request for testing changes in github actions. +## Using images in github actions +To use the image in github actions, you can use it in github actions using code like this +``` +jobs: + my_job_name: + runs-on: ubuntu-22.04 + container: + image: ghcr.io/nhsdigital/eps-devcontainers/:githubactions- + options: --user 1001:1001 + steps: + - name: copy .tool-versions + run: | + cp /home/vscode/.tool-versions "$HOME/.tool-versions" + ... other steps .... +``` +It is important that +- the image uses the tag starting githubactions- +- there is `options: --user 1001:1001` below image +- the first step copies .tool-versions from /home/vscode to $HOME/.tool-versions + ## Generating a .trivyignore file You can generate a .trivyignore file for known vulnerabilities by either downloading the json scan output generated by the build, or by generating it locally using the scanning images commands above with a make target of scan-image-json @@ -238,3 +273,49 @@ poetry run python \ --input .out/scan_results_docker.json \ --output src/projects/fhir_facade_api/.trivyignore.new.yaml ``` + +## Common makefile targets +There are a set of common Makefiles that are defined in `src/base/.devcontainer/Mk` and are included from `common.mk`. These are installed to /usr/local/share/eps/Mk on the base image so are available for all containers. + +This should be added to the end of each projects Makefile to include them +``` +%: + @$(MAKE) -f /usr/local/share/eps/Mk/common.mk $@ +``` +### Targets +The following targets are defined. These are needed for quality checks to run. Some targets are project specific and so should be overridden in the projects Makefile. + +Build targets (`build.mk`) +- `install` - placeholder target - should be overridden locally +- `install-node` - placeholder target - should be overridden locally +- `docker-build` - placeholder target - should be overridden locally +- `compile` - placeholder target - should be overridden locally + +Check targets (`check.mk`) +- `lint` - placeholder target - should be overridden locally +- `test` - placeholder target - should be overridden locally +- `shellcheck` - runs shellcheck on `scripts/*.sh` and `.github/scripts/*.sh` when files exist +- `cfn-lint` - runs `cfn-lint` against `cloudformation/**/*.yml|yaml` and `SAMtemplates/**/*.yml|yaml` +- `cdk-synth` - placeholder target - should be overridden locally +- `cfn-guard-sam-templates` - validates SAM templates against cfn-guard rulesets and writes outputs to `.cfn_guard_out/` +- `cfn-guard-cloudformation` - validates `cloudformation` templates against cfn-guard rulesets and writes outputs to `.cfn_guard_out/` +- `cfn-guard-cdk` - validates `cdk.out` against cfn-guard rulesets and writes outputs to `.cfn_guard_out/` +- `cfn-guard-terraform` - validates `terraform_plans` against cfn-guard rulesets and writes outputs to `.cfn_guard_out/` +- `actionlint` - runs actionlint against github actions +- `secret-scan` - runs git-secrets (including scanning history) against the repo +- `guard-` - checks if an environment variable is set and errors if it is not + +Credentials targets (`credentials.mk`) +- `aws-configure` - configures an AWS sso session +- `aws-login` - Authorizes an sso session with AWS so aws cli tools can be used. You may still need to set AWS_PROFILE before running commands +- `github-login` - Authorizes github cli to github with scope to read packages +- `create-npmrc` - depends on `github-login`, then writes `.npmrc` with a GitHub Packages auth token and `@nhsdigital` registry + +Trivy targets (`trivy.mk`) +- `trivy-license-check` - runs Trivy license scan (HIGH/CRITICAL) and writes `.trivy_out/license_scan.txt` +- `trivy-generate-sbom` - generates CycloneDX SBOM at `.trivy_out/sbom.cdx.json` +- `trivy-scan-python` - scans Python dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_python.txt` +- `trivy-scan-node` - scans Node dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_node.txt` +- `trivy-scan-go` - scans Go dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_go.txt` +- `trivy-scan-java` - scans Java dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_java.txt` +- `trivy-scan-docker` - scans a built image (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_docker.txt` (requires `DOCKER_IMAGE`), for example: diff --git a/src/base/.devcontainer/Dockerfile b/src/base/.devcontainer/Dockerfile index b9cbafb..1d39021 100644 --- a/src/base/.devcontainer/Dockerfile +++ b/src/base/.devcontainer/Dockerfile @@ -2,39 +2,37 @@ FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 ARG SCRIPTS_DIR=/usr/local/share/eps ARG CONTAINER_NAME -ARG MULTI_ARCH_TAG -ARG BASE_VERSION_TAG ARG IMAGE_TAG ARG TARGETARCH ENV SCRIPTS_DIR=${SCRIPTS_DIR} ENV CONTAINER_NAME=${CONTAINER_NAME} -ENV MULTI_ARCH_TAG=${MULTI_ARCH_TAG} -ENV BASE_VERSION_TAG=${BASE_VERSION_TAG} -ENV IMAGE_TAG=${IMAGE_TAG} ENV TARGETARCH=${TARGETARCH} -LABEL org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers -LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}" -LABEL org.opencontainers.image.licenses=MIT -LABEL org.opencontainers.image.version=${IMAGE_TAG} -LABEL org.opencontainers.image.containerName=${CONTAINER_NAME} -LABEL org.opencontainers.image.authors="NHS England EPS Team" -LABEL org.opencontainers.image.base.image="mcr.microsoft.com/devcontainers/base:ubuntu-22.04" - COPY .tool-versions.asdf ${SCRIPTS_DIR}/${CONTAINER_NAME}/.tool-versions.asdf -COPY --chmod=755 scripts ${SCRIPTS_DIR}/${CONTAINER_NAME} +COPY --chmod=755 scripts/root_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/root_install.sh +COPY --chmod=755 Mk ${SCRIPTS_DIR}/Mk WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME} RUN ./root_install.sh +COPY --chmod=755 scripts/vscode_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/vscode_install.sh USER vscode COPY --chown=vscode:vscode .tool-versions.asdf /home/vscode/.tool-versions.asdf COPY --chown=vscode:vscode .tool-versions /home/vscode/.tool-versions -ENV PATH="/home/vscode/.asdf/shims/:$PATH" +ENV PATH="/home/vscode/.asdf/shims/:/home/vscode/.guard/bin/:$PATH" WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME} RUN ./vscode_install.sh # Switch back to root to install the devcontainer CLI globally USER root + +ENV IMAGE_TAG=${IMAGE_TAG} + +LABEL org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers +LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}" +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.containerName=${CONTAINER_NAME} +LABEL org.opencontainers.image.authors="NHS England EPS Team" +LABEL org.opencontainers.image.base.image="mcr.microsoft.com/devcontainers/base:ubuntu-22.04" diff --git a/src/base/.devcontainer/Mk/build.mk b/src/base/.devcontainer/Mk/build.mk new file mode 100644 index 0000000..2e0e0a7 --- /dev/null +++ b/src/base/.devcontainer/Mk/build.mk @@ -0,0 +1,16 @@ +.PHONY: install install-node docker-build compile +install: + echo "Not implemented" + exit 1 + +install-node: + echo "Not implemented" + exit 1 + +docker-build: + echo "Not implemented" + exit 1 + +compile: + echo "Not implemented" + exit 1 diff --git a/src/base/.devcontainer/Mk/check.mk b/src/base/.devcontainer/Mk/check.mk new file mode 100644 index 0000000..10117d1 --- /dev/null +++ b/src/base/.devcontainer/Mk/check.mk @@ -0,0 +1,89 @@ +.PHONY: lint test shellcheck cfn-lint cdk-synth cfn-guard-sam-templates cfn-guard-cloudformation cfn-guard-cdk cfn-guard-terraform +lint: + echo "Not implemented" + exit 1 + +test: + echo "Not implemented" + exit 1 + +shellcheck: + @if find .github/scripts -maxdepth 1 -type f -name "*.sh" | grep -q .; then \ + shellcheck .github/scripts/*.sh; \ + fi + @if find scripts -maxdepth 1 -type f -name "*.sh" | grep -q .; then \ + shellcheck scripts/*.sh; \ + fi + +cfn-lint: + cfn-lint -I "cloudformation/**/*.y*ml" 2>&1 | awk '/Run scan/ { print } /^[EW][0-9]/ { print; getline; print }' + cfn-lint -I "SAMtemplates/**/*.y*ml" 2>&1 | awk '/Run scan/ { print } /^[EW][0-9]/ { print; getline; print }' + +cdk-synth: + echo "Not implemented" + exit 1 + +cfn-guard-sam-templates: + @bash -eu -o pipefail -c '\ + rulesets=("ncsc" "ncsc-cafv3" "wa-Reliability-Pillar" "wa-Security-Pillar"); \ + mkdir -p .cfn_guard_out; \ + for ruleset in "$${rulesets[@]}"; do \ + while IFS= read -r -d "" file; do \ + SAM_OUTPUT=$$(sam validate -t "$$file" --region eu-west-2 --debug 2>&1 | grep -Pazo "(?s)AWSTemplateFormatVersion.*\\n/" | tr -d "\\0"); \ + output_file=".cfn_guard_out/$${file}_$${ruleset}.txt"; \ + mkdir -p "$$(dirname "$$output_file")"; \ + echo "$${SAM_OUTPUT::-1}" | ~/.guard/bin/cfn-guard validate --rules "/usr/local/share/eps/cfnguard_rulesets/output/$$ruleset.guard" --show-summary fail > "$$output_file"; \ + done < <(find ./SAMtemplates -type f \( -name "*.yaml" -o -name "*.yml" \) -print0); \ + done\ + ' + +cfn-guard-cloudformation: + @bash -eu -o pipefail -c '\ + rulesets=("ncsc" "ncsc-cafv3" "wa-Reliability-Pillar" "wa-Security-Pillar"); \ + mkdir -p .cfn_guard_out; \ + for ruleset in "$${rulesets[@]}"; do \ + ~/.guard/bin/cfn-guard validate \ + --data cloudformation \ + --rules "/tmp/ruleset/output/$$ruleset.guard" \ + --show-summary fail \ + > ".cfn_guard_out/cloudformation_$$ruleset.txt"; \ + done\ + ' + +cfn-guard-cdk: + @bash -eu -o pipefail -c '\ + rulesets=("ncsc" "ncsc-cafv3" "wa-Reliability-Pillar" "wa-Security-Pillar"); \ + mkdir -p .cfn_guard_out; \ + for ruleset in "$${rulesets[@]}"; do \ + ~/.guard/bin/cfn-guard validate \ + --data cdk.out \ + --rules "/tmp/ruleset/output/$$ruleset.guard" \ + --show-summary fail \ + > ".cfn_guard_out/cdk_$$ruleset.txt"; \ + done\ + ' + +cfn-guard-terraform: + @bash -eu -o pipefail -c '\ + rulesets=("ncsc" "ncsc-cafv3" "wa-Reliability-Pillar" "wa-Security-Pillar"); \ + mkdir -p .cfn_guard_out; \ + for ruleset in "$${rulesets[@]}"; do \ + ~/.guard/bin/cfn-guard validate \ + --data terraform_plans \ + --rules "/tmp/ruleset/output/$$ruleset.guard" \ + --show-summary fail \ + > ".cfn_guard_out/terraform_$$ruleset.txt"; \ + done\ + ' + +actionlint: + actionlint + +secret-scan: + git-secrets --scan-history . + +guard-%: + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable $* not set"; \ + exit 1; \ + fi diff --git a/src/base/.devcontainer/Mk/common.mk b/src/base/.devcontainer/Mk/common.mk new file mode 100644 index 0000000..7045c74 --- /dev/null +++ b/src/base/.devcontainer/Mk/common.mk @@ -0,0 +1,4 @@ +include /usr/local/share/eps/Mk/build.mk +include /usr/local/share/eps/Mk/check.mk +include /usr/local/share/eps/Mk/trivy.mk +include /usr/local/share/eps/Mk/credentials.mk diff --git a/src/base/.devcontainer/Mk/credentials.mk b/src/base/.devcontainer/Mk/credentials.mk new file mode 100644 index 0000000..ab0c292 --- /dev/null +++ b/src/base/.devcontainer/Mk/credentials.mk @@ -0,0 +1,14 @@ +.PHONY: aws-configure aws-login create-npmrc github-login + +aws-configure: + aws configure sso --region eu-west-2 + +aws-login: + aws sso login --sso-session sso-session + +create-npmrc: github-login + echo "//npm.pkg.github.com/:_authToken=$$(gh auth token)" > .npmrc + echo "@nhsdigital:registry=https://npm.pkg.github.com" >> .npmrc + +github-login: + gh auth login --scopes read:packages diff --git a/src/base/.devcontainer/Mk/trivy.mk b/src/base/.devcontainer/Mk/trivy.mk new file mode 100644 index 0000000..d5a37d8 --- /dev/null +++ b/src/base/.devcontainer/Mk/trivy.mk @@ -0,0 +1,92 @@ +.PHONY: trivy-license-check trivy-generate-sbom trivy-scan-python trivy-scan-node trivy-scan-go trivy-scan-java + +trivy-license-check: + mkdir -p .trivy_out/ + @if [ -f poetry.lock ]; then \ + poetry self add poetry-plugin-export; \ + poetry export -f requirements.txt --with dev --without-hashes --output=requirements.txt; \ + fi + @if [ -f src/go.sum ]; then \ + cd src && go mod vendor; \ + fi + VIRTUAL_ENV=./.venv/ trivy fs . \ + --scanners license \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --pkg-types library \ + --exit-code 1 \ + --output .trivy_out/license_scan.txt \ + --format table + @if [ -f poetry.lock ]; then rm -f requirements.txt; fi + @if [ -f src/go.sum ]; then rm -rf src/vendor; fi + +trivy-generate-sbom: + mkdir -p .trivy_out/ + trivy fs . \ + --scanners vuln \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 0 \ + --output .trivy_out/sbom.cdx.json \ + --format cyclonedx + +trivy-scan-python: + mkdir -p .trivy_out/ + trivy fs . \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 1 \ + --skip-files "**/package-lock.json,**/go.mod,**/pom.xml" \ + --output .trivy_out/dependency_results_python.txt \ + --format table + +trivy-scan-node: + mkdir -p .trivy_out/ + trivy fs . \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 1 \ + --skip-files "**/poetry.lock,**/go.mod,**/pom.xml" \ + --output .trivy_out/dependency_results_node.txt \ + --format table + +trivy-scan-go: + mkdir -p .trivy_out/ + trivy fs . \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 1 \ + --skip-files "**/poetry.lock,**/package-lock.json,**/pom.xml" \ + --output .trivy_out/dependency_results_go.txt \ + --format table + +trivy-scan-java: + mkdir -p .trivy_out/ + trivy fs . \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 1 \ + --skip-files "**/poetry.lock,**/package-lock.json,**/go.mod" \ + --output .trivy_out/dependency_results_java.txt \ + --format table + +trivy-scan-docker: + mkdir -p .trivy_out/ + trivy image $${DOCKER_IMAGE} \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --config trivy.yaml \ + --include-dev-deps \ + --exit-code 1 \ + --pkg-types os,library \ + --output .trivy_out/dependency_results_docker.txt \ + --format table diff --git a/src/base/.devcontainer/scripts/root_install.sh b/src/base/.devcontainer/scripts/root_install.sh index e0014fb..2352747 100755 --- a/src/base/.devcontainer/scripts/root_install.sh +++ b/src/base/.devcontainer/scripts/root_install.sh @@ -27,8 +27,8 @@ apt-get -y install --no-install-recommends htop vim curl git build-essential \ libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev libbz2-dev \ zlib1g-dev unixodbc unixodbc-dev libsecret-1-0 libsecret-1-dev libsqlite3-dev \ jq apt-transport-https ca-certificates gnupg-agent \ - software-properties-common bash-completion make libbz2-dev \ - libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev \ + software-properties-common bash-completion make \ + libreadline-dev wget llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev liblzma-dev netcat-traditional libyaml-dev uuid-runtime xxd unzip # Download correct SAM CLI for arch @@ -67,12 +67,12 @@ mkdir -p /usr/share/secrets-scanner chmod 755 /usr/share/secrets-scanner curl -L https://raw.githubusercontent.com/NHSDigital/software-engineering-quality-framework/main/tools/nhsd-git-secrets/nhsd-rules-deny.txt -o /usr/share/secrets-scanner/nhsd-rules-deny.txt -# fix user and group ids for vscode user to be 1001 so it can be used by github actions -requested_uid=1001 -requested_gid=1001 -current_uid="$(id -u vscode)" -current_gid="$(id -g vscode)" -if [ "${current_gid}" != "${requested_gid}" ]; then groupmod -g "${requested_gid}" vscode; fi -if [ "${current_uid}" != "${requested_uid}" ]; then usermod -u "${requested_uid}" -g "${requested_gid}" vscode; fi +# get cfn-guard ruleset +wget -O /tmp/ruleset.zip https://github.com/aws-cloudformation/aws-guard-rules-registry/releases/download/1.0.2/ruleset-build-v1.0.2.zip >/dev/null 2>&1 +mkdir -p "${SCRIPTS_DIR}/cfnguard_rulesets" +unzip /tmp/ruleset.zip -d "${SCRIPTS_DIR}/cfnguard_rulesets" >/dev/null 2>&1 +rm /tmp/ruleset.zip -chown -R vscode:vscode /home/vscode +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/base/.devcontainer/scripts/vscode_install.sh b/src/base/.devcontainer/scripts/vscode_install.sh index 5f1a123..c0e96f7 100755 --- a/src/base/.devcontainer/scripts/vscode_install.sh +++ b/src/base/.devcontainer/scripts/vscode_install.sh @@ -21,6 +21,9 @@ asdf plugin add terraform https://github.com/asdf-community/asdf-hashicorp.git asdf plugin add trivy https://github.com/zufardhiyaulhaq/asdf-trivy.git asdf plugin add yq https://github.com/sudermanjr/asdf-yq.git +# install cfn-guard +$ curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/aws-cloudformation/cloudformation-guard/main/install-guard.sh | sh + # install base asdf versions of common tools cd /home/vscode asdf install diff --git a/src/common/Dockerfile b/src/common/Dockerfile index fdf7afd..a894e18 100644 --- a/src/common/Dockerfile +++ b/src/common/Dockerfile @@ -12,24 +12,15 @@ ARG BASE_VERSION_TAG ARG IMAGE_TAG ARG TARGETARCH -ENV BASE_IMAGE=${BASE_IMAGE} ENV SCRIPTS_DIR=${SCRIPTS_DIR} -ENV CONTAINER_NAME=${CONTAINER_NAME} -ENV MULTI_ARCH_TAG=${MULTI_ARCH_TAG} -ENV BASE_VERSION_TAG=${BASE_VERSION_TAG} -ENV IMAGE_TAG=${IMAGE_TAG} ENV TARGETARCH=${TARGETARCH} -LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}" -LABEL org.opencontainers.image.version=${IMAGE_TAG} -LABEL org.opencontainers.image.base.name=${BASE_IMAGE} -LABEL org.opencontainers.image.containerName=${CONTAINER_NAME} - USER root -COPY --chmod=755 scripts ${SCRIPTS_DIR}/${CONTAINER_NAME} +COPY --chmod=755 scripts/root_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/root_install.sh WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME} RUN ./root_install.sh +COPY --chmod=755 scripts/vscode_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/vscode_install.sh USER vscode WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME} @@ -40,3 +31,14 @@ RUN ./vscode_install.sh # Switch back to root to install the devcontainer CLI globally USER root + +ENV CONTAINER_NAME=${CONTAINER_NAME} +ENV MULTI_ARCH_TAG=${MULTI_ARCH_TAG} +ENV BASE_VERSION_TAG=${BASE_VERSION_TAG} +ENV IMAGE_TAG=${IMAGE_TAG} +ENV BASE_IMAGE=${BASE_IMAGE} + +LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}" +LABEL org.opencontainers.image.version=${IMAGE_TAG} +LABEL org.opencontainers.image.base.name=${BASE_IMAGE} +LABEL org.opencontainers.image.containerName=${CONTAINER_NAME} diff --git a/src/githubactions/Dockerfile b/src/githubactions/Dockerfile new file mode 100644 index 0000000..78e9eaa --- /dev/null +++ b/src/githubactions/Dockerfile @@ -0,0 +1,19 @@ +ARG BASE_IMAGE_NAME=base +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/nhsdigital/eps-devcontainers/${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} + +# changes needed so that it can be used by github actions +## change vscode user and group id to be 1001 +## change and move vscode home dir to be /github/home + +RUN requested_uid=1001 \ + && requested_gid=1001 \ + && current_uid="$(id -u vscode)" \ + && current_gid="$(id -g vscode)" \ + && if [ "${current_gid}" != "${requested_gid}" ]; then groupmod -g "${requested_gid}" vscode; fi \ + && if [ "${current_uid}" != "${requested_uid}" ]; then usermod -u "${requested_uid}" -g "${requested_gid}" vscode; fi + +USER vscode +ENV PATH="/home/vscode/.asdf/shims/:/home/vscode/.guard/bin/:$PATH:/vscode_path_mod" +ENV ASDF_DATA_DIR=/home/vscode/.asdf +USER root diff --git a/src/languages/node_24_python_3_12/.devcontainer/scripts/root_install.sh b/src/languages/node_24_python_3_12/.devcontainer/scripts/root_install.sh index 0510f2c..52fa2b1 100755 --- a/src/languages/node_24_python_3_12/.devcontainer/scripts/root_install.sh +++ b/src/languages/node_24_python_3_12/.devcontainer/scripts/root_install.sh @@ -1,2 +1,7 @@ #!/usr/bin/env bash set -e +export DEBIAN_FRONTEND=noninteractive + +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/languages/node_24_python_3_12/.devcontainer/scripts/vscode_install.sh b/src/languages/node_24_python_3_12/.devcontainer/scripts/vscode_install.sh index e16905e..7b9883b 100755 --- a/src/languages/node_24_python_3_12/.devcontainer/scripts/vscode_install.sh +++ b/src/languages/node_24_python_3_12/.devcontainer/scripts/vscode_install.sh @@ -7,3 +7,6 @@ asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git asdf install python asdf install + +# install cfn-lint +pip install --user cfn-lint diff --git a/src/languages/node_24_python_3_13/.devcontainer/scripts/root_install.sh b/src/languages/node_24_python_3_13/.devcontainer/scripts/root_install.sh index 0510f2c..52fa2b1 100755 --- a/src/languages/node_24_python_3_13/.devcontainer/scripts/root_install.sh +++ b/src/languages/node_24_python_3_13/.devcontainer/scripts/root_install.sh @@ -1,2 +1,7 @@ #!/usr/bin/env bash set -e +export DEBIAN_FRONTEND=noninteractive + +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/languages/node_24_python_3_13/.devcontainer/scripts/vscode_install.sh b/src/languages/node_24_python_3_13/.devcontainer/scripts/vscode_install.sh index e16905e..7b9883b 100755 --- a/src/languages/node_24_python_3_13/.devcontainer/scripts/vscode_install.sh +++ b/src/languages/node_24_python_3_13/.devcontainer/scripts/vscode_install.sh @@ -7,3 +7,6 @@ asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git asdf install python asdf install + +# install cfn-lint +pip install --user cfn-lint diff --git a/src/languages/node_24_python_3_14/.devcontainer/scripts/root_install.sh b/src/languages/node_24_python_3_14/.devcontainer/scripts/root_install.sh index 0510f2c..52fa2b1 100755 --- a/src/languages/node_24_python_3_14/.devcontainer/scripts/root_install.sh +++ b/src/languages/node_24_python_3_14/.devcontainer/scripts/root_install.sh @@ -1,2 +1,7 @@ #!/usr/bin/env bash set -e +export DEBIAN_FRONTEND=noninteractive + +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/languages/node_24_python_3_14/.devcontainer/scripts/vscode_install.sh b/src/languages/node_24_python_3_14/.devcontainer/scripts/vscode_install.sh index e16905e..7b9883b 100755 --- a/src/languages/node_24_python_3_14/.devcontainer/scripts/vscode_install.sh +++ b/src/languages/node_24_python_3_14/.devcontainer/scripts/vscode_install.sh @@ -7,3 +7,6 @@ asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git asdf install python asdf install + +# install cfn-lint +pip install --user cfn-lint diff --git a/src/languages/python_3_10/.devcontainer/scripts/root_install.sh b/src/languages/python_3_10/.devcontainer/scripts/root_install.sh index 0510f2c..52fa2b1 100755 --- a/src/languages/python_3_10/.devcontainer/scripts/root_install.sh +++ b/src/languages/python_3_10/.devcontainer/scripts/root_install.sh @@ -1,2 +1,7 @@ #!/usr/bin/env bash set -e +export DEBIAN_FRONTEND=noninteractive + +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/projects/fhir_facade_api/.devcontainer/scripts/root_install.sh b/src/projects/fhir_facade_api/.devcontainer/scripts/root_install.sh index 0733658..f8ddcd1 100755 --- a/src/projects/fhir_facade_api/.devcontainer/scripts/root_install.sh +++ b/src/projects/fhir_facade_api/.devcontainer/scripts/root_install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e - +export DEBIAN_FRONTEND=noninteractive # install non snap version of firefox add-apt-repository -y ppa:mozillateam/ppa cat < /etc/apt/preferences.d/mozilla-firefox @@ -11,3 +11,7 @@ Pin-Priority: 1001 EOF apt-get -y install firefox + +# clean up +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*