Merge pull request #77 from uselagoon/feature/insights-scanner

Feature - Adds insights scanner

.github/workflows/insights-scanner-image.yaml

@@ -0,0 +1,109 @@
+name: Publish insights-scanner image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'build-image'
+    paths:
+      - 'insights-scanner/**'
+      - '.github/workflows/insights-scanner-image.yaml'
+    tags:
+      - 'insights-scanner-v*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'insights-scanner/**'
+      - '.github/workflows/insights-scanner-image.yaml'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout PR
+        uses: actions/checkout@v4
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          fetch-depth: "0"
+          ref: ${{ github.event.pull_request.head.sha }}
+      -
+        name: Checkout Branch or Tag
+        uses: actions/checkout@v4
+        if: ${{ github.event_name != 'pull_request' }}
+        with:
+          fetch-depth: "0"
+      -
+        name: Create SERVICE_TAG variable
+        id: service_tag_var
+        run: |
+          RAW_TAG=$(echo $(git describe --abbrev=0 --tags --match 'insights-scanner-*'))
+          SERVICE_TAG=${RAW_TAG#"insights-scanner-"}
+          echo "using insights-scanner tag $SERVICE_TAG"
+          echo "SERVICE_TAG=$SERVICE_TAG" >> $GITHUB_ENV
+      -
+        name: Set version for non-tag build
+        if: "!startsWith(github.ref, 'refs/tags/insights-scanner')"
+        id: version_non-tag_build
+        run: |
+          DOCKER_TAG=${{ env.SERVICE_TAG }}-$(git rev-parse --short=8 HEAD)
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          echo "insights-scanner version $DOCKER_TAG"
+      -
+        name: Set version for tag build
+        if: "startsWith(github.ref, 'refs/tags/insights-scanner')"
+        id: version_tag_build
+        run: |
+          DOCKER_TAG=${{ env.SERVICE_TAG }}
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          echo "insights-scanner version $DOCKER_TAG"
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ${{ github.repository_owner }}/insights-scanner
+            ghcr.io/${{ github.repository_owner }}/insights-scanner
+          tags: |
+            # set edge tag for default branch
+            type=edge,enable={{is_default_branch}}
+            # set tag+build for default branch
+            type=raw,value=${{ env.DOCKER_TAG}},enable={{is_default_branch}}
+            # tag event
+            type=raw,value=${{ env.DOCKER_TAG}},enable=${{ startsWith(github.ref, 'refs/tags/insights-scanner') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/insights-scanner') }}
+            # pull request event
+            type=ref,event=pr
+            # pull request event
+            type=ref,event=branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: insights-scanner
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

insights-scanner/Dockerfile

@@ -0,0 +1,56 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+ARG GO_VER
+FROM ${UPSTREAM_REPO:-uselagoon}/commons:${UPSTREAM_TAG:-latest} AS commons
+FROM aquasec/trivy:0.52.2 AS trivy
+
+FROM docker:20.10.24
+
+LABEL org.opencontainers.image.authors="The Lagoon Authors" maintainer="The Lagoon Authors"
+LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-service-images" repository="https://github.com/uselagoon/lagoon-service-images"
+
+ENV LAGOON=insights-scanner
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
+COPY --from=commons /sbin/tini /sbin/
+COPY --from=commons /home /home
+
+RUN chmod g+w /etc/passwd \
+    && mkdir -p /home
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+# Defining Versions
+ENV KUBECTL_VERSION=v1.30.3
+
+RUN apk add -U --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing aufs-util \
+    && apk add --no-cache curl bash skopeo
+
+RUN architecture=$(case $(uname -m) in x86_64 | amd64) echo "amd64" ;; aarch64 | arm64 | armv8) echo "arm64" ;; *) echo "amd64" ;; esac) \
+    && curl -Lo /usr/bin/kubectl https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/${architecture}/kubectl \
+    && chmod +x /usr/bin/kubectl
+
+COPY --from=trivy /usr/local/bin/trivy /usr/local/bin/trivy
+
+WORKDIR /app
+
+COPY ./*.sh /app
+
+RUN chmod +x /app/run.sh && /bin/fix-permissions /app/run.sh
+
+# We'll set DOCKER_HOST to the lagoon default, but with the assumption that it's overridable at runtime by insights-handler
+ENV DOCKER_HOST=docker-host.lagoon.svc
+
+# bring in entrypoint to kill startup if the appropriate DOCLER_HOST isn't found
+COPY ./dockerhost-entrypoint.sh /lagoon/entrypoints/100-docker-entrypoint.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/run.sh"]

insights-scanner/README.md

@@ -0,0 +1,3 @@
+# Insights-remote scanner image
+
+This image is used by Lagoon Insights Remote to do post-build insights scans, see [this PR](https://github.com/uselagoon/insights-remote/pull/43) for details.

insights-scanner/dockerhost-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# try connect to docker-host 10 times before giving up
+DOCKER_HOST_COUNTER=1
+DOCKER_HOST_TIMEOUT=10
+until docker -H ${DOCKER_HOST} info &> /dev/null
+do
+if [ $DOCKER_HOST_COUNTER -lt $DOCKER_HOST_TIMEOUT ]; then
+    let DOCKER_HOST_COUNTER=DOCKER_HOST_COUNTER+1
+    echo "${DOCKER_HOST} not available yet, waiting for 5 secs"
+    sleep 5
+else
+    echo "could not connect to ${DOCKER_HOST}"
+    exit 1
+fi
+done

insights-scanner/insights-scan.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+TMP_DIR="${TMP_DIR:-/tmp}"
+SBOM_OUTPUT="cyclonedx"
+
+SBOM_OUTPUT_FILE="${TMP_DIR}/${IMAGE_NAME}.cyclonedx.json.gz"
+SBOM_CONFIGMAP="lagoon-insights-sbom-${IMAGE_NAME}"
+IMAGE_INSPECT_CONFIGMAP="lagoon-insights-image-${IMAGE_NAME}"
+IMAGE_INSPECT_OUTPUT_FILE="${TMP_DIR}/${IMAGE_NAME}.image-inspect.json.gz"
+
+set +x
+echo "Running image inspect on: ${IMAGE_FULL}"
+
+skopeo inspect --retry-times 5 docker://${IMAGE_FULL} --tls-verify=false | gzip > ${IMAGE_INSPECT_OUTPUT_FILE}
+
+processImageInspect() {
+  echo "Successfully generated image inspection data for ${IMAGE_FULL}"
+
+  # If lagoon-insights-image-inpsect-[IMAGE] configmap already exists then we need to update, else create new
+  if kubectl -n ${NAMESPACE} get configmap $IMAGE_INSPECT_CONFIGMAP &> /dev/null; then
+      kubectl \
+          -n ${NAMESPACE} \
+          create configmap $IMAGE_INSPECT_CONFIGMAP \
+          --from-file=${IMAGE_INSPECT_OUTPUT_FILE} \
+          -o json \
+          --dry-run=client | kubectl replace -f -
+  else
+      kubectl \
+          -n ${NAMESPACE} \
+          create configmap ${IMAGE_INSPECT_CONFIGMAP} \
+          --from-file=${IMAGE_INSPECT_OUTPUT_FILE}
+  fi
+  kubectl \
+      -n ${NAMESPACE} \
+      label configmap ${IMAGE_INSPECT_CONFIGMAP} \
+      lagoon.sh/insightsProcessed- \
+      lagoon.sh/insightsType=image-gz \
+      lagoon.sh/buildName=${LAGOON_BUILD_NAME} \
+      lagoon.sh/project=${PROJECT} \
+      lagoon.sh/environment=${ENVIRONMENT} \
+      lagoon.sh/service=${IMAGE_NAME} \
+      insights.lagoon.sh/type=inspect
+}
+
+processImageInspect
+
+echo "Running sbom scan using trivy"
+echo "Image being scanned: ${IMAGE_FULL}"
+
+trivy image ${IMAGE_FULL} --format ${SBOM_OUTPUT} | gzip > ${SBOM_OUTPUT_FILE}
+
+FILESIZE=$(stat -c%s "$SBOM_OUTPUT_FILE")
+echo "Size of ${SBOM_OUTPUT_FILE} = $FILESIZE bytes."
+
+processSbom() {
+  if (( $FILESIZE > 950000 )); then
+    echo "$SBOM_OUTPUT_FILE is too large, skipping pushing to configmap"
+    return
+  else
+    echo "Successfully generated SBOM for ${IMAGE_FULL}"
+
+    # If lagoon-insights-sbom-[IMAGE] configmap already exists then we need to update, else create new
+    if kubectl -n ${NAMESPACE} get configmap $SBOM_CONFIGMAP &> /dev/null; then
+        kubectl \
+            -n ${NAMESPACE} \
+            create configmap $SBOM_CONFIGMAP \
+            --from-file=${SBOM_OUTPUT_FILE} \
+            -o json \
+            --dry-run=client | kubectl replace -f -
+    else
+        # Create configmap and add label (#have to add label separately: https://github.com/kubernetes/kubernetes/issues/60295)
+        kubectl \
+            -n ${NAMESPACE} \
+            create configmap ${SBOM_CONFIGMAP} \
+            --from-file=${SBOM_OUTPUT_FILE}
+    fi
+    kubectl \
+        -n ${NAMESPACE} \
+        label configmap ${SBOM_CONFIGMAP} \
+        lagoon.sh/insightsProcessed- \
+        lagoon.sh/insightsType=sbom-gz \
+        lagoon.sh/buildName=${LAGOON_BUILD_NAME} \
+        lagoon.sh/project=${PROJECT} \
+        lagoon.sh/environment=${ENVIRONMENT} \
+        lagoon.sh/service=${IMAGE_NAME} \
+        insights.lagoon.sh/type=sbom
+  fi
+}
+
+processSbom

insights-scanner/run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+DOCKER_HOST="${DOCKER_HOST:-docker-host.lagoon.svc}"
+
+# Read the comma-separated values
+IFS=','
+
+# Iterate over each image in the list
+for image in $INSIGHT_SCAN_IMAGES; do
+    # Populate the variable IMAGE_FULL for each iteration
+    # IMAGE_FULL="$image"
+    IMAGE_NAME=$(echo "$image" | awk -F'/' '{print $NF}' | cut -d':' -f1 | cut -d'@' -f1)
+    IMAGE_FULL="$(echo "$image" | cut -d':' -f1 | cut -d'@' -f1):latest"
+
+    echo "Processing image: $IMAGE_FULL"
+    echo "With image name: $IMAGE_NAME"    
+    . /app/insights-scan.sh
+done