Merge branch 'main' into navarone/add-feature-comparison-table

.buildkite/publish/build-and-push-multiarch-docker.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+########
+# Builds the multiarch docker image and pushes it to the docker registry
+########
+
+set -exu
+set -o pipefail
+
+# Load our common environment variables for publishing
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CURDIR
+
+# shellcheck source=./publish-common.sh
+source "$CURDIR/publish-common.sh"
+
+# Set our tag name as well as the tag names of the individual platform images
+TAG_NAME="${BASE_TAG_NAME}:${VERSION}"
+AMD64_TAG="${BASE_TAG_NAME}:${VERSION}-amd64"
+ARM64_TAG="${BASE_TAG_NAME}:${VERSION}-arm64"
+
+# Pull the images from the registry
+buildah pull "$AMD64_TAG"
+buildah pull "$ARM64_TAG"
+
+# ensure +x is set to avoid writing any sensitive information to the console
+set +x
+
+# Log into Docker
+echo "Logging into docker..."
+DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field "${DOCKER_USER_KEY}" "${VAULT_PATH}")
+vault read -address "${VAULT_ADDR}" -field "${DOCKER_PASS_KEY}" "${VAULT_PATH}" | \
+  buildah login --username="${DOCKER_USER}" --password-stdin docker.elastic.co
+
+# Create the manifest for the multiarch image
+echo "Creating manifest..."
+buildah manifest create "$TAG_NAME" \
+  "$AMD64_TAG" \
+  "$ARM64_TAG"
+
+# ... and push it
+echo "Pushing manifest..."
+buildah manifest push "$TAG_NAME" "docker://$TAG_NAME"
+
+# Write out the final manifest for debugging purposes
+echo "Built and pushed multiarch image... dumping final manifest..."
+buildah manifest inspect "$TAG_NAME"

.buildkite/publish/build-docker.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+########
+# Builds the docker image and saves it to an archive file
+# so it can be stored as an artifact in Buildkite
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CURDIR
+
+# shellcheck source=./publish-common.sh
+source "$CURDIR/publish-common.sh"
+
+pushd "$PROJECT_ROOT"
+
+# set our complete tag name and build the image
+TAG_NAME="$BASE_TAG_NAME:${VERSION}-${ARCHITECTURE}"
+docker build -f "$DOCKERFILE_PATH" -t "$TAG_NAME" .
+
+# save the image to an archive file
+OUTPUT_PATH="$PROJECT_ROOT/.artifacts"
+OUTPUT_FILE="$OUTPUT_PATH/${DOCKER_ARTIFACT_KEY}-${VERSION}-${ARCHITECTURE}.tar.gz"
+mkdir -p "$OUTPUT_PATH"
+docker save "$TAG_NAME" | gzip > "$OUTPUT_FILE"
+
+popd

.buildkite/publish/publish-common.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+if [[ "${CURDIR:-}" == "" ]]; then
+  echo "!! CURDIR is not set. Exiting."
+  exit 2
+fi
+
+function realpath {
+  echo "$(cd "$(dirname "$1")" || exit; pwd)"/"$(basename "$1")";
+}
+
+export SCRIPT_DIR="$CURDIR"
+
+BUILDKITE_DIR=$(realpath "$(dirname "$SCRIPT_DIR")")
+PROJECT_ROOT=$(realpath "$(dirname "$BUILDKITE_DIR")")
+VERSION_PATH="$PROJECT_ROOT/product_version"
+VERSION=$(cat "$VERSION_PATH")
+IS_SNAPSHOT=$(buildkite-agent meta-data get is_snapshot)
+
+export BUILDKITE_DIR
+export PROJECT_ROOT
+export VERSION
+
+if [[ "${IS_SNAPSHOT:-}" == "true" ]]; then
+  echo "Adding SNAPSHOT labeling"
+  export VERSION="${VERSION}-SNAPSHOT"
+fi
+
+export BASE_TAG_NAME="${DOCKER_IMAGE_NAME:-docker.elastic.co/integrations/crawler}"
+export DOCKERFILE_PATH="${DOCKERFILE_PATH:-Dockerfile}"
+export DOCKER_ARTIFACT_KEY="${DOCKER_ARTIFACT_KEY:-elastic-crawler-docker}"
+
+export VAULT_ADDR="${VAULT_ADDR:-https://vault-ci-prod.elastic.dev}"
+export VAULT_PATH="secret/ci/elastic-crawler/docker-ci-admin"
+export DOCKER_PASS_KEY="secret_20240823"
+export DOCKER_USER_KEY="user_20240823"

.buildkite/publish/push-docker.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+########
+# Pushes the docker image to the docker registry
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CURDIR
+
+# shellcheck source=./publish-common.sh
+source "$CURDIR"/publish-common.sh
+
+# Load the image from the artifact created in build-docker.sh
+echo "Loading image from archive file..."
+docker load < "$PROJECT_ROOT/.artifacts/${DOCKER_ARTIFACT_KEY}-${VERSION}-${ARCHITECTURE}.tar.gz"
+
+# ensure +x is set to avoid writing any sensitive information to the console
+set +x
+
+# Log into Docker
+echo "Logging into docker..."
+DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field "${DOCKER_USER_KEY}" "${VAULT_PATH}")
+vault read -address "${VAULT_ADDR}" -field "${DOCKER_PASS_KEY}" "${VAULT_PATH}" | \
+  docker login -u "$DOCKER_USER" --password-stdin docker.elastic.co
+
+# Set our tag name and push the image
+TAG_NAME="$BASE_TAG_NAME:${VERSION}-${ARCHITECTURE}"
+echo "Pushing image to docker with tag: $TAG_NAME"
+docker push "$TAG_NAME"

.buildkite/publish/test-docker.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+########
+# Loads the docker image and tests the structure
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CURDIR
+
+# shellcheck source=./publish-common.sh
+source "$CURDIR/publish-common.sh"
+
+# Detect the platform we are running on (needed for container-structure-test)
+arch_name=$(uname -sr)
+case "$arch_name" in
+    Darwin*)
+        echo "detected MacOS platform"
+        LOCAL_MACHINE_ARCH="MacOS"
+        ;;
+    Linux*)
+        echo "detected Linux platform"
+        LOCAL_MACHINE_ARCH="Linux"
+        ;;
+    *)
+        echo "Unsupported platform: $arch_name"
+        exit 2
+        ;;
+esac
+
+# Load the image from the artifact created in build-docker.sh
+echo "Loading image from archive file..."
+docker load < "$PROJECT_ROOT/.artifacts/${DOCKER_ARTIFACT_KEY}-${VERSION}-${ARCHITECTURE}.tar.gz"
+
+# Ensure we have container-structure-test installed
+echo "Ensuring test environment is set up"
+
+BIN_DIR="$PROJECT_ROOT/bin"
+TEST_EXEC="$BIN_DIR/container-structure-test"
+if [[ ! -f "$TEST_EXEC" ]]; then
+  mkdir -p "$BIN_DIR"
+
+  pushd "$BIN_DIR"
+  if [[ "$LOCAL_MACHINE_ARCH" == "MacOS" ]]; then
+    curl -LO "https://storage.googleapis.com/container-structure-test/latest/container-structure-test-darwin-$ARCHITECTURE"
+    mv "container-structure-test-darwin-$ARCHITECTURE" container-structure-test
+  else
+    curl -LO "https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-$ARCHITECTURE"
+    mv "container-structure-test-linux-$ARCHITECTURE" container-structure-test
+  fi
+
+  chmod +x container-structure-test
+  popd
+fi
+
+# Generate our config file
+TEST_CONFIG_FILE="$PROJECT_ROOT/.buildkite/publish/container-structure-test.yaml"
+
+# Remove -SNAPSHOT if it exists
+CLEAN_VERSION="${VERSION%-SNAPSHOT}"
+# The config file needs escaped dots - we'll do that here
+ESCAPED_VERSION="${CLEAN_VERSION//./\\\\.}"
+
+# Generate the config file text
+TEST_CONFIG_TEXT='
+schemaVersion: "2.0.0"
+
+commandTests:
+  # ensure JRuby 9.4.* is installed
+  - name: "JRuby Installation 9.4.*"
+    command: "ruby"
+    args: ["--version"]
+    expectedOutput: ["jruby\\s9\\.4\\.*"]
+  - name: "Crawler installation"
+    command: "/app/bin/crawler"
+    args: ["--version"]
+    expectedOutput: ["'"${ESCAPED_VERSION}"'*"]
+'
+# ... and save the config file
+printf '%s\n' "$TEST_CONFIG_TEXT" > "$TEST_CONFIG_FILE"
+
+# Finally, run the tests
+echo "Running container-structure-test"
+TAG_NAME="$BASE_TAG_NAME:${VERSION}-${ARCHITECTURE}"
+"$TEST_EXEC" test --image "$TAG_NAME" --config "$TEST_CONFIG_FILE"

.buildkite/release-pipeline.yml

@@ -1,3 +1,125 @@
+## 🏠/.buildkite/pipeline-release.yml
+# Manual triggered pipeline to build and publish Docker images
+
 steps:
-  - label: "Temp step"
-    command: echo "Hello!"
+  - input: "Build information"
+    fields:
+      - select: "Build as SNAPSHOT"
+        key: "is_snapshot"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+  # ----
+  # Docker builds for amd64
+  # ----
+  - group: ":package: amd64 Build and Test"
+    key: "build_and_test_amd64"
+    if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/"
+    steps:
+      - label: "Building amd64 Docker image"
+        agents:
+          provider: aws
+          instanceType: m6i.xlarge
+          imagePrefix: ci-amazonlinux-2
+        env:
+          ARCHITECTURE: "amd64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        command: ".buildkite/publish/build-docker.sh"
+        key: "build_docker_image_amd64"
+        artifact_paths: ".artifacts/*.tar.gz"
+      - label: "Testing amd64 Docker image"
+        agents:
+          provider: aws
+          instanceType: m6i.xlarge
+          imagePrefix: ci-amazonlinux-2
+        env:
+          ARCHITECTURE: "amd64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        depends_on: "build_docker_image_amd64"
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_amd64
+          - ".buildkite/publish/test-docker.sh"
+  # ----
+  # Docker builds for arm64
+  # ----
+  - group: ":package: arm64 Build and Test"
+    key: "build_and_test_arm64"
+    if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/"
+    steps:
+      - label: "Building arm64 Docker image"
+        agents:
+          provider: aws
+          instanceType: m6g.xlarge
+          imagePrefix: ci-amazonlinux-2-aarch64
+          diskSizeGb: 40
+          diskName: '/dev/xvda'
+        env:
+          ARCHITECTURE: "arm64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        command: ".buildkite/publish/build-docker.sh"
+        key: "build_docker_image_arm64"
+        artifact_paths: ".artifacts/*.tar.gz"
+      - label: "Testing arm64 Docker image"
+        agents:
+          provider: aws
+          instanceType: m6g.xlarge
+          imagePrefix: ci-amazonlinux-2-aarch64
+          diskSizeGb: 40
+          diskName: '/dev/xvda'
+        env:
+          ARCHITECTURE: "arm64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        depends_on: "build_docker_image_arm64"
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_arm64
+          - ".buildkite/publish/test-docker.sh"
+  # ----
+  # Multiarch Docker image build and push
+  # ----
+  - group: ":truck: Publish images"
+    if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/"
+    depends_on:
+      - "build_and_test_amd64"
+      - "build_and_test_arm64"
+    steps:
+      - label: "Push amd64 Docker image"
+        key: "push_amd64_docker_image"
+        env:
+          ARCHITECTURE: "amd64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        agents:
+          provider: aws
+          instanceType: m6i.xlarge
+          imagePrefix: ci-amazonlinux-2
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_amd64
+          - ".buildkite/publish/push-docker.sh"
+      - label: "Push arm64 Docker image"
+        key: "push_arm64_docker_image"
+        env:
+          ARCHITECTURE: "arm64"
+          DOCKERFILE_PATH: "Dockerfile.wolfi"
+        agents:
+          provider: aws
+          instanceType: m6g.xlarge
+          imagePrefix: ci-amazonlinux-2-aarch64
+          diskSizeGb: 40
+          diskName: '/dev/xvda'
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_arm64
+          - ".buildkite/publish/push-docker.sh"
+      - label: "Build and push multiarch Docker image"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/drivah:0.25.0"
+          ephemeralStorage: "20G"
+          memory: "4G"
+        command: ".buildkite/publish/build-and-push-multiarch-docker.sh"
+        depends_on:
+          - "push_amd64_docker_image"
+          - "push_arm64_docker_image"

.buildkite/scripts/run_ci_step.sh

@@ -2,8 +2,11 @@
 
 set -euxo pipefail
 
-export RUBY_VERSION=$(cat .ruby-version)
-export JAVA_VERSION=$(cat .java-version)
+RUBY_VERSION="$(cat .ruby-version)"
+JAVA_VERSION="$(cat .java-version)"
+
+export RUBY_VERSION
+export JAVA_VERSION
 
 case $1 in
   lint)