Use gitlab's multi-project pipelines to trigger agent build (#19130)

* Use gitlab's multi-project pipelines to trigger agent build

* fix "only" section

* switch from only to rules

* Fix variables

* fix compare to

* hardcode compare_to target

* explain why we hardcode compare_to

* remove old stages

* fix and speed up logs pipeline validation

* add job to validate agent build

* remove validation step, default pipeline check is enough

* support triggering manually from any branch

* fix rules

* don't require manual build step in order for pipeline to pass

* add job to show pipeline as failed if manual trigger fails

* remove "validation" job, it won't do what we think it should

Author: iliakur

.gitlab-ci.yml

@@ -1,5 +1,6 @@
 include:
   - /.gitlab/software_composition_analysis.yaml
+  - /.gitlab/build_agent.yaml
 
 variables:
   TAGGER_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/integrations-core:tagger
@@ -13,9 +14,8 @@ variables:
 
 stages:
   - build
-  - trigger
-  - release
   - validate
+  - release
   - notify
 
 cache: &slack-cache
@@ -26,6 +26,7 @@ cache: &slack-cache
 
 validate-log-integrations:
   stage: validate
+  needs: []
   image: $VALIDATE_LOG_INTGS
   only:
   - schedules
@@ -46,89 +47,9 @@ validate-log-integrations:
     when: always
   tags: [ "runner:main" ]
 
-trigger-agent-build:
-  stage: trigger
-  image: $VALIDATE_AGENT_BUILD
-  rules:
-    # Never trigger for tag pipelines
-    - if: $CI_COMMIT_TAG
-      when: never
-    # Trigger on non-release branches
-    - if: ($CI_COMMIT_BRANCH !~ /^7.[0-9]{2}.x/)
-      when: always
-  cache:
-    <<: *slack-cache
-  script:
-    - export SLACK_CACHE_DIR="${PWD}/.slack-cache"
-    - DATADOG_AGENT_PIPELINE_URL=$(aws ssm get-parameter --region us-east-1 --name ci.integrations-core.datadog-agent-pipeline-url --with-decryption --query "Parameter.Value" --out text)
-    - export DEPENDENCIES_CHANGED="false"
-    - git fetch origin master
-    - BASE_REF=$(git merge-base HEAD FETCH_HEAD)
-    - git --no-pager diff --exit-code --name-only HEAD $BASE_REF -- .deps || export DEPENDENCIES_CHANGED="true" || true
-    - export RELEASE_BRANCH=""
-    - |
-      if [[ "$DEPENDENCIES_CHANGED" = "true" ]]; then
-          export RELEASE_BRANCH=$(git tag -l '7\.*' --points-at $BASE_REF | cat | sed 's/\..-.*/\.x/')
-          echo "Dependency files under .deps have been modified, triggering an agent build pipeline."
-          DATADOG_AGENT_PIPELINE_URL=$DATADOG_AGENT_PIPELINE_URL RELEASE_BRANCH=$RELEASE_BRANCH python -u /trigger_agent_build.py --output-file pipeline_id.txt
-
-          # Get slack user
-          if [[ $GITLAB_USER_LOGIN = "codesync" ]]; then EMAIL=$(git show -s --format="%ae" "$CI_COMMIT_SHA"); else EMAIL=$GITLAB_USER_EMAIL; fi
-          SLACK_AUTHOR=$(echo $EMAIL | email2slackid)
-          if [ -z "$SLACK_AUTHOR" ]; then echo "${EMAIL} cannot be translated into a Slack user, defaulting to ${AGENT_BUILD_NOTIFICATIONS_SLACK_CHANNEL}"; SLACK_AUTHOR=$AGENT_BUILD_NOTIFICATIONS_SLACK_CHANNEL; fi
-
-          MESSAGE="Starting a build of the agent, watch ${CI_PIPELINE_URL}."
-          postmessage "$SLACK_AUTHOR" "$MESSAGE"
-      else
-          echo "Nothing to run, skipping job."
-      fi
-  artifacts:
-    paths:
-    - pipeline_id.txt
-    expire_in: 1 day
-    when: always
-  tags: [ "runner:main" ]
-
-
-validate-agent-build:
-  stage: validate
-  image: $VALIDATE_AGENT_BUILD
-  timeout: 3 hours
-  cache:
-    <<: *slack-cache
-  script:
-    - export SLACK_CACHE_DIR="${PWD}/.slack-cache"
-    - |
-      if [[ -s ./pipeline_id.txt ]]; then
-          # Fetch secrets from SCM
-          GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.integrations-core.temp_gitlab_token --with-decryption --query "Parameter.Value" --out text)
-          DATADOG_AGENT_PIPELINE_URL=$(aws ssm get-parameter --region us-east-1 --name ci.integrations-core.datadog-agent-pipeline-url --with-decryption --query "Parameter.Value" --out text)
-          PIPELINE_ID=$(cat pipeline_id.txt)
-
-          # Get slack user
-          if [[ $GITLAB_USER_LOGIN = "codesync" ]]; then EMAIL=$(git show -s --format="%ae" "$CI_COMMIT_SHA"); else EMAIL=$GITLAB_USER_EMAIL; fi
-          SLACK_AUTHOR=$(echo $EMAIL | email2slackid)
-          if [ -z "$SLACK_AUTHOR" ]; then echo "${EMAIL} cannot be translated into a Slack user, defaulting to ${AGENT_BUILD_NOTIFICATIONS_SLACK_CHANNEL}"; SLACK_AUTHOR=$AGENT_BUILD_NOTIFICATIONS_SLACK_CHANNEL; fi
-
-          # Wait for the agent build to complete
-          export BUILD_SUCCESS="true"
-          GITLAB_TOKEN=$GITLAB_TOKEN DATADOG_AGENT_PIPELINE_URL=$DATADOG_AGENT_PIPELINE_URL python -u /validate_agent_build.py --pipeline-id $(cat pipeline_id.txt) || export BUILD_SUCCESS="false" || true
-          echo $BUILD_SUCCESS
-          if [[ "$BUILD_SUCCESS" = "true" ]]; then
-              echo "Agent build was successful!"
-              postmessage "$SLACK_AUTHOR" "Build ${CI_JOB_URL} was successful, you can merge your PR!" success
-          else
-              echo "Agent build failed."
-              postmessage "$SLACK_AUTHOR" "The build failed, you might want to retry this job: ${CI_JOB_URL}" alert
-              exit 1
-          fi
-      else
-        echo "A pipeline was not triggered, skipping this job."
-      fi
-
-  tags: [ "runner:main" ]
-
 notify-slack:
+  needs:
+    - validate-log-integrations
   stage: notify
   image: $NOTIFIER_IMAGE
   only:
@@ -232,18 +153,3 @@ validate-log-intgs-builder:
     - cd .gitlab/validate-logs-intgs/
     - docker buildx build --tag $VALIDATE_LOG_INTGS . --push
   tags: [ "arch:amd64" ]
-
-validate-agent-build-builder:
-  stage: build
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
-  only:
-    changes:
-      - .gitlab/validate-agent-build/**/*
-      - .gitlab-ci.yml
-    refs:
-      - master
-  script:
-    - cd .gitlab/validate-agent-build/
-    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/Datadog/devtools.git --depth 1
-    - docker buildx build --tag $VALIDATE_AGENT_BUILD . --push
-  tags: [ "arch:amd64" ]

.gitlab/build_agent.yaml

@@ -0,0 +1,43 @@
+---
+.build-agent-tpl:
+  variables:
+    _AGENT_BRANCH: main # Used only by this job
+    # Pass these to triggered agent build job.
+    RELEASE_VERSION_6: "nightly"
+    RELEASE_VERSION_7: "nightly-a7"
+    BUCKET_BRANCH: "dev"
+    DEPLOY_AGENT: "false"
+    INTEGRATIONS_CORE_VERSION: ${CI_COMMIT_REF_NAME}
+    # disable kitchen and e2e tests
+    RUN_KITCHEN_TESTS: "false"
+    RUN_E2E_TESTS: "off"
+  stage: build
+  trigger:
+    project: DataDog/datadog-agent
+    branch: ${_AGENT_BRANCH}
+    # It's more convenient to directly show if downstream pipeline succeeded.
+    strategy: depend
+
+build-agent-auto:
+  extends: .build-agent-tpl
+  rules:
+    - changes:
+        # We don't yet support variable expansion in `compare_to`. It was added in gitlab 17.2:
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/369916#note_1972773223
+        compare_to: "master"
+        paths:
+          - .deps/*
+          - .gitlab/build_agent.yaml
+
+build-agent-manual:
+  extends: .build-agent-tpl
+  # We don't want to require the manual build job for regular pipelines.
+  allow_failure: true
+  rules:
+    # By default we test with Agent's main branch.
+    # From a release branch we want to use the same release branch in the agent repo.
+    - if: $CI_COMMIT_BRANCH =~ "/^7.\d+.x$/"
+      when: manual
+      variables:
+        _AGENT_BRANCH: ${CI_COMMIT_BRANCH}
+    - when: manual