fix(ci): e2e dd logs failure (#23038)

* wip

* bump to stable debian

* fix bug with docker ARG evaluation

* more dbg statements

* try with more recent docker.io/datadog/fakeintake

* test with bigger timeout

* make test runner usable locally

* add retries

* more tweaks to config

* cleanup

Author: pront

.github/workflows/ci-integration-review.yml

@@ -113,7 +113,7 @@ jobs:
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }}
+          command: bash scripts/int-e2e-test.sh int ${{ matrix.service }}
 
   e2e-tests:
     needs: prep-pr
@@ -136,7 +136,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs
+          command: bash scripts/int-e2e-test.sh e2e datadog-logs
 
       - name: datadog-e2e-metrics
         if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics')
@@ -146,7 +146,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics
+          command: bash scripts/int-e2e-test.sh e2e datadog-metrics
 
   update-pr-status:
     name: Signal result to PR

.github/workflows/e2e.yml

@@ -89,7 +89,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs
+          command: bash scripts/int-e2e-test.sh e2e datadog-logs
 
       - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
@@ -98,7 +98,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics
+          command: bash scripts/int-e2e-test.sh e2e datadog-metrics
 
 
   e2e-test-suite:

.github/workflows/integration.yml

@@ -148,7 +148,7 @@ jobs:
                   "${{ needs.changes.outputs.all-int }}" == "true" || \
                   "$should_run" == "true" ]]; then
               echo "Running test for ${{ matrix.service }}"
-              bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }}
+              bash scripts/int-e2e-test.sh int ${{ matrix.service }}
             else
               echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met."
             fi

scripts/e2e/datadog-logs/compose.yaml

@@ -85,13 +85,13 @@ services:
   # which does the validation of consistency with the other fakeintake service.
   fakeintake-agent:
     # TODO: temporarily pegging the image as latest results in failures
-    image: docker.io/datadog/fakeintake:v77a06f2b
+    image: docker.io/datadog/fakeintake:ved764626
 
   # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner
   # which does the validation of consistency with the other fakeintake service.
   fakeintake-vector:
     # TODO: temporarily pegging the image as latest results in failures
-    image: docker.io/datadog/fakeintake:v77a06f2b
+    image: docker.io/datadog/fakeintake:ved764626
 
 networks:
   default:

scripts/ci-int-e2e-test.sh renamed to scripts/int-e2e-test.sh

@@ -6,11 +6,6 @@
 
 set -u
 
-if [[ -z "${CI:-}" ]]; then
-  echo "Aborted: this script is for use in CI." >&2
-  exit 1
-fi
-
 if [ $# -ne 2 ]
 then
   echo "usage: $0 [int|e2e] TEST_NAME"
@@ -27,5 +22,10 @@ sleep 30
 cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}"
 RET=$?
 cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}"
-./scripts/upload-test-results.sh
+
+# Only upload test results if CI is defined
+if [[ -n "${CI:-}" ]]; then
+  ./scripts/upload-test-results.sh
+fi
+
 exit $RET

tests/data/e2e/datadog/logs/agent_only.yaml

@@ -27,3 +27,6 @@ logs_config:
   logs_no_ssl: true
   force_use_http: true
   batch_wait: 1
+
+# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker
+dd_url: 'http://fakeintake-agent:80'

tests/data/e2e/datadog/logs/agent_vector.yaml

@@ -32,3 +32,6 @@ vector:
   logs:
     enabled: true
     url: "http://vector:8181"
+
+# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker
+dd_url: 'http://fakeintake-agent:80'

tests/e2e/datadog/logs/mod.rs

@@ -1,11 +1,13 @@
 use serde_json::Value;
 use tracing::info;
 
-use vector::test_util::trace_init;
-
 use super::*;
+use std::time::Duration;
+use vector::test_util::trace_init;
 
 const LOGS_ENDPOINT: &str = "/api/v2/logs";
+const MAX_RETRIES: usize = 10;
+const WAIT_INTERVAL: Duration = Duration::from_secs(1);
 
 fn expected_log_events() -> usize {
     std::env::var("EXPECTED_LOG_EVENTS")
@@ -70,25 +72,36 @@ fn reduce_to_data(payloads: Vec<FakeIntakePayload<Value>>) -> Vec<Value> {
 async fn validate() {
     trace_init();
 
-    // Even with configuring docker service dependencies, we need a small buffer of time
-    // to ensure events flow through to fakeintake before asking for them
-    std::thread::sleep(std::time::Duration::from_secs(2));
-
+    // Retry until we have log payloads or hit max retries.
+    // This is to ensure events flow through to fakeintake before asking for them.
     info!("getting log payloads from agent-only pipeline");
-    let mut agent_payloads = get_fakeintake_payloads::<FakeIntakeResponseJson>(
-        &fake_intake_agent_address(),
-        LOGS_ENDPOINT,
-    )
-    .await
-    .payloads;
-
-    // the logs endpoint receives an empty healthcheck payload in the beginning
-    if !agent_payloads.is_empty() {
-        agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty())
+    let mut agent_payloads = Vec::new();
+    for _ in 0..MAX_RETRIES {
+        agent_payloads = get_fakeintake_payloads::<FakeIntakeResponseJson>(
+            &fake_intake_agent_address(),
+            LOGS_ENDPOINT,
+        )
+        .await
+        .payloads;
+
+        if !agent_payloads.is_empty() {
+            break;
+        }
+
+        info!("No valid payloads yet, retrying...");
+        tokio::time::sleep(WAIT_INTERVAL).await;
     }
 
-    let mut agent_payloads = reduce_to_data(agent_payloads);
+    // If we still don't have valid payloads after retries, fail the test
+    assert!(
+        !agent_payloads.is_empty(),
+        "Failed to get valid log payloads from agent pipeline after {MAX_RETRIES} retries"
+    );
+
+    // The logs endpoint receives an empty healthcheck payload in the beginning
+    agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty());
 
+    let mut agent_payloads = reduce_to_data(agent_payloads);
     common_assertions(&mut agent_payloads);
 
     info!("getting log payloads from agent-vector pipeline");