Merge branch 'main' into elser-def-service

szabosteve · Nov 22, 2024 · dd0fafd · dd0fafd
2 parents 1098b48 + f3eb27e
commit dd0fafd
Show file tree

Hide file tree

Showing 2,885 changed files with 82,917 additions and 47,371 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.17.0$" : "8.x",
+    "^v8.18.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }
diff --git a/.buildkite/packer_cache.sh b/.buildkite/packer_cache.sh
@@ -29,6 +29,8 @@ for branch in "${branches[@]}"; do
   fi
 
   export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
-  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
+  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4
+  "checkout/${branch}/gradlew" --stop
+  pkill -f '.*GradleDaemon.*'
   rm -rf "checkout/${branch}"
 done
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml
@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -40,8 +41,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -9,6 +9,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -288,8 +289,8 @@ steps:
         env:
           BWC_VERSION: 8.15.4
 
-      - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
+      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -302,7 +303,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.0
+          BWC_VERSION: 8.16.2
 
       - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
@@ -320,6 +321,22 @@ steps:
         env:
           BWC_VERSION: 8.17.0
 
+      - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.18.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300
@@ -345,8 +362,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -38,8 +39,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             GRADLE_TASK:
               - checkPart1

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.0#bwcTest
+      - label: 8.16.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.0
+          BWC_VERSION: 8.16.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,6 +344,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.18.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.18.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -429,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
@@ -3,68 +3,15 @@ config:
 steps:
   - group: packaging-tests-unix
     steps:
-      - label: "{{matrix.image}} / docker / packaging-tests-unix"
-        key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
-        timeout_in_minutes: 300
-        matrix:
-          setup:
-            image:
-              - debian-11
-              - opensuse-leap-15
-              - oraclelinux-7
-              - oraclelinux-8
-              - sles-12
-              - sles-15
-              - ubuntu-1804
-              - ubuntu-2004
-              - ubuntu-2204
-              - rocky-8
-              - rocky-9
-              - rhel-7
-              - rhel-8
-              - rhel-9
-              - almalinux-8
-        agents:
-          provider: gcp
-          image: family/elasticsearch-{{matrix.image}}
-          diskSizeGb: 350
-          machineType: custom-16-32768
-      - label: "{{matrix.image}} / packages / packaging-tests-unix"
-        key: "packaging-tests-unix-packages"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.packages
-        timeout_in_minutes: 300
-        matrix:
-          setup:
-            image:
-              - debian-11
-              - opensuse-leap-15
-              - oraclelinux-7
-              - oraclelinux-8
-              - sles-12
-              - sles-15
-              - ubuntu-1804
-              - ubuntu-2004
-              - ubuntu-2204
-              - rocky-8
-              - rocky-9
-              - rhel-7
-              - rhel-8
-              - rhel-9
-              - almalinux-8
-        agents:
-          provider: gcp
-          image: family/elasticsearch-{{matrix.image}}
-          diskSizeGb: 350
-          machineType: custom-16-32768
-      - label: "{{matrix.image}} / archives / packaging-tests-unix"
-        key: "packaging-tests-unix-archives"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.archives
+      - label: "{{matrix.image}} / {{matrix.PACKAGING_TASK}} / packaging-tests-unix"
+        key: "packaging-tests-unix"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.{{matrix.PACKAGING_TASK}}
         timeout_in_minutes: 300
         matrix:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -79,6 +26,11 @@ steps:
               - rhel-8
               - rhel-9
               - almalinux-8
+            PACKAGING_TASK:
+              - docker
+              - docker-cloud-ess
+              - packages
+              - archives
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

diff --git a/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml b/.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml
@@ -12,7 +12,7 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2019
+              - windows-2022
             PACKAGING_TASK:
               - default-windows-archive
         agents:

diff --git a/.buildkite/pipelines/pull-request/packaging-tests-windows.yml b/.buildkite/pipelines/pull-request/packaging-tests-windows.yml
@@ -10,8 +10,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             PACKAGING_TASK:
               - default-windows-archive

diff --git a/.buildkite/scripts/cloud-deploy.sh b/.buildkite/scripts/cloud-deploy.sh
@@ -2,11 +2,11 @@
 
 set -euo pipefail
 
-.ci/scripts/run-gradle.sh buildCloudDockerImage
+.ci/scripts/run-gradle.sh buildCloudEssDockerImage
 
 ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
-DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
-docker tag elasticsearch-cloud:test "$DOCKER_TAG"
+DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud-ess:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
+docker tag elasticsearch-cloud-ess:test "$DOCKER_TAG"
 
 echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
 unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD

diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh
@@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}"
 BRANCH="${BUILDKITE_BRANCH:-}"
 
 # Don't publish main branch to staging
-if [[ "$BRANCH" == "main" && "$WORKFLOW" == "staging" ]]; then
+if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then
   exit 0
 fi
 

diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh
@@ -2,18 +2,17 @@
 
 set -euo pipefail
 
-# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
-./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+# This is a workaround for https://github.com/gradle/gradle/issues/28159
+.ci/scripts/run-gradle.sh --no-daemon precommit
 
-./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-# TODO run-gradle.sh script causes issues because of init script handling
-./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -15,6 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.4"
-  - "8.16.0"
+  - "8.16.2"
   - "8.17.0"
+  - "8.18.0"
   - "9.0.0"
diff --git a/.ci/dockerOnLinuxExclusions b/.ci/dockerOnLinuxExclusions
@@ -15,6 +15,7 @@ sles-15.2
 sles-15.3
 sles-15.4
 sles-15.5
+sles-15.6
 
 # These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
 # for PR checks

diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.15.4"
-  - "8.16.0"
+  - "8.16.2"
   - "8.17.0"
+  - "8.18.0"
   - "9.0.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -65,6 +65,30 @@ server/src/main/java/org/elasticsearch/bootstrap @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/node @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/plugins @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/breaker @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/bytes @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/cli @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/collect @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/component @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/compress @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/document @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/file @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/hash @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/io @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/logging @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/metrics @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/network @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/path @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/recycler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/regex @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/scheduler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/settings @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/text @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/time @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/transport @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/unit @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/util @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/xcontent @elastic/es-core-infra
 
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security