Merge branch 'main' into load_many

Author: nik9000

.buildkite/pipelines/periodic-micro-benchmarks.yml

@@ -0,0 +1,10 @@
+steps:
+  - label: periodic-micro-benchmarks
+    command: |
+      .ci/scripts/run-gradle.sh -p benchmarks/ run --args 'org.elasticsearch.benchmark._nightly -rf json -rff build/result.json'
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.esql;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
@@ -67,9 +67,9 @@ public class QueryPlanningBenchmark {
     }
 
     private PlanTelemetry telemetry;
-    private EsqlParser parser;
-    private Analyzer analyzer;
-    private LogicalPlanOptimizer optimizer;
+    private EsqlParser defaultParser;
+    private Analyzer manyFieldsAnalyzer;
+    private LogicalPlanOptimizer defaultOptimizer;
 
     @Setup
     public void setup() {
@@ -100,8 +100,8 @@ public void setup() {
         var functionRegistry = new EsqlFunctionRegistry();
 
         telemetry = new PlanTelemetry(functionRegistry);
-        parser = new EsqlParser();
-        analyzer = new Analyzer(
+        defaultParser = new EsqlParser();
+        manyFieldsAnalyzer = new Analyzer(
             new AnalyzerContext(
                 config,
                 functionRegistry,
@@ -112,10 +112,10 @@ public void setup() {
             ),
             new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
         );
-        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+        defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
     }
 
-    private LogicalPlan plan(String query) {
+    private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
         var parsed = parser.createStatement(query, new QueryParams(), telemetry);
         var analyzed = analyzer.analyze(parsed);
         var optimized = optimizer.optimize(analyzed);
@@ -124,6 +124,6 @@ private LogicalPlan plan(String query) {
 
     @Benchmark
     public void manyFields(Blackhole blackhole) {
-        blackhole.consume(plan("FROM test | LIMIT 10"));
+        blackhole.consume(plan(defaultParser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -120,6 +120,7 @@ public void execute(Task t) {
                 "--add-opens=java.base/java.nio.file=ALL-UNNAMED",
                 "--add-opens=java.base/java.time=ALL-UNNAMED",
                 "--add-opens=java.management/java.lang.management=ALL-UNNAMED",
+                "--enable-native-access=ALL-UNNAMED",
                 "-XX:+HeapDumpOnOutOfMemoryError"
             );
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy

@@ -5,10 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
-import java.nio.file.Path
-
 class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
-    def "works"() {
+    def "basic functionality"() {
         given:
         file("src/main/java/com/example/Example.java") << """
             package com.example;
@@ -54,12 +52,70 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
 
         def location = Map.of(
             "module", "com.example",
-            "representative_class", Path.of("com", "example", "Example.class").toString()
+            "representative_class", "com/example/Example.class"
         )
         def expectedOutput = Map.of(
             "component", "example-component",
             "locations", List.of(location)
         )
         new ObjectMapper().readValue(output, Map.class) == expectedOutput
     }
+
+    def "dependencies"() {
+        buildFile << """
+        import org.elasticsearch.gradle.plugin.GenerateTestBuildInfoTask;
+
+        plugins {
+            id 'java'
+            id 'elasticsearch.test-build-info'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+
+        dependencies {
+            // We pin to specific versions here because they are known to have the properties we want to test.
+            // We're not actually running this code.
+            implementation "org.ow2.asm:asm:9.7.1" // has module-info.class
+            implementation "junit:junit:4.13" // has Automatic-Module-Name, and brings in hamcrest which does not
+        }
+
+        tasks.withType(GenerateTestBuildInfoTask.class) {
+            componentName = 'example-component'
+            outputFile = new File('build/generated-build-info/plugin-test-build-info.json')
+        }
+        """
+
+        when:
+        def result = gradleRunner('generateTestBuildInfo').build()
+        def task = result.task(":generateTestBuildInfo")
+
+
+        then:
+        task.outcome == TaskOutcome.SUCCESS
+
+        def output = file("build/generated-build-info/plugin-test-build-info.json")
+        output.exists() == true
+
+        def locationFromModuleInfo = Map.of(
+            "module", "org.objectweb.asm",
+            "representative_class", 'org/objectweb/asm/AnnotationVisitor.class'
+        )
+        def locationFromManifest = Map.of(
+            "module", "junit",
+            "representative_class", 'junit/textui/TestRunner.class'
+        )
+        def locationFromJarFileName = Map.of(
+            "module", "hamcrest.core",
+            "representative_class", 'org/hamcrest/BaseDescription.class'
+        )
+        def expectedOutput = Map.of(
+            "component", "example-component",
+            "locations", List.of(locationFromModuleInfo, locationFromManifest, locationFromJarFileName)
+        )
+
+        def value = new ObjectMapper().readValue(output, Map.class)
+        value == expectedOutput
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateTestBuildInfoTask.java

@@ -41,7 +41,7 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.CodeSource;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
@@ -209,17 +209,16 @@ private String extractModuleNameFromJar(File file, JarFile jarFile) throws IOExc
      * @return a {@link StringBuilder} with the {@code META-INF/versions/<version number>} if it exists; otherwise null
      */
     private static StringBuilder versionDirectoryIfExists(JarFile jarFile) {
+        Comparator<Integer> numericOrder = Integer::compareTo;
         List<Integer> versions = jarFile.stream()
             .filter(je -> je.getName().startsWith(META_INF_VERSIONS_PREFIX) && je.getName().endsWith("/module-info.class"))
             .map(
                 je -> Integer.parseInt(
                     je.getName().substring(META_INF_VERSIONS_PREFIX.length(), je.getName().length() - META_INF_VERSIONS_PREFIX.length())
                 )
             )
+            .sorted(numericOrder.reversed())
             .toList();
-        versions = new ArrayList<>(versions);
-        versions.sort(Integer::compareTo);
-        versions = versions.reversed();
         int major = Runtime.version().feature();
         StringBuilder path = new StringBuilder(META_INF_VERSIONS_PREFIX);
         for (int version : versions) {
@@ -300,7 +299,10 @@ private String extractClassNameFromDirectory(File dir) throws IOException {
             public @NotNull FileVisitResult visitFile(@NotNull Path candidate, @NotNull BasicFileAttributes attrs) {
                 String name = candidate.getFileName().toString(); // Just the part after the last dir separator
                 if (name.endsWith(".class") && (name.equals("module-info.class") || name.contains("$")) == false) {
-                    result = candidate.toAbsolutePath().toString().substring(dir.getAbsolutePath().length() + 1);
+                    result = candidate.toAbsolutePath()
+                        .toString()
+                        .substring(dir.getAbsolutePath().length() + 1)
+                        .replace(File.separatorChar, '/');
                     return TERMINATE;
                 } else {
                     return CONTINUE;
@@ -316,20 +318,24 @@ private String extractClassNameFromDirectory(File dir) throws IOException {
      * if it exists or the preset one derived from the jar task
      */
     private String extractModuleNameFromDirectory(File dir) throws IOException {
-        List<File> files = new ArrayList<>(List.of(dir));
-        while (files.isEmpty() == false) {
-            File find = files.removeFirst();
-            if (find.exists()) {
-                if (find.getName().equals("module-info.class")) {
-                    try (InputStream inputStream = new FileInputStream(find)) {
-                        return extractModuleNameFromModuleInfo(inputStream);
+        var visitor = new SimpleFileVisitor<Path>() {
+            private String result = getModuleName().getOrNull();
+
+            @Override
+            public @NotNull FileVisitResult visitFile(@NotNull Path candidate, @NotNull BasicFileAttributes attrs) throws IOException {
+                String name = candidate.getFileName().toString(); // Just the part after the last dir separator
+                if (name.equals("module-info.class")) {
+                    try (InputStream inputStream = new FileInputStream(candidate.toFile())) {
+                        result = extractModuleNameFromModuleInfo(inputStream);
+                        return TERMINATE;
                     }
-                } else if (find.isDirectory()) {
-                    files.addAll(Arrays.asList(find.listFiles()));
+                } else {
+                    return CONTINUE;
                 }
             }
-        }
-        return getModuleName().getOrNull();
+        };
+        Files.walkFileTree(dir.toPath(), visitor);
+        return visitor.result;
     }
 
     /**

catalog-info.yaml

@@ -263,3 +263,41 @@ spec:
         Daily:
           branch: main
           cronline: "0 12 * * * America/New_York"
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-periodic-micro-benchmarks
+  description: Runs periodic micro benchmarks fom the main branch
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-periodic-micro-benchmarks
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: ":elasticsearch: Runs nightly micro benchmarks fom the main branch"
+      name: elasticsearch / periodic / micro-benchmarks
+    spec:
+      repository: elastic/elasticsearch
+      pipeline_file: .buildkite/pipelines/periodic-micro-benchmarks.yml
+      branch_configuration: main
+      teams:
+        elasticsearch-team: {}
+        ml-core: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+      schedules:
+        Daily:
+          branch: main
+          cronline: "@daily"

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a9e07d97c55f5b854b1c203509e57b6f466eb26318540b2e1b59734355f747b6 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:68e0781cd592beda39880428985d5fecca1cf2abb18365da73bf1f7ebd994974 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a9e07d97c55f5b854b1c203509e57b6f466eb26318540b2e1b59734355f747b6
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:68e0781cd592beda39880428985d5fecca1cf2abb18365da73bf1f7ebd994974
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:55ee1dca9780931b0929d6eb734f455790c06ddbb59f55008e0cddebfbfd1e2e AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:3d19648819612728a676ab4061edfb3283bd7117a22c6c4479ee1c1d51831832 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -79,7 +79,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:55ee1dca9780931b0929d6eb734f455790c06ddbb59f55008e0cddebfbfd1e2e
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:3d19648819612728a676ab4061edfb3283bd7117a22c6c4479ee1c1d51831832
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/changelog/125408.yaml

@@ -0,0 +1,6 @@
+pr: 125408
+summary: Prevent ML data retention logic from failing when deleting documents in read-only
+  indices
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/126581.yaml

@@ -0,0 +1,10 @@
+pr: 126581
+summary: "Optimize shared blob cache evictions on shard removal
+  Shared blob cache evictions occur on the cluster applier thread when shards are
+  removed from a node. These can be expensive if a large number of shards are
+  being removed. This change uses the context of the removal to avoid unnecessary
+  evictions that might hold up the applier thread.
+  "
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/127583.yaml

@@ -0,0 +1,6 @@
+pr: 127583
+summary: Add optimization to purge join on null merge key
+area: ES|QL
+type: enhancement
+issues:
+ - 125577

docs/changelog/128241.yaml

@@ -0,0 +1,5 @@
+pr: 128241
+summary: Adding VoyageAI's v3.5 models
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/128259.yaml

@@ -0,0 +1,6 @@
+pr: 128259
+summary: Added geometry validation for GEO types to exit early on invalid latitudes
+area: Geo
+type: bug
+issues:
+ - 128234

docs/changelog/128263.yaml

@@ -0,0 +1,5 @@
+pr: 128263
+summary: Allow lookup join on mixed numeric fields
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/128320.yaml

@@ -0,0 +1,5 @@
+pr: 128320
+summary: Use new source loader when lower `docId` is accessed
+area: Codec
+type: bug
+issues: []

docs/changelog/128449.yaml

@@ -0,0 +1,5 @@
+pr: 128449
+summary: "[Draft] Support concurrent multipart uploads in Azure"
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/128464.yaml

@@ -0,0 +1,5 @@
+pr: 128464
+summary: Add support for parameters in LIMIT command
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/128473.yaml

@@ -0,0 +1,5 @@
+pr: 128473
+summary: Conditionally force sequential reading in `LuceneSyntheticSourceChangesSnapshot`
+area: Logs
+type: enhancement
+issues: []