Merge branch 'main' into half_pushable

Author: nik9000

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
+      - label: "{{matrix.image}} / 9.0.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.1
+          BWC_VERSION: 9.0.2
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.yml

@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
+      - label: 9.0.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.1
+          BWC_VERSION: 9.0.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -19,5 +19,5 @@ BWC_VERSION:
   - "8.17.7"
   - "8.18.2"
   - "8.19.0"
-  - "9.0.1"
+  - "9.0.2"
   - "9.1.0"

.ci/snapshotBwcVersions

@@ -2,5 +2,5 @@ BWC_VERSION:
   - "8.17.7"
   - "8.18.2"
   - "8.19.0"
-  - "9.0.1"
+  - "9.0.2"
   - "9.1.0"

build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java

@@ -183,11 +183,7 @@ private static CopySpec createBundleSpec(
     ) {
         var bundleSpec = project.copySpec();
         bundleSpec.from(buildProperties);
-        bundleSpec.from(pluginMetadata, copySpec -> {
-            // metadata (eg custom security policy)
-            // the codebases properties file is only for tests and not needed in production
-            copySpec.exclude("plugin-security.codebases");
-        });
+        bundleSpec.from(pluginMetadata);
         bundleSpec.from(
             (Callable<TaskProvider<Task>>) () -> project.getPluginManager().hasPlugin("com.gradleup.shadow")
                 ? project.getTasks().named("shadowJar")

distribution/archives/integ-test-zip/src/javaRestTest/resources/plugin-security.policy

@@ -922,7 +922,7 @@ void jarHellCheck(PluginDescriptor candidateInfo, Path candidateDir, Path plugin
     private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoot, List<Path> deleteOnFailure) throws Exception {
         final PluginDescriptor info = loadPluginInfo(tmpRoot);
 
-        Path legacyPolicyFile = tmpRoot.resolve(PluginDescriptor.ES_PLUGIN_POLICY);
+        Path legacyPolicyFile = tmpRoot.resolve("plugin-security.policy");
         if (Files.exists(legacyPolicyFile)) {
             terminal.errorPrintln(
                 "WARNING: this plugin contains a legacy Security Policy file. Starting with version 8.18, "

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -0,0 +1,5 @@
+pr: 126876
+summary: Improve HNSW filtered search speed through new heuristic
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/125694.yaml

@@ -249,6 +249,12 @@ $$$index-final-pipeline$$$
 $$$index-hidden$$$ `index.hidden`
 :   Indicates whether the index should be hidden by default. Hidden indices are not returned by default when using a wildcard expression. This behavior is controlled per request through the use of the `expand_wildcards` parameter. Possible values are `true` and `false` (default).
 
+$$$index-dense-vector-hnsw-filter-heuristic$$$ `index.dense_vector.hnsw_filter_heuristic`
+:   The heuristic to utilize when executing a filtered search against vectors in an HNSW graph. This setting is in technical preview may be changed or removed in a future release. It can be set to:
+
+* `acorn` (default) - Only vectors that match the filter criteria are searched. This is the fastest option, and generally provides faster searches at similar recall to `fanout`, but `num_candidates` might need to be increased for exceptionally high recall requirements.
+* `fanout` - All vectors are compared with the query vector, but only those passing the criteria are added to the search results. Can be slower than `acorn`, but may yield higher recall.
+
 $$$index-esql-stored-fields-sequential-proportion$$$
 
 `index.esql.stored_fields_sequential_proportion`

docs/changelog/126342.yaml

@@ -9,11 +9,11 @@
 
 package org.elasticsearch.core.internal.provider;
 
+import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.internal.provider.EmbeddedImplClassLoader.CompoundEnumeration;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.PrivilegedOperations;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
 import org.elasticsearch.test.jar.JarUtils;
 
@@ -195,13 +195,10 @@ private Object newFooBar(boolean enableMulti, int... versions) throws Exception
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntries(outerJar, jarEntries);
         URL[] urls = new URL[] { outerJar.toUri().toURL() };
-        URLClassLoader parent = URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
-        try {
+        try (URLClassLoader parent = loader(urls)) {
             EmbeddedImplClassLoader loader = EmbeddedImplClassLoader.getInstance(parent, "x-foo");
             Class<?> c = loader.loadClass("p.FooBar");
             return c.getConstructor().newInstance();
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -245,8 +242,7 @@ public void testResourcesBasic() throws Exception {
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntriesUTF(outerJar, jarEntries);
         URL[] urls = new URL[] { outerJar.toUri().toURL() };
-        URLClassLoader parent = URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
-        try {
+        try (URLClassLoader parent = loader(urls)) {
             EmbeddedImplClassLoader loader = EmbeddedImplClassLoader.getInstance(parent, "res");
             // resource in a valid java package dir
             URL url = loader.findResource("p/res.txt");
@@ -274,8 +270,6 @@ public void testResourcesBasic() throws Exception {
                     hasToString(endsWith("impl.jar!/IMPL-JARS/res/zoo-impl.jar/A-C/res.txt"))
                 )
             );
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -326,9 +320,7 @@ private void testResourcesParent(String resourcePath) throws Exception {
                 containsInAnyOrder("Parent Resource", "Embedded Resource")
             );
         } finally {
-            for (URLClassLoader closeable : closeables) {
-                PrivilegedOperations.closeURLClassLoader(closeable);
-            }
+            IOUtils.close(closeables);
         }
     }
 
@@ -463,9 +455,7 @@ private void testResourcesVersioned(String resourcePath, boolean enableMulti, in
                 assertThat(new String(is.readAllBytes(), UTF_8), is("Hello World" + expectedVersion));
             }
         } finally {
-            for (URLClassLoader closeable : closeables) {
-                PrivilegedOperations.closeURLClassLoader(closeable);
-            }
+            IOUtils.close(closeables);
         }
     }
 
@@ -493,8 +483,7 @@ public void testIDontHaveIt() throws Exception {
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntriesUTF(outerJar, jarEntries);
         URL[] urls = new URL[] { outerJar.toUri().toURL() };
-        URLClassLoader parent = URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
-        try {
+        try (URLClassLoader parent = loader(urls)) {
             embedLoader = EmbeddedImplClassLoader.getInstance(parent, "res");
 
             Class<?> c = embedLoader.loadClass("java.lang.Object");
@@ -514,8 +503,6 @@ public void testIDontHaveIt() throws Exception {
             expectThrows(NPE, () -> embedLoader.getResourceAsStream(null));
             expectThrows(NPE, () -> embedLoader.resources(null));
             expectThrows(NPE, () -> embedLoader.loadClass(null));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -542,8 +529,7 @@ public void testLoadWithJarDependencies() throws Exception {
         JarUtils.createJarWithEntries(outerJar, jarEntries);
         URL[] urls = new URL[] { outerJar.toUri().toURL() };
 
-        URLClassLoader parent = URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
-        try {
+        try (URLClassLoader parent = loader(urls)) {
             EmbeddedImplClassLoader loader = EmbeddedImplClassLoader.getInstance(parent, "blah");
             Class<?> c = loader.loadClass("p.Foo");
             Object obj = c.getConstructor().newInstance();
@@ -555,8 +541,6 @@ public void testLoadWithJarDependencies() throws Exception {
             expectThrows(CNFE, () -> loader.loadClass("p.Unknown"));
             expectThrows(CNFE, () -> loader.loadClass("q.Unknown"));
             expectThrows(CNFE, () -> loader.loadClass("r.Unknown"));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -577,18 +561,20 @@ public void testResourcesWithMultipleJars() throws Exception {
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntriesUTF(outerJar, jarEntries);
         URL[] urls = new URL[] { outerJar.toUri().toURL() };
-        URLClassLoader parent = URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
-        try {
+
+        try (URLClassLoader parent = loader(urls)) {
             EmbeddedImplClassLoader loader = EmbeddedImplClassLoader.getInstance(parent, "blah");
             var res = Collections.list(loader.getResources("res.txt"));
             assertThat(res, hasSize(3));
             List<String> l = res.stream().map(EmbeddedImplClassLoaderTests::urlToString).toList();
             assertThat(l, containsInAnyOrder("fooRes", "barRes", "bazRes"));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
+    private static URLClassLoader loader(URL[] urls) {
+        return URLClassLoader.newInstance(urls, EmbeddedImplClassLoaderTests.class.getClassLoader());
+    }
+
     @SuppressForbidden(reason = "file urls")
     static String urlToString(URL url) {
         try {

docs/changelog/126605.yaml

@@ -10,11 +10,11 @@
 package org.elasticsearch.core.internal.provider;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.PrivilegedOperations;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
 import org.elasticsearch.test.jar.JarUtils;
 
 import java.lang.module.ModuleDescriptor;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.file.Files;
@@ -117,12 +117,8 @@ public class FooIntSupplier implements java.util.function.IntSupplier {
         Path topLevelDir = createTempDir(getTestName());
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntries(outerJar, jarEntries);
-        URLClassLoader parent = URLClassLoader.newInstance(
-            new URL[] { outerJar.toUri().toURL() },
-            ProviderLocatorTests.class.getClassLoader()
-        );
 
-        try {
+        try (URLClassLoader parent = loader(outerJar)) {
             // test scenario
             ProviderLocator<IntSupplier> locator = new ProviderLocator<>("x-foo", IntSupplier.class, parent, "x.foo.impl", Set.of(), true);
             IntSupplier impl = locator.get();
@@ -139,8 +135,6 @@ public class FooIntSupplier implements java.util.function.IntSupplier {
             assertThat(md.exports(), containsInAnyOrder(exportsOf("p")));
             assertThat(md.opens(), containsInAnyOrder(opensOf("q")));
             assertThat(md.packages(), containsInAnyOrder(equalTo("p"), equalTo("q"), equalTo("r")));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -172,21 +166,15 @@ public class FooLongSupplier implements java.util.function.LongSupplier {
         Path topLevelDir = createTempDir(getTestName());
         Path outerJar = topLevelDir.resolve("impl.jar");
         JarUtils.createJarWithEntries(outerJar, jarEntries);
-        URLClassLoader parent = URLClassLoader.newInstance(
-            new URL[] { outerJar.toUri().toURL() },
-            ProviderLocatorTests.class.getClassLoader()
-        );
 
-        try {
+        try (URLClassLoader parent = loader(outerJar)) {
             // test scenario
             ProviderLocator<LongSupplier> locator = new ProviderLocator<>("x-foo", LongSupplier.class, parent, "", Set.of(), false);
             LongSupplier impl = locator.get();
             assertThat(impl.getAsLong(), is(55L));
             assertThat(impl.toString(), equalTo("Hello from FooLongSupplier - non-modular!"));
             assertThat(impl.getClass().getName(), equalTo("p.FooLongSupplier"));
             assertThat(impl.getClass().getModule().isNamed(), is(false));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
 
@@ -215,21 +203,18 @@ public class BarIntSupplier implements java.util.function.IntSupplier {
         Path pb = Files.createDirectories(barRoot.resolve("pb"));
         Files.write(pb.resolve("BarIntSupplier.class"), classToBytes.get("pb.BarIntSupplier"));
 
-        URLClassLoader parent = URLClassLoader.newInstance(
-            new URL[] { topLevelDir.toUri().toURL() },
-            ProviderLocatorTests.class.getClassLoader()
-        );
-
-        try {
+        try (URLClassLoader parent = loader(topLevelDir)) {
             // test scenario
             ProviderLocator<IntSupplier> locator = new ProviderLocator<>("y-bar", IntSupplier.class, parent, "", Set.of(), false);
             IntSupplier impl = locator.get();
             assertThat(impl.getAsInt(), is(42));
             assertThat(impl.toString(), equalTo("Hello from BarIntSupplier - exploded non-modular!"));
             assertThat(impl.getClass().getName(), equalTo("pb.BarIntSupplier"));
             assertThat(impl.getClass().getModule().isNamed(), is(false));
-        } finally {
-            PrivilegedOperations.closeURLClassLoader(parent);
         }
     }
+
+    private static URLClassLoader loader(Path jar) throws MalformedURLException {
+        return URLClassLoader.newInstance(new URL[] { jar.toUri().toURL() }, ProviderLocatorTests.class.getClassLoader());
+    }
 }