Vector test tools (#128934)

This adds some testing tools for verifying vector recall and latency
directly without having to spin up an entire ES node and running a rally
track.

Its pretty barebones and takes inspiration from lucene-util, but I
wanted access to our own formats and tooling to make our lives easier.

Here is an example config file. This will build the initial index, run
queries at num_candidates: 50, then again at num_candidates 100 (without
reindexing, and re-using the cached nearest neighbors).

```
[{
  "doc_vectors" : "path",
  "query_vectors" : "path",
  "num_docs" : 10000,
  "num_queries" : 10,
  "index_type" : "hnsw",
  "num_candidates" : 50,
  "k" : 10,
  "hnsw_m" : 16,
  "hnsw_ef_construction" : 200,
  "index_threads" : 4,
  "reindex" : true,
  "force_merge" : false,
  "vector_space" : "maximum_inner_product",
  "dimensions" : 768
},
{
"doc_vectors" : "path",
"query_vectors" : "path",
"num_docs" : 10000,
"num_queries" : 10,
"index_type" : "hnsw",
"num_candidates" : 100,
"k" : 10,
"hnsw_m" : 16,
"hnsw_ef_construction" : 200,
"vector_space" : "maximum_inner_product",
"dimensions" : 768
}
]
```

To execute:

```
./gradlew :qa:vector:checkVec --args="/Path/to/knn_tester_config.json"
```

Calling `./gradlew :qa:vector:checkVecHelp` gives some guidance on how
to use it, additionally providing a way to run it via java directly
(useful to bypass gradlew guff).

Author: benwtrent

qa/vector/build.gradle

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.java'
+apply plugin: 'elasticsearch.build'
+
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /lucene-.*/, to: 'lucene'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  enabled = false
+}
+
+dependencies {
+  api "org.apache.lucene:lucene-core:${versions.lucene}"
+  api "org.apache.lucene:lucene-queries:${versions.lucene}"
+  api "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  implementation project(':libs:logging')
+  implementation project(':server')
+}
+/**
+ * Task to run the KnnIndexTester with the provided parameters.
+ */
+tasks.register("checkVec", JavaExec) {
+  group = "Execution"
+  description = "Runs KnnIndexTester with the provided parameters to validate recall and performance."
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass.set("org.elasticsearch.test.knn.KnnIndexTester")
+  // Configure logging to console
+  systemProperty "es.logger.out", "console"
+  systemProperty "es.logger.level", "INFO"  // Change to DEBUG if needed
+
+  if (buildParams.getRuntimeJavaVersion().map { it.majorVersion.toInteger() }.get() >= 21) {
+    jvmArgs '-Xms4g', '-Xmx4g', '--add-modules=jdk.incubator.vector', '--enable-native-access=ALL-UNNAMED', '-Djava.util.concurrent.ForkJoinPool.common.parallelism=8', '-XX:+UnlockDiagnosticVMOptions', '-XX:+DebugNonSafepoints', '-XX:+HeapDumpOnOutOfMemoryError'
+  }
+}
+
+tasks.register("checkVecHelp", JavaExec) {
+  group = "Help"
+  description = "Prints help for the KnnIndexTester task."
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass.set("org.elasticsearch.test.knn.KnnIndexTester")
+  args = ["--help"]
+  doLast {
+    println """
+    =============================================================================
+    KnnIndexTester Help
+    =============================================================================
+
+    Run with Gradle:
+    ----------------
+    # Using default configuration file
+    ./gradlew :qa:vector:checkVec
+
+    # Using custom configuration file
+    ./gradlew :qa:vector:checkVec --args="path/to/your/config.json"
+
+    # Adjust heap size
+    ./gradlew :qa:vector:checkVec -Dorg.gradle.jvmargs="-Xmx8g" --args="path/to/your/config.json"
+
+    # Set environment variable for more extensive JVM options
+    export GRADLE_OPTS="-Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=100"
+    ./gradlew :qa:vector:checkVec
+
+
+    Run directly with Java:
+    ----------------------
+    # Generate classpath (run once to create the file)
+    ./gradlew :qa:vector:printClasspath
+
+    # Then use the classpath file with java
+    java -cp "\$(cat build/vector_classpath.txt)" \\
+         --add-modules=jdk.incubator.vector \\
+         --enable-native-access=ALL-UNNAMED \\
+         -Djava.util.concurrent.ForkJoinPool.common.parallelism=8 \\
+         -Xmx4g \\
+         -Xms4g \\\\
+         org.elasticsearch.test.knn.KnnIndexTester path/to/your/config.json
+    """
+  }
+}
+
+tasks.register("printClasspath") {
+  group = "Help"
+  description = "Prints the classpath needed to run KnnIndexTester directly with java"
+
+  doLast {
+    def classpathFile = new File("${buildDir}/vector_classpath.txt")
+    classpathFile.parentFile.mkdirs()
+    classpathFile.text = sourceSets.main.runtimeClasspath.asPath
+    println "Classpath written to: ${classpathFile.absolutePath}"
+  }
+}